Models

Own model discovery, compilation and registry selection through explicit model metadata and geometry contracts.

Co-authored-by: whoisdomi <dcantu83@gmail.com>

scripts/model_compiler.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Compile staged ONNX sources without installing or selecting the result."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import tempfile
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+COMPONENTS = {
+  "supercombo": ("driving_supercombo", "supercombo"),
+  "off-policy": ("driving_off_policy", "off_policy", "offpolicy"),
+  "on-policy": ("driving_on_policy", "on_policy", "onpolicy"),
+  "policy": ("driving_policy", "policy"),
+  "vision": ("driving_vision", "vision"),
+}
+
+
+def source_files(directory: Path, model: str) -> dict[str, Path]:
+  selected = directory / model if (directory / model).is_dir() else directory
+  found = {}
+  for path in sorted(selected.glob("*.onnx")):
+    stem = path.stem.lower()
+    for component, aliases in COMPONENTS.items():
+      if any(stem == alias or stem == f"{model}_{alias}" for alias in aliases):
+        if component in found:
+          raise ValueError(f"Ambiguous {component} source: {found[component]} and {path}")
+        found[component] = path.resolve()
+        break
+  return found
+
+
+def driving_compile_args(files: dict[str, Path], requested: str) -> tuple[str, list[str]]:
+  selected = "supercombo" if requested == "auto" and "supercombo" in files else requested
+  if selected == "supercombo":
+    if "supercombo" not in files:
+      raise ValueError("Supercombo input requires driving_supercombo.onnx")
+    return "supercombo", ["--supercombo-onnx", str(files["supercombo"])]
+  if "vision" not in files or not ({"policy", "on-policy"} & files.keys()):
+    raise ValueError("Split input requires driving_vision.onnx and driving_policy.onnx or driving_on_policy.onnx")
+  result = ["--vision-onnx", str(files["vision"])]
+  primary = files.get("on-policy", files.get("policy"))
+  if "off-policy" in files:
+    return "vision_multi_policy", [*result, "--on-policy-onnx", str(primary), "--off-policy-onnx", str(files["off-policy"])]
+  return "vision_policy", [*result, "--policy-onnx", str(primary)]
+
+
+def sha256_file(path: Path) -> str:
+  digest = hashlib.sha256()
+  with path.open("rb") as source:
+    for block in iter(lambda: source.read(1024 * 1024), b""):
+      digest.update(block)
+  return digest.hexdigest()
+
+
+def compile_env(gpu: bool = False) -> dict[str, str]:
+  env = dict(os.environ)
+  env["PYTHONPATH"] = os.pathsep.join((str(REPO_ROOT), str(REPO_ROOT / "tinygrad_repo"), env.get("PYTHONPATH", "")))
+  defaults = {"DEV": "QCOM", "IMAGE": "1", "FLOAT16": "1", "NOLOCALS": "1", "JIT_BATCH_SIZE": "0", "OPENPILOT_HACKS": "1"}
+  for key, value in defaults.items():
+    env.setdefault(key, value)
+  if gpu:
+    for key in ("IMAGE", "NOLOCALS", "OPENPILOT_HACKS"):
+      env.pop(key, None)
+    env.update(DEV="USB+AMD:LLVM", WARP_DEV="QCOM", GMMU="0", TC_OPT="2", TC_MIN_GLOBALS="32")
+  return env
+
+
+def parse_args(argv=None):
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument("--model", help="Stable output model ID")
+  parser.add_argument("--version", required=True, choices=tuple(f"v{value}" for value in range(8, 17)))
+  parser.add_argument("--input-dir", type=Path, default=Path("/data/openpilot/uncompiledmodels"))
+  parser.add_argument("--output-dir", type=Path, default=Path("/data/openpilot/compiledmodels"))
+  parser.add_argument("--input-format", choices=("auto", "supercombo", "split"), default="auto")
+  parser.add_argument("--gpu", action="store_true")
+  parser.add_argument("--image-history-pipeline", choices=("policy", "warp"), default="policy")
+  parser.add_argument("--frame-skip", type=int)
+  args, unknown = parser.parse_known_args(argv)
+  if unknown:
+    if len(unknown) != 1 or not unknown[0].startswith("--") or args.model:
+      parser.error("Use --model ID or one --ID flag")
+    args.model = unknown[0][2:]
+  if not args.model or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", args.model):
+    parser.error("A lowercase model ID containing letters, digits, hyphens or underscores is required")
+  if args.frame_skip is not None and args.frame_skip < 1:
+    parser.error("--frame-skip must be positive")
+  return args
+
+
+def main(argv=None):
+  args = parse_args(argv)
+  model_type, source_args = driving_compile_args(source_files(args.input_dir, args.model), args.input_format)
+  sources = [Path(source_args[index]) for index in range(1, len(source_args), 2)]
+  for source in sources:
+    with source.open("rb") as file:
+      prefix = file.read(128)
+    if not prefix or prefix.startswith(b"version https://git-lfs.github.com/spec/v1"):
+      raise ValueError(f"Expected ONNX model bytes, not an empty file or Git LFS pointer: {source}")
+  before = {path.name: sha256_file(path) for path in sources}
+  output_dir = args.output_dir.resolve()
+  output_dir.mkdir(parents=True, exist_ok=True)
+  output = output_dir / f"{args.model}_driving_tinygrad.pkl"
+  env = compile_env(args.gpu)
+  with tempfile.TemporaryDirectory(prefix=f".{args.model}-compile-", dir=output_dir) as temporary:
+    env["TMPDIR"] = temporary
+    staged = Path(temporary) / output.name
+    command = [sys.executable, "-m", "openpilot.starpilot.models.compile", "--model-type", model_type,
+               "--model-size", "512x256", "--camera-resolutions", "1928x1208", "1344x760",
+               "--behavior-version", args.version, "--output", str(staged), "--out-of-band",
+               "--image-history-pipeline", args.image_history_pipeline, *source_args]
+    if args.frame_skip is not None:
+      command += ["--frame-skip", str(args.frame_skip)]
+    subprocess.run(command, cwd=REPO_ROOT, env=env, check=True)
+    if {path.name: sha256_file(path) for path in sources} != before:
+      raise ValueError("ONNX source changed during compilation")
+    revision = next(item["commit"] for item in json.loads((REPO_ROOT / "upstream-sync.json").read_text())["dependencies"]
+                    if item["path"] == "tinygrad_repo")
+    receipt = {"model_id": args.model, "behavior_version": args.version, "model_type": model_type,
+               "artifact_abi": "tinygrad_single_v1_arena", "compiler_revision": revision,
+               "source_sha256": before, "artifact_sha256": sha256_file(staged), "artifact_size": staged.stat().st_size,
+               "execution_device": env["DEV"], "warp_device": env.get("WARP_DEV", env["DEV"]),
+               "image_history_pipeline": args.image_history_pipeline, "camera_resolutions": [[1928, 1208], [1344, 760]]}
+    staged.replace(output)
+    receipt_path = output.with_suffix(".build.json")
+    temporary_receipt = Path(temporary) / receipt_path.name
+    temporary_receipt.write_text(json.dumps(receipt, indent=2) + "\n")
+    temporary_receipt.replace(receipt_path)
+  print(f"Compiled {output}; build receipt {receipt_path}")
+  return 0
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())

scripts/model_rebuild.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""Rebuild a model generation on an explicitly selected compiler device."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shlex
+import subprocess
+import sys
+import time
+from urllib.request import urlopen
+from urllib.error import HTTPError
+
+REPO = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO))
+from openpilot.starpilot.models.catalog import ARTIFACT_ABI, CATALOG_PATH, COMPILER_REVISION, GENERATION
+from openpilot.starpilot.models.manager import atomic_json, validate_manifest
+
+BUCKET = "StarPilot-Driving/StarPilot-Resources"
+REMOTE_WORK = "/data/starpilot-v26-rebuild"
+CHUNK_BYTES = 45 * 1024 * 1024
+
+
+def digest(path: Path) -> str:
+  with path.open("rb") as stream:
+    return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def run(args: list[str], *, output=None) -> None:
+  subprocess.run(args, check=True, stdout=output, stderr=subprocess.STDOUT if output is not None else None)
+
+
+def remote(device: str, command: list[str], *, output=None) -> None:
+  run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o", "ServerAliveInterval=30",
+       device, shlex.join(command)], output=output)
+
+
+def inventory() -> list[dict]:
+  result = []
+  url = f"https://huggingface.co/api/buckets/{BUCKET}/tree?recursive=true&limit=1000"
+  while url:
+    with urlopen(url, timeout=40) as response:
+      result.extend(json.load(response))
+      links = response.headers.get("Link", "")
+    url = next((p.split(">")[0].strip("< ") for p in links.split(",") if 'rel="next"' in p), "")
+  return result
+
+
+def source_plan(model_id: str, source_map: dict, objects: list[dict]) -> list[dict]:
+  source_id = source_map[model_id]["source_id"]
+  files = [obj for obj in objects if obj["type"] == "file" and obj["path"].startswith(f"onnx/{source_id}/") and
+           obj["path"].endswith(".onnx")]
+  if not files:
+    raise ValueError(f"No archived ONNX source for {model_id} ({source_id})")
+  return files
+
+
+def prepare_sources(model_id: str, files: list[dict], workspace: Path, hf: str) -> Path:
+  directory = workspace / "onnx" / model_id
+  directory.mkdir(parents=True, exist_ok=True)
+  for obj in files:
+    name = Path(obj["path"]).name
+    if "driving_" not in name:
+      raise ValueError(f"Unrecognized model source filename: {name}")
+    name = model_id + "_" + name[name.index("driving_"):]
+    destination = directory / name
+    if not destination.is_file() or destination.stat().st_size != obj["size"]:
+      run([hf, "buckets", "cp", f"hf://buckets/{BUCKET}/{obj['path']}", str(destination), "--format", "quiet"])
+    if destination.stat().st_size != obj["size"]:
+      raise ValueError("Source download size mismatch")
+  return directory
+
+
+def split_artifact(artifact: Path, directory: Path) -> tuple[list[Path], int]:
+  directory.mkdir(parents=True, exist_ok=True)
+  count = (artifact.stat().st_size + CHUNK_BYTES - 1) // CHUNK_BYTES
+  if count == 1:
+    return [artifact], 0
+  parts = []
+  with artifact.open("rb") as stream:
+    for index in range(1, count + 1):
+      part = directory / f"{artifact.name}.chunk{index:02d}of{count:02d}"
+      part.write_bytes(stream.read(CHUNK_BYTES))
+      parts.append(part)
+  return parts, count
+
+
+def publish_artifact(model_id: str, artifact: Path, receipt: dict, workspace: Path, hf: str) -> dict:
+  if receipt["compiler_revision"] != COMPILER_REVISION or receipt["artifact_abi"] != ARTIFACT_ABI or digest(artifact) != receipt["artifact_sha256"]:
+    raise ValueError("Compiled artifact identity changed")
+  parts, count = split_artifact(artifact, workspace / "release" / model_id)
+  for part in parts:
+    run([hf, "buckets", "cp", str(part), f"hf://buckets/{BUCKET}/models/{GENERATION}/{model_id}/{part.name}", "--format", "quiet"])
+  if count:
+    marker = workspace / "release" / model_id / f"{artifact.name}.chunkmanifest"
+    marker.write_text(str(count))
+    run([hf, "buckets", "cp", str(marker), f"hf://buckets/{BUCKET}/models/{GENERATION}/{model_id}/{marker.name}", "--format", "quiet"])
+  return {"artifact_format": ARTIFACT_ABI, "artifact_size": artifact.stat().st_size,
+          "artifact_sha256": receipt["artifact_sha256"], "artifact_chunk_count": count,
+          "source_sha256": receipt["source_sha256"]}
+
+
+def compile_one(model: dict, args, source_map: dict, objects: list[dict]) -> dict:
+  mid = model["id"]
+  work = args.workspace
+  result_path = work / "results" / f"{mid}.json"
+  output_dir = work / "compiled" / mid
+  artifact = output_dir / f"{mid}_driving_tinygrad.pkl"
+  if result_path.is_file():
+    previous = json.loads(result_path.read_text())
+    if (previous.get("status") == "passed" and previous.get("compiler_revision") == COMPILER_REVISION and artifact.is_file() and
+        digest(artifact) == previous["artifact_sha256"] and (not args.publish or previous.get("published"))):
+      return previous
+  sources = prepare_sources(mid, source_plan(mid, source_map, objects), work, args.hf)
+  remote_dir = f"{REMOTE_WORK}/work/{mid}"
+  remote(args.device, ["mkdir", "-p", f"{remote_dir}/onnx", f"{remote_dir}/compiled"])
+  run(["rsync", "-az", str(sources) + "/", f"{args.device}:{remote_dir}/onnx/"])
+  command = ["/usr/local/venv/bin/python", "/data/openpilot/scripts/model_compiler.py", "--model", mid,
+             "--version", model["version"], "--input-format", source_map[mid]["input_format"],
+             "--input-dir", f"{remote_dir}/onnx", "--output-dir", f"{remote_dir}/compiled"]
+  if model.get("uses_external_gpu"):
+    command.append("--gpu")
+  with (work / "logs" / f"{mid}.log").open("ab") as log:
+    remote(args.device, ["env", "PYTHONUNBUFFERED=1", *command], output=log)
+    validate = ["/usr/local/venv/bin/python", "/data/openpilot/scripts/validate_driving_model.py",
+                f"{remote_dir}/compiled/{artifact.name}", "--version", model["version"],
+                "--source-dir", f"{remote_dir}/onnx", "--report", f"{remote_dir}/compiled/validation.json", "--max-p95-ms", "50"]
+    if model.get("uses_external_gpu"):
+      validate.append("--gpu")
+    remote(args.device, validate, output=log)
+  output_dir.mkdir(parents=True, exist_ok=True)
+  run(["rsync", "-az", f"{args.device}:{remote_dir}/compiled/", str(output_dir) + "/"])
+  receipt = json.loads(artifact.with_suffix(".build.json").read_text())
+  validation = json.loads((output_dir / "validation.json").read_text())
+  if (digest(artifact) != receipt["artifact_sha256"] or not validation.get("passed") or
+      validation.get("artifact_sha256") != receipt["artifact_sha256"]):
+    raise ValueError("Compiled model failed runtime validation")
+  result = {**receipt, "status": "passed", "published": False, "validation": validation}
+  if args.publish:
+    result["release"] = publish_artifact(mid, artifact, receipt, work, args.hf)
+    result["published"] = True
+  atomic_json(result_path, result)
+  # Only this invocation's sources and output are removed after the verified host copy exists.
+  remote(args.device, ["/usr/local/venv/bin/python", "-c", "import shutil,sys; shutil.rmtree(sys.argv[1])", remote_dir])
+  return result
+
+
+def publish_manifest(workspace: Path, hf: str) -> None:
+  manifest = json.loads(CATALOG_PATH.read_text())
+  try:
+    with urlopen(f"https://huggingface.co/buckets/{BUCKET}/resolve/manifests/model_names_{GENERATION}.json", timeout=30) as response:
+      live = json.load(response)
+    existing = validate_manifest(live)
+    for row in manifest["models"]:
+      if row["id"] in existing:
+        row.update(existing[row["id"]])
+  except HTTPError as error:
+    if error.code != 404:
+      raise
+  for model in manifest["models"]:
+    path = workspace / "results" / f"{model['id']}.json"
+    if path.is_file():
+      result = json.loads(path.read_text())
+      if result.get("status") == "passed" and result.get("published") and result.get("compiler_revision") == COMPILER_REVISION:
+        model.update(result["release"])
+  path = workspace / "manifests" / f"model_names_{GENERATION}.json"
+  atomic_json(path, manifest)
+  run([hf, "buckets", "cp", str(path), f"hf://buckets/{BUCKET}/manifests/{path.name}", "--format", "quiet"])
+
+
+def main():
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument("--device", required=True, help="Explicit SSH target, for example comma@comma-device.local")
+  parser.add_argument("--workspace", required=True, type=Path)
+  parser.add_argument("--models", nargs="+", help="Stable model IDs; defaults to the full catalog, favorites first")
+  parser.add_argument("--hf", default=str(Path.home()/".local/bin/hf"))
+  parser.add_argument("--publish", action="store_true", help="Upload validated v26 artifacts and manifest to Hugging Face")
+  args = parser.parse_args()
+  args.workspace = args.workspace.resolve()
+  for directory in ("results", "logs", "manifests"):
+    (args.workspace / directory).mkdir(parents=True, exist_ok=True)
+  rows = json.loads(CATALOG_PATH.read_text())["models"]
+  source_map = json.loads((REPO / "scripts/model_source_map_v26.json").read_text())
+  if args.models:
+    by_id = {row["id"]: row for row in rows}
+    rows = [by_id[mid] for mid in args.models]
+  else:
+    rows.sort(key=lambda row: (not row.get("community_favorite", False), row["id"] != "cinquev3"))
+  objects = inventory()
+  atomic_json(args.workspace / "source-inventory.json", {"objects": objects})
+  failures = []
+  for model in rows:
+    mid = model["id"]
+    print(f"START {mid}", flush=True)
+    started = time.monotonic()
+    try:
+      compile_one(model, args, source_map, objects)
+      if args.publish:
+        publish_manifest(args.workspace, args.hf)
+      print(f"PASS {mid} {time.monotonic()-started:.0f}s", flush=True)
+    except (OSError, ValueError, subprocess.CalledProcessError) as error:
+      failures.append(mid)
+      atomic_json(args.workspace / "results" / f"{mid}.failure.json", {"model_id": mid, "error": str(error)})
+      print(f"FAIL {mid}: {error}", flush=True)
+  return int(bool(failures))
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())

scripts/model_source_map_v26.json

@@ -0,0 +1,495 @@
+{
+  "tr14223": {
+    "source_id": "tr1422",
+    "input_format": "split",
+    "ref": "64fd3f986081ba6cb488d02e799845367add29e2"
+  },
+  "tr15223": {
+    "source_id": "tr1522",
+    "input_format": "split",
+    "ref": "ed38ca8cc998f0a2b47cde29da11b782e4551e8b"
+  },
+  "tr16223": {
+    "source_id": "tr16",
+    "input_format": "split",
+    "ref": "e4b6a2736580dc427473bcf8a6e3b08eb352edd4"
+  },
+  "letr223": {
+    "source_id": "letr22",
+    "input_format": "split",
+    "ref": "c2fa07b82c2e09964e4b6dccb3e4465a2e4ad138"
+  },
+  "spacelab22223": {
+    "source_id": "spacelab2222",
+    "input_format": "split",
+    "ref": "d3a0378ae601f61307a53d1173c251ee36c56317"
+  },
+  "vikander223": {
+    "source_id": "vikander22",
+    "input_format": "split",
+    "ref": "d134fdd7d51ba84848866c0265e37508c38f9366"
+  },
+  "fof223": {
+    "source_id": "fof22",
+    "input_format": "split",
+    "ref": "d807b5c476658dba7760d30949baa619074c4a4e"
+  },
+  "vfof223": {
+    "source_id": "vfof22",
+    "input_format": "split",
+    "ref": "c790d3f58ad32cf15f384e6018d942b2348f0c5b"
+  },
+  "dtr223": {
+    "source_id": "dtr22",
+    "input_format": "split",
+    "ref": "5b405920ec9511e146432de08171d6f93295248b"
+  },
+  "dtr6223": {
+    "source_id": "dtr622",
+    "input_format": "split",
+    "ref": "a6cf39c07a67dbc346af03010093fe4788c402d0"
+  },
+  "uvdtr6223": {
+    "source_id": "uvdtr622",
+    "input_format": "split",
+    "ref": "079d0461604632719e73dc5b6a5eb9897c624da2"
+  },
+  "sp2223": {
+    "source_id": "sp222",
+    "input_format": "split",
+    "ref": "f1b8f510b2c554191ca15cebe35f2c5b304cf23c"
+  },
+  "fp223": {
+    "source_id": "fp22",
+    "input_format": "split",
+    "ref": "95aff72e351e935221ec965aa0384dfd502d383d"
+  },
+  "kv223": {
+    "source_id": "kv22",
+    "input_format": "split",
+    "ref": "16e87d4c72e5efec18032f6b4a5f6ec35d0df4e4"
+  },
+  "gwm3223": {
+    "source_id": "gwm322",
+    "input_format": "split",
+    "ref": "93b26a91a62b69b809162ad2bf65e36be2158dc1"
+  },
+  "gwm5223": {
+    "source_id": "gwm522",
+    "input_format": "split",
+    "ref": "e3ee440f7d5ac12c50db6381e292a55a69b745a7"
+  },
+  "gwm6223": {
+    "source_id": "gwm622",
+    "input_format": "split",
+    "ref": "91a1cea814f76d605d5b15de9634a3c8f79df509"
+  },
+  "gwm8223": {
+    "source_id": "gwm822",
+    "input_format": "split",
+    "ref": "96e7b310b164b55fb73abdd505b66fb36630718e"
+  },
+  "gwm9223": {
+    "source_id": "gwm922",
+    "input_format": "split",
+    "ref": "f2413040a8560c0c17c18353a3c75124a2f09d17"
+  },
+  "cgwm223": {
+    "source_id": "cgwm22",
+    "input_format": "split",
+    "ref": "94dd3bd9107cd413ed2418f9e6c0cf805a9ca495"
+  },
+  "bd223": {
+    "source_id": "bd22",
+    "input_format": "split",
+    "ref": "b74f5189a74446015c0cf78a4a9f0134a347ae3b"
+  },
+  "nr223": {
+    "source_id": "nr22",
+    "input_format": "split",
+    "ref": "266b642180f0e9278e1db46ccf8c1a2a8dae4bb3"
+  },
+  "tcp2223": {
+    "source_id": "tcp222",
+    "input_format": "split",
+    "ref": "19084132cd3956413a0fdfcdc18971a55df5e0ff"
+  },
+  "tcp3223": {
+    "source_id": "tcp322",
+    "input_format": "split",
+    "ref": "5eb912c025a2207c7d428deee74ded49e5905527"
+  },
+  "fbw223": {
+    "source_id": "fbw22",
+    "input_format": "split",
+    "ref": "c4488e3411285f6fc3d008bca885e05731fd75c2"
+  },
+  "nevada223": {
+    "source_id": "nevada22",
+    "input_format": "split",
+    "ref": "3193eac5e385aa010694a8ac192ff38ffe000193"
+  },
+  "wmi4223": {
+    "source_id": "wmi422",
+    "input_format": "split",
+    "ref": "2d8e596a0208dafe4ed2d945b07d6a7d72f70fb8"
+  },
+  "wmi523": {
+    "source_id": "wmi52",
+    "input_format": "split",
+    "ref": "545e0ed13f85e4c744fa1f69095863ea9dbe6d6b"
+  },
+  "wmi623": {
+    "source_id": "wmi62",
+    "input_format": "split",
+    "ref": "54c6c5776a37dc56fde626314f53ec3316d88a48"
+  },
+  "wmi723": {
+    "source_id": "wmi72",
+    "input_format": "split",
+    "ref": "aa3d90c92a7bd7a4e7db1b3fdee3d3e522358823"
+  },
+  "wmi7223": {
+    "source_id": "wmi722",
+    "input_format": "split",
+    "ref": "a2ace1ed6b84d7c7a2dbf990c1b566f9a9df167f"
+  },
+  "wmi823": {
+    "source_id": "wmi82",
+    "input_format": "split",
+    "ref": "4c438f59e7dcf5fddc769032882b62d0cb805e9b"
+  },
+  "wmi923": {
+    "source_id": "wmi92",
+    "input_format": "split",
+    "ref": "8950897d7e4d2ba2426b2b31cf29f25e87a3d4ba"
+  },
+  "wmi1023": {
+    "source_id": "wmi102",
+    "input_format": "split",
+    "ref": "855f5e4ddefd69a20cc4e9da004eb53f3e00d950"
+  },
+  "wmi1123": {
+    "source_id": "wmi112",
+    "input_format": "split",
+    "ref": "7e3fd7a63c5a09c3fabe108b1b62bba3cf684878"
+  },
+  "cd21023": {
+    "source_id": "cd2102",
+    "input_format": "split",
+    "ref": "55f66e2246359c6593605399a0199d94d13ad90d"
+  },
+  "op223": {
+    "source_id": "op22",
+    "input_format": "split",
+    "ref": "ae34a24c59d85df7efad5fde2b760fb6d0b9dd6e"
+  },
+  "op323": {
+    "source_id": "op32",
+    "input_format": "split",
+    "ref": "7e548dd765873bed301c0a19cfe10c3ca6be2bbe"
+  },
+  "op423": {
+    "source_id": "op42",
+    "input_format": "split",
+    "ref": "25abcc49fce2f6268d8cabf759e31c626edbf584"
+  },
+  "op523": {
+    "source_id": "op52",
+    "input_format": "split",
+    "ref": "b390b98c5a359c293fdc2501af79b19867fcdac4"
+  },
+  "op623": {
+    "source_id": "op62",
+    "input_format": "split",
+    "ref": "831f13da61d937bb40ea0f50590cdc3b5380f313"
+  },
+  "opv73": {
+    "source_id": "opv7",
+    "input_format": "split",
+    "ref": "cb327933002bc1a00bf60a3c20af2eb7a5f653e5"
+  },
+  "opv83": {
+    "source_id": "opv8",
+    "input_format": "split",
+    "ref": "052692b25d63c5ddda276b5c2271383b6aff129f"
+  },
+  "opv93": {
+    "source_id": "opv9",
+    "input_format": "split",
+    "ref": "eae2a73e0ac600d7f0342fc4397568e0733fe6e2"
+  },
+  "opv103": {
+    "source_id": "opv10",
+    "input_format": "split",
+    "ref": "5faf14e04e4038ec4673d119c950b46051630205"
+  },
+  "opv113": {
+    "source_id": "opv11",
+    "input_format": "split",
+    "ref": "bbde5ddb5a0ee35c98a962029355f9a95403a825"
+  },
+  "opv123": {
+    "source_id": "opv12",
+    "input_format": "split",
+    "ref": "7d4c295c27bc3f72727c49ec73b98e45745b588d"
+  },
+  "opv133": {
+    "source_id": "opv13",
+    "input_format": "split",
+    "ref": "faeeaad3a5841694b4c3bd6d37e41e3db4b4873e"
+  },
+  "op163": {
+    "source_id": "op16",
+    "input_format": "split",
+    "ref": "72101c6e50ea594f24e75d3b605abf8b5cab1d6b"
+  },
+  "op16d3": {
+    "source_id": "op16d",
+    "input_format": "split",
+    "ref": "0a628146d1a9a314e329e2c0b5de20e6223211e4"
+  },
+  "op16dv23": {
+    "source_id": "op16dv2",
+    "input_format": "split",
+    "ref": "0a628146d1a9a314e329e2c0b5de20e6223211e4"
+  },
+  "rlv1dl3": {
+    "source_id": "rlv1dl",
+    "input_format": "split",
+    "ref": "38c0e3c95de5b63b56ecbbe507abc49b9b0091a3"
+  },
+  "deeprl33": {
+    "source_id": "deeprl3",
+    "input_format": "split",
+    "ref": "a1060427b9b3ac1d11cbad05380929ea8235840a"
+  },
+  "deeprl333": {
+    "source_id": "deeprl33",
+    "input_format": "supercombo",
+    "ref": "4604c98f21c9a22230e417d17d03c402b9b5f869"
+  },
+  "rl343": {
+    "source_id": "rl34",
+    "input_format": "supercombo",
+    "ref": "4e6b072404110056ccf9bcc232c718ac71d45478"
+  },
+  "drl3": {
+    "source_id": "drl",
+    "input_format": "supercombo",
+    "ref": "a881acfbd59cb9ed18c54535b835721e2bc96def"
+  },
+  "nopp3": {
+    "source_id": "nopp",
+    "input_format": "supercombo",
+    "ref": "c740fe5f58faefcb9184c6f9f1fb45130b83cfe8"
+  },
+  "tobyrl3": {
+    "source_id": "tobyrl",
+    "input_format": "supercombo",
+    "ref": "2d4acf10e34c4c8e68043d60b9437f42d89ca746"
+  },
+  "michael-rl3": {
+    "source_id": "michael-rl",
+    "input_format": "supercombo",
+    "ref": "968fd3c989b993cf03187aa0e05cd2088e275551"
+  },
+  "michael-rl23": {
+    "source_id": "michael-rl2",
+    "input_format": "supercombo",
+    "ref": "37b38bf738edfb1daa6875255f581e5fc9b0a258"
+  },
+  "karnbir3": {
+    "source_id": "karnbir",
+    "input_format": "supercombo",
+    "ref": "c0fbd2b13030df95ad249b6e253156a4d7ddb7bb"
+  },
+  "karnbir23": {
+    "source_id": "karnbir2",
+    "input_format": "supercombo",
+    "ref": "682d33d0414d7cef5a7286c24298003503ea13f4"
+  },
+  "ms23": {
+    "source_id": "ms2",
+    "input_format": "split",
+    "ref": "d70b13931793bc0e4d8efd36128dd66f227a3f81"
+  },
+  "pp2223": {
+    "source_id": "pp222",
+    "input_format": "split",
+    "ref": "50c78a9dd670a305ca898b96245cadbc2ecdc1a6"
+  },
+  "ds2223": {
+    "source_id": "ds222",
+    "input_format": "split",
+    "ref": "5ff0e48f90d662d8c9d5061adc63e90c198c9b81"
+  },
+  "nn2223": {
+    "source_id": "nn222",
+    "input_format": "split",
+    "ref": "83b81b83cdf76a577946a1567f4644e1a018443c"
+  },
+  "sc23": {
+    "source_id": "sc2",
+    "input_format": "split",
+    "ref": "e4a4b4b1adf2d19fedab4d195faa382b061fa754"
+  },
+  "pop23": {
+    "source_id": "pop2",
+    "input_format": "split",
+    "ref": "6f71783a8a8faa07ddaeef5bbb6809b4f4f44a15"
+  },
+  "pop223": {
+    "source_id": "pop22",
+    "input_format": "split",
+    "ref": "62bf6fb072880905a4c490f0f4f4a6b3c23346ec"
+  },
+  "nid223": {
+    "source_id": "nid22",
+    "input_format": "split",
+    "ref": "13e79e9fad60c19e751e4f9ab0538d39f1bb54dd"
+  },
+  "kerrygold223": {
+    "source_id": "kerrygold22",
+    "input_format": "split",
+    "ref": "dc7a92ea630e4f6053082fc25eca83938927b91e"
+  },
+  "deeprl3v23": {
+    "source_id": "deeprl3v2",
+    "input_format": "supercombo",
+    "ref": "702fa71ad4dd8de08425eb11a1a42aaeb64892c9"
+  },
+  "rh3": {
+    "source_id": "rh",
+    "input_format": "supercombo",
+    "ref": "93f5aa469a72b7621aef7da7901c100e0113e4d9"
+  },
+  "gyhu3": {
+    "source_id": "gyhu",
+    "input_format": "supercombo",
+    "ref": "574735edc6e1aafdc2a69395f9a32e7f5cc4b62b"
+  },
+  "rdfv23": {
+    "source_id": "rdf",
+    "input_format": "supercombo",
+    "ref": "a95e2c25cae5fbf1afba7628bfb7acc4af59e0cc"
+  },
+  "rdf23": {
+    "source_id": "rdf2",
+    "input_format": "supercombo",
+    "ref": "0f8b4248a2e8bdd63b6ea4c6e1bbb32119cb2620"
+  },
+  "rdf43": {
+    "source_id": "rdf4",
+    "input_format": "supercombo",
+    "ref": "a5a6412d08474cffb49a69afb910756afdee123e"
+  },
+  "rdf33": {
+    "source_id": "rdf3",
+    "input_format": "supercombo",
+    "ref": "ea2151ba4b82854277f37f03b949f15fe2733dc8"
+  },
+  "rdf53": {
+    "source_id": "rdf5",
+    "input_format": "supercombo",
+    "ref": "7fb03ca474f03e95e59ec0c8a6c5fba831bd5fd1"
+  },
+  "rdf63": {
+    "source_id": "rdf6",
+    "input_format": "supercombo",
+    "ref": "35703097905a122c9f3ddf0d12889b4873d7e2a2"
+  },
+  "tsf": {
+    "source_id": "tsf",
+    "input_format": "supercombo",
+    "ref": "4d911346cde4e0d2978a625f31679808284cc19d"
+  },
+  "tsfdo": {
+    "source_id": "tsfdo",
+    "input_format": "supercombo",
+    "ref": "0394d7284cfc229f772ee4fa73ff724ba8245110"
+  },
+  "lebowski3": {
+    "source_id": "lebowski",
+    "input_format": "supercombo",
+    "ref": "fa0c6876d3cf070e91e25e5353ceadc68a5b3285"
+  },
+  "tt-driving": {
+    "source_id": "tt-driving",
+    "input_format": "supercombo"
+  },
+  "tee-time": {
+    "source_id": "tee-time",
+    "input_format": "supercombo"
+  },
+  "bmrlnapv2": {
+    "source_id": "bmrlnapv2",
+    "input_format": "supercombo",
+    "ref": "ba37c02c7c64225b8aec79411d1a8c2f5c1aa343"
+  },
+  "happybirthday": {
+    "source_id": "happybirthday",
+    "input_format": "supercombo",
+    "ref": "db06f5f36caa379e112bafd38423e37b66fa4149"
+  },
+  "berightthere": {
+    "source_id": "berightthere",
+    "input_format": "supercombo",
+    "ref": "bf74ce5448306088b4e9bc843472c39402930bc7"
+  },
+  "remove-avgpoolv4": {
+    "source_id": "remove-avgpoolv4",
+    "input_format": "supercombo"
+  },
+  "remove-avgpoolv3": {
+    "source_id": "remove-avgpoolv3",
+    "input_format": "supercombo"
+  },
+  "remove-avgpoolv2": {
+    "source_id": "remove-avgpoolv2",
+    "input_format": "supercombo"
+  },
+  "remove-avgpoolv1": {
+    "source_id": "remove-avgpoolv1",
+    "input_format": "supercombo"
+  },
+  "rdf-driving": {
+    "source_id": "rdf-driving",
+    "input_format": "supercombo"
+  },
+  "sad": {
+    "source_id": "sad",
+    "input_format": "supercombo",
+    "ref": "30de303d5ffb63957f8acbdb256fd4c7d360455a"
+  },
+  "bmrlnapv6": {
+    "source_id": "bmrlnapv6",
+    "input_format": "supercombo",
+    "ref": "9d683c06518c0358fb402f38468a7030700c38ac"
+  },
+  "deeprl3v2": {
+    "source_id": "deeprl3v2",
+    "input_format": "supercombo",
+    "ref": "702fa71ad4dd8de08425eb11a1a42aaeb64892c9"
+  },
+  "cinque-driving": {
+    "source_id": "cinque-driving",
+    "input_format": "supercombo"
+  },
+  "cdm": {
+    "source_id": "cdm",
+    "input_format": "supercombo"
+  },
+  "cinquev3": {
+    "source_id": "cinquev3",
+    "input_format": "supercombo"
+  },
+  "resfit": {
+    "source_id": "resfit",
+    "input_format": "supercombo"
+  },
+  "mountaindewv1": {
+    "source_id": "mountaindewv1",
+    "input_format": "supercombo"
+  }
+}

scripts/validate_driving_model.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""Exercise a compiled catalog model on synthetic NV12 frames without starting modeld."""
+import argparse
+import gc
+import hashlib
+import json
+import math
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+
+import numpy as np
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+  sys.path.insert(0, str(REPO_ROOT))
+
+CAMERAS = ((1928, 1208), (1344, 760))
+REQUIRED_SHAPES = {"plan": (1, 33, 15), "lane_lines": (1, 4, 33, 2), "road_edges": (1, 2, 33, 2),
+                   "lead": (1, 3, 6, 4), "pose": (1, 6)}
+REQUIRED_KEYS = frozenset((*REQUIRED_SHAPES, "lane_lines_prob", "lead_prob", "meta", "desire_state", "desire_pred",
+                           "wide_from_device_euler", "road_transform"))
+
+
+def sha256_file(path: Path) -> str:
+  digest = hashlib.sha256()
+  with path.open("rb") as file:
+    for block in iter(lambda: file.read(1024 * 1024), b""):
+      digest.update(block)
+  return digest.hexdigest()
+
+
+def check_outputs(outputs: dict[str, np.ndarray], version: str) -> dict[str, str]:
+  required = REQUIRED_KEYS | ({"action"} if version in ("v14", "v15", "v16") else set())
+  if missing := required - outputs.keys():
+    raise ValueError(f"Missing parsed outputs: {sorted(missing)}")
+  shapes = REQUIRED_SHAPES | ({"action": (1, 2)} if "action" in outputs else {})
+  hashes = {}
+  for name, value in outputs.items():
+    array = np.asarray(value)
+    if not array.size or not np.isfinite(array).all():
+      raise ValueError(f"Empty or non-finite parsed output: {name}")
+    if name in shapes and array.shape != shapes[name]:
+      raise ValueError(f"Incorrect parsed {name} shape: {array.shape}, expected {shapes[name]}")
+    hashes[name] = hashlib.sha256(array.tobytes()).hexdigest()
+  return hashes
+
+
+def fill_nv12(frame: np.ndarray, width: int, height: int, step: int, seed: int) -> None:
+  from openpilot.system.camerad.cameras.nv12_info import get_nv12_info
+
+  stride, y_height, uv_height, _ = get_nv12_info(width, height)
+  frame.fill(16)
+  y = frame[:stride * y_height].reshape(y_height, stride)
+  x_coordinate = np.arange(width, dtype=np.uint32)[None, :]
+  y_coordinate = np.arange(height, dtype=np.uint32)[:, None]
+  y[:height, :width] = 16 + ((x_coordinate * 3 + y_coordinate * 5 + step * 17 + seed) % 220)
+  uv = frame[stride * y_height:stride * (y_height + uv_height)].reshape(uv_height, stride)
+  uv.fill(128)
+  uv[:height // 2, :width:2] = 96 + ((step * 3 + seed) % 64)
+  uv[:height // 2, 1:width:2] = 96 + ((step * 7 + seed) % 64)
+
+
+def frame_transform(width: int, height: int, step: int) -> np.ndarray:
+  return np.array([[0.8 * width / 512, 0, 0.1 * width + math.sin(step) * 2],
+                   [0, 0.8 * height / 256, 0.1 * height + math.cos(step) * 2], [0, 0, 1]], dtype=np.float32)
+
+
+def upload_frames(runner, frames: dict[str, np.ndarray]) -> None:
+  from tinygrad import Device, Tensor
+
+  for name, frame in frames.items():
+    runner._blob_cache[(name, frame.ctypes.data)] = Tensor(frame, device=runner.warp_device).realize()
+  Device[runner.warp_device].synchronize()
+  if runner.queue_device != runner.warp_device:
+    Device[runner.queue_device].synchronize()
+
+
+def validate_camera(artifact: Path, version: str, camera: tuple[int, int], steps: int, seed: int,
+                    gpu: bool, max_p95_ms: float | None = None, runner_factory=None, action_update=None, initial_action=None) -> dict:
+  device_frames = runner_factory is None
+  if runner_factory is None:
+    from openpilot.starpilot.models.runner import CatalogModelState, action_from_outputs
+    from openpilot.cereal import log
+    runner_factory, action_update = CatalogModelState, action_from_outputs
+    initial_action = log.ModelDataV2.Action()
+  width, height = camera
+  start = time.perf_counter()
+  runner = runner_factory(width, height, artifact, version, gpu)
+  load_ms = (time.perf_counter() - start) * 1000
+  start = time.perf_counter()
+  runner.warmup()
+  warmup_ms = (time.perf_counter() - start) * 1000
+  frames = {name: np.zeros(runner.frame_buf_size, dtype=np.uint8) for name in runner.vision_input_names}
+  previous_action = initial_action
+  latencies, changed, baseline = [], set(), None
+  output_shapes = {}
+  actions = []
+  for step in range(steps):
+    for index, frame in enumerate(frames.values()):
+      fill_nv12(frame, width, height, step, seed + index * 97)
+    transforms = {name: frame_transform(width, height, step + index) for index, name in enumerate(frames)}
+    desire = np.zeros(8, dtype=np.float32)
+    if step % 4 == 0:
+      desire[1 + (step // 4) % 2] = 1
+    speed = 5.0 + step % 20
+    inputs = {"desire_pulse": desire, "traffic_convention": np.array([step % 2, 1 - step % 2], dtype=np.float32),
+              "action_t": np.array([.2, .4], dtype=np.float32),
+              "prev_action": np.array([previous_action.desiredCurvature * max(1., speed) ** 2,
+                                        previous_action.desiredAcceleration], dtype=np.float32),
+              "lateral_control_params": np.array([speed, .2], dtype=np.float32)}
+    if device_frames:
+      upload_frames(runner, frames)
+    start = time.perf_counter()
+    outputs = runner.run(frames, transforms, inputs)
+    latencies.append((time.perf_counter() - start) * 1000)
+    hashes = check_outputs(outputs, version)
+    if baseline is None:
+      baseline = hashes
+      output_shapes = {name: list(np.asarray(value).shape) for name, value in outputs.items()}
+    else:
+      if hashes.keys() != baseline.keys():
+        raise ValueError("Parsed output names changed between frames")
+      changed.update(name for name, value in hashes.items() if value != baseline[name])
+    previous_action = action_update(outputs, version, previous_action, .2, .4, speed)
+    action = [float(previous_action.desiredCurvature), float(previous_action.desiredAcceleration)]
+    if not np.isfinite(action).all():
+      raise ValueError("Non-finite derived driving action")
+    actions.append(action)
+  if not changed.intersection(REQUIRED_KEYS | {"action"}):
+    raise ValueError("Driving outputs stayed unchanged across changing frames and inputs")
+  timing = {"min": float(np.min(latencies)), "median": float(np.median(latencies)),
+            "p95": float(np.percentile(latencies, 95)), "max": float(np.max(latencies))}
+  if max_p95_ms is not None and timing["p95"] > max_p95_ms:
+    raise ValueError(f"p95 latency {timing['p95']:.2f} ms exceeds {max_p95_ms:.2f} ms")
+  result = {"passed": True, "camera": [width, height], "steps": steps, "seed": seed,
+            "load_ms": load_ms, "warmup_ms": warmup_ms, "inference_ms": timing,
+            "changed_outputs": sorted(changed), "output_shapes": output_shapes, "last_output_sha256": hashes,
+            "last_action": {"desired_curvature": actions[-1][0], "desired_acceleration": actions[-1][1]},
+            "execution_device": getattr(runner, "queue_device", None), "warp_device": getattr(runner, "warp_device", None)}
+  del runner
+  gc.collect()
+  return result
+
+
+def parse_args(argv=None):
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument("artifact", type=Path)
+  parser.add_argument("--version", required=True, choices=tuple(f"v{number}" for number in range(8, 17)))
+  parser.add_argument("--steps", type=int, default=24)
+  parser.add_argument("--seed", type=int, default=42)
+  parser.add_argument("--gpu", action="store_true")
+  parser.add_argument("--max-p95-ms", type=float)
+  parser.add_argument("--build-receipt", type=Path)
+  parser.add_argument("--source-dir", type=Path, help="Also verify the build receipt's ONNX source hashes")
+  parser.add_argument("--report", type=Path)
+  parser.add_argument("--camera", choices=("1928x1208", "1344x760"), help=argparse.SUPPRESS)
+  args = parser.parse_args(argv)
+  if args.steps < 2:
+    parser.error("--steps must be at least 2")
+  if args.max_p95_ms is not None and (not math.isfinite(args.max_p95_ms) or args.max_p95_ms <= 0):
+    parser.error("--max-p95-ms must be positive and finite")
+  if args.seed < 0:
+    parser.error("--seed must be nonnegative")
+  return args
+
+
+def main(argv=None):
+  args = parse_args(argv)
+  artifact = args.artifact.resolve()
+  report_path = args.report or artifact.with_suffix(".validation.json")
+  receipt_path = args.build_receipt or artifact.with_suffix(".build.json")
+  protected = {artifact, receipt_path.resolve()}
+  report = {"passed": False, "validation": "synthetic_nv12_catalog_runner", "artifact": str(artifact),
+            "behavior_version": args.version, "steps": args.steps, "seed": args.seed, "cameras": []}
+  try:
+    if args.camera:
+      camera = tuple(map(int, args.camera.split("x")))
+      report.update(validate_camera(artifact, args.version, camera, args.steps, args.seed, args.gpu, args.max_p95_ms))
+    else:
+      before = sha256_file(artifact)
+      report.update(artifact_sha256=before, artifact_size=artifact.stat().st_size, sources_verified=False)
+      if receipt_path.is_file():
+        receipt = json.loads(receipt_path.read_text())
+        if receipt.get("artifact_sha256") != before or receipt.get("behavior_version") != args.version:
+          raise ValueError("Build receipt does not match artifact hash/behavior version")
+        source_hashes = receipt.get("source_sha256", {})
+        report.update(build_receipt_sha256=sha256_file(receipt_path), source_sha256=source_hashes,
+                      compiler_revision=receipt.get("compiler_revision"), artifact_abi=receipt.get("artifact_abi"))
+        if args.source_dir:
+          if not source_hashes or any(Path(name).name != name for name in source_hashes):
+            raise ValueError("Build receipt has no bounded source filenames")
+          protected.update((args.source_dir / name).resolve() for name in source_hashes)
+          if {name: sha256_file(args.source_dir / name) for name in source_hashes} != source_hashes:
+            raise ValueError("Source hashes do not match build receipt")
+          report["sources_verified"] = True
+      elif args.build_receipt or args.source_dir:
+        raise ValueError("Build receipt required for requested source verification")
+      # Separate processes release all device allocations between camera formats.
+      with tempfile.TemporaryDirectory(prefix="starpilot-model-validation-") as temporary:
+        for width, height in CAMERAS:
+          child_report = Path(temporary) / f"{width}x{height}.json"
+          command = [sys.executable, str(Path(__file__).resolve()), str(artifact), "--version", args.version,
+                     "--steps", str(args.steps), "--seed", str(args.seed), "--camera", f"{width}x{height}",
+                     "--report", str(child_report)]
+          if args.gpu:
+            command.append("--gpu")
+          if args.max_p95_ms is not None:
+            command += ["--max-p95-ms", str(args.max_p95_ms)]
+          completed = subprocess.run(command, cwd=REPO_ROOT, check=False)
+          result = json.loads(child_report.read_text()) if child_report.is_file() else {
+            "passed": False, "camera": [width, height], "error": "Validation process exited without a report"}
+          if completed.returncode:
+            result["passed"] = False
+          report["cameras"].append(result)
+      if sha256_file(artifact) != before:
+        raise ValueError("Artifact changed during validation")
+      report["passed"] = all(camera["passed"] for camera in report["cameras"])
+  except Exception as error:
+    report.update(passed=False, error=f"{type(error).__name__}: {error}")
+  if report_path.resolve() in protected:
+    print("Refusing to overwrite a validation input with its report", file=sys.stderr)
+    return 1
+  report_path.parent.mkdir(parents=True, exist_ok=True)
+  report_path.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n")
+  print(f"{'PASS' if report['passed'] else 'FAIL'}: {report_path}")
+  return 0 if report["passed"] else 1
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())