model stuff

docs/MODEL_REBUILD.md

@@ -47,7 +47,10 @@ python3 scripts/model_rebuild_pipeline.py extract \
   --base-manifest /path/to/model_names_v21.json
 ```
 
-Source commits are defined in `scripts/model_source_map_v22.json`.
+The original catalog sources are defined in `scripts/model_source_map_v22.json`.
+Recovered late-model and supercombo sources, including RDF2, are defined in
+`scripts/model_source_map_v23.json`. The v23 map is intentionally separate so
+adding a recovered iteration cannot alter the older model source history.
 
 ## Compile
 
@@ -175,6 +178,18 @@ compiled and upload-ready files, and writes an ID map. It preserves display
 names and behavioral versions. The current model manager requests v23 only;
 the manifest is fetched from `Models/model_names_v23.json`, while v22 remains
 available for devices that have not updated yet.
+
+After importing newly compiled sources, normalize the release namespace before
+copying files into either resource repository:
+
+```bash
+python3 scripts/reconcile_v23_artifacts.py \
+  --workspace /Volumes/T5/StarPilot-Model-Rebuild-2026-06-22
+```
+
+This maps recovered source IDs to their v23 release IDs, removes duplicate
+macOS metadata files, and adds `rdf23` for Regret Driven Framework V2. It does
+not overwrite a conflicting artifact.
 Repository-hosted multipart files are discovered by naming convention, so no
 size, hash, format, or part-count metadata is required.
 

scripts/model_compiler.py

@@ -552,14 +552,16 @@ def compile_driving(
 ) -> Path:
   model_type, source_args = driving_compile_args(files, input_format)
   output_path = output_dir / f"{model_key}_driving_tinygrad.pkl"
+  # A rebuild queue may compile several models into the same directory. Only
+  # replace the selected model; deleting every driving artifact here loses
+  # models that were successfully compiled earlier in the queue.
   removed = remove_paths(sorted({
     output_path,
     *multipart_output_paths(output_path, output_dir),
-    *output_dir.glob("*_driving_tinygrad.pkl"),
-    *output_dir.glob("*_driving_tinygrad.pkl.p[0-9][0-9]"),
-    *output_dir.glob("*_driving_tinygrad.pkl.sha256"),
-    *output_dir.glob("*_driving_*_tinygrad.pkl"),
-    *output_dir.glob("*_driving_*_metadata.pkl"),
+    *output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl"),
+    *output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl.p[0-9][0-9]"),
+    *output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl.sha256"),
+    *output_dir.glob(f"{model_key}_driving_*_metadata.pkl"),
   }))
   if removed:
     print(f"  cleared {removed} existing driving output entries")

scripts/model_source_map_v23.json

@@ -1,12 +1,13 @@
 {
   "karnbir": {"ref": "c0fbd2b13030df95ad249b6e253156a4d7ddb7bb", "input_format": "supercombo"},
   "karnbir2": {"ref": "682d33d0414d7cef5a7286c24298003503ea13f4", "input_format": "supercombo"},
-  "michael-rl": {"ref": "968fd3c98ed7f5b0b4ef0ea1cf4c58a7c3e4b77c", "input_format": "supercombo"},
+  "michael-rl": {"ref": "968fd3c989b993cf03187aa0e05cd2088e275551", "input_format": "supercombo"},
   "michael-rl2": {"ref": "37b38bf738edfb1daa6875255f581e5fc9b0a258", "input_format": "supercombo"},
   "tobyrl": {"ref": "2d4acf10e34c4c8e68043d60b9437f42d89ca746", "input_format": "supercombo"},
   "nopp": {"ref": "c740fe5f58faefcb9184c6f9f1fb45130b83cfe8", "input_format": "supercombo"},
-  "drl": {"ref": "a881acfbdc7b3e5d5e1ff0b1ee1a8dcf43b11f3b", "input_format": "supercombo"},
+  "drl": {"ref": "a881acfbd59cb9ed18c54535b835721e2bc96def", "input_format": "supercombo"},
   "deeprl33": {"ref": "4604c98f21c9a22230e417d17d03c402b9b5f869", "input_format": "supercombo"},
-  "rl34": {"ref": "4e6b07240f5f1b3d9a4b182e7d4fb42ae2bd37e5", "input_format": "supercombo"},
-  "gyhu": {"ref": "574735edc6e1aafdc2a69395f9a32e7f5cc4b62b", "input_format": "supercombo"}
+  "rl34": {"ref": "4e6b072404110056ccf9bcc232c718ac71d45478", "input_format": "supercombo"},
+  "gyhu": {"ref": "574735edc6e1aafdc2a69395f9a32e7f5cc4b62b", "input_format": "supercombo"},
+  "rdf2": {"ref": "0f8b4248a2e8bdd63b6ea4c6e1bbb32119cb2620", "input_format": "supercombo"}
 }

scripts/reconcile_v23_artifacts.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Normalize rebuilt artifacts and the v23 manifest into one release namespace."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+
+SOURCE_TO_RELEASE = {
+  "opv8": "opv83",
+  "opv9": "opv93",
+  "opv10": "opv103",
+  "opv11": "opv113",
+  "opv12": "opv123",
+  "opv13": "opv133",
+  "op16": "op163",
+  "op16d": "op16d3",
+  "op16dv2": "op16dv23",
+  "rlv1dl": "rlv1dl3",
+  "deeprl3": "deeprl33",
+  "deeprl3v2": "deeprl3v23",
+  "ms2": "ms23",
+  "pp222": "pp2223",
+  "nn222": "nn2223",
+  "pop2": "pop23",
+  "nid22": "nid223",
+  "kerrygold22": "kerrygold223",
+  "karnbir": "karnbir3",
+  "karnbir2": "karnbir23",
+  "michael-rl": "michael-rl3",
+  "michael-rl2": "michael-rl23",
+  "tobyrl": "tobyrl3",
+  "nopp": "nopp3",
+  "drl": "drl3",
+  "deeprl33": "deeprl333",
+  "rl34": "rl343",
+  "gyhu": "gyhu3",
+  "rdf2": "rdf23",
+}
+
+
+def sha256(path: Path) -> str:
+  digest = hashlib.sha256()
+  with path.open("rb") as source:
+    for chunk in iter(lambda: source.read(1024 * 1024), b""):
+      digest.update(chunk)
+  return digest.hexdigest()
+
+
+def renamed_name(filename: str, old_id: str, new_id: str) -> str:
+  prefix = f"{old_id}_driving_tinygrad.pkl"
+  if filename == prefix or filename.startswith(f"{prefix}."):
+    return f"{new_id}_driving_tinygrad.pkl{filename[len(prefix):]}"
+  return filename
+
+
+def normalize_artifact_names(directory: Path) -> None:
+  if not directory.is_dir():
+    return
+  for path in directory.glob("._*"):
+    path.unlink(missing_ok=True)
+
+  # Snapshot the input names. Some source IDs are also release IDs (for
+  # example deeprl3 -> deeprl33 and deeprl33 -> deeprl333); scanning the live
+  # directory would apply those mappings twice.
+  original_paths = list(directory.iterdir())
+  moves = []
+  for old_id, new_id in SOURCE_TO_RELEASE.items():
+    sources = [path for path in original_paths if path.exists() and renamed_name(path.name, old_id, new_id) != path.name]
+    for source in sources:
+      destination = directory / renamed_name(source.name, old_id, new_id)
+      moves.append((source, destination))
+
+  source_paths = {source for source, _ in moves}
+  temporary_moves = []
+  for index, (source, destination) in enumerate(moves):
+    if destination.exists() and destination not in source_paths:
+      if source.is_file() and destination.is_file() and source.stat().st_size == destination.stat().st_size and sha256(source) == sha256(destination):
+        source.unlink()
+        continue
+      raise FileExistsError(f"Refusing to overwrite conflicting artifact: {destination}")
+    temporary = directory / f".__reconcile_{index}__{source.name}"
+    if temporary.exists():
+      raise FileExistsError(f"Temporary reconciliation path already exists: {temporary}")
+    source.rename(temporary)
+    temporary_moves.append((temporary, destination, source.name))
+
+  for temporary, destination, source_name in temporary_moves:
+    if destination.exists():
+      raise FileExistsError(f"Refusing to overwrite conflicting artifact: {destination}")
+    if destination.name.endswith(".pkl.sha256"):
+      checksum_text = temporary.read_text(encoding="utf-8")
+      checksum_text = checksum_text.replace(source_name.removesuffix(".sha256"), destination.name.removesuffix(".sha256"))
+      temporary.write_text(checksum_text, encoding="utf-8")
+    temporary.rename(destination)
+
+
+def normalize_manifest(path: Path) -> None:
+  payload = json.loads(path.read_text(encoding="utf-8"))
+  models = payload.get("models", [])
+  normalized = []
+  seen = set()
+  duplicate_base_removed = False
+  for model in models:
+    entry = dict(model)
+    if entry.get("id") == "deeprl3v23_":
+      if duplicate_base_removed:
+        continue
+      entry["id"] = "deeprl33"
+      duplicate_base_removed = True
+    if entry["id"] in seen:
+      raise ValueError(f"Duplicate v23 model ID: {entry['id']}")
+    seen.add(entry["id"])
+    normalized.append(entry)
+
+  if "rdf23" not in seen:
+    insert_at = next((index + 1 for index, model in enumerate(normalized) if model.get("id") == "rdfv23"), len(normalized))
+    normalized.insert(insert_at, {
+      "id": "rdf23",
+      "name": "Regret Driven Framework V2 👀📡",
+      "version": "v15",
+      "series": "OP Series",
+      "released": "2026-08-10",
+      "community_favorite": False,
+    })
+
+  payload["models"] = normalized
+  path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def main() -> int:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--workspace", type=Path, required=True)
+  args = parser.parse_args()
+  normalize_artifact_names(args.workspace / "compiled")
+  normalize_artifact_names(args.workspace / "ready-for-resources")
+  normalize_manifest(args.workspace / "manifests/model_names_v23.json")
+  return 0
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())

starpilot/assets/tests/test_model_pipeline.py

@@ -5,6 +5,7 @@ import sys
 from pathlib import Path
 
 from scripts import model_compiler
+from scripts import reconcile_v23_artifacts
 from scripts.model_compiler import split_oversized_artifact
 from openpilot.common import file_chunker
 from openpilot.selfdrive.modeld import compile_modeld
@@ -86,6 +87,32 @@ def test_external_gpu_compilation_is_opt_in(tmp_path, monkeypatch):
   assert all(flag not in external_kwargs["env"] for flag in ("IMAGE", "NOLOCALS", "OPENPILOT_HACKS"))
 
 
+def test_compile_clears_only_selected_model_outputs(tmp_path, monkeypatch):
+  monkeypatch.setattr(model_compiler, "build_compile_env", lambda: {})
+  monkeypatch.setattr(model_compiler.subprocess, "run", lambda *args, **kwargs: None)
+  (tmp_path / "normal_driving_tinygrad.pkl").write_bytes(b"old")
+  (tmp_path / "normal_driving_tinygrad.pkl.p00").write_bytes(b"old")
+  (tmp_path / "other_driving_tinygrad.pkl").write_bytes(b"keep")
+
+  model_compiler.compile_driving(
+    "normal", {"driving_supercombo": tmp_path / "model.onnx"}, "supercombo", "v15", tmp_path, "policy",
+  )
+
+  assert not (tmp_path / "normal_driving_tinygrad.pkl").exists()
+  assert not (tmp_path / "normal_driving_tinygrad.pkl.p00").exists()
+  assert (tmp_path / "other_driving_tinygrad.pkl").read_bytes() == b"keep"
+
+
+def test_v23_namespace_mapping_does_not_cascade(tmp_path):
+  (tmp_path / "deeprl3_driving_tinygrad.pkl.p00").write_bytes(b"base")
+  (tmp_path / "deeprl33_driving_tinygrad.pkl.p00").write_bytes(b"v3")
+
+  reconcile_v23_artifacts.normalize_artifact_names(tmp_path)
+
+  assert (tmp_path / "deeprl33_driving_tinygrad.pkl.p00").read_bytes() == b"base"
+  assert (tmp_path / "deeprl333_driving_tinygrad.pkl.p00").read_bytes() == b"v3"
+
+
 def test_gpu_is_external_gpu_cli_alias(monkeypatch):
   monkeypatch.setattr(sys, "argv", ["models", "--model", "large", "--gpu"])
   args = model_compiler.parse_args()