diff --git a/docs/MODEL_REBUILD.md b/docs/MODEL_REBUILD.md index cfe5f7953..af1a25de9 100644 --- a/docs/MODEL_REBUILD.md +++ b/docs/MODEL_REBUILD.md @@ -47,7 +47,10 @@ python3 scripts/model_rebuild_pipeline.py extract \ --base-manifest /path/to/model_names_v21.json ``` -Source commits are defined in `scripts/model_source_map_v22.json`. +The original catalog sources are defined in `scripts/model_source_map_v22.json`. +Recovered late-model and supercombo sources, including RDF2, are defined in +`scripts/model_source_map_v23.json`. The v23 map is intentionally separate so +adding a recovered iteration cannot alter the older model source history. ## Compile @@ -175,6 +178,18 @@ compiled and upload-ready files, and writes an ID map. It preserves display names and behavioral versions. The current model manager requests v23 only; the manifest is fetched from `Models/model_names_v23.json`, while v22 remains available for devices that have not updated yet. + +After importing newly compiled sources, normalize the release namespace before +copying files into either resource repository: + +```bash +python3 scripts/reconcile_v23_artifacts.py \ + --workspace /Volumes/T5/StarPilot-Model-Rebuild-2026-06-22 +``` + +This maps recovered source IDs to their v23 release IDs, removes duplicate +macOS metadata files, and adds `rdf23` for Regret Driven Framework V2. It does +not overwrite a conflicting artifact. Repository-hosted multipart files are discovered by naming convention, so no size, hash, format, or part-count metadata is required. diff --git a/scripts/model_compiler.py b/scripts/model_compiler.py index 6ed0688b5..c5d11fe52 100644 --- a/scripts/model_compiler.py +++ b/scripts/model_compiler.py @@ -552,14 +552,16 @@ def compile_driving( ) -> Path: model_type, source_args = driving_compile_args(files, input_format) output_path = output_dir / f"{model_key}_driving_tinygrad.pkl" + # A rebuild queue may compile several models into the same directory. Only + # replace the selected model; deleting every driving artifact here loses + # models that were successfully compiled earlier in the queue. removed = remove_paths(sorted({ output_path, *multipart_output_paths(output_path, output_dir), - *output_dir.glob("*_driving_tinygrad.pkl"), - *output_dir.glob("*_driving_tinygrad.pkl.p[0-9][0-9]"), - *output_dir.glob("*_driving_tinygrad.pkl.sha256"), - *output_dir.glob("*_driving_*_tinygrad.pkl"), - *output_dir.glob("*_driving_*_metadata.pkl"), + *output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl"), + *output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl.p[0-9][0-9]"), + *output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl.sha256"), + *output_dir.glob(f"{model_key}_driving_*_metadata.pkl"), })) if removed: print(f" cleared {removed} existing driving output entries") diff --git a/scripts/model_source_map_v23.json b/scripts/model_source_map_v23.json index 8c253ecb8..b715d80b2 100644 --- a/scripts/model_source_map_v23.json +++ b/scripts/model_source_map_v23.json @@ -1,12 +1,13 @@ { "karnbir": {"ref": "c0fbd2b13030df95ad249b6e253156a4d7ddb7bb", "input_format": "supercombo"}, "karnbir2": {"ref": "682d33d0414d7cef5a7286c24298003503ea13f4", "input_format": "supercombo"}, - "michael-rl": {"ref": "968fd3c98ed7f5b0b4ef0ea1cf4c58a7c3e4b77c", "input_format": "supercombo"}, + "michael-rl": {"ref": "968fd3c989b993cf03187aa0e05cd2088e275551", "input_format": "supercombo"}, "michael-rl2": {"ref": "37b38bf738edfb1daa6875255f581e5fc9b0a258", "input_format": "supercombo"}, "tobyrl": {"ref": "2d4acf10e34c4c8e68043d60b9437f42d89ca746", "input_format": "supercombo"}, "nopp": {"ref": "c740fe5f58faefcb9184c6f9f1fb45130b83cfe8", "input_format": "supercombo"}, - "drl": {"ref": "a881acfbdc7b3e5d5e1ff0b1ee1a8dcf43b11f3b", "input_format": "supercombo"}, + "drl": {"ref": "a881acfbd59cb9ed18c54535b835721e2bc96def", "input_format": "supercombo"}, "deeprl33": {"ref": "4604c98f21c9a22230e417d17d03c402b9b5f869", "input_format": "supercombo"}, - "rl34": {"ref": "4e6b07240f5f1b3d9a4b182e7d4fb42ae2bd37e5", "input_format": "supercombo"}, - "gyhu": {"ref": "574735edc6e1aafdc2a69395f9a32e7f5cc4b62b", "input_format": "supercombo"} + "rl34": {"ref": "4e6b072404110056ccf9bcc232c718ac71d45478", "input_format": "supercombo"}, + "gyhu": {"ref": "574735edc6e1aafdc2a69395f9a32e7f5cc4b62b", "input_format": "supercombo"}, + "rdf2": {"ref": "0f8b4248a2e8bdd63b6ea4c6e1bbb32119cb2620", "input_format": "supercombo"} } diff --git a/scripts/reconcile_v23_artifacts.py b/scripts/reconcile_v23_artifacts.py new file mode 100644 index 000000000..239137d6e --- /dev/null +++ b/scripts/reconcile_v23_artifacts.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Normalize rebuilt artifacts and the v23 manifest into one release namespace.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +SOURCE_TO_RELEASE = { + "opv8": "opv83", + "opv9": "opv93", + "opv10": "opv103", + "opv11": "opv113", + "opv12": "opv123", + "opv13": "opv133", + "op16": "op163", + "op16d": "op16d3", + "op16dv2": "op16dv23", + "rlv1dl": "rlv1dl3", + "deeprl3": "deeprl33", + "deeprl3v2": "deeprl3v23", + "ms2": "ms23", + "pp222": "pp2223", + "nn222": "nn2223", + "pop2": "pop23", + "nid22": "nid223", + "kerrygold22": "kerrygold223", + "karnbir": "karnbir3", + "karnbir2": "karnbir23", + "michael-rl": "michael-rl3", + "michael-rl2": "michael-rl23", + "tobyrl": "tobyrl3", + "nopp": "nopp3", + "drl": "drl3", + "deeprl33": "deeprl333", + "rl34": "rl343", + "gyhu": "gyhu3", + "rdf2": "rdf23", +} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def renamed_name(filename: str, old_id: str, new_id: str) -> str: + prefix = f"{old_id}_driving_tinygrad.pkl" + if filename == prefix or filename.startswith(f"{prefix}."): + return f"{new_id}_driving_tinygrad.pkl{filename[len(prefix):]}" + return filename + + +def normalize_artifact_names(directory: Path) -> None: + if not directory.is_dir(): + return + for path in directory.glob("._*"): + path.unlink(missing_ok=True) + + # Snapshot the input names. Some source IDs are also release IDs (for + # example deeprl3 -> deeprl33 and deeprl33 -> deeprl333); scanning the live + # directory would apply those mappings twice. + original_paths = list(directory.iterdir()) + moves = [] + for old_id, new_id in SOURCE_TO_RELEASE.items(): + sources = [path for path in original_paths if path.exists() and renamed_name(path.name, old_id, new_id) != path.name] + for source in sources: + destination = directory / renamed_name(source.name, old_id, new_id) + moves.append((source, destination)) + + source_paths = {source for source, _ in moves} + temporary_moves = [] + for index, (source, destination) in enumerate(moves): + if destination.exists() and destination not in source_paths: + if source.is_file() and destination.is_file() and source.stat().st_size == destination.stat().st_size and sha256(source) == sha256(destination): + source.unlink() + continue + raise FileExistsError(f"Refusing to overwrite conflicting artifact: {destination}") + temporary = directory / f".__reconcile_{index}__{source.name}" + if temporary.exists(): + raise FileExistsError(f"Temporary reconciliation path already exists: {temporary}") + source.rename(temporary) + temporary_moves.append((temporary, destination, source.name)) + + for temporary, destination, source_name in temporary_moves: + if destination.exists(): + raise FileExistsError(f"Refusing to overwrite conflicting artifact: {destination}") + if destination.name.endswith(".pkl.sha256"): + checksum_text = temporary.read_text(encoding="utf-8") + checksum_text = checksum_text.replace(source_name.removesuffix(".sha256"), destination.name.removesuffix(".sha256")) + temporary.write_text(checksum_text, encoding="utf-8") + temporary.rename(destination) + + +def normalize_manifest(path: Path) -> None: + payload = json.loads(path.read_text(encoding="utf-8")) + models = payload.get("models", []) + normalized = [] + seen = set() + duplicate_base_removed = False + for model in models: + entry = dict(model) + if entry.get("id") == "deeprl3v23_": + if duplicate_base_removed: + continue + entry["id"] = "deeprl33" + duplicate_base_removed = True + if entry["id"] in seen: + raise ValueError(f"Duplicate v23 model ID: {entry['id']}") + seen.add(entry["id"]) + normalized.append(entry) + + if "rdf23" not in seen: + insert_at = next((index + 1 for index, model in enumerate(normalized) if model.get("id") == "rdfv23"), len(normalized)) + normalized.insert(insert_at, { + "id": "rdf23", + "name": "Regret Driven Framework V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-10", + "community_favorite": False, + }) + + payload["models"] = normalized + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", type=Path, required=True) + args = parser.parse_args() + normalize_artifact_names(args.workspace / "compiled") + normalize_artifact_names(args.workspace / "ready-for-resources") + normalize_manifest(args.workspace / "manifests/model_names_v23.json") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/starpilot/assets/tests/test_model_pipeline.py b/starpilot/assets/tests/test_model_pipeline.py index 5409f4ffb..34e6ada51 100644 --- a/starpilot/assets/tests/test_model_pipeline.py +++ b/starpilot/assets/tests/test_model_pipeline.py @@ -5,6 +5,7 @@ import sys from pathlib import Path from scripts import model_compiler +from scripts import reconcile_v23_artifacts from scripts.model_compiler import split_oversized_artifact from openpilot.common import file_chunker from openpilot.selfdrive.modeld import compile_modeld @@ -86,6 +87,32 @@ def test_external_gpu_compilation_is_opt_in(tmp_path, monkeypatch): assert all(flag not in external_kwargs["env"] for flag in ("IMAGE", "NOLOCALS", "OPENPILOT_HACKS")) +def test_compile_clears_only_selected_model_outputs(tmp_path, monkeypatch): + monkeypatch.setattr(model_compiler, "build_compile_env", lambda: {}) + monkeypatch.setattr(model_compiler.subprocess, "run", lambda *args, **kwargs: None) + (tmp_path / "normal_driving_tinygrad.pkl").write_bytes(b"old") + (tmp_path / "normal_driving_tinygrad.pkl.p00").write_bytes(b"old") + (tmp_path / "other_driving_tinygrad.pkl").write_bytes(b"keep") + + model_compiler.compile_driving( + "normal", {"driving_supercombo": tmp_path / "model.onnx"}, "supercombo", "v15", tmp_path, "policy", + ) + + assert not (tmp_path / "normal_driving_tinygrad.pkl").exists() + assert not (tmp_path / "normal_driving_tinygrad.pkl.p00").exists() + assert (tmp_path / "other_driving_tinygrad.pkl").read_bytes() == b"keep" + + +def test_v23_namespace_mapping_does_not_cascade(tmp_path): + (tmp_path / "deeprl3_driving_tinygrad.pkl.p00").write_bytes(b"base") + (tmp_path / "deeprl33_driving_tinygrad.pkl.p00").write_bytes(b"v3") + + reconcile_v23_artifacts.normalize_artifact_names(tmp_path) + + assert (tmp_path / "deeprl33_driving_tinygrad.pkl.p00").read_bytes() == b"base" + assert (tmp_path / "deeprl333_driving_tinygrad.pkl.p00").read_bytes() == b"v3" + + def test_gpu_is_external_gpu_cli_alias(monkeypatch): monkeypatch.setattr(sys, "argv", ["models", "--model", "large", "--gpu"]) args = model_compiler.parse_args()