diff --git a/scripts/model_compiler.py b/scripts/model_compiler.py index 5a1c92f20..4f3b658f5 100644 --- a/scripts/model_compiler.py +++ b/scripts/model_compiler.py @@ -5,6 +5,7 @@ import hashlib import json import os import pickle +import shutil import subprocess import sys from pathlib import Path @@ -19,6 +20,7 @@ COMPILE_SCRIPT = REPO_ROOT / "tinygrad_repo/examples/openpilot/compile3.py" DRIVING_COMPILE_SCRIPT = REPO_ROOT / "selfdrive/modeld/compile_modeld.py" DM_WARP_COMPILE_SCRIPT = REPO_ROOT / "selfdrive/modeld/compile_dm_warp.py" MODEL_VERSIONS_CACHE = Path("/data/models/.model_versions.json") +MODELS_PATH = MODEL_VERSIONS_CACHE.parent # runtime dir modeld loads from: /data/models DM_MODEL_KEY = "dm" DM_MODEL_NAME = "dmonitoring_model" @@ -84,6 +86,11 @@ def parse_args() -> argparse.Namespace: help="Compile the driving artifact for the USB AMD GPU.") parser.add_argument("--split-artifact", type=Path, help="Split an existing oversized PKL without compiling.") parser.add_argument("--chunk-size-mib", type=int, default=95, help="Multipart size in MiB; must be below 100.") + parser.add_argument("--no-split", action="store_true", + help="Keep a single .pkl even if >100 MiB (for local installs, which need one " + "file). Auto-enabled for local- model IDs.") + parser.add_argument("--no-install", action="store_true", + help="Do not auto-copy a local- model into /data/models after compiling.") parser.add_argument( "--image-history-pipeline", choices=("policy", "warp"), @@ -302,6 +309,48 @@ def multipart_output_paths(artifact: Path, output_dir: Path | None = None) -> li ] +def install_local_artifact(artifact: Path, model_key: str, version: str) -> None: + """Copy a freshly compiled local- model into the runtime dir modeld loads from, + and ensure its .json sidecar carries the correct version. + + The sidecar version is NOT cosmetic: without it _discover_local_models() records + an empty version, which downstream parses on the wrong contract (a v15 model then + drives like v11). Since we know the build version here, we write it so the local + install is correct by default. Local models must be a single is_file() in + /data/models to show in the picker. No-ops off-device (no /data/models). + """ + if not MODELS_PATH.is_dir(): + print(f" skipped auto-install: {MODELS_PATH} not present (not on device?)") + return + dest = MODELS_PATH / artifact.name + shutil.copy2(artifact, dest) + print(f" installed -> {dest}") + + sidecar = MODELS_PATH / f"{model_key}.json" + info: dict = {} + if sidecar.is_file(): + try: + loaded = json.loads(sidecar.read_text()) + if isinstance(loaded, dict): + info = loaded + except Exception as error: + print(f" WARN: existing sidecar {sidecar.name} is malformed, rewriting: {error}") + if not version: + if not str(info.get("version") or "").strip(): + print(f" WARN: could not determine version -- set it by hand in {sidecar.name} " + "or the model may drive on the wrong version contract") + return + # keep any user-set name/series; only guarantee a correct, non-empty version + if str(info.get("version") or "").strip() == version and sidecar.is_file(): + print(f" sidecar ok: {sidecar.name} (version {version})") + return + info.setdefault("name", model_key[len("local-"):].replace("_", " ").replace("-", " ").strip()) + info.setdefault("series", "Local") + info["version"] = version + sidecar.write_text(json.dumps(info, indent=2) + "\n") + print(f" wrote sidecar {sidecar.name} (version {version})") + + def split_oversized_artifact( artifact: Path, output_dir: Path | None = None, @@ -507,17 +556,34 @@ def main() -> int: if not version and input_format == "supercombo": version = "v15" version_label = version or "unspecified behavior" - print(f"Compiling {model_key} ({input_format}, {version_label}) from {args.input_dir} -> {args.output_dir}") + will_install = model_key.startswith("local-") and not args.no_install + target = f"{args.output_dir}" + (f" -> {MODELS_PATH} (auto-install)" if will_install else "") + print(f"Compiling {model_key} ({input_format}, {version_label}) from {args.input_dir} -> {target}") output = compile_driving(model_key, files, input_format, version, args.output_dir, args.image_history_pipeline, args.external_gpu) print(f" saved {output.name}") - multipart_outputs = split_oversized_artifact(output) - if multipart_outputs: - print(" artifact exceeds 100 MiB; created repository-safe multipart files:") - for multipart_output in multipart_outputs: - print(f" {multipart_output.name} ({multipart_output.stat().st_size} bytes)") - output.unlink() - print(f" removed oversized source artifact {output.name}") + # Local models install as a single is_file() and never go to GitHub, so the >100 MiB + # repo split is pointless for them (you'd only have to reassemble it). Keep one .pkl. + keep_single = args.no_split or model_key.startswith("local-") + if keep_single: + is_local = model_key.startswith("local-") + if output.stat().st_size > REPOSITORY_FILE_LIMIT: + size_mb = output.stat().st_size / 1e6 + if is_local: + print(f" local model: kept as one {size_mb:.1f} MB file (repo split not needed)") + else: + print(f" --no-split: kept one {size_mb:.1f} MB file; over 100 MB, so split it " + "before committing to a repo (re-run without --no-split, or --split-artifact)") + if is_local and not args.no_install: + install_local_artifact(output, model_key, version) + else: + multipart_outputs = split_oversized_artifact(output) + if multipart_outputs: + print(" artifact exceeds 100 MiB; created repository-safe multipart files:") + for multipart_output in multipart_outputs: + print(f" {multipart_output.name} ({multipart_output.stat().st_size} bytes)") + output.unlink() + print(f" removed oversized source artifact {output.name}") print("Done.") return 0 diff --git a/starpilot/assets/model_manager.py b/starpilot/assets/model_manager.py index f79f6012e..af2318eb7 100644 --- a/starpilot/assets/model_manager.py +++ b/starpilot/assets/model_manager.py @@ -25,6 +25,8 @@ from openpilot.starpilot.common.starpilot_variables import MODELS_PATH MANIFEST_CANDIDATES = ("v22",) DEFAULT_MODEL_KEY = "sc2" +LOCAL_MODEL_PREFIX = "local-" +LOCAL_MODEL_SERIES = "Local Series" ARTIFACT_URLS_CACHE = ".model_artifact_urls.json" ARTIFACT_METADATA_CACHE = ".model_artifacts.json" MODEL_KEY_CANONICAL_MAP = { @@ -51,6 +53,11 @@ def is_builtin_model_key(model_key: str) -> bool: return canonical_model_key(model_key) == DEFAULT_MODEL_KEY +def is_local_model_key(model_key: str) -> bool: + """Hand-installed models live outside the manifest and are never downloaded or pruned.""" + return canonical_model_key(model_key).startswith(LOCAL_MODEL_PREFIX) + + def model_key_aliases(model_key: str) -> list[str]: canonical_key = canonical_model_key(model_key) aliases = [canonical_key] @@ -443,6 +450,8 @@ class ModelManager: valid_keys = set(self.available_models) for model_file in MODELS_PATH.glob("*_driving_*"): model_key = model_file.name.split("_driving_", 1)[0] + if is_local_model_key(model_key): + continue if model_key not in valid_keys: delete_file(model_file, print_error=False) @@ -472,7 +481,51 @@ class ModelManager: self._set_model_param_keys(replacement, replacement_name, None) self._sync_selected_model_version() + def _discover_local_models(self) -> list[dict]: + """Synthesize manifest entries for hand-installed models found in MODELS_PATH. + + A local model is any "_driving_*" artifact. An optional ".json" + sidecar supplies name/version/series; without one the version stays empty so + modeld falls back to the version embedded in the artifact itself. + """ + try: + entries = sorted(MODELS_PATH.glob(f"{LOCAL_MODEL_PREFIX}*_driving_*")) + except Exception as error: + print(f"Failed to scan for local models: {error}") + return [] + + discovered: dict[str, dict] = {} + for model_file in entries: + model_key = model_file.name.split("_driving_", 1)[0] + # Keys and names land in comma-joined params, so a comma would corrupt the catalog. + if not model_key.startswith(LOCAL_MODEL_PREFIX) or "," in model_key or model_key in discovered: + continue + + info: dict = {} + sidecar = MODELS_PATH / f"{model_key}.json" + if sidecar.is_file(): + try: + loaded = json.loads(sidecar.read_text()) + if isinstance(loaded, dict): + info = loaded + except Exception as error: + print(f"Ignoring malformed local model sidecar {sidecar.name}: {error}") + + fallback_name = model_key[len(LOCAL_MODEL_PREFIX):].replace("_", " ").replace("-", " ").strip() + discovered[model_key] = { + "id": model_key, + "name": _clean_model_name(info.get("name") or fallback_name or model_key).replace(",", " "), + "version": str(info.get("version") or "").strip(), + "series": str(info.get("series") or LOCAL_MODEL_SERIES).strip().replace(",", " "), + "released": str(info.get("released") or "2100-01-01").strip(), + "community_favorite": False, + "artifact_format": UNIFIED_ARTIFACT_FORMAT, + } + + return list(discovered.values()) + def update_model_params(self, model_info: list[dict], manifest_version: str): + model_info = list(model_info) + self._discover_local_models() self.available_models = [str(model.get("id") or "").strip() for model in model_info] self.available_model_names = [_clean_model_name(model.get("name")) for model in model_info] self.model_versions = [str(model.get("version") or "").strip() for model in model_info] @@ -516,6 +569,8 @@ class ModelManager: def _migrate_to_unified_artifacts(self, selected_model: str): removed = 0 for model_file in MODELS_PATH.glob("*_driving_*"): + if is_local_model_key(model_file.name.split("_driving_", 1)[0]): + continue if model_file.is_file() or model_file.is_symlink(): delete_file(model_file, print_error=False) removed += 1 @@ -576,6 +631,14 @@ class ModelManager: self.downloading_model = False return + # Local models have no upstream URL; a download attempt would 404 and then + # delete_file() the artifact on verification failure. + if is_local_model_key(model_to_download): + self.params_memory.put(DOWNLOAD_PROGRESS_PARAM, "Local model, nothing to download.") + self.params_memory.remove(MODEL_DOWNLOAD_PARAM) + self.downloading_model = False + return + repo_url = get_repository_url() if not repo_url: handle_error(None, "GitHub and GitLab are offline...", "Repository unavailable", MODEL_DOWNLOAD_PARAM, DOWNLOAD_PROGRESS_PARAM, self.params_memory) @@ -679,6 +742,9 @@ class ModelManager: handle_error(None, "Download cancelled...", "Download cancelled...", MODEL_DOWNLOAD_ALL_PARAM, DOWNLOAD_PROGRESS_PARAM, self.params_memory) return + if is_local_model_key(model_key): + continue + artifact_format = artifact_format_map.get(model_key, "") if self._is_model_downloaded(model_key, artifact_format): continue @@ -697,6 +763,8 @@ class ModelManager: self.params_memory.put(DOWNLOAD_PROGRESS_PARAM, "Updating...") for model_file in MODELS_PATH.glob("*_driving_*"): + if is_local_model_key(model_file.name.split("_driving_", 1)[0]): + continue if model_file.is_file(): delete_file(model_file, print_error=False)