The smallest promotion

scripts/model_compiler.py

@@ -42,7 +42,9 @@ MEDMODEL_INPUT_SIZE = (512, 256)
 DM_INPUT_SIZE = (1440, 960)
 MODEL_RUN_FREQ = 20
 MODEL_CONTEXT_FREQ = 5
-REPOSITORY_FILE_LIMIT = 100 * 1024 * 1024
+# GitHub/GitLab advertise a 100 MB per-file limit. Use the decimal limit so
+# artifacts such as a 104.4 MB PKL are split before they reach the remote.
+REPOSITORY_FILE_LIMIT = 100_000_000
 DEFAULT_MULTIPART_SIZE = 95 * 1024 * 1024
 USBGPU_PROBE_ATTEMPTS = 10
 USBGPU_PROBE_TIMEOUT = 2
@@ -61,14 +63,15 @@ def build_compile_env(*, supercombo: bool = False) -> dict[str, str]:
   env = os.environ.copy()
   existing_pythonpath = env.get("PYTHONPATH", "")
   env["PYTHONPATH"] = f"{REPO_ROOT}{os.pathsep}{existing_pythonpath}" if existing_pythonpath else str(REPO_ROOT)
-  defaults = {} if supercombo else {
-    "DEBUG": "0",
+  defaults = {
     "FLOAT16": "1",
-    "IMAGE": "2",
+    "IMAGE": "1" if supercombo else "2",
     "JIT_BATCH_SIZE": "0",
     "NOLOCALS": "1",
     "OPENPILOT_HACKS": "1",
-  }
+  } | ({} if supercombo else {
+    "DEBUG": "0",
+  })
   for key, default in defaults.items():
     try:
       int(str(env.get(key)), 0)
@@ -720,7 +723,7 @@ def main() -> int:
   else:
     multipart_outputs = split_oversized_artifact(output)
     if multipart_outputs:
-      print("  artifact exceeds 100 MiB; created repository-safe multipart files:")
+      print("  artifact exceeds 100 MB; created repository-safe multipart files:")
       for multipart_output in multipart_outputs:
         print(f"    {multipart_output.name} ({multipart_output.stat().st_size} bytes)")
       output.unlink()

scripts/model_rebuild_pipeline.py

@@ -14,14 +14,15 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
 if str(REPO_ROOT / "scripts") not in sys.path:
   sys.path.insert(0, str(REPO_ROOT / "scripts"))
 
-from model_compiler import split_oversized_artifact
+from model_compiler import REPOSITORY_FILE_LIMIT, split_oversized_artifact
 
 DEFAULT_OPENPILOT = Path.home() / "openpilot"
 DEFAULT_WORKSPACE = Path("/Volumes/T5/StarPilot-Model-Rebuild-2026-06-22")
 DEFAULT_SOURCE_MAP = REPO_ROOT / "scripts/model_source_map_v22.json"
 DEFAULT_MANIFEST = DEFAULT_WORKSPACE / "manifests/model_names_v22.json"
-REMOTE = "comma@192.168.3.110"
+REMOTE = os.environ.get("STAR_PILOT_MODEL_REMOTE", "comma@192.168.3.109")
 REMOTE_ROOT = Path("/data/openpilot")
+SSH_OPTIONS = ("-o", "ConnectTimeout=10", "-o", "ConnectionAttempts=1")
 
 MODEL_FILENAMES = (
   "driving_supercombo.onnx",
@@ -100,7 +101,17 @@ def resolve_lfs(repo: Path, pointer_path: Path, ref: str, git_path: str) -> None
   oid, expected_size = pointer
   object_path = git_object_path(repo, oid)
   if not object_path.is_file():
-    run(["git", "-C", str(repo), "lfs", "fetch", "origin", ref, "--include", git_path])
+    fetch_refs = [ref]
+    if not ref.startswith("refs/"):
+      fetch_refs.extend((f"refs/remotes/origin/{ref}", f"refs/heads/{ref}"))
+    for fetch_ref in fetch_refs:
+      result = subprocess.run(
+        ["git", "-C", str(repo), "lfs", "fetch", "origin", fetch_ref, "--include", git_path],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+      )
+      if result.returncode == 0 and object_path.is_file():
+        break
   if not object_path.is_file():
     raise FileNotFoundError(f"Missing LFS object {oid} for {ref}:{git_path}")
   if object_path.stat().st_size != expected_size or sha256_file(object_path) != oid:
@@ -198,7 +209,7 @@ def extract_model(model_id: str, source: dict, repo: Path, workspace: Path) -> d
 
 def remote(command: str, *, capture: bool = False):
   result = subprocess.run(
-    ["ssh", REMOTE, command],
+    ["ssh", *SSH_OPTIONS, REMOTE, command],
     check=False,
     text=capture,
     capture_output=capture,
@@ -221,6 +232,38 @@ def stage_ready_artifact(artifact: Path, workspace: Path) -> None:
     ready_path.chmod(0o644)
 
 
+def pull_remote_artifact(remote_output: str, local_output: Path) -> None:
+  """Pull either a single artifact or the compiler's repository-safe parts."""
+  local_output.unlink(missing_ok=True)
+  for stale in local_output.parent.glob(f"{local_output.name}.p[0-9][0-9]"):
+    stale.unlink()
+  local_output.parent.joinpath(f"{local_output.name}.sha256").unlink(missing_ok=True)
+  local_output.parent.mkdir(parents=True, exist_ok=True)
+  run(["rsync", "-az", "-e", "ssh -o ConnectTimeout=10 -o ConnectionAttempts=1", f"{REMOTE}:{remote_output}*", f"{local_output.parent}/"])
+
+  parts = sorted(local_output.parent.glob(f"{local_output.name}.p[0-9][0-9]"))
+  if local_output.is_file():
+    for part in parts:
+      part.unlink()
+    local_output.parent.joinpath(f"{local_output.name}.sha256").unlink(missing_ok=True)
+    return
+  checksum_path = local_output.parent / f"{local_output.name}.sha256"
+  if not parts or not checksum_path.is_file():
+    raise FileNotFoundError(f"Remote compiler returned neither {local_output.name} nor verified parts")
+  with open(local_output, "wb") as destination:
+    for part in parts:
+      with open(part, "rb") as source:
+        shutil.copyfileobj(source, destination, length=1024 * 1024)
+  expected = checksum_path.read_text().split()[0]
+  actual = sha256_file(local_output)
+  if actual != expected:
+    local_output.unlink(missing_ok=True)
+    raise ValueError(f"Reassembled {local_output.name} checksum mismatch: {actual} != {expected}")
+  for part in parts:
+    part.unlink()
+  checksum_path.unlink()
+
+
 def compile_model(model_id: str, source: dict, version: str, workspace: Path, force: bool) -> dict:
   local_output = workspace / "compiled" / f"{model_id}_driving_tinygrad.pkl"
   if local_output.is_file() and not force:
@@ -233,7 +276,7 @@ def compile_model(model_id: str, source: dict, version: str, workspace: Path, fo
   remote_input = f"{REMOTE_ROOT}/uncompiledmodels/{model_id}"
   remote_output = f"{REMOTE_ROOT}/compiledmodels/{model_id}_driving_tinygrad.pkl"
   remote(f"rm -rf {remote_input} && mkdir -p {remote_input} {REMOTE_ROOT}/compiledmodels")
-  run(["rsync", "-az", "--exclude=._*", f"{source_dir}/", f"{REMOTE}:{remote_input}/"])
+  run(["rsync", "-az", "-e", "ssh -o ConnectTimeout=10 -o ConnectionAttempts=1", "--exclude=._*", f"{source_dir}/", f"{REMOTE}:{remote_input}/"])
 
   log_path = workspace / "logs" / f"{model_id}.log"
   command_parts = [
@@ -245,11 +288,11 @@ def compile_model(model_id: str, source: dict, version: str, workspace: Path, fo
     command_parts.append("--external-gpu")
   command = " ".join(command_parts)
   with open(log_path, "wb") as log_file:
-    process = subprocess.run(["ssh", REMOTE, command], stdout=log_file, stderr=subprocess.STDOUT)
+    process = subprocess.run(["ssh", *SSH_OPTIONS, REMOTE, command], stdout=log_file, stderr=subprocess.STDOUT)
   if process.returncode != 0:
     raise RuntimeError(f"Compilation failed for {model_id}; see {log_path}")
 
-  run(["rsync", "-az", f"{REMOTE}:{remote_output}", str(local_output)])
+  pull_remote_artifact(remote_output, local_output)
   local_output.chmod(0o644)
   stage_ready_artifact(local_output, workspace)
   result = artifact_result(model_id, local_output, "compiled")
@@ -264,7 +307,7 @@ def artifact_result(model_id: str, path: Path, status: str) -> dict:
     "path": str(path),
     "size": path.stat().st_size,
     "sha256": sha256_file(path),
-    "multipart": path.stat().st_size > 100 * 1024 * 1024,
+    "multipart": path.stat().st_size > REPOSITORY_FILE_LIMIT,
   }
 
 
@@ -272,10 +315,11 @@ def validate_model(model_id: str, version: str, workspace: Path) -> dict:
   artifact = workspace / "compiled" / f"{model_id}_driving_tinygrad.pkl"
   if not artifact.is_file():
     raise FileNotFoundError(artifact)
-  run(["rsync", "-az", str(artifact), f"{REMOTE}:/data/models/{artifact.name}"])
+  run(["rsync", "-az", "-e", "ssh -o ConnectTimeout=10 -o ConnectionAttempts=1", str(artifact), f"{REMOTE}:/data/models/{artifact.name}"])
   run([
     "rsync",
     "-az",
+    "-e", "ssh -o ConnectTimeout=10 -o ConnectionAttempts=1",
     str(REPO_ROOT / "scripts/validate_model_artifact.py"),
     f"{REMOTE}:{REMOTE_ROOT}/scripts/validate_model_artifact.py",
   ])
@@ -313,7 +357,7 @@ def update_manifest(base_manifest: Path, workspace: Path, source_map: dict) -> d
     model.pop("artifact_size", None)
     model.pop("artifact_sha256", None)
     model.pop("artifact_urls", None)
-    if not artifact.is_file() or artifact.stat().st_size <= 100 * 1024 * 1024:
+    if not artifact.is_file() or artifact.stat().st_size <= REPOSITORY_FILE_LIMIT:
       model.pop("artifact_url", None)
     else:
       multipart_handoff.append({

scripts/model_source_map_v23.json

@@ -0,0 +1,12 @@
+{
+  "karnbir": {"ref": "c0fbd2b13030df95ad249b6e253156a4d7ddb7bb", "input_format": "supercombo"},
+  "karnbir2": {"ref": "682d33d0414d7cef5a7286c24298003503ea13f4", "input_format": "supercombo"},
+  "michael-rl": {"ref": "968fd3c98ed7f5b0b4ef0ea1cf4c58a7c3e4b77c", "input_format": "supercombo"},
+  "michael-rl2": {"ref": "37b38bf738edfb1daa6875255f581e5fc9b0a258", "input_format": "supercombo"},
+  "tobyrl": {"ref": "2d4acf10e34c4c8e68043d60b9437f42d89ca746", "input_format": "supercombo"},
+  "nopp": {"ref": "c740fe5f58faefcb9184c6f9f1fb45130b83cfe8", "input_format": "supercombo"},
+  "drl": {"ref": "a881acfbdc7b3e5d5e1ff0b1ee1a8dcf43b11f3b", "input_format": "supercombo"},
+  "deeprl33": {"ref": "4604c98f21c9a22230e417d17d03c402b9b5f869", "input_format": "supercombo"},
+  "rl34": {"ref": "4e6b07240f5f1b3d9a4b182e7d4fb42ae2bd37e5", "input_format": "supercombo"},
+  "gyhu": {"ref": "574735edc6e1aafdc2a69395f9a32e7f5cc4b62b", "input_format": "supercombo"}
+}

scripts/namespace_model_artifacts.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""Publish rebuilt artifacts under a manifest namespace without changing sources."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def load_manifest(path: Path) -> tuple[dict, list[dict]]:
+  payload = json.loads(path.read_text(encoding="utf-8"))
+  if not isinstance(payload, dict) or not isinstance(payload.get("models"), list):
+    raise ValueError(f"Expected an object containing a models list: {path}")
+  return payload, payload["models"]
+
+
+def renamed_filename(filename: str, old_id: str, new_id: str) -> str:
+  old_prefix = f"{old_id}_driving_tinygrad.pkl"
+  if filename == old_prefix or filename.startswith(f"{old_prefix}."):
+    return f"{new_id}_driving_tinygrad.pkl{filename[len(old_prefix):]}"
+  return filename
+
+
+def rename_model_files(directory: Path, old_id: str, new_id: str) -> None:
+  if not directory.is_dir():
+    return
+
+  candidates = [
+    path for path in directory.iterdir()
+    if renamed_filename(path.name, old_id, new_id) != path.name
+  ]
+  if not candidates:
+    return
+
+  staged: list[tuple[Path, Path]] = []
+  for source in candidates:
+    temporary = source.with_name(f".{source.name}.namespace-tmp")
+    if temporary.exists():
+      temporary.unlink()
+    source.rename(temporary)
+    staged.append((temporary, directory / renamed_filename(source.name, old_id, new_id)))
+
+  for temporary, destination in staged:
+    if destination.exists():
+      destination.unlink()
+    temporary.rename(destination)
+
+  checksum = directory / f"{new_id}_driving_tinygrad.pkl.sha256"
+  if checksum.is_file():
+    text = checksum.read_text(encoding="utf-8")
+    text = text.replace(f"{old_id}_driving_tinygrad.pkl", f"{new_id}_driving_tinygrad.pkl")
+    checksum.write_text(text, encoding="utf-8")
+
+
+def remap_multipart_handoff(path: Path, id_map: dict[str, str]) -> None:
+  if not path.is_file():
+    return
+  payload = json.loads(path.read_text(encoding="utf-8"))
+  if not isinstance(payload, list):
+    raise ValueError(f"Expected a multipart handoff list: {path}")
+  for entry in payload:
+    if not isinstance(entry, dict):
+      continue
+    old_id = str(entry.get("id") or "")
+    new_id = id_map.get(old_id)
+    if not new_id:
+      continue
+    entry["id"] = new_id
+    for key in ("filename",):
+      if entry.get(key):
+        entry[key] = renamed_filename(str(entry[key]), old_id, new_id)
+    entry["parts"] = [renamed_filename(str(part), old_id, new_id) for part in entry.get("parts", [])]
+  path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+
+
+def build_id_map(models: list[dict], suffix: str, manifest_version: str) -> dict[str, str]:
+  """Append the namespace suffix without colliding with an existing source ID."""
+  source_ids = [str(model["id"]) for model in models]
+  source_id_set = set(source_ids)
+  id_map: dict[str, str] = {}
+  for old_id in source_ids:
+    candidate = f"{old_id}{suffix}"
+    if candidate in source_id_set:
+      candidate = f"{old_id}{manifest_version}"
+    while candidate in id_map.values():
+      candidate += "_"
+    id_map[old_id] = candidate
+  return id_map
+
+
+def namespace_workspace(
+  workspace: Path,
+  base_manifest: Path,
+  manifest_version: str,
+  suffix: str,
+  completed_ids: list[str] | None = None,
+) -> Path:
+  payload, models = load_manifest(base_manifest)
+  if not suffix or any(not str(model.get("id") or "") for model in models):
+    raise ValueError("Every manifest model must have an ID and the suffix must be non-empty")
+
+  id_map = build_id_map(models, suffix, manifest_version)
+  if len(set(id_map.values())) != len(id_map):
+    raise ValueError("The requested model namespace contains ID collisions")
+
+  compiled = workspace / "compiled"
+  ready = workspace / "ready-for-resources"
+  ids_to_namespace = set(completed_ids) if completed_ids is not None else set(id_map)
+  unknown_ids = sorted(ids_to_namespace - set(id_map))
+  if unknown_ids:
+    raise KeyError(f"IDs are not present in the base manifest: {', '.join(unknown_ids)}")
+  missing = [
+    old_id for old_id in ids_to_namespace
+    if not (compiled / f"{old_id}_driving_tinygrad.pkl").is_file()
+    and not (compiled / f"{id_map[old_id]}_driving_tinygrad.pkl").is_file()
+  ]
+  if missing:
+    raise FileNotFoundError(f"Missing compiled artifacts for: {', '.join(sorted(missing))}")
+
+  for old_id, new_id in id_map.items():
+    if old_id not in ids_to_namespace:
+      continue
+    rename_model_files(compiled, old_id, new_id)
+    rename_model_files(ready, old_id, new_id)
+
+  namespaced_models = []
+  for model in models:
+    namespaced = dict(model)
+    namespaced["id"] = id_map[str(model["id"])]
+    namespaced_models.append(namespaced)
+
+  output = dict(payload)
+  output["models"] = namespaced_models
+  output_path = workspace / "manifests" / f"model_names_{manifest_version}.json"
+  output_path.parent.mkdir(parents=True, exist_ok=True)
+  output_path.write_text(json.dumps(output, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+  remap_multipart_handoff(ready / "multipart.json", id_map)
+  (workspace / "source-maps" / f"model_id_namespace_{manifest_version}.json").write_text(
+    json.dumps(id_map, indent=2) + "\n", encoding="utf-8",
+  )
+  return output_path
+
+
+def main() -> int:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--workspace", type=Path, required=True)
+  parser.add_argument("--base-manifest", type=Path, required=True)
+  parser.add_argument("--manifest-version", default="v23")
+  parser.add_argument("--suffix", default="3")
+  parser.add_argument(
+    "--completed-ids",
+    nargs="+",
+    help="Only namespace these confirmed rebuilt IDs; still write the full manifest.",
+  )
+  args = parser.parse_args()
+
+  output = namespace_workspace(args.workspace, args.base_manifest, args.manifest_version, args.suffix, args.completed_ids)
+  print(output)
+  return 0
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())

scripts/validate_model_artifact.py

@@ -15,7 +15,6 @@ if str(REPO_ROOT) not in sys.path:
 from tinygrad.tensor import Tensor
 
 from openpilot.common.params import Params
-from openpilot.selfdrive.modeld.compile_modeld import WARP_INPUTS
 from openpilot.selfdrive.modeld.modeld import ModelState
 
 
@@ -48,16 +47,16 @@ def main() -> int:
   if "action_t" in model.npy:
     model.npy["action_t"][:] = [0.15, 0.25]
 
-  img, big_img = model.warp_enqueue(
-    **{key: model.input_queues[key] for key in WARP_INPUTS},
+  warped = model.warp_enqueue(
+    **{key: model.input_queues[key] for key in model.warp_input_keys},
     frame=frames[0],
     big_frame=frames[1],
   )
-  outputs = model.run_policy(
-    **{key: model.input_queues[key] for key in model.policy_input_keys},
-    img=img,
-    big_img=big_img,
-  )
+  policy_inputs = {key: model.input_queues[key] for key in model.policy_input_keys}
+  if model.image_history_pipeline == "policy":
+    outputs = model.run_policy(**policy_inputs, warped=warped)
+  else:
+    outputs = model.run_policy(**policy_inputs, img=warped[0], big_img=warped[1])
   arrays = [output.numpy().flatten() for output in outputs]
   if model.model_type == "supercombo":
     parsed = model.parser.parse_outputs(model.slice_outputs(arrays[0], model.output_slices))