models: add big model ONNX hash tracking (#1982)

* Update Python packages

* bump tg

* bump

* ci: route build_model runner by target_hardware instead of hardcoding chestnut

* hack, remove before merge

* Revert build-model runner hack and uv.lock update

* why were they hard coded

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Jason Wen <haibin.wen3@gmail.com>

.github/workflows/sunnypilot-build-model.yaml

@@ -121,7 +121,7 @@ jobs:
           if-no-files-found: error
 
   build_model:
-    runs-on: [self-hosted, chestnut]
+    runs-on: [self-hosted, "${{ inputs.target_hardware == 'chestnut' && 'chestnut' || 'tici' }}"]
     needs: get_model
     env:
       MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})

openpilot/selfdrive/ui/sunnypilot/mici/onroad/hud_renderer.py

@@ -8,24 +8,17 @@ import pyray as rl
 
 from openpilot.selfdrive.ui.mici.onroad.hud_renderer import HudRenderer
 from openpilot.selfdrive.ui.sunnypilot.onroad.blind_spot_indicators import BlindSpotIndicators
-from openpilot.selfdrive.ui.ui_state import ui_state
 
 
 class HudRendererSP(HudRenderer):
   def __init__(self):
     super().__init__()
     self.blind_spot_indicators = BlindSpotIndicators()
-    self._sp_engaged: bool = False
 
   def _update_state(self) -> None:
     super()._update_state()
     self.blind_spot_indicators.update()
 
-    sp_engaged = ui_state.engaged
-    if sp_engaged != self._sp_engaged:
-      self._chestnut_fade_time = rl.get_time() if sp_engaged else 0
-    self._sp_engaged = sp_engaged
-
   def _render(self, rect: rl.Rectangle) -> None:
     super()._render(rect)
     self.blind_spot_indicators.render(rect)

openpilot/sunnypilot/modeld_v2/modeld.py

@@ -8,22 +8,22 @@ See the LICENSE.md file in the root directory for more details.
 
 import os
 os.environ['GMMU'] = '0'
+import numpy as np
+import threading
+import time
+from setproctitle import setproctitle
+from tinygrad.tensor import Tensor
+
+import openpilot.cereal.messaging as messaging
 from openpilot.common.hardware import COMMA_HARDWARE
 from openpilot.selfdrive.modeld.helpers import chestnut_present, load_oob
-import time
-import numpy as np
-import openpilot.cereal.messaging as messaging
 from openpilot.cereal import log
 from opendbc.car.structs import car
 from openpilot.cereal.services import SERVICE_LIST
-from setproctitle import setproctitle
 from openpilot.cereal.messaging import PubMaster, SubMaster
 from openpilot.cereal.visionipc import VisionStreamType
 from msgq.visionipc import VisionIpcClient, VisionBuf
 from opendbc.car.car_helpers import get_demo_car_params
-
-from tinygrad.tensor import Tensor
-
 from openpilot.common.file_chunker import open_file_chunked
 from openpilot.common.swaglog import cloudlog
 from openpilot.common.params import Params
@@ -42,13 +42,13 @@ from openpilot.sunnypilot.modeld_v2.constants import Plan
 from openpilot.sunnypilot.modeld_v2.meta_helper import load_meta_constants
 from openpilot.sunnypilot.modeld_v2.camera_offset_helper import CameraOffsetHelper
 from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, make_split_input_queues, make_supercombo_input_queues, WARP_INPUTS, POLICY_INPUTS
-
 from openpilot.sunnypilot.livedelay.helpers import get_lat_delay
 from openpilot.sunnypilot.modeld_v2.modeld_base import ModelStateBase
 from openpilot.sunnypilot.models.helpers import get_active_bundle
 from openpilot.sunnypilot.selfdrive.controls.lib.relc import RoadEdgeLaneChangeController
 
 PROCESS_NAME = "openpilot.selfdrive.modeld.modeld_tinygrad"
+BIG_MODEL_TIMEOUT = 60
 
 
 def _pkl_exists(path):
@@ -68,6 +68,7 @@ def _find_driving_pkl(bundle):
   pkl_path = os.path.join(model_root, pkl_name)
   if _pkl_exists(pkl_path):
     return pkl_path
+  return None
 
 
 class FrameMeta:
@@ -102,7 +103,7 @@ class ModelState(ModelStateBase):
     self.chestnut = chestnut
 
     pkl_path = _find_driving_pkl(model_bundle)
-    assert pkl_path is not None, "No driving pkl found — all models must be compiled with compile_modeld.py"
+    assert pkl_path is not None, f"No driving pkl found for {'chestnut' if chestnut else 'small model'} — all models must be compiled with compile_modeld.py"
     self._init_combined(pkl_path, cam_w, cam_h, model_bundle)
 
   def _init_combined(self, pkl_path, cam_w, cam_h, bundle):
@@ -185,9 +186,6 @@ class ModelState(ModelStateBase):
     else:
       self.warp(**{k: self.input_queues[k] for k in WARP_INPUTS}, frame=frame_tensor, big_frame=big_frame_tensor)
 
-    if self.chestnut:
-      self.warmup()
-
   def warmup(self) -> None:
     dummy_frames = {k: np.zeros(self.frame_buf_params[k][3], dtype=np.uint8) for k in self._vision_input_names}
     transforms = {k: np.eye(3, dtype=np.float32) for k in [self._road_key, self._wide_key] if k}
@@ -288,8 +286,7 @@ class ModelState(ModelStateBase):
       buf[0, -1, :] = outputs['desired_curvature'][0, :] if not self.mlsim else 0
 
     if self.chestnut and not np.all(np.isfinite(outputs.get('plan', np.array([0.])))):
-      cloudlog.error("model output not finite, dropping frame")
-      return None
+      raise RuntimeError("model output not finite")
 
     return outputs
 
@@ -363,21 +360,26 @@ def main(demo=False):
 
   model = None
   if CHESTNUT:
-    import threading
-    def load():
-      nonlocal model
-      model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, chestnut=True)
-    t = threading.Thread(target=load, daemon=True)
-    t.start()
-    t.join(60)
-    if model is None:
-      params.put_bool("ChestnutActive", False)
-      raise RuntimeError("chestnut model load failed or timed out (60s)")
-    params.put_bool("ChestnutActive", True)
-  else:
-    model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, chestnut=False)
+    big_model = None
+    def load_big():
+      nonlocal big_model
+      try:
+        m = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, chestnut=True)
+        m.warmup()
+        big_model = m
+      except Exception:
+        cloudlog.exception("chestnut load failed")
+    loader = threading.Thread(target=load_big, daemon=True)
+    loader.start()
+    loader.join(BIG_MODEL_TIMEOUT)
+    model = big_model
+    params.put_bool("ChestnutActive", model is not None)
 
+  small_model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, chestnut=False) if model is None or CHESTNUT else None
+  if model is None:
+    model = small_model
   params.put_bool("ChestnutLoading", False)
+  assert model is not None
   cloudlog.warning(f"models loaded in {time.monotonic() - st:.1f}s, modeld starting")
 
   # messaging
@@ -386,7 +388,7 @@ def main(demo=False):
   sm = SubMaster(["deviceState", "carState", "narrowRoadCameraState", "extrinsicsCalibration", "driverMonitoringState", "carControl", "lateralDelay"])
 
   publish_state = PublishState()
-  chestnut_state = ChestnutState(pm, CHESTNUT) if CHESTNUT else None
+  chestnut_state = ChestnutState(pm, model.chestnut) if CHESTNUT else None
 
   # setup filter to track dropped frames
   frame_dropped_filter = FirstOrderFilter(0., 10., 1. / model.constants.MODEL_FREQ)
@@ -509,7 +511,19 @@ def main(demo=False):
       inputs['action_t'] = np.array([lat_action_t, long_action_t], dtype=np.float32)
 
     mt1 = time.perf_counter()
-    model_output = model.run(bufs, transforms, inputs, prepare_only)
+    try:
+      model_output = model.run(bufs, transforms, inputs, prepare_only)
+    except Exception:
+      if not params.get_bool("ChestnutActive"):
+        raise
+      cloudlog.exception("chestnut failed, falling back to small")
+      params.put_bool("ChestnutActive", False)
+      assert small_model is not None
+      model = small_model
+      if chestnut_state is not None:
+        chestnut_state.big = False
+      run_count = 0
+      model_output = None
     mt2 = time.perf_counter()
     model_execution_time = mt2 - mt1
 

openpilot/sunnypilot/modeld_v2/tests/test_fallback.py

@@ -0,0 +1,62 @@
+"""
+Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
+
+This file is part of sunnypilot and is licensed under the MIT License.
+See the LICENSE.md file in the root directory for more details.
+"""
+
+import io
+import requests
+
+from openpilot.common.file_chunker import get_chunk_name
+from openpilot.common.hardware import hw
+from openpilot.common.test import OpenpilotTestCase
+from openpilot.selfdrive.modeld.helpers import dump_oob
+import openpilot.sunnypilot.modeld_v2.modeld as modeld_module
+from openpilot.sunnypilot.modeld_v2.tests import helpers as tests_helpers
+from openpilot.sunnypilot.modeld_v2.tests.helpers import DummyModel, DummyBundle, CAM_W, CAM_H
+from openpilot.sunnypilot.models.fetcher import ModelParser, ModelFetcher
+
+tmp_path = tests_helpers.tmp_path
+
+
+class TestFallback(OpenpilotTestCase):
+  def test_find_dual_model_in_bundle(self, tmp_path, monkeypatch):
+    lebowski_file = 'driving_lebowski.pkl'
+    tsfdo_file = 'driving_tsfdo.pkl'
+    (tmp_path / lebowski_file).write_bytes(b'fkasdjfkljf')
+    (tmp_path / tsfdo_file).write_bytes(b'dskfajklsdjlsfka')
+
+    monkeypatch.setattr(hw.Paths, 'model_root', staticmethod(lambda: str(tmp_path)))
+    big_bundle = DummyBundle(models=[DummyModel('supercombo', lebowski_file)])
+    small_bundle = DummyBundle(models=[DummyModel('supercombo', tsfdo_file)])
+    big_pkl = modeld_module._find_driving_pkl(big_bundle)
+    small_pkl = modeld_module._find_driving_pkl(small_bundle)
+
+    assert big_pkl is not None and lebowski_file in big_pkl
+    assert small_pkl is not None and tsfdo_file in small_pkl
+
+  def test_download_models_and_init_modelstate_fallback(self, tmp_path, monkeypatch):
+    monkeypatch.setattr(hw.Paths, 'model_root', staticmethod(lambda: str(tmp_path)))
+    big_json = requests.get(ModelFetcher.MODEL_URL_CHESTNUT).json()
+    big_bundle = ModelParser.parse_models(big_json)[-1]
+    small_json = requests.get(ModelFetcher.MODEL_URL).json()
+    small_bundle = ModelParser.parse_models(small_json)[-1]
+
+    buf = io.BytesIO()
+    dump_oob(tests_helpers.make_pkl_data(tests_helpers.ARCHETYPES['supercombo_non20hz']), buf)
+    oob_bytes = buf.getvalue()
+
+    for bundle in (big_bundle, small_bundle):
+      artifact = bundle.models[0].artifact
+      for i in range(len(artifact.chunks)):
+        (tmp_path / get_chunk_name(artifact.fileName, i, len(artifact.chunks))).write_bytes(oob_bytes if i == 0 else b"")
+
+    monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None, *, chestnut=None: small_bundle)
+    assert modeld_module.ModelState(CAM_W, CAM_H, chestnut=False).chestnut is False
+
+    monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None, *, chestnut=None: big_bundle)
+    try:
+      assert modeld_module.ModelState(CAM_W, CAM_H, chestnut=True).chestnut is True
+    except Exception as e:
+      assert "AMD" in str(e) or "device" in str(e).lower()

openpilot/sunnypilot/models/default_model.py

@@ -1,11 +1,14 @@
 import argparse
 import os
 import hashlib
+import requests
+import re
 
 from openpilot.common.basedir import BASEDIR
 from openpilot.selfdrive.ui.ui_state import ui_state
 from openpilot.sunnypilot import get_file_hash
 from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL, DEFAULT_BIG_MODEL
+from openpilot.sunnypilot.models.fetcher import ModelFetcher
 
 
 def get_default_model() -> str:
@@ -17,7 +20,9 @@ def get_default_model() -> str:
 
 DEFAULT_MODEL_NAME_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "model_name.py")
 MODEL_HASH_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "tests", "model_hash")
+BIG_MODEL_HASH_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "tests", "big_model_hash")
 SUPERCOMBO_ONNX_PATH = os.path.join(BASEDIR, "openpilot", "selfdrive", "modeld", "models", "driving_supercombo.onnx")
+BIG_SUPERCOMBO_ONNX_PATH = os.path.join(BASEDIR, "openpilot", "selfdrive", "modeld", "models", "big_driving_supercombo.onnx")
 
 
 def update_model_hash():
@@ -29,15 +34,42 @@ def update_model_hash():
 
   print(f"Generated and updated new combined model hash to {MODEL_HASH_PATH}")
 
+  if os.path.exists(BIG_SUPERCOMBO_ONNX_PATH):
+    import subprocess
+    rel = os.path.relpath(BIG_SUPERCOMBO_ONNX_PATH, os.getcwd())
+    pointer = subprocess.check_output(["git", "show", f"HEAD:{rel}"], text=True)
+    oid = next(l.split(":", 1)[1] for l in pointer.splitlines() if l.startswith("oid sha256:"))
+    big_combined_hash = hashlib.sha256(oid.encode()).hexdigest()
+
+    with open(BIG_MODEL_HASH_PATH, "w") as f:
+      f.write(big_combined_hash)
+
+    print(f"Generated and updated new big model hash to {BIG_MODEL_HASH_PATH}")
+
+
+def get_ref_for_name(url: str, name: str) -> str:
+  response = requests.get(url, timeout=10)
+  if response.status_code == 200:
+    bundles = response.json()["bundles"]
+    matching = [b for b in bundles if re.search(name, f"{b['short_name']} {b['display_name']}", re.IGNORECASE)]
+    if matching:
+      return max(matching, key=lambda b: int(b["index"]))["ref"]
+  return ""
+
 
 def update_default_model_names(default_model_name: str, default_big_model_name: str):
   print("[CHANGE DEFAULT MODEL NAMES]")
+  small_ref = get_ref_for_name(ModelFetcher.MODEL_URL, default_model_name)
+  big_ref = get_ref_for_name(ModelFetcher.MODEL_URL_CHESTNUT, default_big_model_name)
+
   with open(DEFAULT_MODEL_NAME_PATH, "w") as f:
     f.write(f'DEFAULT_MODEL = "{default_model_name}"\n')
+    f.write(f'DEFAULT_MODEL_REF = "{small_ref}"\n')
     f.write(f'DEFAULT_BIG_MODEL = "{default_big_model_name}"\n')
+    f.write(f'DEFAULT_BIG_MODEL_REF = "{big_ref}"\n')
 
-  print(f'New default small model name: "{default_model_name}"')
-  print(f'New default big model name: "{default_big_model_name}"')
+  print(f'New default small model name: "{default_model_name}" (ref: {small_ref})')
+  print(f'New default big model name: "{default_big_model_name}" (ref: {big_ref})')
   print("[DONE]")
 
 

openpilot/sunnypilot/models/fetcher.py

@@ -138,8 +138,8 @@ class ModelCache:
 
 class ModelFetcher:
   """Handles fetching and caching of model data from remote source"""
-  MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v21.json"
-  MODEL_URL_CHESTNUT = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_chestnut_v22.json"
+  MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v22.json"
+  MODEL_URL_CHESTNUT = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_chestnut_v23.json"
 
   MODEL_SOURCES = {
     "qcom": (MODEL_URL, ""),

openpilot/sunnypilot/models/helpers.py

@@ -19,7 +19,7 @@ from openpilot.common.hardware.hw import Paths
 from openpilot.selfdrive.modeld.helpers import chestnut_present
 
 # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO
-REQUIRED_JSON_VERSION = 18
+REQUIRED_JSON_VERSION = 19
 
 CUSTOM_MODEL_PATH = Paths.model_root()
 METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl'

openpilot/sunnypilot/models/manager.py

@@ -328,6 +328,12 @@ class ModelManagerSP:
         validate_active_bundles(self.params, self.source_models)
         self.active_bundle = get_active_bundle(self.params, chestnut=self.chestnut_present)
 
+        if get_selected_bundle(self.params, "chestnut") is not None and get_selected_bundle(self.params, "qcom") is None:
+          if self.params.get("ModelManager_DownloadRef") is None:
+            from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL_REF
+            if DEFAULT_MODEL_REF:
+              self.params.put("ModelManager_DownloadRef", DEFAULT_MODEL_REF)
+
         self._process_download_requests()
 
         if self.params.get("ModelManager_ClearCache"):

openpilot/sunnypilot/models/model_name.py

@@ -1,2 +1,4 @@
 DEFAULT_MODEL = "CD210"
+DEFAULT_MODEL_REF = "5b6436a90cf6902b8aaa71c2b6f3d7164d8ae391"
 DEFAULT_BIG_MODEL = "Lebowski"
+DEFAULT_BIG_MODEL_REF = "fa0c6876d3cf070e91e25e5353ceadc68a5b3285"

openpilot/sunnypilot/models/tests/big_model_hash

@@ -0,0 +1 @@
+8bba37156aa17d49210cad028744c839ea9ed7b1f19428a8ecfafdc1e07a73b6

openpilot/sunnypilot/models/tests/test_default_model.py

@@ -5,12 +5,25 @@ This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
 
+import os
+import subprocess
+
 from openpilot.sunnypilot import get_file_hash
-from openpilot.sunnypilot.models.default_model import MODEL_HASH_PATH, SUPERCOMBO_ONNX_PATH
+from openpilot.sunnypilot.models.default_model import MODEL_HASH_PATH, SUPERCOMBO_ONNX_PATH, BIG_MODEL_HASH_PATH, \
+                                                      BIG_SUPERCOMBO_ONNX_PATH
 import hashlib
 from openpilot.common.test import OpenpilotTestCase
 
 
+def _get_lfs_oid(path: str) -> str:
+  """Extract the LFS OID (SHA256 of actual content) from git, works whether the file is smudged or not."""
+  pointer = subprocess.check_output(["git", "show", f"HEAD:{path}"], text=True)
+  for line in pointer.splitlines():
+    if line.startswith("oid sha256:"):
+      return line.split(":", 1)[1]
+  raise ValueError(f"No LFS OID found for {path}")
+
+
 class TestDefaultModel(OpenpilotTestCase):
   def test_compare_onnx_hashes(self):
     supercombo_hash = get_file_hash(SUPERCOMBO_ONNX_PATH)
@@ -21,3 +34,15 @@ class TestDefaultModel(OpenpilotTestCase):
       current_hash = f.read().strip()
 
     assert combined_hash == current_hash, "Run openpilot/sunnypilot/models/default_model.py to update the default model name and hash"
+
+  def test_compare_big_onnx_hashes(self):
+    if not os.path.exists(BIG_SUPERCOMBO_ONNX_PATH):
+      self.skipTest("big_driving_supercombo.onnx not present")
+
+    oid = _get_lfs_oid(os.path.relpath(BIG_SUPERCOMBO_ONNX_PATH, os.getcwd()))
+    combined_hash = hashlib.sha256(oid.encode()).hexdigest()
+
+    with open(BIG_MODEL_HASH_PATH) as f:
+      current_hash = f.read().strip()
+
+    assert combined_hash == current_hash, "Run openpilot/sunnypilot/models/default_model.py to update the default model name and hash"

openpilot/sunnypilot/models/tests/test_manager_download.py

@@ -386,7 +386,7 @@ class TestManagerDownload(ManagerDownloadTestBase):
     def body():
       artifact = self.make_artifact(chunked=True)
       self._bundle.ref = "test-ref"
-      self._bundle.minimumSelectorVersion = 18
+      self._bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
       params, store = self._make_params_with_store()
       self.manager.params = params
       asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "qcom"))
@@ -406,7 +406,7 @@ class TestManagerDownload(ManagerDownloadTestBase):
     def body():
       self.make_artifact(chunked=True)
       self._bundle.ref = "big-ref"
-      self._bundle.minimumSelectorVersion = 18
+      self._bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
       params, store = self._make_params_with_store()
       self.manager.params = params
       asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "chestnut"))
@@ -469,7 +469,7 @@ def manifest_bundle(short_name: str, ref: str, index: int = 0, is_big: bool = Fa
     "environment": "release",
     "runner": "tinygrad",
     "is_big": is_big,
-    "minimum_selector_version": "18",
+    "minimum_selector_version": str(helpers.REQUIRED_JSON_VERSION),
     "ref": ref,
     "models": [{
       "type": "supercombo",
@@ -655,7 +655,7 @@ class TestActiveBundleValidation(OpenpilotTestCase):
   def _raw_bundle(ref: str, runner: int | None = None) -> dict:
     bundle = custom.ModelManagerSP.ModelBundle.new_message()
     bundle.ref = ref
-    bundle.minimumSelectorVersion = 18
+    bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
     if runner is not None:
       bundle.runner = runner
     return bundle.to_dict()
@@ -697,7 +697,7 @@ class TestActiveBundleSelection(OpenpilotTestCase):
   def _raw_bundle(ref: str) -> dict:
     bundle = custom.ModelManagerSP.ModelBundle.new_message()
     bundle.ref = ref
-    bundle.minimumSelectorVersion = 18
+    bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
     return bundle.to_dict()
 
   def _params(self, qcom=None, chestnut=None):
@@ -744,7 +744,7 @@ class TestEffectiveSource(OpenpilotTestCase):
   def _raw_bundle(ref: str) -> dict:
     bundle = custom.ModelManagerSP.ModelBundle.new_message()
     bundle.ref = ref
-    bundle.minimumSelectorVersion = 18
+    bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
     return bundle.to_dict()
 
   def test_runtime_no_gpu(self):