models: add big model ONNX hash tracking (#1982)

* Update Python packages

* bump tg

* bump

* ci: route build_model runner by target_hardware instead of hardcoding chestnut

* hack, remove before merge

* Revert build-model runner hack and uv.lock update

* why were they hard coded

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Jason Wen <haibin.wen3@gmail.com>

openpilot/common/params_keys.h

@@ -132,7 +132,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
     {"ChestnutActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
     {"ChestnutLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
-    {"AuxPowerSave", {PERSISTENT | BACKUP, BOOL}},
     {"Version", {PERSISTENT, STRING}},
 
     // --- sunnypilot params --- //

openpilot/selfdrive/ui/ui_state.py

@@ -220,7 +220,6 @@ class UIState(UIStateSP):
                              ChestnutState.UNCOMPILED if detected else ChestnutState.DISCONNECTED)
       return
 
-    self.chestnut_present = self.chestnut_present or detected
     model_seen = self.sm.recv_frame["modelV2"] > self.started_frame
     if not self.chestnut_present:
       self.chestnut_state = ChestnutState.DISCONNECTED

openpilot/sunnypilot/models/default_model.py

@@ -20,7 +20,9 @@ def get_default_model() -> str:
 
 DEFAULT_MODEL_NAME_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "model_name.py")
 MODEL_HASH_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "tests", "model_hash")
+BIG_MODEL_HASH_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "tests", "big_model_hash")
 SUPERCOMBO_ONNX_PATH = os.path.join(BASEDIR, "openpilot", "selfdrive", "modeld", "models", "driving_supercombo.onnx")
+BIG_SUPERCOMBO_ONNX_PATH = os.path.join(BASEDIR, "openpilot", "selfdrive", "modeld", "models", "big_driving_supercombo.onnx")
 
 
 def update_model_hash():
@@ -32,6 +34,18 @@ def update_model_hash():
 
   print(f"Generated and updated new combined model hash to {MODEL_HASH_PATH}")
 
+  if os.path.exists(BIG_SUPERCOMBO_ONNX_PATH):
+    import subprocess
+    rel = os.path.relpath(BIG_SUPERCOMBO_ONNX_PATH, os.getcwd())
+    pointer = subprocess.check_output(["git", "show", f"HEAD:{rel}"], text=True)
+    oid = next(l.split(":", 1)[1] for l in pointer.splitlines() if l.startswith("oid sha256:"))
+    big_combined_hash = hashlib.sha256(oid.encode()).hexdigest()
+
+    with open(BIG_MODEL_HASH_PATH, "w") as f:
+      f.write(big_combined_hash)
+
+    print(f"Generated and updated new big model hash to {BIG_MODEL_HASH_PATH}")
+
 
 def get_ref_for_name(url: str, name: str) -> str:
   response = requests.get(url, timeout=10)

openpilot/sunnypilot/models/fetcher.py

@@ -138,8 +138,8 @@ class ModelCache:
 
 class ModelFetcher:
   """Handles fetching and caching of model data from remote source"""
-  MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v21.json"
-  MODEL_URL_CHESTNUT = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_chestnut_v22.json"
+  MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v22.json"
+  MODEL_URL_CHESTNUT = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_chestnut_v23.json"
 
   MODEL_SOURCES = {
     "qcom": (MODEL_URL, ""),

openpilot/sunnypilot/models/helpers.py

@@ -19,7 +19,7 @@ from openpilot.common.hardware.hw import Paths
 from openpilot.selfdrive.modeld.helpers import chestnut_present
 
 # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO
-REQUIRED_JSON_VERSION = 18
+REQUIRED_JSON_VERSION = 19
 
 CUSTOM_MODEL_PATH = Paths.model_root()
 METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl'

openpilot/sunnypilot/models/tests/big_model_hash

@@ -0,0 +1 @@
+8bba37156aa17d49210cad028744c839ea9ed7b1f19428a8ecfafdc1e07a73b6

openpilot/sunnypilot/models/tests/test_default_model.py

@@ -5,12 +5,25 @@ This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
 
+import os
+import subprocess
+
 from openpilot.sunnypilot import get_file_hash
-from openpilot.sunnypilot.models.default_model import MODEL_HASH_PATH, SUPERCOMBO_ONNX_PATH
+from openpilot.sunnypilot.models.default_model import MODEL_HASH_PATH, SUPERCOMBO_ONNX_PATH, BIG_MODEL_HASH_PATH, \
+                                                      BIG_SUPERCOMBO_ONNX_PATH
 import hashlib
 from openpilot.common.test import OpenpilotTestCase
 
 
+def _get_lfs_oid(path: str) -> str:
+  """Extract the LFS OID (SHA256 of actual content) from git, works whether the file is smudged or not."""
+  pointer = subprocess.check_output(["git", "show", f"HEAD:{path}"], text=True)
+  for line in pointer.splitlines():
+    if line.startswith("oid sha256:"):
+      return line.split(":", 1)[1]
+  raise ValueError(f"No LFS OID found for {path}")
+
+
 class TestDefaultModel(OpenpilotTestCase):
   def test_compare_onnx_hashes(self):
     supercombo_hash = get_file_hash(SUPERCOMBO_ONNX_PATH)
@@ -21,3 +34,15 @@ class TestDefaultModel(OpenpilotTestCase):
       current_hash = f.read().strip()
 
     assert combined_hash == current_hash, "Run openpilot/sunnypilot/models/default_model.py to update the default model name and hash"
+
+  def test_compare_big_onnx_hashes(self):
+    if not os.path.exists(BIG_SUPERCOMBO_ONNX_PATH):
+      self.skipTest("big_driving_supercombo.onnx not present")
+
+    oid = _get_lfs_oid(os.path.relpath(BIG_SUPERCOMBO_ONNX_PATH, os.getcwd()))
+    combined_hash = hashlib.sha256(oid.encode()).hexdigest()
+
+    with open(BIG_MODEL_HASH_PATH) as f:
+      current_hash = f.read().strip()
+
+    assert combined_hash == current_hash, "Run openpilot/sunnypilot/models/default_model.py to update the default model name and hash"

openpilot/sunnypilot/models/tests/test_manager_download.py

@@ -386,7 +386,7 @@ class TestManagerDownload(ManagerDownloadTestBase):
     def body():
       artifact = self.make_artifact(chunked=True)
       self._bundle.ref = "test-ref"
-      self._bundle.minimumSelectorVersion = 18
+      self._bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
       params, store = self._make_params_with_store()
       self.manager.params = params
       asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "qcom"))
@@ -406,7 +406,7 @@ class TestManagerDownload(ManagerDownloadTestBase):
     def body():
       self.make_artifact(chunked=True)
       self._bundle.ref = "big-ref"
-      self._bundle.minimumSelectorVersion = 18
+      self._bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
       params, store = self._make_params_with_store()
       self.manager.params = params
       asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "chestnut"))
@@ -469,7 +469,7 @@ def manifest_bundle(short_name: str, ref: str, index: int = 0, is_big: bool = Fa
     "environment": "release",
     "runner": "tinygrad",
     "is_big": is_big,
-    "minimum_selector_version": "18",
+    "minimum_selector_version": str(helpers.REQUIRED_JSON_VERSION),
     "ref": ref,
     "models": [{
       "type": "supercombo",
@@ -655,7 +655,7 @@ class TestActiveBundleValidation(OpenpilotTestCase):
   def _raw_bundle(ref: str, runner: int | None = None) -> dict:
     bundle = custom.ModelManagerSP.ModelBundle.new_message()
     bundle.ref = ref
-    bundle.minimumSelectorVersion = 18
+    bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
     if runner is not None:
       bundle.runner = runner
     return bundle.to_dict()
@@ -697,7 +697,7 @@ class TestActiveBundleSelection(OpenpilotTestCase):
   def _raw_bundle(ref: str) -> dict:
     bundle = custom.ModelManagerSP.ModelBundle.new_message()
     bundle.ref = ref
-    bundle.minimumSelectorVersion = 18
+    bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
     return bundle.to_dict()
 
   def _params(self, qcom=None, chestnut=None):
@@ -744,7 +744,7 @@ class TestEffectiveSource(OpenpilotTestCase):
   def _raw_bundle(ref: str) -> dict:
     bundle = custom.ModelManagerSP.ModelBundle.new_message()
     bundle.ref = ref
-    bundle.minimumSelectorVersion = 18
+    bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
     return bundle.to_dict()
 
   def test_runtime_no_gpu(self):

openpilot/sunnypilot/sunnylink/settings_ui.json

@@ -1675,12 +1675,6 @@
               "widget": "toggle",
               "title": "Onroad Uploads"
             },
-            {
-              "key": "AuxPowerSave",
-              "widget": "toggle",
-              "title": "Disable Aux Port When Offroad",
-              "description": "Power off the aux USB-C port while offroad to save power. It powers back on automatically when you go onroad."
-            },
             {
               "key": "MaxTimeOffroad",
               "widget": "option",

openpilot/sunnypilot/sunnylink/settings_ui_src/pages/device.yaml

@@ -30,10 +30,6 @@ sections:
   - key: OnroadUploads
     widget: toggle
     title: Onroad Uploads
-  - key: AuxPowerSave
-    widget: toggle
-    title: Disable Aux Port When Offroad
-    description: Power off the aux USB-C port while offroad to save power. It powers back on automatically when you go onroad.
   - key: MaxTimeOffroad
     widget: option
     title: Max Time Offroad

openpilot/system/hardware/hardwared.py

@@ -21,7 +21,6 @@ from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
 from openpilot.common.hardware import HARDWARE, COMMA_HARDWARE
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.hardware.usb import CHESTNUT_FW_VERSION, CHESTNUT_ROM_USB_IDS, CHESTNUT_USB_IDS, get_usb_state, get_usb_topology, set_usb_state
-from openpilot.system.hardware.chestnut.flash import VBUS_PATH
 from openpilot.common.linux import LinuxSystemStats
 from openpilot.system.loggerd.config import get_available_percent
 from openpilot.common.swaglog import cloudlog
@@ -50,10 +49,6 @@ class Chestnut:
     self.attempts = 0
     self.last_attempt = 0.
     self.flashed = False
-    self.vbus_on = None
-    self.params = Params()
-    self.powersave = False
-    self.last_offroad = None
 
   def flash(self) -> None:
     ret = subprocess.run(["sudo", sys.executable, os.path.join(BASEDIR, "openpilot/system/hardware/chestnut/flash.py"), CHESTNUT_FW_VERSION],
@@ -61,19 +56,9 @@ class Chestnut:
     cloudlog.event("chestnut flash done", returncode=ret.returncode, output=ret.stdout[-1000:], error=ret.returncode != 0)
     self.flashed = ret.returncode == 0
 
-  def set_vbus(self, on: bool) -> None:
-    if on == self.vbus_on:
-      return
-    subprocess.run(["sudo", "tee", VBUS_PATH], input=b"1" if on else b"0", stdout=subprocess.DEVNULL, check=False)
-    self.vbus_on = on
-
   def update(self, offroad: bool, usb_state: list[dict]) -> None:
     mismatch = any((d["vendorId"], d["productId"]) in CHESTNUT_USB_IDS + CHESTNUT_ROM_USB_IDS and
                    d["product"] != f"custom {CHESTNUT_FW_VERSION}-CLEAN" for d in usb_state)
-    if offroad != self.last_offroad:
-      self.powersave = self.params.get_bool("AuxPowerSave")
-      self.last_offroad = offroad
-    self.set_vbus((not offroad or mismatch) or not self.powersave)
     if not mismatch:
       self.flashed = False
       return