init

* helpers.py get_lat_delay fix

* fix trailing whitespace

* lint

---------

Co-authored-by: Nayan <nayan8teen@gmail.com>

.github/workflows/sunnypilot-build-model.yaml

@@ -103,10 +103,10 @@ jobs:
       - run: |
           cd ${{ github.workspace }}/openpilot/openpilot
           if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then
-            git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx" -X "**/selfdrive/modeld/models/dmonitoring_*.onnx"
+            git lfs pull -X "selfdrive/modeld/models/big_*.onnx" -X "selfdrive/modeld/models/dmonitoring_*.onnx"
             rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx
           else
-            git lfs pull -I "**/selfdrive/modeld/models/big_*.onnx"
+            git lfs pull -I "selfdrive/modeld/models/big_*.onnx"
             find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete
           fi
       - name: 'Upload Artifact'
@@ -116,7 +116,7 @@ jobs:
           path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
 
   build_model:
-    runs-on: [self-hosted, usbgpu]
+    runs-on: [self-hosted, tici]
     needs: get_model
     env:
       MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
@@ -127,6 +127,7 @@ jobs:
           fetch-depth: 1
           submodules: recursive
 
+      - run: git lfs pull
 
       - name: Set environment variables
         id: set-env
@@ -159,7 +160,7 @@ jobs:
           fi
           source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
-          rm -rf ${{ env.MODELS_DIR }}/*.onnx*
+          rm -rf ${{ env.MODELS_DIR }}/*.onnx
 
       - name: Download model artifacts
         uses: actions/download-artifact@v4
@@ -179,7 +180,6 @@ jobs:
           MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
           CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
 
-          TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
           if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
             echo "USBGPU build"
             export USBGPU=1
@@ -187,40 +187,27 @@ jobs:
             OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
           else
             echo "QCOM build"
-            TG_FLAGS="$TG_FLAGS_QCOM"
+            TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
             OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
           fi
 
           # Generate metadata for all ONNX files
           find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
             echo "Generating metadata: $onnx_file"
-            env ${TG_FLAGS_QCOM} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
+            env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
           done
 
           # Detect model type and build compile args
-          VISION_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do
-            [ -f "$f" ] && VISION_ONNX="$f" && break
-          done
-
-          POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do
-            [ -f "$f" ] && POLICY_ONNX="$f" && break
-          done
-
-          OFF_POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do
-            [ -f "$f" ] && OFF_POLICY_ONNX="$f" && break
-          done
-
-          ON_POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do
-            [ -f "$f" ] && ON_POLICY_ONNX="$f" && break
-          done
-
+          VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
+          POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
+          OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
+          ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
           SUPERCOMBO_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do
-            [ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break
+          for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do
+            if [ -f "$f" ]; then
+              SUPERCOMBO_ONNX="$f"
+              break
+            fi
           done
 
           MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""

openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py

@@ -4,11 +4,13 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
 This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
+from collections.abc import Callable
 import pyray as rl
 
 from openpilot.cereal import custom
 from openpilot.sunnypilot.models.default_model import DEFAULT_MODEL
 from openpilot.selfdrive.ui.mici.widgets.button import BigButton
+from openpilot.selfdrive.ui.mici.widgets.dialog import BigConfirmationDialog
 from openpilot.selfdrive.ui.sunnypilot.layouts.settings.models import ModelsLayout
 from openpilot.selfdrive.ui.ui_state import ui_state, device
 from openpilot.system.ui.lib.application import FontWeight, gui_app
@@ -46,6 +48,71 @@ class CurrentModelInfo(Widget):
     self.info_text.set_position(self._rect.x + 20, self._rect.y + 161 - 25)
     self.info_text.render()
 
+
+class FolderSelectionMici(NavScroller):
+  def __init__(self, select_default_cb: Callable, select_folder_cb: Callable):
+    super().__init__()
+
+    favs = ui_state.params.get("ModelManager_Favs")
+    favorites = set(favs.split(';')) if favs else set()
+
+    manager = ui_state.sm["modelManagerSP"]
+    bundles = manager.availableBundles
+    folders = {}
+    for bundle in bundles:
+      folder = next((override.value for override in bundle.overrides if override.key == "folder"), "")
+      folders.setdefault(folder, []).append(bundle)
+
+    if favorites:
+      for fav_bundle in [bundle for bundle in bundles if bundle.ref in favorites]:
+        folders.setdefault("favorites", []).append(fav_bundle)
+
+    folder_buttons = []
+    default_btn = BigButton(f"{DEFAULT_MODEL} (Default)".lower())
+    default_btn.set_click_callback(select_default_cb)
+    folder_buttons.append(default_btn)
+
+    for folder in sorted(folders.keys(), key=lambda f: max((bundle.index for bundle in folders[f]), default=-1), reverse=True):
+      btn = BigButton(folder.lower())
+      btn.set_click_callback(lambda f=folder: select_folder_cb(f))
+      if folder.lower() == "favorites":
+        folder_buttons.insert(0, btn)
+      else:
+        folder_buttons.append(btn)
+
+    self._scroller.add_widgets(folder_buttons)
+
+
+class ModelSelectionMici(NavScroller):
+  def __init__(self, folder_name: str, select_model_cb: Callable):
+    super().__init__()
+
+    favs = ui_state.params.get("ModelManager_Favs")
+    favorites = set(favs.split(';')) if favs else set()
+
+    manager = ui_state.sm["modelManagerSP"]
+    bundles = manager.availableBundles
+    folders = {}
+    for bundle in bundles:
+      folder = next((override.value for override in bundle.overrides if override.key == "folder"), "")
+      folders.setdefault(folder, []).append(bundle)
+
+    if favorites:
+      for fav_bundle in [bundle for bundle in bundles if bundle.ref in favorites]:
+        folders.setdefault("favorites", []).append(fav_bundle)
+
+    bundles_in_folder = sorted(folders.get(folder_name, []), key=lambda b: b.index, reverse=True)
+
+    btns = []
+    for bundle in bundles_in_folder:
+      txt = bundle.displayName.lower()
+      btn = BigButton(txt)
+      btn.set_click_callback(lambda b=bundle: select_model_cb(b))
+      btns.append(btn)
+
+    self._scroller.add_widgets(btns)
+
+
 class ModelsLayoutMici(NavScroller):
   def __init__(self):
     super().__init__()
@@ -59,81 +126,49 @@ class ModelsLayoutMici(NavScroller):
     self.select_model_btn = BigButton(tr("select model"))
     self.select_model_btn.set_click_callback(self._show_folders)
 
+    self.clear_cache_btn = BigButton(tr("clear cache"), "")
+    self.clear_cache_btn.set_click_callback(self._clear_cache)
+
     self.cancel_download_btn = BigButton(tr("cancel download"))
     self.cancel_download_btn.set_click_callback(lambda: ui_state.params.remove("ModelManager_DownloadIndex"))
 
-    self.main_items = [self.current_model_info, self.select_model_btn, self.cancel_download_btn]
+    self.main_items = [self.current_model_info, self.select_model_btn, self.clear_cache_btn, self.cancel_download_btn]
     self._scroller.add_widgets(self.main_items)
 
   @property
   def model_manager(self):
     return ui_state.sm["modelManagerSP"]
 
-  def _get_grouped_bundles(self, favorites = None):
-    bundles = self.model_manager.availableBundles
-    folders = {}
-    for bundle in bundles:
-      folder = next((override.value for override in bundle.overrides if override.key == "folder"), "")
-      folders.setdefault(folder, []).append(bundle)
-
-    if favorites:
-      for fav_bundle in [bundle for bundle in bundles if bundle.ref in favorites]:
-        folders.setdefault("favorites", []).append(fav_bundle)
-
-    return folders
-
-  def _push_selection_view(self, items):
-    scroller = NavScroller()
-    scroller._scroller.add_widgets(items)
-    gui_app.push_widget(scroller)
-
   def _show_folders(self):
     self.focused_widget = self.select_model_btn
 
-    favs = ui_state.params.get("ModelManager_Favs")
-    favorites = set(favs.split(';')) if favs else set()
+    def select_default():
+      ui_state.params.remove("ModelManager_ActiveBundle")
+      gui_app.pop_widgets_to(self, instant=True)
+      self._scroller.scroll_panel.set_offset(0)
+      self._scroller.scroll_to(0)
 
-    folders = self._get_grouped_bundles(favorites)
-    folder_buttons = []
-    default_btn = BigButton(f"{DEFAULT_MODEL} (Default)".lower())
-    default_btn.set_click_callback(self._select_default)
-    folder_buttons.append(default_btn)
+    def select_folder(folder_name):
+      def select_model(bundle):
+        ui_state.params.put("ModelManager_DownloadIndex", bundle.index)
+        gui_app.pop_widgets_to(self, instant=True)
+        self._scroller.scroll_panel.set_offset(0)
+        self._scroller.scroll_to(0)
 
-    for folder in sorted(folders.keys(), key=lambda f: max((bundle.index for bundle in folders[f]), default=-1), reverse=True):
-      if folder.lower() in ["release models", "master models", "favorites"]:
-        btn = BigButton(folder.lower())
-        btn.set_click_callback(lambda f=folder: self._select_folder(f))
-        if folder.lower() == "favorites":
-          folder_buttons.insert(0, btn)
-        else:
-          folder_buttons.append(btn)
-    self._push_selection_view(folder_buttons)
+      model_selection = ModelSelectionMici(folder_name, select_model)
+      gui_app.push_widget(model_selection)
 
-  def _pop_to_main(self):
-    gui_app.pop_widgets_to(self)
+    folder_selection = FolderSelectionMici(select_default, select_folder)
+    gui_app.push_widget(folder_selection)
 
-  def _select_model(self, bundle):
-    ui_state.params.put("ModelManager_DownloadIndex", bundle.index)
-    self._pop_to_main()
+  def _clear_cache(self):
+    def confirm_callback():
+      ui_state.params.put_bool("ModelManager_ClearCache", True)
 
-  def _select_default(self):
-    ui_state.params.remove("ModelManager_ActiveBundle")
-    self._pop_to_main()
-
-  def _select_folder(self, folder_name):
-    favs = ui_state.params.get("ModelManager_Favs")
-    favorites = set(favs.split(';')) if favs else set()
-
-    folders = self._get_grouped_bundles(favorites)
-    bundles = sorted(folders.get(folder_name, []), key=lambda b: b.index, reverse=True)
-
-    btns = []
-    for bundle in bundles:
-      txt = bundle.displayName.lower()
-      btn = BigButton(txt)
-      btn.set_click_callback(lambda b=bundle: self._select_model(b))
-      btns.append(btn)
-    self._push_selection_view(btns)
+    lbl = tr("slide to clear cache")
+    icon = gui_app.texture("icons_mici/settings/device/update.png", 64, 64)
+    dlg = BigConfirmationDialog(lbl, icon, confirm_callback=confirm_callback, red=True)
+    gui_app.push_widget(dlg)
 
   def hide_event(self):
     super().hide_event()
@@ -145,6 +180,7 @@ class ModelsLayoutMici(NavScroller):
     super()._update_state()
 
     self.select_model_btn.set_enabled(ui_state.is_offroad())
+    self.clear_cache_btn.set_enabled(ui_state.is_offroad())
     self.cancel_download_btn.set_visible(False)
     self.current_model_info.current_model_header._shimmer = False
     self.current_model_info.info_header._shimmer = False
@@ -191,4 +227,3 @@ class ModelsLayoutMici(NavScroller):
       self.current_model_info.info_header.set_text(tr("progress") + self._download_progress)
       self.current_model_info.info_header._shimmer = True
       self.current_model_info.info_text.set_text(f"{progress/count:.2f}%")
-

openpilot/sunnypilot/livedelay/helpers.py

@@ -8,7 +8,10 @@ from openpilot.common.params import Params
 
 
 def get_lat_delay(params: Params, stock_lat_delay: float) -> float:
-  if params.get_bool("LagdToggle"):
-    return float(params.get("LagdValueCache", return_default=True))
+# live learning on: use what lagd publishes.
+# off: use the fixed steerActuatorDelay + software delay sum that LagdToggle caches.
 
-  return stock_lat_delay
+  if params.get_bool("LagdToggle"):
+    return stock_lat_delay
+
+  return float(params.get("LagdValueCache", return_default=True))

openpilot/sunnypilot/modeld_v2/compile_modeld.py

@@ -272,17 +272,18 @@ def _parse_size(size_str: str) -> tuple[int, int]:
   return int(width), int(height)
 
 
-def read_file_chunked_to_disk(path):
+def read_file_chunked_to_shm(path):
   if not path:
     return None
   import atexit
   import shutil
   from openpilot.common.file_chunker import open_file_chunked
-  tmp_path = f'{path}.unchunked'
-  with open(tmp_path, 'wb') as f, open_file_chunked(path) as src:
-    shutil.copyfileobj(src, f)
-  atexit.register(lambda: os.path.exists(tmp_path) and os.remove(tmp_path))
-  return tmp_path
+  from openpilot.common.hardware.hw import Paths
+  shm_path = os.path.join(Paths.shm_path(), os.path.basename(path))
+  atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path))
+  with open(shm_path, 'wb') as dst, open_file_chunked(path) as src:
+    shutil.copyfileobj(src, dst)
+  return shm_path
 
 
 def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]:
@@ -326,11 +327,11 @@ if __name__ == "__main__":
   model_w, model_h = args.model_size
   output_data = {}
 
-  args.vision_onnx = read_file_chunked_to_disk(args.vision_onnx)
-  args.policy_onnx = read_file_chunked_to_disk(args.policy_onnx)
-  args.off_policy_onnx = read_file_chunked_to_disk(args.off_policy_onnx)
-  args.on_policy_onnx = read_file_chunked_to_disk(args.on_policy_onnx)
-  args.supercombo_onnx = read_file_chunked_to_disk(args.supercombo_onnx)
+  args.vision_onnx = read_file_chunked_to_shm(args.vision_onnx)
+  args.policy_onnx = read_file_chunked_to_shm(args.policy_onnx)
+  args.off_policy_onnx = read_file_chunked_to_shm(args.off_policy_onnx)
+  args.on_policy_onnx = read_file_chunked_to_shm(args.on_policy_onnx)
+  args.supercombo_onnx = read_file_chunked_to_shm(args.supercombo_onnx)
 
   vision_runner = OnnxRunner(args.vision_onnx) if args.vision_onnx else None
 

openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py

@@ -5,15 +5,10 @@ This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
 
-import os
-import tempfile
-from pathlib import Path
-
 import numpy as np
 from openpilot.common.parameterized import parameterized
 
-from openpilot.common.file_chunker import chunk_file, get_chunk_targets
-from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key, read_file_chunked_to_disk
+from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key
 from openpilot.common.test import OpenpilotTestCase
 
 
@@ -165,33 +160,3 @@ class TestOutputSlicePreservation(OpenpilotTestCase):
     policy_slices = {'plan': slice(0, 495), 'meta': slice(495, 550)}
     assert set(vision_slices.keys()) & set(policy_slices.keys()) == set(), \
       "vision and policy slices should not overlap in keys"
-
-
-class TestReadFileChunkedToDisk(OpenpilotTestCase):
-  def test_none_passthrough(self):
-    assert read_file_chunked_to_disk(None) is None
-
-  def test_unchunked_source_staged_on_disk(self):
-    with tempfile.TemporaryDirectory() as d:
-      src = Path(d) / "driving_supercombo.onnx"
-      payload = os.urandom(1024)
-      src.write_bytes(payload)
-
-      out = Path(read_file_chunked_to_disk(str(src)))
-
-      assert out.parent == Path(d)
-      assert out.name == "driving_supercombo.onnx.unchunked"
-      assert out.read_bytes() == payload
-
-  def test_chunked_source_reassembled_on_disk(self):
-    with tempfile.TemporaryDirectory() as d:
-      src = Path(d) / "driving_supercombo.onnx"
-      payload = os.urandom(4096)
-      src.write_bytes(payload)
-      chunk_file(str(src), get_chunk_targets(str(src), len(payload)))
-      assert not src.exists()
-
-      out = Path(read_file_chunked_to_disk(str(src)))
-
-      assert out.parent == Path(d)
-      assert out.read_bytes() == payload

release/ci/model_generator.py

@@ -53,28 +53,24 @@ def create_pkl_name(full_name: str) -> str:
   return pkl
 
 
-def _hash_pkl(pkl_path: Path) -> str:
+def _read_pkl_bytes(pkl_path: Path) -> bytes:
   manifest = Path(f"{pkl_path}.chunkmanifest")
   if manifest.exists():
     num_chunks = int(manifest.read_text().strip())
-    paths = [Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}") for i in range(num_chunks)]
-  else:
-    paths = [pkl_path]
-
-  digest = hashlib.sha256()
-  for path in paths:
-    with path.open('rb') as f:
-      while block := f.read(1024 * 1024):
-        digest.update(block)
-  return digest.hexdigest()
+    parts = []
+    for i in range(num_chunks):
+      chunk = Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}")
+      parts.append(chunk.read_bytes())
+    return b''.join(parts)
+  return pkl_path.read_bytes()
 
 
 def _find_driving_pkl(output_path: Path) -> Path | None:
-  for pattern in ('*driving_tinygrad.pkl', '*driving_*_tinygrad.pkl'):
+  for pattern in ('driving_tinygrad.pkl', 'driving_*_tinygrad.pkl'):
     matches = sorted(output_path.glob(pattern))
     if matches:
       return matches[0]
-  for pattern in ('*driving_tinygrad.pkl.chunkmanifest', '*driving_*_tinygrad.pkl.chunkmanifest'):
+  for pattern in ('driving_tinygrad.pkl.chunkmanifest', 'driving_*_tinygrad.pkl.chunkmanifest'):
     matches = sorted(output_path.glob(pattern))
     if matches:
       return Path(str(matches[0]).removesuffix('.chunkmanifest'))
@@ -91,7 +87,7 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path:
 
 
 def generate_chunked_model(driving_pkl: Path) -> dict:
-  tinygrad_hash = _hash_pkl(driving_pkl)
+  tinygrad_hash = hashlib.sha256(_read_pkl_bytes(driving_pkl)).hexdigest()
 
   chunks_config = []
   manifest_file = Path(f"{driving_pkl}.chunkmanifest")