ci: Dedicated Model Runner (#1922)

* ci: Dedicated Model Runner

* recurse

* not needed

* fix wrapper

* whoops

* bypass

* modeld_v2: restore chestnut link check before big model build

* modeld_v2: stage onnx to disk instead of shared memory

* ci: clear unchunked onnx temps before model build

* ci: stream the pkl hash instead of loading it into memory

---------

Co-authored-by: Jason Wen <haibin.wen3@gmail.com>

.github/workflows/sunnypilot-build-model.yaml

@@ -103,10 +103,10 @@ jobs:
       - run: |
           cd ${{ github.workspace }}/openpilot/openpilot
           if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then
-            git lfs pull -X "selfdrive/modeld/models/big_*.onnx" -X "selfdrive/modeld/models/dmonitoring_*.onnx"
+            git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx" -X "**/selfdrive/modeld/models/dmonitoring_*.onnx"
             rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx
           else
-            git lfs pull -I "selfdrive/modeld/models/big_*.onnx"
+            git lfs pull -I "**/selfdrive/modeld/models/big_*.onnx"
             find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete
           fi
       - name: 'Upload Artifact'
@@ -116,7 +116,7 @@ jobs:
           path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
 
   build_model:
-    runs-on: [self-hosted, tici]
+    runs-on: [self-hosted, usbgpu]
     needs: get_model
     env:
       MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
@@ -127,7 +127,6 @@ jobs:
           fetch-depth: 1
           submodules: recursive
 
-      - run: git lfs pull
 
       - name: Set environment variables
         id: set-env
@@ -160,7 +159,7 @@ jobs:
           fi
           source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
-          rm -rf ${{ env.MODELS_DIR }}/*.onnx
+          rm -rf ${{ env.MODELS_DIR }}/*.onnx*
 
       - name: Download model artifacts
         uses: actions/download-artifact@v4
@@ -180,6 +179,7 @@ jobs:
           MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
           CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
 
+          TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
           if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
             echo "USBGPU build"
             export USBGPU=1
@@ -187,27 +187,40 @@ jobs:
             OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
           else
             echo "QCOM build"
-            TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
+            TG_FLAGS="$TG_FLAGS_QCOM"
             OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
           fi
 
           # Generate metadata for all ONNX files
           find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
             echo "Generating metadata: $onnx_file"
-            env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
+            env ${TG_FLAGS_QCOM} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
           done
 
           # Detect model type and build compile args
-          VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
-          POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
-          OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
-          ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
+          VISION_ONNX=""
+          for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do
+            [ -f "$f" ] && VISION_ONNX="$f" && break
+          done
+
+          POLICY_ONNX=""
+          for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do
+            [ -f "$f" ] && POLICY_ONNX="$f" && break
+          done
+
+          OFF_POLICY_ONNX=""
+          for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do
+            [ -f "$f" ] && OFF_POLICY_ONNX="$f" && break
+          done
+
+          ON_POLICY_ONNX=""
+          for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do
+            [ -f "$f" ] && ON_POLICY_ONNX="$f" && break
+          done
+
           SUPERCOMBO_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do
-            if [ -f "$f" ]; then
-              SUPERCOMBO_ONNX="$f"
-              break
-            fi
+          for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do
+            [ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break
           done
 
           MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""

openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py

@@ -4,13 +4,11 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
 This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
-from collections.abc import Callable
 import pyray as rl
 
 from openpilot.cereal import custom
 from openpilot.sunnypilot.models.default_model import DEFAULT_MODEL
 from openpilot.selfdrive.ui.mici.widgets.button import BigButton
-from openpilot.selfdrive.ui.mici.widgets.dialog import BigConfirmationDialog
 from openpilot.selfdrive.ui.sunnypilot.layouts.settings.models import ModelsLayout
 from openpilot.selfdrive.ui.ui_state import ui_state, device
 from openpilot.system.ui.lib.application import FontWeight, gui_app
@@ -19,24 +17,6 @@ from openpilot.system.ui.widgets import Widget
 from openpilot.system.ui.widgets.label import UnifiedLabel
 from openpilot.system.ui.widgets.scroller import NavScroller
 
-def _build_folders() -> dict[str, list]:
-  manager = ui_state.sm["modelManagerSP"]
-  bundles = manager.availableBundles
-  folders = {}
-  for bundle in bundles:
-    folder = next((override.value for override in bundle.overrides if override.key == "folder"), "")
-    folders.setdefault(folder, []).append(bundle)
-
-  favs = ui_state.params.get("ModelManager_Favs")
-  favorites = set(favs.split(';')) if favs else set()
-
-  if favorites:
-    for fav_bundle in [bundle for bundle in bundles if bundle.ref in favorites]:
-      folders.setdefault("favorites", []).append(fav_bundle)
-
-  return folders
-
-
 class CurrentModelInfo(Widget):
   def __init__(self):
     super().__init__()
@@ -66,41 +46,6 @@ class CurrentModelInfo(Widget):
     self.info_text.set_position(self._rect.x + 20, self._rect.y + 161 - 25)
     self.info_text.render()
 
-
-class FolderSelectionMici(NavScroller):
-
-  def __init__(self, folder_name: str | None = None,
-               select_default_callback: Callable | None = None,
-               select_folder_callback: Callable | None = None,
-               select_model_callback: Callable | None = None):
-    super().__init__()
-
-    folders = _build_folders()
-
-    btns = []
-    if folder_name is None:
-      assert select_default_callback is not None and select_folder_callback is not None
-      default_btn = BigButton(f"{DEFAULT_MODEL} (Default)".lower())
-      default_btn.set_click_callback(select_default_callback)
-      btns.append(default_btn)
-
-      for folder in sorted(folders.keys(), key=lambda f: max((bundle.index for bundle in folders[f]), default=-1), reverse=True):
-        btn = BigButton(folder.lower())
-        btn.set_click_callback(lambda f=folder: select_folder_callback(f))
-        if folder.lower() == "favorites":
-          btns.insert(0, btn)
-        else:
-          btns.append(btn)
-    else:
-      assert select_model_callback is not None
-      for bundle in sorted(folders.get(folder_name, []), key=lambda b: b.index, reverse=True):
-        btn = BigButton(bundle.displayName.lower())
-        btn.set_click_callback(lambda b=bundle: select_model_callback(b))
-        btns.append(btn)
-
-    self._scroller.add_widgets(btns)
-
-
 class ModelsLayoutMici(NavScroller):
   def __init__(self):
     super().__init__()
@@ -114,47 +59,81 @@ class ModelsLayoutMici(NavScroller):
     self.select_model_btn = BigButton(tr("select model"))
     self.select_model_btn.set_click_callback(self._show_folders)
 
-    self.clear_cache_btn = BigButton(tr("clear cache"), "")
-    self.clear_cache_btn.set_click_callback(self._clear_cache)
-
     self.cancel_download_btn = BigButton(tr("cancel download"))
     self.cancel_download_btn.set_click_callback(lambda: ui_state.params.remove("ModelManager_DownloadIndex"))
 
-    self.main_items = [self.current_model_info, self.select_model_btn, self.clear_cache_btn, self.cancel_download_btn]
+    self.main_items = [self.current_model_info, self.select_model_btn, self.cancel_download_btn]
     self._scroller.add_widgets(self.main_items)
 
   @property
   def model_manager(self):
     return ui_state.sm["modelManagerSP"]
 
+  def _get_grouped_bundles(self, favorites = None):
+    bundles = self.model_manager.availableBundles
+    folders = {}
+    for bundle in bundles:
+      folder = next((override.value for override in bundle.overrides if override.key == "folder"), "")
+      folders.setdefault(folder, []).append(bundle)
+
+    if favorites:
+      for fav_bundle in [bundle for bundle in bundles if bundle.ref in favorites]:
+        folders.setdefault("favorites", []).append(fav_bundle)
+
+    return folders
+
+  def _push_selection_view(self, items):
+    scroller = NavScroller()
+    scroller._scroller.add_widgets(items)
+    gui_app.push_widget(scroller)
+
   def _show_folders(self):
     self.focused_widget = self.select_model_btn
 
-    def select_default():
-      ui_state.params.remove("ModelManager_ActiveBundle")
-      gui_app.pop_widgets_to(self, instant=True)
-      self._scroller.scroll_panel.set_offset(0)
-      self._scroller.scroll_to(0)
+    favs = ui_state.params.get("ModelManager_Favs")
+    favorites = set(favs.split(';')) if favs else set()
 
-    def select_model(bundle):
-      ui_state.params.put("ModelManager_DownloadIndex", bundle.index)
-      gui_app.pop_widgets_to(self, instant=True)
-      self._scroller.scroll_panel.set_offset(0)
-      self._scroller.scroll_to(0)
+    folders = self._get_grouped_bundles(favorites)
+    folder_buttons = []
+    default_btn = BigButton(f"{DEFAULT_MODEL} (Default)".lower())
+    default_btn.set_click_callback(self._select_default)
+    folder_buttons.append(default_btn)
 
-    def select_folder(folder_name):
-      gui_app.push_widget(FolderSelectionMici(folder_name, select_model_callback=select_model))
+    for folder in sorted(folders.keys(), key=lambda f: max((bundle.index for bundle in folders[f]), default=-1), reverse=True):
+      if folder.lower() in ["release models", "master models", "favorites"]:
+        btn = BigButton(folder.lower())
+        btn.set_click_callback(lambda f=folder: self._select_folder(f))
+        if folder.lower() == "favorites":
+          folder_buttons.insert(0, btn)
+        else:
+          folder_buttons.append(btn)
+    self._push_selection_view(folder_buttons)
 
-    gui_app.push_widget(FolderSelectionMici(select_default_callback=select_default, select_folder_callback=select_folder))
+  def _pop_to_main(self):
+    gui_app.pop_widgets_to(self)
 
-  def _clear_cache(self):
-    def confirm_callback():
-      ui_state.params.put_bool("ModelManager_ClearCache", True)
+  def _select_model(self, bundle):
+    ui_state.params.put("ModelManager_DownloadIndex", bundle.index)
+    self._pop_to_main()
 
-    lbl = tr("slide to clear cache")
-    icon = gui_app.texture("icons_mici/settings/device/uninstall.png", 64, 64)
-    dlg = BigConfirmationDialog(lbl, icon, confirm_callback=confirm_callback, red=True)
-    gui_app.push_widget(dlg)
+  def _select_default(self):
+    ui_state.params.remove("ModelManager_ActiveBundle")
+    self._pop_to_main()
+
+  def _select_folder(self, folder_name):
+    favs = ui_state.params.get("ModelManager_Favs")
+    favorites = set(favs.split(';')) if favs else set()
+
+    folders = self._get_grouped_bundles(favorites)
+    bundles = sorted(folders.get(folder_name, []), key=lambda b: b.index, reverse=True)
+
+    btns = []
+    for bundle in bundles:
+      txt = bundle.displayName.lower()
+      btn = BigButton(txt)
+      btn.set_click_callback(lambda b=bundle: self._select_model(b))
+      btns.append(btn)
+    self._push_selection_view(btns)
 
   def hide_event(self):
     super().hide_event()
@@ -166,7 +145,6 @@ class ModelsLayoutMici(NavScroller):
     super()._update_state()
 
     self.select_model_btn.set_enabled(ui_state.is_offroad())
-    self.clear_cache_btn.set_enabled(ui_state.is_offroad())
     self.cancel_download_btn.set_visible(False)
     self.current_model_info.current_model_header._shimmer = False
     self.current_model_info.info_header._shimmer = False
@@ -213,3 +191,4 @@ class ModelsLayoutMici(NavScroller):
       self.current_model_info.info_header.set_text(tr("progress") + self._download_progress)
       self.current_model_info.info_header._shimmer = True
       self.current_model_info.info_text.set_text(f"{progress/count:.2f}%")
+

openpilot/sunnypilot/modeld_v2/compile_modeld.py

@@ -272,18 +272,17 @@ def _parse_size(size_str: str) -> tuple[int, int]:
   return int(width), int(height)
 
 
-def read_file_chunked_to_shm(path):
+def read_file_chunked_to_disk(path):
   if not path:
     return None
   import atexit
   import shutil
   from openpilot.common.file_chunker import open_file_chunked
-  from openpilot.common.hardware.hw import Paths
-  shm_path = os.path.join(Paths.shm_path(), os.path.basename(path))
-  atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path))
-  with open(shm_path, 'wb') as dst, open_file_chunked(path) as src:
-    shutil.copyfileobj(src, dst)
-  return shm_path
+  tmp_path = f'{path}.unchunked'
+  with open(tmp_path, 'wb') as f, open_file_chunked(path) as src:
+    shutil.copyfileobj(src, f)
+  atexit.register(lambda: os.path.exists(tmp_path) and os.remove(tmp_path))
+  return tmp_path
 
 
 def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]:
@@ -327,11 +326,11 @@ if __name__ == "__main__":
   model_w, model_h = args.model_size
   output_data = {}
 
-  args.vision_onnx = read_file_chunked_to_shm(args.vision_onnx)
-  args.policy_onnx = read_file_chunked_to_shm(args.policy_onnx)
-  args.off_policy_onnx = read_file_chunked_to_shm(args.off_policy_onnx)
-  args.on_policy_onnx = read_file_chunked_to_shm(args.on_policy_onnx)
-  args.supercombo_onnx = read_file_chunked_to_shm(args.supercombo_onnx)
+  args.vision_onnx = read_file_chunked_to_disk(args.vision_onnx)
+  args.policy_onnx = read_file_chunked_to_disk(args.policy_onnx)
+  args.off_policy_onnx = read_file_chunked_to_disk(args.off_policy_onnx)
+  args.on_policy_onnx = read_file_chunked_to_disk(args.on_policy_onnx)
+  args.supercombo_onnx = read_file_chunked_to_disk(args.supercombo_onnx)
 
   vision_runner = OnnxRunner(args.vision_onnx) if args.vision_onnx else None
 

openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py

@@ -5,10 +5,15 @@ This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
 
+import os
+import tempfile
+from pathlib import Path
+
 import numpy as np
 from openpilot.common.parameterized import parameterized
 
-from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key
+from openpilot.common.file_chunker import chunk_file, get_chunk_targets
+from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key, read_file_chunked_to_disk
 from openpilot.common.test import OpenpilotTestCase
 
 
@@ -160,3 +165,33 @@ class TestOutputSlicePreservation(OpenpilotTestCase):
     policy_slices = {'plan': slice(0, 495), 'meta': slice(495, 550)}
     assert set(vision_slices.keys()) & set(policy_slices.keys()) == set(), \
       "vision and policy slices should not overlap in keys"
+
+
+class TestReadFileChunkedToDisk(OpenpilotTestCase):
+  def test_none_passthrough(self):
+    assert read_file_chunked_to_disk(None) is None
+
+  def test_unchunked_source_staged_on_disk(self):
+    with tempfile.TemporaryDirectory() as d:
+      src = Path(d) / "driving_supercombo.onnx"
+      payload = os.urandom(1024)
+      src.write_bytes(payload)
+
+      out = Path(read_file_chunked_to_disk(str(src)))
+
+      assert out.parent == Path(d)
+      assert out.name == "driving_supercombo.onnx.unchunked"
+      assert out.read_bytes() == payload
+
+  def test_chunked_source_reassembled_on_disk(self):
+    with tempfile.TemporaryDirectory() as d:
+      src = Path(d) / "driving_supercombo.onnx"
+      payload = os.urandom(4096)
+      src.write_bytes(payload)
+      chunk_file(str(src), get_chunk_targets(str(src), len(payload)))
+      assert not src.exists()
+
+      out = Path(read_file_chunked_to_disk(str(src)))
+
+      assert out.parent == Path(d)
+      assert out.read_bytes() == payload

release/ci/model_generator.py

@@ -53,24 +53,28 @@ def create_pkl_name(full_name: str) -> str:
   return pkl
 
 
-def _read_pkl_bytes(pkl_path: Path) -> bytes:
+def _hash_pkl(pkl_path: Path) -> str:
   manifest = Path(f"{pkl_path}.chunkmanifest")
   if manifest.exists():
     num_chunks = int(manifest.read_text().strip())
-    parts = []
-    for i in range(num_chunks):
-      chunk = Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}")
-      parts.append(chunk.read_bytes())
-    return b''.join(parts)
-  return pkl_path.read_bytes()
+    paths = [Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}") for i in range(num_chunks)]
+  else:
+    paths = [pkl_path]
+
+  digest = hashlib.sha256()
+  for path in paths:
+    with path.open('rb') as f:
+      while block := f.read(1024 * 1024):
+        digest.update(block)
+  return digest.hexdigest()
 
 
 def _find_driving_pkl(output_path: Path) -> Path | None:
-  for pattern in ('driving_tinygrad.pkl', 'driving_*_tinygrad.pkl'):
+  for pattern in ('*driving_tinygrad.pkl', '*driving_*_tinygrad.pkl'):
     matches = sorted(output_path.glob(pattern))
     if matches:
       return matches[0]
-  for pattern in ('driving_tinygrad.pkl.chunkmanifest', 'driving_*_tinygrad.pkl.chunkmanifest'):
+  for pattern in ('*driving_tinygrad.pkl.chunkmanifest', '*driving_*_tinygrad.pkl.chunkmanifest'):
     matches = sorted(output_path.glob(pattern))
     if matches:
       return Path(str(matches[0]).removesuffix('.chunkmanifest'))
@@ -87,7 +91,7 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path:
 
 
 def generate_chunked_model(driving_pkl: Path) -> dict:
-  tinygrad_hash = hashlib.sha256(_read_pkl_bytes(driving_pkl)).hexdigest()
+  tinygrad_hash = _hash_pkl(driving_pkl)
 
   chunks_config = []
   manifest_file = Path(f"{driving_pkl}.chunkmanifest")