modeld_v2: spatial features (PR-1934)

* big needs small

* no download

* actually

* send it

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -211,6 +211,8 @@ jobs:
     needs: [ prepare_strategy ]
     runs-on: ubuntu-24.04
     if: ${{ needs.prepare_strategy.outputs.include_big_model == 'true' }}
+    outputs:
+      onnx_sha256: ${{ steps.resolve.outputs.onnx_sha256 }}
     env:
       HF_REPO: sunnypilot/sunnypilot_models_v1
       HF_DEFAULTS_PATH: models/defaults/big
@@ -226,6 +228,7 @@ jobs:
         run: |
           ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
           echo "Repo ONNX hash: $ACTUAL_ONNX_HASH"
+          echo "onnx_sha256=$ACTUAL_ONNX_HASH" >> $GITHUB_OUTPUT
 
           JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
 
@@ -267,36 +270,6 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Download big model chunks
-        run: |
-          ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
-          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
-          DEFAULTS=$(curl -fsSL "$JSON_URL")
-          BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
-
-          mkdir -p big_model_chunks
-          ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
-          BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
-          NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
-
-          CANONICAL="big_driving_tinygrad.pkl"
-          echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
-            CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+')
-            CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
-            ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
-            echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
-            curl -fsSL -o "big_model_chunks/${CANONICAL_CHUNK}" "$ENCODED_URL"
-          done
-
-          echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest"
-
-      - name: Upload big model chunks
-        uses: actions/upload-artifact@v4
-        with:
-          name: big-model-chunks
-          path: big_model_chunks/
-          compression-level: 0
-
       - name: Cancel run on failure
         if: failure()
         run: gh run cancel ${{ github.run_id }}
@@ -339,12 +312,32 @@ jobs:
           mkdir -p "${{ github.workspace }}/chestnut_output"
           tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output"
 
-      - name: Download big model chunks
+      - name: Download big model chunks from HF
         if: ${{ needs.prepare_chestnut.result == 'success' }}
-        uses: actions/download-artifact@v4
-        with:
-          name: big-model-chunks
-          path: big_model_chunks
+        env:
+          HF_REPO: sunnypilot/sunnypilot_models_v1
+          HF_DEFAULTS_PATH: models/defaults/big
+        run: |
+          ONNX_HASH="${{ needs.prepare_chestnut.outputs.onnx_sha256 }}"
+          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
+          DEFAULTS=$(curl -fsSL "$JSON_URL")
+          BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
+
+          mkdir -p big_model_chunks
+          ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
+          BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
+          NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
+
+          CANONICAL="big_driving_tinygrad.pkl"
+          echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
+            CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+')
+            CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
+            ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
+            echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
+            curl -fsSL -o "big_model_chunks/${CANONICAL_CHUNK}" "$ENCODED_URL"
+          done
+
+          echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest"
 
       - name: Inject big model into chestnut
         if: ${{ needs.prepare_chestnut.result == 'success' }}

openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py

@@ -178,27 +178,14 @@ class ModelsLayout(Widget):
     # circled_slash is authored grey; tinting it again only darkens it
     return {"name": name, "text_color": rl.GRAY, "icon": "icons/circled_slash.png", "icon_color": rl.WHITE}
 
-  @staticmethod
-  def _show_reset_params_dialog():
-    def _callback(response):
-      if response == DialogResult.CONFIRM:
-        ui_state.params.remove("CalibrationParams")
-        ui_state.params.remove("LiveTorqueParameters")
-    msg = tr("Model download has started in the background. We suggest resetting calibration. Would you like to do that now?")
-    dialog = ConfirmDialog(msg, tr("Reset Calibration"), callback=_callback)
-    gui_app.push_widget(dialog)
-
   def _on_model_selected(self, result):
     if result != DialogResult.CONFIRM:
       return
     selected_ref = self.model_dialog.selection_ref
     if selected_ref == "Default":
       ui_state.params.remove("ModelManager_ActiveBundle")
-      self._show_reset_params_dialog()
     elif selected_bundle := next((bundle for bundle in self.model_manager.availableBundles if bundle.ref == selected_ref), None):
       ui_state.params.put("ModelManager_DownloadIndex", selected_bundle.index)
-      if self.model_manager.activeBundle and selected_bundle.generation != self.model_manager.activeBundle.generation:
-        self._show_reset_params_dialog()
     self.model_dialog = None
 
   @staticmethod

openpilot/sunnypilot/modeld_v2/compile_modeld.py

@@ -7,6 +7,7 @@ See the LICENSE.md file in the root directory for more details.
 """
 
 import argparse
+import math
 import os
 import tempfile
 import time
@@ -66,14 +67,15 @@ def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tu
   if desire_key:
     shapes['desire'] = (input_shapes[desire_key][2],)
 
-  if is_supercombo and 'features_buffer' in input_shapes:
-    fb = input_shapes['features_buffer']
-    shapes['prev_feat'] = (fb[0], fb[2])
-
   for key, shape in input_shapes.items():
     if key not in (desire_key, 'features_buffer') and 'img' not in key:
       shapes[key] = tuple(shape)
 
+  if is_supercombo and 'features_buffer' in input_shapes:
+    fb = input_shapes['features_buffer']
+    feat_dim = math.prod(fb[2:])
+    shapes['prev_feat'] = (fb[0], feat_dim)
+
   sizes = [int(np.prod(size)) for size in shapes.values()]
   return shapes, sizes
 
@@ -117,8 +119,9 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D
   }
 
   if features_buffer:
+    feat_dim = math.prod(features_buffer[2:])
     feat_q_len = frame_skip * features_buffer[1] if is_supercombo else frame_skip * (features_buffer[1] - 1) + 1
-    queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], features_buffer[2]),
+    queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], feat_dim),
                        dtype=np.float32), device=device).contiguous().realize()
 
   queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')})
@@ -183,14 +186,14 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
     warped_dev = warped.to(Device.DEFAULT)
     Tensor.realize(packed_npy_inputs_dev, warped_dev)
 
-    img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn).realize()
-    big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn).realize()
+    img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn)
+    big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn)
 
     unpacked_tensors = [tensor.reshape(shape) for tensor, shape in zip(packed_npy_inputs_dev.split(npy_sizes), npy_shapes.values(), strict=True)]
     unpacked_dict = dict(zip(npy_shapes.keys(), unpacked_tensors, strict=True))
 
     desire_dev = unpacked_dict['desire']
-    desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn).realize()
+    desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn)
 
     inputs = {desire_key: desire_buf}
     for key, tensor_val in unpacked_dict.items():
@@ -199,19 +202,22 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
 
     if 'prev_feat' in unpacked_dict:
       prev_feat_dev = unpacked_dict['prev_feat']
-      inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).realize()
+      feat_buf = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn)
+      inputs['features_buffer'] = feat_buf if len(fb := input_shapes['features_buffer']) <= 3 else feat_buf.reshape(fb)
 
     if vision_runner:
       vision_out_cast = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32').realize()
       if 'features_buffer' not in inputs:
         new_feat = vision_out_cast[:, features_slice].reshape(1, -1).unsqueeze(0)
-        inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize()
+        feat_buf = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize()
+        inputs['features_buffer'] = feat_buf if len(fb := input_shapes['features_buffer']) <= 3 else feat_buf.reshape(fb)
       policy_outs = [next(iter(pol_runner(inputs).values())).cast('float32').realize() for pol_runner in policy_runners]
       return (vision_out_cast, *policy_outs) if len(policy_outs) > 1 else (vision_out_cast, policy_outs[0])
 
     inputs.update({road_key: img, wide_key: big_img})
     if 'features_buffer' not in inputs:
-      inputs['features_buffer'] = sample_skip_fn(feat_q)
+      feat_buf = sample_skip_fn(feat_q)
+      inputs['features_buffer'] = feat_buf if len(fb := input_shapes['features_buffer']) <= 3 else feat_buf.reshape(fb)
 
     policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32').realize()
     if 'features_buffer' not in inputs and features_slice is not None:

openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py

@@ -195,3 +195,85 @@ class TestReadFileChunkedToDisk(OpenpilotTestCase):
 
       assert out.parent == Path(d)
       assert out.read_bytes() == payload
+
+
+class Test4DFeaturesBuffer(OpenpilotTestCase):
+  def test_get_policy_npy_shapes_4d(self):
+    from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes
+    input_shapes = {
+      'desire_pulse': (1, 25, 8),
+      'features_buffer': (1, 24, 32, 512),  # compare 4d to 3d for regression
+      'traffic_convention': (1, 2),
+      'action_t': (1, 2)
+    }
+    shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True)
+    assert shapes['prev_feat'] == (1, 16384)
+    assert sizes == [8, 2, 2, 16384]
+
+  def test_get_policy_npy_shapes_3d(self):
+    from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes
+    input_shapes = {
+      'desire_pulse': (1, 25, 8),
+      'features_buffer': (1, 24, 512),
+      'traffic_convention': (1, 2),
+      'action_t': (1, 2)
+    }
+    shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True)
+    assert shapes['prev_feat'] == (1, 512)
+    assert sizes == [8, 2, 2, 512]
+
+
+class TestStockCompileModeldEquivalence(OpenpilotTestCase):
+  def test_get_policy_npy_shapes_matches_stock(self):
+    from openpilot.selfdrive.modeld.compile_modeld import get_policy_npy_shapes as stock_get_policy_npy_shapes
+    from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes as sunny_get_policy_npy_shapes
+
+    stock_input_shapes = {
+      'desire_pulse': (1, 25, 8),
+      'features_buffer': (1, 24, 512), # see below comment
+      'traffic_convention': (1, 2),
+      'action_t': (1, 2),
+    }
+
+    stock_shapes, stock_sizes = stock_get_policy_npy_shapes(stock_input_shapes)
+    sunny_shapes, sunny_sizes = sunny_get_policy_npy_shapes(stock_input_shapes, is_supercombo=True)
+
+    assert sunny_shapes == stock_shapes
+    assert sunny_sizes == stock_sizes
+    assert sunny_shapes['prev_feat'] == (1, 512)
+
+  def test_make_input_queues_full_stock_equivalence(self):
+    from openpilot.selfdrive.modeld.compile_modeld import make_input_queues as stock_make_input_queues
+    from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues as sunny_make_supercombo_input_queues
+    input_shapes = {
+      'img': (1, 12, 128, 256),
+      'desire_pulse': (1, 25, 8),
+      'features_buffer': (1, 24, 512),  # when https://github.com/commaai/openpilot/pull/38681 merges, update to 1,24,32,512
+      'traffic_convention': (1, 2),
+      'action_t': (1, 2),
+    }
+    frame_skip = 4
+
+    stock_queues, stock_npy = stock_make_input_queues(input_shapes, frame_skip, device='NPY')
+    sunny_queues, sunny_npy = sunny_make_supercombo_input_queues(input_shapes, frame_skip, device='NPY')
+    assert set(sunny_queues.keys()) == set(stock_queues.keys())
+    for key in stock_queues:
+      assert sunny_queues[key].shape == stock_queues[key].shape, \
+        f"Queue shape mismatch for {key}: sunny {sunny_queues[key].shape} != stock {stock_queues[key].shape}"
+    assert set(sunny_npy.keys()) == set(stock_npy.keys())
+    for key in stock_npy:
+      assert sunny_npy[key].shape == stock_npy[key].shape, \
+        f"Numpy array shape mismatch for {key}: sunny {sunny_npy[key].shape} != stock {stock_npy[key].shape}"
+
+  def test_make_warp_queues_stock_equivalence(self):
+    from openpilot.selfdrive.modeld.compile_modeld import make_warp_input_queues as stock_make_warp_queues
+    from openpilot.sunnypilot.modeld_v2.compile_modeld import make_warp_queues as sunny_make_warp_queues
+    stock_vision_shapes = {'img': (1, 12, 128, 256)}  # for now?
+    stock_queues, stock_npy = stock_make_warp_queues(stock_vision_shapes, frame_skip=4, device='NPY')
+    sunny_queues, sunny_npy = sunny_make_warp_queues(device='NPY')
+
+    assert set(sunny_npy.keys()) == set(stock_npy.keys()) == {'tfm', 'big_tfm'}
+    for key in sunny_npy:
+      assert sunny_npy[key].shape == stock_npy[key].shape == (3, 3)
+
+

openpilot/sunnypilot/models/fetcher.py

@@ -141,7 +141,7 @@ class ModelCache:
 class ModelFetcher:
   """Handles fetching and caching of model data from remote source"""
   MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v20.json"
-  MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v21.json"
+  MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v22.json"
 
   def __init__(self, params: Params):
     self.params = params

openpilot/sunnypilot/models/manager.py

@@ -143,13 +143,17 @@ class ModelManagerSP:
       is_cached = False
       if len(artifact.chunks) > 0:
         from openpilot.common.file_chunker import get_chunk_name
+        num_chunks = len(artifact.chunks)
         chunks_valid = True
         for i, chunk in enumerate(artifact.chunks):
-          chunk_path = get_chunk_name(full_path, i, len(artifact.chunks))
+          chunk_path = get_chunk_name(full_path, i, num_chunks)
           if not await verify_file(chunk_path, chunk.sha256):
             chunks_valid = False
             break
-        if chunks_valid and len(artifact.chunks) > 0:
+          artifact.downloadProgress.progress = ((i + 1) / num_chunks) * 100
+          self._sync_artifact_progress(artifact)
+          self._report_status()
+        if chunks_valid and num_chunks > 0:
           is_cached = True
       else:
         if await verify_file(full_path, expected_hash):
@@ -216,6 +220,9 @@ class ModelManagerSP:
     """Downloads all models in a bundle"""
     self.selected_bundle = model_bundle
     self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloading
+    for model in self.selected_bundle.models:
+      model.artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloading
+    self._report_status()
     os.makedirs(destination_path, exist_ok=True)
 
     try:
@@ -260,7 +267,9 @@ class ModelManagerSP:
         self.active_bundle = get_active_bundle(self.params)
 
         if (index_to_download := self.params.get("ModelManager_DownloadIndex")) is not None:
-          if model_to_download := next((model for model in self.available_models if model.index == index_to_download), None):
+          if self.active_bundle and self.active_bundle.index == index_to_download:
+            self.params.remove("ModelManager_DownloadIndex")
+          elif model_to_download := next((model for model in self.available_models if model.index == index_to_download), None):
             try:
               self.download(model_to_download, Paths.model_root())
             except Exception as e:

openpilot/sunnypilot/selfdrive/locationd/tests/test_locationd.py

@@ -83,9 +83,14 @@ class TestLocationdProc(OpenpilotTestCase):
       self.pm.send(msg.which(), msg)
       if msg.which() == "cameraOdometry":
         self.pm.wait_for_readers_to_update(msg.which(), timeout=1, dt=0.005)
-    time.sleep(1)  # wait for async params write
+    for _ in range(50):
+      val = self.params.get('LastGPSPositionLLK')
+      if val is not None:
+        break
+      time.sleep(0.1)
 
-    lastGPS = json.loads(self.params.get('LastGPSPositionLLK'))
+    self.assertIsNotNone(val, "LastGPSPositionLLK not written within 5s")
+    lastGPS = json.loads(val)
     self.assertAlmostEqual(lastGPS['latitude'], self.lat, delta=0.001)
     self.assertAlmostEqual(lastGPS['longitude'], self.lon, delta=0.001)
     self.assertAlmostEqual(lastGPS['altitude'], self.alt, delta=0.001)

openpilot/sunnypilot/sunnylink/athena/sunnylinkd.py

@@ -28,7 +28,8 @@ from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutExce
                        create_connection, WebSocketConnectionClosedException)
 
 import openpilot.cereal.messaging as messaging
-from openpilot.sunnypilot.models.default_model import get_default_model
+from openpilot.selfdrive.modeld.helpers import usbgpu_present, usbgpu_compiled
+from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL, DEFAULT_BIG_MODEL
 from openpilot.sunnypilot.selfdrive.car.sync_sunnylink_params import update_car_list_param
 from openpilot.sunnypilot.sunnylink.api import SunnylinkApi
 from openpilot.sunnypilot.sunnylink.utils import sunnylink_need_register, sunnylink_ready, get_param_as_byte, save_param_from_base64_encoded_string
@@ -181,7 +182,10 @@ def getParamsMetadata() -> str:
     schema = generate_schema()
     schema["capabilities"] = generate_capabilities()
     schema["capability_labels"] = CAPABILITY_LABELS
-    schema["default_model"] = get_default_model()
+    # mirrors get_default_model() — ui_state unavailable in sunnylinkd process
+    show_big = (usbgpu_present() and usbgpu_compiled()
+                and (params.get_bool("UsbGpuActive") or params.get_bool("UsbGpuLoading") or params.get_bool("IsOffroad")))
+    schema["default_model"] = DEFAULT_BIG_MODEL if show_big else DEFAULT_MODEL
     raw = json.dumps(schema, separators=(",", ":")).encode("utf-8")
     return base64.b64encode(gzip.compress(raw)).decode("utf-8")
   except Exception: