no dance

This reverts commit b244f5d0ae.

.github/workflows/build-default-big-model.yaml

@@ -8,17 +8,35 @@ env:
   HF_DEFAULTS_PATH: models/defaults/big
 
 jobs:
+  resolve_name:
+    runs-on: ubuntu-24.04
+    outputs:
+      model_name: ${{ steps.name.outputs.model_name }}
+      onnx_ref: ${{ steps.name.outputs.onnx_ref }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: name
+        run: |
+          NAME=$(PYTHONPATH=${{ github.workspace }} python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)")
+          ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx)
+          echo "model_name=${NAME}" >> $GITHUB_OUTPUT
+          echo "onnx_ref=$ONNX_REF" >> $GITHUB_OUTPUT
+
   build_model:
+    needs: resolve_name
     uses: ./.github/workflows/sunnypilot-build-model.yaml
     with:
-      upstream_branch: ${{ github.sha }}
-      custom_name: default-big-model
+      upstream_branch: ${{ needs.resolve_name.outputs.onnx_ref }}
+      custom_name: ${{ needs.resolve_name.outputs.model_name }}
       target_hardware: usbgpu
     secrets: inherit
 
   upload_defaults:
-    needs: build_model
+    needs: [ resolve_name, build_model ]
     runs-on: ubuntu-24.04
+    permissions:
+      id-token: write
+      contents: write
     steps:
       - uses: actions/checkout@v4
         with:
@@ -31,7 +49,7 @@ jobs:
       - name: Download artifact name
         uses: actions/download-artifact@v4
         with:
-          name: artifact-name-default-big-model
+          name: artifact-name-${{ needs.resolve_name.outputs.model_name }}
           path: artifact_name
 
       - name: Read artifact name
@@ -46,33 +64,20 @@ jobs:
           name: ${{ steps.artifact.outputs.artifact_name }}
           path: output
 
-      - name: Upload model to HF defaults
+      - name: Upload to HF and update default_models.json
         env:
           HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
           ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
         run: |
           rm -f output/artifact_name.txt
-          hf upload ${{ env.HF_REPO }} \
-            output/ \
-            "${HF_DEFAULTS_PATH}/${ARTIFACT_NAME}/" \
-            --repo-type=dataset
-
-      - name: Get tinygrad ref and ONNX hash
-        id: meta
-        run: |
           export PYTHONPATH=$(pwd)
-          echo "tinygrad_ref=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)" >> $GITHUB_OUTPUT
-          echo "onnx_sha256=$(sha256sum openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx | cut -d' ' -f1)" >> $GITHUB_OUTPUT
-
-      - name: Update default_models.json on HF
-        env:
-          HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
-          ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
-        run: |
           python3 release/ci/upload_default_model.py \
             --hf-repo "${{ env.HF_REPO }}" \
             --hf-defaults-path "${{ env.HF_DEFAULTS_PATH }}" \
             --artifact-name "$ARTIFACT_NAME" \
-            --metadata-path "output/metadata.json" \
-            --onnx-sha256 "${{ steps.meta.outputs.onnx_sha256 }}" \
-            --tinygrad-ref "${{ steps.meta.outputs.tinygrad_ref }}"
+            --model-dir output \
+            --onnx-path "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" \
+            --onnx-ref "${{ needs.resolve_name.outputs.onnx_ref }}" \
+            --model-name "${{ needs.resolve_name.outputs.model_name }}" \
+            --tinygrad-ref "$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)" \
+            --run-number "${{ github.run_number }}"

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -36,6 +36,7 @@ jobs:
       publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
       is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
       build: ${{ steps.strategy.outputs.build }}
+      include_big_model: ${{ steps.strategy.outputs.include_big_model }}
     steps:
       - uses: actions/checkout@v4
       - name: Extract deploy strategy
@@ -78,6 +79,9 @@ jobs:
             stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
             echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
             echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
+
+            include_big_model="$(echo "$CONFIG" | jq -r '.include_big_model // false')";
+            echo "include_big_model=$include_big_model" >> $GITHUB_OUTPUT
           fi
           echo "build=$BUILD" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
@@ -203,6 +207,101 @@ jobs:
           source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
 
+  prepare_chestnut:
+    needs: [ prepare_strategy ]
+    runs-on: ubuntu-24.04
+    if: ${{ needs.prepare_strategy.outputs.include_big_model == 'true' }}
+    env:
+      HF_REPO: sunnypilot/sunnypilot_models_v1
+      HF_DEFAULTS_PATH: models/defaults/big
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+
+      - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
+
+      - name: Check HF defaults and build if needed
+        id: resolve
+        run: |
+          ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
+          echo "Repo ONNX hash: $ACTUAL_ONNX_HASH"
+
+          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
+
+          check_hash() {
+            DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
+            BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
+            [ -n "$BUNDLE" ] && [ "$BUNDLE" != "null" ]
+          }
+
+          if check_hash; then
+            echo "HF defaults match repo ONNX"
+          else
+            echo "No matching model on HF — triggering build"
+            gh workflow run build-default-big-model.yaml --ref "${{ github.head_ref || github.ref_name }}"
+
+            echo "Waiting for build to start..."
+            sleep 120
+
+            RUN_ID=$(gh run list --workflow=build-default-big-model.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId')
+            if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
+              echo "::error::Failed to find build-default-big-model run"
+              exit 1
+            fi
+
+            echo "Waiting for run $RUN_ID..."
+            gh run watch "$RUN_ID"
+
+            CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
+            if [ "$CONCLUSION" != "success" ]; then
+              echo "::error::build-default-big-model failed: $CONCLUSION"
+              exit 1
+            fi
+
+            if ! check_hash; then
+              echo "::error::HF defaults still don't match after build"
+              exit 1
+            fi
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download big model chunks
+        run: |
+          ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
+          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
+          DEFAULTS=$(curl -fsSL "$JSON_URL")
+          BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
+
+          mkdir -p big_model_chunks
+          ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
+          BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
+          NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
+
+          CANONICAL="big_driving_tinygrad.pkl"
+          echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
+            CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+')
+            CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
+            ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
+            echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
+            curl -fsSL -o "big_model_chunks/${CANONICAL_CHUNK}" "$ENCODED_URL"
+          done
+
+          echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest"
+
+      - name: Upload big model chunks
+        uses: actions/upload-artifact@v4
+        with:
+          name: big-model-chunks
+          path: big_model_chunks/
+          compression-level: 0
+
+      - name: Cancel run on failure
+        if: failure()
+        run: gh run cancel ${{ github.run_id }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   publish:
     concurrency:
@@ -211,14 +310,20 @@ jobs:
       # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
       group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
       cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
-    if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
-    needs: [ build, prepare_strategy ]
+    if: ${{
+      always() && !cancelled() &&
+      needs.build.result == 'success' &&
+      needs.prepare_strategy.result == 'success' &&
+      (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
+      (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
+      }}
+    needs: [ build, prepare_strategy, prepare_chestnut ]
     runs-on: ubuntu-24.04
     environment: ${{ needs.prepare_strategy.outputs.environment }}
     steps:
       - uses: actions/checkout@v4
 
-      - name: Download build artifacts
+      - name: Download prebuilt artifact
         uses: actions/download-artifact@v4
         with:
           name: prebuilt
@@ -228,6 +333,24 @@ jobs:
           mkdir -p ${{ env.OUTPUT_DIR }}
           tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
 
+      - name: Prepare chestnut output
+        if: ${{ needs.prepare_chestnut.result == 'success' }}
+        run: |
+          mkdir -p "${{ github.workspace }}/chestnut_output"
+          tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output"
+
+      - name: Download big model chunks
+        if: ${{ needs.prepare_chestnut.result == 'success' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: big-model-chunks
+          path: big_model_chunks
+
+      - name: Inject big model into chestnut
+        if: ${{ needs.prepare_chestnut.result == 'success' }}
+        run: |
+          cp big_model_chunks/* "${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models/"
+
       - name: Configure Git
         run: |
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
@@ -248,6 +371,22 @@ jobs:
             "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
             "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
 
+      - name: Publish chestnut branch
+        if: ${{ needs.prepare_chestnut.result == 'success' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
+          CHESTNUT_DIR="${{ github.workspace }}/chestnut_output"
+
+          ${{ env.CI_DIR }}/publish.sh \
+            "${{ github.workspace }}" \
+            "$CHESTNUT_DIR" \
+            "$CHESTNUT_BRANCH" \
+            "${{ needs.prepare_strategy.outputs.version }}" \
+            "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
+            "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
+
       - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
         if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
         run: |
@@ -260,6 +399,7 @@ jobs:
     - prepare_strategy
     - build
     - publish
+    - prepare_chestnut
     runs-on: ubuntu-24.04
     if: ${{ (always() && !cancelled() && !failure())
       && needs.publish.result == 'success'
@@ -279,6 +419,7 @@ jobs:
           export commit_short_sha="${commit_short_sha:0:7}"
           export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
           export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
+          export chestnut_branch="${{ needs.prepare_chestnut.result == 'success' && format('{0}-chestnut', needs.prepare_strategy.outputs.new_branch) || '' }}"
 
           MESSAGE=$(cat << 'EOF' | envsubst
           ${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}

openpilot/common/params_keys.h

@@ -132,6 +132,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
     {"UsbGpuActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
     {"UsbGpuLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
+    {"UsbGpuLoadProgress", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, INT, "0"}},
     {"Version", {PERSISTENT, STRING}},
 
     // --- sunnypilot params --- //

openpilot/common/version.py

@@ -16,6 +16,15 @@ MASTER_SP_BRANCHES = ['master']
 RELEASE_BRANCHES = ['release-tizi-staging', 'release-mici-staging', 'release-tizi', 'release-mici', 'nightly']
 TESTED_BRANCHES = RELEASE_BRANCHES + ['devel-staging', 'nightly-dev'] + RELEASE_SP_BRANCHES + TESTED_SP_BRANCHES
 
+CHESTNUT_BRANCHES = {
+  "staging": "staging-chestnut",
+  "dev": "dev-chestnut",
+  "release-mici": "release-chestnut",
+  "release-tizi": "release-chestnut",
+  "release-mici-staging": "release-chestnut-staging",
+  "release-tizi-staging": "release-chestnut-staging",
+}
+
 SP_BRANCH_MIGRATIONS = {
   ("tici", "staging-c3-new"): "staging-tici",
   ("tici", "dev-c3-new"): "staging-tici",

openpilot/selfdrive/modeld/load_progress.py

@@ -0,0 +1,45 @@
+import os
+from openpilot.common.file_chunker import open_file_chunked, get_existing_chunks
+from openpilot.common.params import Params
+
+
+class ProgressReader:
+  # wraps a chunked stream, writes byte-read % to a param (throttled to whole percent)
+  def __init__(self, inner, total, param):
+    self._inner, self._total, self._param = inner, total, param
+    self._params = Params()
+    self._read = 0
+    self._pct = -1
+    # sub-read size ~ 1% of total, so big weight buffers advance the % smoothly (not in jumps)
+    self._step = max(64 * 1024, total // 100)
+
+  def _bump(self, n):
+    self._read += n
+    if self._total:
+      pct = min(100, self._read * 100 // self._total)
+      if pct != self._pct:
+        self._pct = pct
+        self._params.put(self._param, pct)
+
+  def read(self, size=-1):
+    data = self._inner.read(size)
+    self._bump(len(data))
+    return data
+
+  def readinto(self, b):
+    # sub-chunk big buffers so the % advances smoothly instead of jumping per weight
+    view = memoryview(b)
+    done = 0
+    while done < len(view):
+      n = self._inner.readinto(view[done:done + self._step])
+      if not n:
+        break
+      done += n
+      self._bump(n)
+    return done
+
+
+def open_with_progress(pkl_path, param="UsbGpuLoadProgress"):
+  # chunked reader that reports load progress to `param`
+  total = sum(os.path.getsize(p) for p in get_existing_chunks(pkl_path))
+  return ProgressReader(open_file_chunked(pkl_path), total, param)

openpilot/selfdrive/modeld/modeld.py

@@ -29,6 +29,7 @@ from openpilot.selfdrive.modeld.parse_model_outputs import Parser
 from openpilot.selfdrive.modeld.compile_modeld import make_input_queues, WARP_INPUTS, POLICY_INPUTS
 from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_driving_model_data, fill_pose_msg, PublishState
 from openpilot.common.file_chunker import open_file_chunked
+from openpilot.selfdrive.modeld.load_progress import open_with_progress
 from openpilot.selfdrive.modeld.constants import ModelConstants, Plan
 from openpilot.selfdrive.modeld.helpers import usbgpu_present, usbgpu_compiled, modeld_pkl_path, get_tg_input_devices, load_oob
 
@@ -145,7 +146,9 @@ class ModelState(ModelStateBase):
     ModelStateBase.__init__(self)
     input_devices = get_tg_input_devices(PROCESS_NAME, usbgpu)
     self.WARP_DEV, self.QUEUE_DEV = input_devices['WARP_DEV'], input_devices['QUEUE_DEV']
-    jits = load_oob(open_file_chunked(modeld_pkl_path(usbgpu)))
+    pkl_path = modeld_pkl_path(usbgpu)
+    stream = open_with_progress(pkl_path) if usbgpu else open_file_chunked(pkl_path)
+    jits = load_oob(stream)
     metadata = jits['metadata']
     self.input_shapes = metadata['input_shapes']
     self.vision_input_names = [k for k in self.input_shapes if 'img' in k]

openpilot/selfdrive/selfdrived/alerts_offroad.json

@@ -18,7 +18,7 @@
     "_comment": "Set extra field to the failed reason."
   },
   "Offroad_ChestnutBranch": {
-    "text": "Chestnut detected! Switch to the release-chestnut branch to use chestnut-class models.",
+    "text": "Chestnut detected! Switch to the %1 branch to use chestnut-class models.",
     "severity": 0
   },
   "Offroad_UnregisteredHardware": {

openpilot/selfdrive/ui/mici/onroad/hud_renderer.py

@@ -223,14 +223,21 @@ class HudRenderer(Widget):
     if icon is not self._egpu_icon:
       self._egpu_fade_time = rl.get_time()
       self._egpu_icon = icon
-    alpha = self._egpu_alpha_filter.update(loading or 0 < rl.get_time() - self._egpu_fade_time < SET_SPEED_PERSISTENCE)
-    if alpha < 1e-2:
-      return
+    # alpha = self._egpu_alpha_filter.update(loading or 0 < rl.get_time() - self._egpu_fade_time < SET_SPEED_PERSISTENCE)
+    # if alpha < 1e-2:
+    #   return
+    alpha = 1.0
 
     pos = rl.Vector2(rect.x + rect.width - 10 - icon.width,
                      rect.y + rect.height - 14 - (self._txt_wheel.height + icon.height) / 2)
     rl.draw_texture_ex(icon, pos, 0.0, 1.0, rl.Color(255, 255, 255, int(255 * opacity * alpha)))
 
+    if loading:
+      pct_text = f"{ui_state.usbgpu_load_progress}%"
+      pct_size = measure_text_cached(self._font_bold, pct_text, FONT_SIZES.max_speed)
+      pct_pos = rl.Vector2(pos.x - 8 - pct_size.x, pos.y + (icon.height - pct_size.y) / 2)
+      rl.draw_text_ex(self._font_bold, pct_text, pct_pos, FONT_SIZES.max_speed, 0, rl.WHITE)
+
   def _draw_steering_wheel(self, rect: rl.Rectangle) -> None:
     wheel_txt = self._txt_wheel_critical if self._show_wheel_critical else self._txt_wheel
 

openpilot/selfdrive/ui/ui_state.py

@@ -86,6 +86,7 @@ class UIState(UIStateSP):
     self.usbgpu_compiled: bool = usbgpu_compiled()
     self.usbgpu_active: bool | None = self.params.get("UsbGpuActive")
     self.usbgpu_loading: bool = self.params.get_bool("UsbGpuLoading")
+    self.usbgpu_load_progress: int = self.params.get("UsbGpuLoadProgress", return_default=True)
     self.started: bool = False
     self.ignition: bool = False
     self.recording_audio: bool = False
@@ -164,6 +165,10 @@ class UIState(UIStateSP):
     # Update started state
     self.started = self.sm["deviceState"].started and self.ignition
 
+    # Poll gpu load progress per-frame (not 5Hz) so the % counter updates smoothly
+    if self.usbgpu_loading:
+      self.usbgpu_load_progress = self.params.get("UsbGpuLoadProgress", return_default=True)
+
     # Update body state
     if self.CP is not None and self.is_body != self.CP.notCar:
       self.is_body = self.CP.notCar
@@ -222,6 +227,7 @@ class UIState(UIStateSP):
       self.usbgpu_compiled = usbgpu_compiled()
     self.usbgpu_active = self.params.get("UsbGpuActive")
     self.usbgpu_loading = self.params.get_bool("UsbGpuLoading")
+    self.usbgpu_load_progress = self.params.get("UsbGpuLoadProgress", return_default=True)
 
     UIStateSP.update_params(self)
 

openpilot/sunnypilot/modeld_v2/compile_modeld.py

@@ -7,7 +7,6 @@ See the LICENSE.md file in the root directory for more details.
 """
 
 import argparse
-import math
 import os
 import tempfile
 import time
@@ -67,15 +66,14 @@ def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tu
   if desire_key:
     shapes['desire'] = (input_shapes[desire_key][2],)
 
+  if is_supercombo and 'features_buffer' in input_shapes:
+    fb = input_shapes['features_buffer']
+    shapes['prev_feat'] = (fb[0], fb[2])
+
   for key, shape in input_shapes.items():
     if key not in (desire_key, 'features_buffer') and 'img' not in key:
       shapes[key] = tuple(shape)
 
-  if is_supercombo and 'features_buffer' in input_shapes:
-    fb = input_shapes['features_buffer']
-    feat_dim = math.prod(fb[2:])
-    shapes['prev_feat'] = (fb[0], feat_dim)
-
   sizes = [int(np.prod(size)) for size in shapes.values()]
   return shapes, sizes
 
@@ -119,9 +117,8 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D
   }
 
   if features_buffer:
-    feat_dim = math.prod(features_buffer[2:])
     feat_q_len = frame_skip * features_buffer[1] if is_supercombo else frame_skip * (features_buffer[1] - 1) + 1
-    queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], feat_dim),
+    queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], features_buffer[2]),
                        dtype=np.float32), device=device).contiguous().realize()
 
   queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')})
@@ -186,14 +183,14 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
     warped_dev = warped.to(Device.DEFAULT)
     Tensor.realize(packed_npy_inputs_dev, warped_dev)
 
-    img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn)
-    big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn)
+    img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn).realize()
+    big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn).realize()
 
     unpacked_tensors = [tensor.reshape(shape) for tensor, shape in zip(packed_npy_inputs_dev.split(npy_sizes), npy_shapes.values(), strict=True)]
     unpacked_dict = dict(zip(npy_shapes.keys(), unpacked_tensors, strict=True))
 
     desire_dev = unpacked_dict['desire']
-    desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn)
+    desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn).realize()
 
     inputs = {desire_key: desire_buf}
     for key, tensor_val in unpacked_dict.items():
@@ -202,22 +199,19 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
 
     if 'prev_feat' in unpacked_dict:
       prev_feat_dev = unpacked_dict['prev_feat']
-      feat_buf = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn)
-      inputs['features_buffer'] = feat_buf if len(fb := input_shapes['features_buffer']) <= 3 else feat_buf.reshape(fb)
+      inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).realize()
 
     if vision_runner:
       vision_out_cast = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32').realize()
       if 'features_buffer' not in inputs:
         new_feat = vision_out_cast[:, features_slice].reshape(1, -1).unsqueeze(0)
-        feat_buf = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize()
-        inputs['features_buffer'] = feat_buf if len(fb := input_shapes['features_buffer']) <= 3 else feat_buf.reshape(fb)
+        inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize()
       policy_outs = [next(iter(pol_runner(inputs).values())).cast('float32').realize() for pol_runner in policy_runners]
       return (vision_out_cast, *policy_outs) if len(policy_outs) > 1 else (vision_out_cast, policy_outs[0])
 
     inputs.update({road_key: img, wide_key: big_img})
     if 'features_buffer' not in inputs:
-      feat_buf = sample_skip_fn(feat_q)
-      inputs['features_buffer'] = feat_buf if len(fb := input_shapes['features_buffer']) <= 3 else feat_buf.reshape(fb)
+      inputs['features_buffer'] = sample_skip_fn(feat_q)
 
     policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32').realize()
     if 'features_buffer' not in inputs and features_slice is not None:

openpilot/sunnypilot/modeld_v2/modeld.py

@@ -25,6 +25,7 @@ from opendbc.car.car_helpers import get_demo_car_params
 from tinygrad.tensor import Tensor
 
 from openpilot.common.file_chunker import open_file_chunked
+from openpilot.selfdrive.modeld.load_progress import open_with_progress
 from openpilot.common.swaglog import cloudlog
 from openpilot.common.params import Params
 from openpilot.common.filter_simple import FirstOrderFilter
@@ -107,7 +108,8 @@ class ModelState(ModelStateBase):
 
   def _init_combined(self, pkl_path, cam_w, cam_h, bundle):
     cloudlog.warning(f"loading combined pkl: {pkl_path}")
-    jits = load_oob(open_file_chunked(pkl_path))
+    stream = open_with_progress(pkl_path) if self.usbgpu else open_file_chunked(pkl_path)
+    jits = load_oob(stream)
 
     self.WARP_DEV = 'QCOM' if COMMA_HARDWARE else 'CPU'
     self.DEV = 'AMD' if self.usbgpu else self.WARP_DEV
@@ -333,6 +335,7 @@ def main(demo=False):
 
   params = Params()
   params.put_bool("UsbGpuLoading", USBGPU)
+  params.put("UsbGpuLoadProgress", 0)
   params.remove("UsbGpuActive")
 
   # visionipc clients

openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py

@@ -195,85 +195,3 @@ class TestReadFileChunkedToDisk(OpenpilotTestCase):
 
       assert out.parent == Path(d)
       assert out.read_bytes() == payload
-
-
-class Test4DFeaturesBuffer(OpenpilotTestCase):
-  def test_get_policy_npy_shapes_4d(self):
-    from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes
-    input_shapes = {
-      'desire_pulse': (1, 25, 8),
-      'features_buffer': (1, 24, 32, 512),  # compare 4d to 3d for regression
-      'traffic_convention': (1, 2),
-      'action_t': (1, 2)
-    }
-    shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True)
-    assert shapes['prev_feat'] == (1, 16384)
-    assert sizes == [8, 2, 2, 16384]
-
-  def test_get_policy_npy_shapes_3d(self):
-    from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes
-    input_shapes = {
-      'desire_pulse': (1, 25, 8),
-      'features_buffer': (1, 24, 512),
-      'traffic_convention': (1, 2),
-      'action_t': (1, 2)
-    }
-    shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True)
-    assert shapes['prev_feat'] == (1, 512)
-    assert sizes == [8, 2, 2, 512]
-
-
-class TestStockCompileModeldEquivalence(OpenpilotTestCase):
-  def test_get_policy_npy_shapes_matches_stock(self):
-    from openpilot.selfdrive.modeld.compile_modeld import get_policy_npy_shapes as stock_get_policy_npy_shapes
-    from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes as sunny_get_policy_npy_shapes
-
-    stock_input_shapes = {
-      'desire_pulse': (1, 25, 8),
-      'features_buffer': (1, 24, 512), # see below comment
-      'traffic_convention': (1, 2),
-      'action_t': (1, 2),
-    }
-
-    stock_shapes, stock_sizes = stock_get_policy_npy_shapes(stock_input_shapes)
-    sunny_shapes, sunny_sizes = sunny_get_policy_npy_shapes(stock_input_shapes, is_supercombo=True)
-
-    assert sunny_shapes == stock_shapes
-    assert sunny_sizes == stock_sizes
-    assert sunny_shapes['prev_feat'] == (1, 512)
-
-  def test_make_input_queues_full_stock_equivalence(self):
-    from openpilot.selfdrive.modeld.compile_modeld import make_input_queues as stock_make_input_queues
-    from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues as sunny_make_supercombo_input_queues
-    input_shapes = {
-      'img': (1, 12, 128, 256),
-      'desire_pulse': (1, 25, 8),
-      'features_buffer': (1, 24, 512),  # when https://github.com/commaai/openpilot/pull/38681 merges, update to 1,24,32,512
-      'traffic_convention': (1, 2),
-      'action_t': (1, 2),
-    }
-    frame_skip = 4
-
-    stock_queues, stock_npy = stock_make_input_queues(input_shapes, frame_skip, device='NPY')
-    sunny_queues, sunny_npy = sunny_make_supercombo_input_queues(input_shapes, frame_skip, device='NPY')
-    assert set(sunny_queues.keys()) == set(stock_queues.keys())
-    for key in stock_queues:
-      assert sunny_queues[key].shape == stock_queues[key].shape, \
-        f"Queue shape mismatch for {key}: sunny {sunny_queues[key].shape} != stock {stock_queues[key].shape}"
-    assert set(sunny_npy.keys()) == set(stock_npy.keys())
-    for key in stock_npy:
-      assert sunny_npy[key].shape == stock_npy[key].shape, \
-        f"Numpy array shape mismatch for {key}: sunny {sunny_npy[key].shape} != stock {stock_npy[key].shape}"
-
-  def test_make_warp_queues_stock_equivalence(self):
-    from openpilot.selfdrive.modeld.compile_modeld import make_warp_input_queues as stock_make_warp_queues
-    from openpilot.sunnypilot.modeld_v2.compile_modeld import make_warp_queues as sunny_make_warp_queues
-    stock_vision_shapes = {'img': (1, 12, 128, 256)}  # for now?
-    stock_queues, stock_npy = stock_make_warp_queues(stock_vision_shapes, frame_skip=4, device='NPY')
-    sunny_queues, sunny_npy = sunny_make_warp_queues(device='NPY')
-
-    assert set(sunny_npy.keys()) == set(stock_npy.keys()) == {'tfm', 'big_tfm'}
-    for key in sunny_npy:
-      assert sunny_npy[key].shape == stock_npy[key].shape == (3, 3)
-
-

openpilot/sunnypilot/models/fetcher.py

@@ -141,7 +141,7 @@ class ModelCache:
 class ModelFetcher:
   """Handles fetching and caching of model data from remote source"""
   MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v20.json"
-  MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v22.json"
+  MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v21.json"
 
   def __init__(self, params: Params):
     self.params = params

openpilot/system/hardware/hardwared.py

@@ -27,7 +27,7 @@ from openpilot.common.swaglog import cloudlog
 from openpilot.sunnypilot.system.statsd import statlog
 from openpilot.system.hardware.power_monitoring import PowerMonitoring
 from openpilot.system.hardware.fan_controller import FanController
-from openpilot.common.version import terms_version, training_version, get_build_metadata, terms_version_sp
+from openpilot.common.version import terms_version, training_version, get_build_metadata, terms_version_sp, CHESTNUT_BRANCHES
 
 
 ThermalStatus = log.DeviceState.ThermalStatus
@@ -301,7 +301,11 @@ def hardware_thread(end_event, hw_queue) -> None:
 
     set_usb_state(msg.deviceState, last_hw_state.usb_state)
     chestnut.update(started_ts is None, last_hw_state.usb_state)
-    set_offroad_alert_if_changed("Offroad_ChestnutBranch", msg.deviceState.chestnutPresent and not big_model_available)
+    current_channel = get_build_metadata().channel
+    chestnut_target = CHESTNUT_BRANCHES.get(current_channel)
+    chestnut_needs_switch = msg.deviceState.chestnutPresent and not big_model_available and chestnut_target is not None
+    set_offroad_alert_if_changed("Offroad_ChestnutBranch", chestnut_needs_switch,
+                                 extra_text=chestnut_target if chestnut_needs_switch else None)
 
     # this subset is only used for offroad
     temp_sources = [

release/ci/model_generator.py

@@ -90,6 +90,18 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path:
   return old_pkl.rename(new_pkl)
 
 
+def _hash_onnx_files(model_dir: Path) -> str | None:
+  onnx_files = sorted(model_dir.glob("*.onnx"))
+  if not onnx_files:
+    return None
+  digest = hashlib.sha256()
+  for f in onnx_files:
+    with f.open('rb') as fh:
+      while block := fh.read(1024 * 1024):
+        digest.update(block)
+  return digest.hexdigest()
+
+
 def generate_chunked_model(driving_pkl: Path) -> dict:
   tinygrad_hash = _hash_pkl(driving_pkl)
 
@@ -123,7 +135,8 @@ def generate_chunked_model(driving_pkl: Path) -> dict:
   }
 
 
-def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown") -> None:
+def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown",
+                         onnx_sha256=None) -> None:
   bundle_json = {
     "short_name": short_name,
     "display_name": custom_name or upstream_branch,
@@ -139,6 +152,9 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short
     "models": models,
   }
 
+  if onnx_sha256:
+    bundle_json["onnx_sha256"] = onnx_sha256
+
   # Write metadata to output_dir
   metadata_json = {
     "bundles": [bundle_json]
@@ -178,4 +194,6 @@ if __name__ == "__main__":
       _driving_pkl = new_pkl
 
   _model_metadata = generate_chunked_model(_driving_pkl)
-  create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch)
+  _onnx_sha256 = _hash_onnx_files(Path(args.model_dir))
+  create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch,
+                       onnx_sha256=_onnx_sha256)

release/ci/upload_default_model.py

@@ -7,35 +7,66 @@ See the LICENSE.md file in the root directory for more details.
 """
 
 import argparse
+import hashlib
 import json
-import sys
 import tempfile
 
 from huggingface_hub import HfApi, hf_hub_download
 
 
+def hash_file(path: str) -> str:
+  digest = hashlib.sha256()
+  with open(path, 'rb') as f:
+    while block := f.read(1024 * 1024):
+      digest.update(block)
+  return digest.hexdigest()
+
+
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument("--hf-repo", required=True)
   parser.add_argument("--hf-defaults-path", required=True)
   parser.add_argument("--artifact-name", required=True)
-  parser.add_argument("--metadata-path", required=True)
-  parser.add_argument("--onnx-sha256", required=True)
+  parser.add_argument("--model-dir", required=True)
+  parser.add_argument("--onnx-path", required=True)
+  parser.add_argument("--onnx-ref", required=True)
+  parser.add_argument("--model-name", required=True)
   parser.add_argument("--tinygrad-ref", required=True)
+  parser.add_argument("--run-number", required=True)
   args = parser.parse_args()
 
-  with open(args.metadata_path) as f:
+  api = HfApi()
+  onnx_sha256 = hash_file(args.onnx_path)
+  short_ref = args.onnx_ref[:8]
+  folder_name = f"model-{args.model_name}-{short_ref}-{args.run_number}"
+
+  print(f"ONNX hash: {onnx_sha256}")
+  print(f"ONNX ref: {args.onnx_ref} (short: {short_ref})")
+  print(f"Folder: {folder_name}")
+
+  metadata_path = f"{args.model_dir}/metadata.json"
+  with open(metadata_path) as f:
     metadata = json.load(f)
 
   bundle = metadata['bundles'][0]
-  bundle['onnx_sha256'] = args.onnx_sha256
+  bundle['display_name'] = args.model_name
+  bundle['onnx_sha256'] = onnx_sha256
+  bundle['onnx_ref'] = args.onnx_ref
 
   artifact = bundle['models'][0]['artifact']
-  hf_base = f"https://huggingface.co/datasets/{args.hf_repo}/resolve/main/{args.hf_defaults_path}/{args.artifact_name}"
+  hf_base = f"https://huggingface.co/datasets/{args.hf_repo}/resolve/main/{args.hf_defaults_path}/{folder_name}"
   artifact['download_uri']['url'] = f"{hf_base}/{artifact['file_name']}"
   for chunk in artifact.get('chunks', []):
     chunk['url'] = f"{hf_base}/{chunk['file_name']}"
 
+  print(f"Uploading model to {args.hf_defaults_path}/{folder_name}/")
+  api.upload_folder(
+    folder_path=args.model_dir,
+    path_in_repo=f"{args.hf_defaults_path}/{folder_name}",
+    repo_id=args.hf_repo,
+    repo_type="dataset",
+  )
+
   json_filename = f"{args.hf_defaults_path}/default_models.json"
   try:
     local_path = hf_hub_download(repo_id=args.hf_repo, repo_type='dataset', filename=json_filename)
@@ -47,7 +78,7 @@ def main():
   defaults_json['tinygrad_ref'] = args.tinygrad_ref
 
   existing_idx = next((i for i, b in enumerate(defaults_json['bundles'])
-                       if b.get('display_name') == bundle.get('display_name')), None)
+                       if b.get('onnx_sha256') == onnx_sha256), None)
   if existing_idx is not None:
     defaults_json['bundles'][existing_idx] = bundle
   else:
@@ -55,7 +86,6 @@ def main():
 
   print(json.dumps(defaults_json, indent=2))
 
-  api = HfApi()
   with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
     json.dump(defaults_json, f, indent=2)
     tmp_path = f.name