Merge branch 'master' into auxpowersave

This reverts commit 01420fc08488374ec8fe3d17b757ccd1564d1328.

.github/release-drafter.yml

@@ -0,0 +1,43 @@
+exclude-labels:
+  - 'no-changelog'
+categories:
+  - title: '🚀 Features'
+    labels:
+      - 'feature'
+      - 'enhancement'
+  - title: '🐛 Bug Fixes'
+    collapse-after: 5
+    labels:
+      - 'fix'
+      - 'bugfix'
+      - 'bug'
+  - title: '🧰 Maintenance'
+    collapse-after: 5
+    label: 'chore'
+change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
+change-title-escapes: '\<*_&'
+replacers:
+  - search: '/[Ss][Uu][Nn][Nn][Yy][Pp][Ii][Ll][Oo][Tt]/g'
+    replace: 'sunnypilot'
+  - search: '/\b[Ss][Pp]\b/g'
+    replace: 'SP'
+version-resolver:
+  major:
+    labels:
+      - 'major'
+  minor:
+    labels:
+      - 'minor'
+  patch:
+    labels:
+      - 'patch'
+  default: patch
+name-template: 'v$RESOLVED_VERSION 🚀'
+tag-template: 'v$RESOLVED_VERSION'
+version-template: "0.$MAJOR.$MINOR.$PATCH" # The day OP becomes v1, we need to bump this
+tag-prefix: "v0." # The day OP becomes v1, we need to bump this
+prerelease-identifier: "staging"
+template: |
+  ## Changes
+
+  $CHANGES

.github/workflows/build-all-tinygrad-models.yaml

@@ -20,11 +20,6 @@ on:
         required: false
         type: string
         default: 'sunnypilot/sunnypilot_models_v1'
-      docs_repo:
-        description: 'GitHub repo holding the driving_models JSON on its gh-pages branch'
-        required: false
-        type: string
-        default: 'sunnypilot/sunnypilot-models'
 
 jobs:
   setup:
@@ -39,6 +34,7 @@ jobs:
       - name: Checkout sunnypilot repo
         uses: actions/checkout@v4
         with:
+          repository: sunnypilot/sunnypilot
           path: sunnypilot
           submodules: recursive
 
@@ -51,10 +47,10 @@ jobs:
           echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
           echo "tinygrad_ref is $ref"
 
-      - name: Checkout docs repo (gh-pages)
+      - name: Checkout docs repo (sunnypilot-models, gh-pages)
         uses: actions/checkout@v4
         with:
-          repository: ${{ inputs.docs_repo }}
+          repository: sunnypilot/sunnypilot-models
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
@@ -122,7 +118,6 @@ jobs:
       json_version: ${{ needs.setup.outputs.json_version }}
       target_hardware: ${{ github.event.inputs.target_hardware }}
       hf_repo: ${{ github.event.inputs.hf_repo }}
-      docs_repo: ${{ inputs.docs_repo }}
       set_min_version: ${{ github.event.inputs.set_min_version }}
       tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
     secrets: inherit
@@ -167,7 +162,6 @@ jobs:
       target_hardware: ${{ github.event.inputs.target_hardware }}
       artifact_suffix: -retry
       hf_repo: ${{ github.event.inputs.hf_repo }}
-      docs_repo: ${{ inputs.docs_repo }}
       set_min_version: ${{ github.event.inputs.set_min_version }}
       tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
     secrets: inherit

.github/workflows/build-single-tinygrad-model.yaml

@@ -39,11 +39,6 @@ on:
         required: false
         type: string
         default: 'sunnypilot/sunnypilot_models_v1'
-      docs_repo:
-        description: 'GitHub repo holding the driving_models JSON on its gh-pages branch'
-        required: false
-        type: string
-        default: 'sunnypilot/sunnypilot-models'
       set_min_version:
         description: 'Minimum selector version'
         required: false
@@ -112,11 +107,6 @@ on:
         required: false
         type: string
         default: 'sunnypilot/sunnypilot_models_v1'
-      docs_repo:
-        description: 'GitHub repo holding the driving_models JSON on its gh-pages branch'
-        required: false
-        type: string
-        default: 'sunnypilot/sunnypilot-models'
 env:
   RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
   JSON_FILE: docs/docs/driving_models_${{ inputs.target_hardware == 'chestnut' && 'chestnut_v' || 'v' }}${{ inputs.json_version }}.json
@@ -146,7 +136,7 @@ jobs:
       - name: Checkout docs repo
         uses: actions/checkout@v4
         with:
-          repository: ${{ inputs.docs_repo }}
+          repository: sunnypilot/sunnypilot-models
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}

.github/workflows/release-drafter.yml

@@ -0,0 +1,28 @@
+name: Release Drafter
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*'
+  pull_request_target:
+    types: [opened, reopened, synchronize]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  update_release_draft:
+    permissions:
+      contents: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: release-drafter/release-drafter@v6
+        with:
+          config-name: release-drafter.yml
+          prerelease: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-discourse.yaml.yml

@@ -0,0 +1,78 @@
+name: Debug Discourse Posting
+
+on:
+  push:
+
+jobs:
+  test-discourse-post:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Post test message to Discourse
+        uses: ./.github/workflows/post-to-discourse
+        with:
+          discourse-url: ${{ vars.DISCOURSE_URL }}
+          api-key: ${{ secrets.DISCOURSE_API_KEY }}
+          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
+          topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
+          message: |
+            ## 🧪 Test Post from GitHub Actions
+            
+            **This is a test post to verify Discourse integration**
+            
+            - **Workflow**: ${{ github.workflow }}
+            - **Run Number**: #${{ github.run_number }}
+            - **Branch**: `${{ github.ref_name }}`
+            - **Commit**: ${{ github.sha }}
+            - **Actor**: @${{ github.actor }}
+            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
+            
+            ---
+            
+            ### Fake Build Info (for testing)
+            - **Version**: 0.9.8-test
+            - **Build**: #42
+            - **Branch**: release-test
+            
+            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+            
+            *This is an automated test message. Drive safe! 🚗💨*
+      
+      
+      - name: Create topic on Discourse
+        uses: ./.github/workflows/post-to-discourse
+        with:
+          discourse-url: ${{ vars.DISCOURSE_URL }}
+          api-key: ${{ secrets.DISCOURSE_API_KEY }}
+          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
+          #topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
+          category-id: 4
+          title: "This is a test of a new topic instead of a reply"
+          message: |
+            ## 🧪 Test Post from GitHub Actions
+            
+            **This is a test post to verify Discourse integration**
+            
+            - **Workflow**: ${{ github.workflow }}
+            - **Run Number**: #${{ github.run_number }}
+            - **Branch**: `${{ github.ref_name }}`
+            - **Commit**: ${{ github.sha }}
+            - **Actor**: @${{ github.actor }}
+            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
+            
+            ---
+            
+            ### Fake Build Info (for testing)
+            - **Version**: 0.9.8-test
+            - **Build**: #42
+            - **Branch**: release-test
+            
+            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+            
+            *This is an automated test message. Drive safe! 🚗💨*
+      - name: Display results
+        if: always()
+        run: |
+          echo "::notice::Discourse post test completed"
+          echo "Check your Discourse topic to verify the post appeared correctly"

.github/workflows/test_models.yml

@@ -1,79 +0,0 @@
-name: Test Models Compatibility With Tinygrad Changes
-on:
-  pull_request:
-    paths:
-      - 'tinygrad_repo'
-  workflow_dispatch:
-
-jobs:
-  generate-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      models: ${{ steps.set-matrix.outputs.models }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Fetch and Parse json
-        id: set-matrix
-        run: |
-          python3 -c '
-          import json, urllib.request, os, re
-
-          with open("openpilot/sunnypilot/models/fetcher.py", "r") as f:
-            urls = re.findall(r"MODEL_URL(?:_CHESTNUT)?\s*=\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", f.read())
-
-          artifacts = []
-          for url in urls:
-            data = json.loads(urllib.request.urlopen(url).read())
-            for bundle in data.get("bundles", []):
-              for model in bundle.get("models", []):
-                if "artifact" in model:
-                  artifacts.append(model["artifact"])
-
-          with open(os.environ["GITHUB_OUTPUT"], "a") as f:
-            f.write(f"models={json.dumps(artifacts)}\n")
-          '
-
-  test-model:
-    name: Test ${{ matrix.artifact.file_name }}
-    needs: generate-matrix
-    runs-on: ubuntu-latest
-    container: ghcr.io/commaai/openpilot-base:latest
-    strategy:
-      fail-fast: false
-      matrix:
-        artifact: ${{ fromJson(needs.generate-matrix.outputs.models) }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-
-      - name: Download Model Chunks in Parallel
-        run: |
-          mkdir -p /tmp/model_chunks
-          echo '${{ toJson(matrix.artifact.chunks) }}' > chunks.json
-
-          BASE_URL="${{ matrix.artifact.download_uri.url }}"
-          export BASE_DIR=$(dirname "$BASE_URL")
-
-          python3 -c '
-          import json, os
-          with open("chunks.json") as f:
-            chunks = json.load(f)
-          manifest_path = f"/tmp/model_chunks/${{ matrix.artifact.file_name }}.chunkmanifest"
-          with open(manifest_path, "w") as f:
-            f.write(str(len(chunks)))
-          base_dir = os.environ["BASE_DIR"]
-          with open("/tmp/curl_config.txt", "w") as f:
-            for c in chunks:
-              fn = c["file_name"]
-              f.write(f"url = \"{base_dir}/{fn}\"\noutput = \"/tmp/model_chunks/{fn}\"\n")
-          '
-          curl -Z --parallel-immediate --parallel-max 16 -s -S -f -L -K /tmp/curl_config.txt
-
-      - name: Run Model Compatibility Test
-        env:
-          MODEL_BASE_NAME: ${{ matrix.artifact.file_name }}
-          MODEL_CHUNK_DIR: "/tmp/model_chunks"
-          PYTHONPATH: ".:./tinygrad_repo"
-        run: |
-          python3 -m pytest openpilot/sunnypilot/modeld_v2/tests/test_models.py

openpilot/common/params_keys.h

@@ -139,6 +139,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"ChestnutActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
     {"ChestnutLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
     {"ChestnutModelError", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
+    {"AuxPowerSave", {PERSISTENT | BACKUP, BOOL}},
     {"Version", {PERSISTENT, STRING}},
 
     // --- sunnypilot params --- //

openpilot/selfdrive/modeld/compile_modeld.py

@@ -37,6 +37,7 @@ from tinygrad.engine.jit import TinyJit
 
 
 NV12Frame = namedtuple("NV12Frame", ['width', 'height', 'stride', 'y_height', 'uv_height', 'size'])
+MODELD_INPUTS = ['img_q', 'big_img_q', 'feat_q', 'desire_q', 'packed_npy_inputs']
 
 
 def nv12_copy_size(stride: int, y_height: int, uv_height: int) -> int:
@@ -112,26 +113,58 @@ def make_frame_prepare(nv12: NV12Frame, model_w, model_h):
   return frame_prepare_tinygrad
 
 
-def get_npy_shapes(input_shapes, state_pairs):
-  shapes = {'tfm': (3, 3), 'big_tfm': (3, 3)} | {
-    name: shape for name, (shape, _) in input_shapes.items() if name not in state_pairs and name != 'new_img'}
+def get_policy_npy_shapes(input_shapes):
+  dp = input_shapes['desire_pulse']  # (1, 25, 8)
+  tc = input_shapes['traffic_convention']  # (1, 2)
+  at = input_shapes['action_t']  # (1, 2)
+  fb = input_shapes['features_buffer']  # (1, T-1, ...) e.g. (1, 24, 32, 512) with spatial features
+  feat_dim = math.prod(fb[2:])
+  # TODO prev_feat shouldn't exist and be handled inside the JIT, but corrupt on QCOM for now
+  shapes = {'desire': (dp[2],), 'traffic_convention': tuple(tc), 'action_t': tuple(at), 'prev_feat': (fb[0], feat_dim)}
   return shapes, [math.prod(s) for s in shapes.values()]
 
 
-def make_input_queues(input_shapes, state_pairs, device, frame_copy_size):
-  shapes, sizes = get_npy_shapes(input_shapes, state_pairs)
+def make_input_queues(input_shapes, frame_skip, device, frame_copy_size):
+  img = input_shapes['img']  # (1, 12, 128, 256)
+  fb = input_shapes['features_buffer']  # (1, T-1, ...), past features only; the model appends the current frame's feature
+  feat_dim = math.prod(fb[2:])
+  dp = input_shapes['desire_pulse']  # (1, 25, 8)
+  n_frames = img[1] // 6
+  img_buf_shape = (frame_skip * (n_frames - 1) + 1, 6, img[2], img[3])
+
+  policy_shapes, _ = get_policy_npy_shapes(input_shapes)
+  shapes = {'tfm': (3, 3), 'big_tfm': (3, 3)} | policy_shapes
+  sizes = [math.prod(s) for s in shapes.values()]
   packed_npy_size = sum(sizes) * np.dtype(np.float32).itemsize
   packed_input = np.zeros(packed_npy_size + 2 * frame_copy_size, dtype=np.uint8)
   packed_npy_inputs = packed_input[:packed_npy_size].view(np.float32)
   frames = packed_input[packed_npy_size:]
   frame_views = {'img': frames[:frame_copy_size], 'big_img': frames[frame_copy_size:]}
+  # views into the packed inputs, to be refilled at runtime
   npy = {k: v.reshape(s) for (k, s), v in zip(shapes.items(), np.split(packed_npy_inputs, np.cumsum(sizes[:-1])), strict=True)}
-  input_queues = {name: Tensor(np.zeros(shape, dtype=dtype.fmt), device=device).realize()
-                  for name, (shape, dtype) in input_shapes.items() if name in state_pairs}
-  input_queues['packed_npy_inputs'] = Tensor(packed_input, device='NPY').realize()
+  input_queues = {
+    'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(),
+    'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(),
+    'feat_q': Tensor(np.zeros((frame_skip * fb[1], fb[0], feat_dim), dtype=np.float32), device=device).contiguous().realize(),
+    'desire_q': Tensor(np.zeros((frame_skip * dp[1], dp[0], dp[2]), dtype=np.float32), device=device).contiguous().realize(),
+    'packed_npy_inputs': Tensor(packed_input, device='NPY').realize(),
+  }
   return input_queues, npy, frame_views
 
 
+def shift_and_sample(buf, new_val, sample_fn):
+  buf.assign(buf[1:].cat(new_val, dim=0).contiguous())
+  return sample_fn(buf)
+
+
+def sample_skip(buf, frame_skip):
+  return buf[::frame_skip].contiguous().flatten(0, 1).unsqueeze(0)
+
+
+def sample_desire(buf, frame_skip):
+  return buf.reshape(-1, frame_skip, *buf.shape[1:]).max(1).flatten(0, 1).unsqueeze(0)
+
+
 def make_warp(nv12, model_w, model_h):
   frame_prepare = make_frame_prepare(nv12, model_w, model_h)
 
@@ -149,27 +182,54 @@ def make_warp(nv12, model_w, model_h):
   return warp
 
 
-def make_run_model(warp, model_runner, input_shapes, state_pairs, frame_copy_size):
-  shapes, sizes = get_npy_shapes(input_shapes, state_pairs)
-  packed_npy_size = sum(sizes) * np.dtype(np.float32).itemsize
+def make_run_policy(model_runner, model_metadata, frame_skip):
+  sample_desire_fn = partial(sample_desire, frame_skip=frame_skip)
+  sample_skip_fn = partial(sample_skip, frame_skip=frame_skip)
+  npy_shapes, npy_sizes = get_policy_npy_shapes(model_metadata['input_shapes'])
+  model_input_dtypes = {name: spec.dtype for name, spec in model_runner.graph_inputs.items()}
 
-  def run_model(packed_npy_inputs, **state_inputs):
-    packed_input = packed_npy_inputs.to(Device.DEFAULT).realize()
+  def run_policy(warped, img_q, big_img_q, feat_q, desire_q, packed_npy_inputs):
+    packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT)
+    Tensor.realize(packed_npy_inputs, warped)
+
+    img = shift_and_sample(img_q, warped[0:1], sample_skip_fn)
+    big_img = shift_and_sample(big_img_q, warped[1:2], sample_skip_fn)
+
+    desire, traffic_convention, action_t, prev_feat = (t.reshape(s) for t, s in zip(packed_npy_inputs.split(npy_sizes), npy_shapes.values(), strict=True))
+    desire_buf = shift_and_sample(desire_q, desire.reshape(1, 1, -1), sample_desire_fn)
+    feat_buf = shift_and_sample(feat_q, prev_feat.reshape(1, 1, -1), sample_skip_fn)
+
+    inputs = {
+      'img': img,
+      'big_img': big_img,
+      'features_buffer': feat_buf.reshape(model_metadata['input_shapes']['features_buffer']),
+      'desire_pulse': desire_buf,
+      'traffic_convention': traffic_convention,
+      'action_t': action_t,
+    }
+    inputs = {name: value.cast(model_input_dtypes[name]) for name, value in inputs.items()}
+    out = next(iter(model_runner(inputs).values())).cast('float32')
+    return out,
+  return run_policy
+
+
+def make_run_model(warp, run_policy, model_metadata, frame_copy_size):
+  _, policy_sizes = get_policy_npy_shapes(model_metadata['input_shapes'])
+  packed_npy_size = (18 + sum(policy_sizes)) * np.dtype(np.float32).itemsize
+
+  def run_model(img_q, big_img_q, feat_q, desire_q, packed_npy_inputs):
+    packed_input = packed_npy_inputs.to(Device.DEFAULT)
+    Tensor.realize(packed_input)
     packed_npy_inputs = packed_input[:packed_npy_size].bitcast('float32')
-    inputs = {name: t.reshape(s) for (name, s), t in zip(shapes.items(), packed_npy_inputs.split(sizes), strict=True)}
     frame = packed_input[packed_npy_size:packed_npy_size + frame_copy_size]
     big_frame = packed_input[packed_npy_size + frame_copy_size:]
-    inputs['new_img'] = warp(inputs.pop('tfm'), inputs.pop('big_tfm'), frame, big_frame)
-    inputs = {name: value.cast(input_shapes[name][1]) for name, value in inputs.items()}
-    outputs = {name: value.contiguous() for name, value in model_runner(inputs | state_inputs).items()}
-    Tensor.realize(*outputs.values())
-    if state_pairs:
-      Tensor.realize(*(state_inputs[name].assign(outputs[next_name]) for name, next_name in state_pairs.items()))
-    return tuple(value for name, value in outputs.items() if name not in state_pairs.values())
+    tfm, big_tfm, policy_inputs = packed_npy_inputs.split([9, 9, sum(policy_sizes)])
+    warped = warp(tfm.reshape(3, 3), big_tfm.reshape(3, 3), frame, big_frame)
+    return run_policy(warped, img_q, big_img_q, feat_q, desire_q, policy_inputs)
   return run_model
 
 
-def compile_jit(jit, make_queues, benchmark_runs):
+def compile_jit(jit, input_keys, make_queues, benchmark_runs):
   if benchmark_runs < 1:
     raise ValueError("benchmark_runs must be at least 1")
 
@@ -185,7 +245,7 @@ def compile_jit(jit, make_queues, benchmark_runs):
         v[:] = rng.integers(0, 256, size=v.shape, dtype=np.uint8)
       Device.default.synchronize()
       st = time.perf_counter()
-      outs = fn(**input_queues)
+      outs = fn(**{k: input_queues[k] for k in input_keys})
       mt = time.perf_counter()
       Device.default.synchronize()
       et = time.perf_counter()
@@ -240,6 +300,7 @@ if __name__ == "__main__":
                  help='camera resolutions WxH (one or more)')
   p.add_argument('--onnx', required=True)
   p.add_argument('--output', required=True)
+  p.add_argument('--frame-skip', type=int, required=True)
   p.add_argument('--benchmark-runs', type=int, default=1,
                  help='timed loaded-JIT runs for each correctness seed')
   args = p.parse_args()
@@ -248,28 +309,24 @@ if __name__ == "__main__":
   model_w, model_h = args.model_size
 
   model_runner = OnnxRunner(model_path)
-  input_shapes = {name: (spec.shape, spec.dtype) for name, spec in model_runner.graph_inputs.items()}
-  state_pairs = {name: f'next_{name}' for name in input_shapes if f'next_{name}' in model_runner.graph_outputs}
   out = {
     'metadata': make_metadata_dict(model_path),
-    'input_shapes': input_shapes,
-    'state_pairs': state_pairs,
     'input_devices': {'model': Device.DEFAULT},
     'run_model': {},
   }
 
+  run_policy = make_run_policy(model_runner, out['metadata'], args.frame_skip)
+
   for cam_w, cam_h in args.camera_resolutions:
     nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h))
     frame_copy_size = nv12_copy_size(nv12.stride, nv12.y_height, nv12.uv_height)
-    make_model_queues = partial(make_input_queues, input_shapes, state_pairs,
+    make_model_queues = partial(make_input_queues, out['metadata']['input_shapes'], args.frame_skip,
                                 frame_copy_size=frame_copy_size)
     warp = make_warp(nv12, model_w, model_h)
-    run_model_jit = TinyJit(make_run_model(warp, model_runner, input_shapes, state_pairs, frame_copy_size), prune=True)
-    out['run_model'][(cam_w,cam_h)] = compile_jit(run_model_jit, make_model_queues, args.benchmark_runs)
+    run_model_jit = TinyJit(make_run_model(warp, run_policy, out['metadata'], frame_copy_size), prune=True)
+    out['run_model'][(cam_w,cam_h)] = compile_jit(run_model_jit, MODELD_INPUTS, make_model_queues,
+                                                  args.benchmark_runs)
 
   with open(args.output, "wb") as f:
     dump_oob(out, f)
-  with open(args.output, "rb") as f:
-    load_oob(f)
-    assert not f.read(1), "unexpected model buffer data"
   print(f"Saved JITs to {args.output} ({os.path.getsize(args.output) / 1e6:.2f} MB)")

openpilot/selfdrive/modeld/modeld.py

@@ -5,6 +5,8 @@ from functools import cached_property
 import os
 os.environ['GMMU'] = '0' # for chestnut fast loading, noop for qcom
 from tinygrad.device import Device
+import usb1
+import struct
 import threading
 import time
 import numpy as np
@@ -26,11 +28,12 @@ from openpilot.common.transformations.model import get_warp_matrix
 from openpilot.selfdrive.controls.lib.desire_helper import DesireHelper
 from openpilot.selfdrive.controls.lib.drive_helpers import get_accel_from_plan, should_stop, smooth_value, get_curvature_from_plan
 from openpilot.selfdrive.modeld.parse_model_outputs import Parser
-from openpilot.selfdrive.modeld.compile_modeld import make_input_queues, nv12_copy_size
+from openpilot.selfdrive.modeld.compile_modeld import make_input_queues, nv12_copy_size, MODELD_INPUTS
 from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_driving_model_data, fill_pose_msg, PublishState
 from openpilot.common.file_chunker import open_file_chunked
+from openpilot.common.hardware.usb import CHESTNUT_USB_IDS
 from openpilot.selfdrive.modeld.constants import ModelConstants, Plan
-from openpilot.selfdrive.modeld.helpers import chestnut_present, chestnut_compiled, modeld_pkl_path, load_oob
+from openpilot.selfdrive.modeld.helpers import chestnut_present, chestnut_compiled, chestnut_ready, modeld_pkl_path, load_oob
 
 from openpilot.sunnypilot.livedelay.helpers import get_lat_delay
 from openpilot.sunnypilot.modeld_v2.modeld_base import ModelStateBase
@@ -72,14 +75,45 @@ def get_action_from_model(model_output: dict[str, np.ndarray], prev_action: log.
                                 shouldStop=bool(stop))
 
 
-class ChestnutGpuState:
-  # GPU metrics require modeld's GPU context
+class ChestnutState:
+  # only modeld can access chestnut
   def __init__(self, pm: PubMaster, big: bool):
     self.pm = pm
     self.big = big
     self.valid = True
     self.sends = 0
     self.metrics = {}
+    self._asm_usb = None
+
+  def _close_asm_usb(self) -> None:
+    if self._asm_usb is not None:
+      self._asm_usb.close()
+      self._asm_usb = None
+
+  def _open_asm_usb(self):
+    context = usb1.USBContext()
+    for vendor_id, product_id in CHESTNUT_USB_IDS:
+      if (handle := context.openByVendorIDAndProductID(vendor_id, product_id, skip_on_error=True)) is not None:
+        return handle
+    context.close()
+
+  def _read_ina(self) -> tuple[int, int, bool]:
+    if "AMD" in Device._opened_devices and self._asm_usb is None:
+      try:
+        raw = Device["AMD"].iface.pci_dev.usb.usb.control_read(0xC0, 5)
+        return struct.unpack('<Hh?', bytes(raw))
+      except Exception:
+        pass
+    if self._asm_usb is None:
+      self._asm_usb = self._open_asm_usb()
+    if self._asm_usb is None:
+      raise usb1.USBErrorNoDevice
+    try:
+      raw = self._asm_usb.controlRead(0xC0, 0xC0, 0, 0, 5, timeout=100)
+    except usb1.USBError:
+      self._close_asm_usb()
+      raise
+    return struct.unpack('<Hh?', bytes(raw))
 
   @cached_property
   def power_limit(self) -> int:
@@ -87,8 +121,8 @@ class ChestnutGpuState:
     return smu._send_msg(smu.smu_mod.PPSMC_MSG_GetPptLimit, 0, read_back_arg=True, timeout=100)
 
   def send(self) -> None:
-    msg = messaging.new_message('chestnutGpuState')
-    state = msg.chestnutGpuState
+    msg = messaging.new_message('chestnutState')
+    state = msg.chestnutState
     self.sends += 1
     if self.big and "AMD" in Device._opened_devices and self.sends % 100 == 1:
       try:
@@ -114,8 +148,21 @@ class ChestnutGpuState:
       for k, v in self.metrics.items():
         setattr(state, k, v)
 
-    msg.valid = not self.big or (self.valid and bool(self.metrics))
-    self.pm.send('chestnutGpuState', msg)
+    asm_valid = False
+    try:
+      # ASM runs on USB-C power, these still read without a gpu
+      state.supplyVoltage, state.supplyCurrent, state.supplyFault = self._read_ina()
+      asm_valid = True
+    except Exception:
+      pass
+    if "AMD" in Device._opened_devices:
+      try:
+        state.pcieLtssm = Device["AMD"].iface.pci_dev.usb.read(0xB450, 1)[0]
+      except Exception:
+        pass
+
+    msg.valid = asm_valid and (not self.big or self.valid)
+    self.pm.send('chestnutState', msg)
 
 
 class FrameMeta:
@@ -137,17 +184,17 @@ class ModelState(ModelStateBase):
     input_devices = jits['input_devices']
     self.model_device = input_devices['model']
     metadata = jits['metadata']
-    self.input_shapes = jits['input_shapes']
-    self.state_pairs = jits['state_pairs']
-    self.vision_input_names = ('img', 'big_img')
+    self.input_shapes = metadata['input_shapes']
+    self.vision_input_names = [k for k in self.input_shapes if 'img' in k]
     self.output_slices = metadata['output_slices']
 
     self.prev_desire = np.zeros(ModelConstants.DESIRE_LEN, dtype=np.float32)
     self.chestnut = chestnut
 
+    self.frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ
     self.frame_copy_size = nv12_copy_size(*get_nv12_info(cam_w, cam_h)[:3])
     self.input_queues, self.npy, self.frame_views = make_input_queues(
-      self.input_shapes, self.state_pairs, device=self.model_device, frame_copy_size=self.frame_copy_size)
+      self.input_shapes, self.frame_skip, device=self.model_device, frame_copy_size=self.frame_copy_size)
     self.parser = Parser()
     self.run_model = jits['run_model'][(cam_w,cam_h)]
 
@@ -169,13 +216,14 @@ class ModelState(ModelStateBase):
     self.npy['tfm'][:,:] = transforms['img'][:,:]
     self.npy['big_tfm'][:,:] = transforms['big_img'][:,:]
 
-    outs, = self.run_model(**self.input_queues)
+    outs, = self.run_model(**{k: self.input_queues[k] for k in MODELD_INPUTS})
     if after_enqueue is not None:
       after_enqueue()
     model_output = outs.numpy()[0]
     if self.chestnut and not np.all(np.isfinite(model_output)):
       raise RuntimeError("model output not finite")
     outputs_dict = self.parser.parse_outputs(self.slice_outputs(model_output, self.output_slices))
+    self.npy['prev_feat'][:] = model_output[self.output_slices['hidden_state']]
 
     if SEND_RAW_PRED:
       outputs_dict['raw_pred'] = model_output.copy()
@@ -187,19 +235,32 @@ class ModelState(ModelStateBase):
     dims = {'desire_pulse': ModelConstants.DESIRE_LEN, 'traffic_convention': 2, 'action_t': 2}
     self.run(dummy_frames, dict.fromkeys(self.vision_input_names, eye), {k: np.zeros(v, dtype=np.float32) for k, v in dims.items()})
     self.input_queues, self.npy, self.frame_views = make_input_queues(
-      self.input_shapes, self.state_pairs, device=self.model_device, frame_copy_size=self.frame_copy_size)
+      self.input_shapes, self.frame_skip, device=self.model_device, frame_copy_size=self.frame_copy_size)
     self.prev_desire[:] = 0
 
 
 def main(demo=False):
   cloudlog.warning("modeld init")
 
-  CHESTNUT = chestnut_present() and chestnut_compiled()
+  chestnut_available = chestnut_present() and chestnut_compiled()
+  CHESTNUT = False
+  if chestnut_available:
+    poller = messaging.Poller()
+    sock = messaging.sub_sock("chestnutState", poller=poller, conflate=True)
+    deadline = time.monotonic() + 4. / SERVICE_LIST['deviceState'].frequency
+    while not CHESTNUT and (remaining := deadline - time.monotonic()) > 0.:
+      if not poller.poll(round(remaining * 1000)):
+        break
+      msg = messaging.recv_one_or_none(sock)
+      CHESTNUT = msg is not None and msg.valid and chestnut_ready(msg.chestnutState)
   if CHESTNUT:
     os.environ['HCQDEV_WAIT_TIMEOUT_MS'] = '3000'
   params = Params()
   params.put_bool("ChestnutLoading", CHESTNUT)
-  params.remove("ChestnutActive")
+  if chestnut_available and not CHESTNUT:
+    params.put_bool("ChestnutActive", False)
+  else:
+    params.remove("ChestnutActive")
 
   config_realtime_process(7, 54)
 
@@ -243,7 +304,11 @@ def main(demo=False):
     loader.start()
     loader.join(BIG_MODEL_TIMEOUT)
     model = big_model
+    if model is None:
+      params.put_bool("ChestnutModelError", True)
     params.put_bool("ChestnutActive", model is not None)
+    if model is not None:
+      params.remove("ChestnutModelError")
 
   small_model = ModelState(vipc_client_main.width, vipc_client_main.height, False) if model is None or CHESTNUT else None
   if model is None:
@@ -253,13 +318,13 @@ def main(demo=False):
   cloudlog.warning(f"models loaded in {time.monotonic() - st:.1f}s, modeld starting")
 
   # messaging
-  pub_socks = ["modelV2", "drivingModelData", "cameraOdometry", "modelDataV2SP"] + (["chestnutGpuState"] if CHESTNUT else [])
+  pub_socks = ["modelV2", "drivingModelData", "cameraOdometry", "modelDataV2SP"] + (["chestnutState"] if CHESTNUT else [])
   pm = PubMaster(pub_socks)
   sm = SubMaster(["deviceState", "carState", "narrowRoadCameraState", "extrinsicsCalibration", "driverMonitoringState", "carControl", "lateralDelay"])
 
   publish_state = PublishState()
   params = Params()
-  chestnut_state = ChestnutGpuState(pm, model.chestnut) if CHESTNUT else None
+  chestnut_state = ChestnutState(pm, model.chestnut) if CHESTNUT else None
 
   # setup filter to track dropped frames
   frame_dropped_filter = FirstOrderFilter(0., 10., 1. / ModelConstants.MODEL_RUN_FREQ)
@@ -370,13 +435,14 @@ def main(demo=False):
     mt1 = time.perf_counter()
     try:
       send_chestnut = (chestnut_state is not None and
-                       run_count % round(ModelConstants.MODEL_RUN_FREQ / SERVICE_LIST['chestnutGpuState'].frequency) == 0)
+                       run_count % round(ModelConstants.MODEL_RUN_FREQ / SERVICE_LIST['chestnutState'].frequency) == 0)
       model_output = model.run(bufs, transforms, inputs, chestnut_state.send if send_chestnut else None)
     except Exception:
       if not params.get_bool("ChestnutActive"):
         raise
       # fallback to small model
       cloudlog.exception("big model failed, fall back to small")
+      params.put_bool("ChestnutModelError", True)
       params.put_bool("ChestnutActive", False)
       assert small_model is not None
       model = small_model
@@ -408,7 +474,6 @@ def main(demo=False):
       DH.update(sm['carState'], sm['carControl'].latActive, lane_change_prob, left_edge, right_edge)
       modelv2_send.modelV2.meta.laneChangeState = DH.lane_change_state
       modelv2_send.modelV2.meta.laneChangeDirection = DH.lane_change_direction
-      mdv2sp_send.valid = modelv2_send.valid
       mdv2sp_send.modelDataV2SP.laneTurnDirection = DH.lane_turn_direction
 
       fill_driving_model_data(drivingdata_send, modelv2_send)

openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py

@@ -4,6 +4,7 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
 This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
+import os
 import re
 import time
 import pyray as rl
@@ -12,8 +13,7 @@ from openpilot.cereal import custom
 from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_selected_bundle, resolve_bundle_by_ref
 from openpilot.common.constants import CV
 from openpilot.selfdrive.ui.ui_state import device, ui_state
-from openpilot.selfdrive.ui.sunnypilot.model_info import (big_model_state, bundles_for_source, carrying_model, default_model_name,
-                                                           model_cache_size_mb, queued_name, refresh_in_progress, refresh_model_list)
+from openpilot.selfdrive.ui.sunnypilot.model_info import big_model_state, bundles_for_source, carrying_model, default_model_name, queued_name
 from openpilot.system.ui.lib.multilang import tr
 from openpilot.system.ui.lib.application import gui_app
 from openpilot.system.ui.widgets import DialogResult, Widget
@@ -21,6 +21,7 @@ from openpilot.system.ui.widgets.confirm_dialog import alert_dialog, ConfirmDial
 from openpilot.system.ui.widgets.scroller_tici import Scroller
 from openpilot.system.ui.widgets.toggle import ON_COLOR
 
+from openpilot.sunnypilot.models.runners.constants import CUSTOM_MODEL_PATH
 from openpilot.system.ui.sunnypilot.lib.styles import style
 from openpilot.system.ui.sunnypilot.lib.utils import NoElideButtonAction, ScrollingButtonAction
 from openpilot.system.ui.sunnypilot.widgets.list_view import ListItemSP, toggle_item_sp, option_item_sp
@@ -39,9 +40,6 @@ class ModelsLayout(Widget):
     self._selection_source = None
     self._downloading = False
     self._verifying = False
-    self._clearing = False
-    self._refreshing = False
-    self._refresh_start: float | None = None
     self._last_note = None
     self.last_cache_calc_time = 0
 
@@ -69,14 +67,15 @@ class ModelsLayout(Widget):
 
     self.download_item = download_status_item(lambda: tr("Download") if self._downloading else tr("Model Status"))
 
-    self.refresh_item = button_item(tr("Refresh Model List"),
-                                    lambda: tr("FETCHING...") if self._refreshing else tr("REFRESH"), "",
-                                    self._refresh_models)
+    self.refresh_item = button_item(tr("Refresh Model List"), tr("REFRESH"), "",
+                                    lambda: (ui_state.params.put("ModelManager_LastSyncTime", 0),
+                                             ui_state.params.put("ModelManager_LastSyncTime_Chestnut", 0),
+                                             gui_app.push_widget(alert_dialog(tr("Fetching Latest Models")))))
 
     self.clear_cache_item = ListItemSP(
       title=tr("Clear Model Cache"),
       description="",
-      action_item=NoElideButtonAction(lambda: tr("CLEARING...") if self._clearing else tr("CLEAR")),
+      action_item=NoElideButtonAction(tr("CLEAR")),
       callback=self._clear_cache
     )
 
@@ -116,38 +115,39 @@ class ModelsLayout(Widget):
     if lagd_toggle:
       desc += f"<br>{tr('Live Steer Delay:')} {ui_state.sm['lateralDelay'].lateralDelay:.3f} s"
     elif ui_state.CP is not None:
-      sw = float(ui_state.params.get("LagdToggleDelay", return_default=True))
+      sw = float(ui_state.params.get("LagdToggleDelay", "0.2"))
       cp = ui_state.CP.steerActuatorDelay
       desc += f"<br>{tr('Actuator Delay:')} {cp:.2f} s + {tr('Software Delay:')} {sw:.2f} s = {tr('Total Delay:')} {cp + sw:.2f} s"
     self.lagd_toggle.set_description(desc)
 
   @staticmethod
   def calculate_cache_size():
-    return model_cache_size_mb()
+    cache_size = 0.0
+    if os.path.exists(CUSTOM_MODEL_PATH):
+      for file in os.listdir(CUSTOM_MODEL_PATH):
+        try:
+          cache_size += os.path.getsize(os.path.join(CUSTOM_MODEL_PATH, file))
+        except OSError:
+          continue
+    return cache_size / (1024**2)
 
   def _clear_cache(self):
     def _callback(response):
       if response == DialogResult.CONFIRM:
         ui_state.params.put_bool("ModelManager_ClearCache", True)
+        self.clear_cache_item.action_item.set_value(f"{self.calculate_cache_size():.2f} MB")
 
     dialog = ConfirmDialog(tr("This will delete ALL downloaded models from the cache except the currently active model. Are you sure?"),
                            tr("Clear Cache"), callback=_callback)
     gui_app.push_widget(dialog)
 
-  def _refresh_models(self):
-    refresh_model_list()
-    self._refresh_start = time.monotonic()
-
   def _handle_bundle_download_progress(self):
     self.cancel_download_item.set_visible(False)
     self._downloading = False
     self._verifying = False
     self.download_item.set_visible(True)
 
-    self._clearing = ui_state.params.get_bool("ModelManager_ClearCache")
-    if self._clearing:
-      self.last_cache_calc_time = 0.0  # refresh the size as soon as clearing finishes
-    elif (current_time := time.monotonic()) - self.last_cache_calc_time > 0.5:
+    if (current_time := time.monotonic()) - self.last_cache_calc_time > 0.5:
       self.last_cache_calc_time = current_time
       self.clear_cache_item.action_item.set_value(f"{self.calculate_cache_size():.2f} MB")
 
@@ -345,13 +345,6 @@ class ModelsLayout(Widget):
     self.big_model_item.action_item.set_enabled(offroad)
     self.small_model_item.set_description("" if offroad else tr("Only available when vehicle is off, or always offroad mode is on"))
 
-    # manager is offroad-only, so an onroad clear would never be serviced
-    self.clear_cache_item.action_item.set_enabled(offroad and not self._downloading and not self._clearing)
-
-    # manager is offroad-only, so a refresh queued onroad would never be serviced
-    self._refreshing = refresh_in_progress(self._refresh_start)
-    self.refresh_item.action_item.set_enabled(offroad and not self._downloading and not self._refreshing)
-
   def _render(self, rect):
     self._scroller.render(rect)
 

openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py

@@ -4,18 +4,15 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
 This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
-import time
-
 import pyray as rl
 
 from openpilot.cereal import custom
-from openpilot.selfdrive.ui.mici.widgets.dialog import BigConfirmationDialog, BigDialog
+from openpilot.selfdrive.ui.mici.widgets.dialog import BigDialog
 from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_selected_bundle
 from openpilot.selfdrive.ui.mici.widgets.button import BigButton
 from openpilot.selfdrive.ui.ui_state import ui_state, device
 from openpilot.selfdrive.ui.sunnypilot.model_info import (active_source, big_model_state, bundles_for_source, carrying_model,
-                                                           default_model_name, model_cache_size_mb, model_info, queued_name,
-                                                           refresh_in_progress, refresh_model_list)
+                                                           default_model_name, model_info, queued_name)
 from openpilot.system.ui.lib.application import FontWeight, gui_app
 from openpilot.system.ui.lib.multilang import tr
 from openpilot.system.ui.widgets import Widget
@@ -84,18 +81,10 @@ class ModelsLayoutMici(NavScroller):
     self.select_model_btn = BigButton(tr("select model"))
     self.select_model_btn.set_click_callback(self._show_folders)
 
-    self.refresh_btn = BigButton(tr("refresh models"))
-    self.refresh_btn.set_click_callback(self._refresh_models)
-    self._refresh_start: float | None = None
-
     self.cancel_download_btn = BigButton(tr("cancel download"))
     self.cancel_download_btn.set_click_callback(lambda: ui_state.params.remove("ModelManager_DownloadRef"))
 
-    self.clear_cache_btn = BigButton(tr("clear cache"), value=f"{model_cache_size_mb():.1f} MB")
-    self.clear_cache_btn.set_click_callback(self._confirm_clear_cache)
-    self._cache_size_time = 0.0
-
-    self.main_items = [self.current_model_info, self.select_model_btn, self.cancel_download_btn, self.refresh_btn, self.clear_cache_btn]
+    self.main_items = [self.current_model_info, self.select_model_btn, self.cancel_download_btn]
     self._scroller.add_widgets(self.main_items)
 
   @property
@@ -173,15 +162,6 @@ class ModelsLayoutMici(NavScroller):
     ui_state.params.remove(ACTIVE_BUNDLE_KEYS[source])
     self._pop_to_main()
 
-  def _confirm_clear_cache(self):
-    icon = gui_app.texture("icons_mici/settings/network/new/trash.png", 54, 64)
-    gui_app.push_widget(BigConfirmationDialog(f"{tr('slide to')}\n{tr('clear cache')}", icon,
-                                              lambda: ui_state.params.put_bool("ModelManager_ClearCache", True), red=True))
-
-  def _refresh_models(self):
-    refresh_model_list()
-    self._refresh_start = time.monotonic()
-
   def _select_folder(self, folder_name):
     source = self._selection_source
     if source is None:  # folders are only reachable after picking a hardware
@@ -225,21 +205,6 @@ class ModelsLayoutMici(NavScroller):
       device.set_override_interactive_timeout(None)
     self._was_downloading = is_downloading
 
-    # manager is offroad-only, so an onroad clear would never be serviced
-    clearing = ui_state.params.get_bool("ModelManager_ClearCache")
-    self.clear_cache_btn.set_enabled(ui_state.is_offroad() and not is_downloading and not clearing)
-    if clearing:
-      self.clear_cache_btn.set_value(tr("clearing..."))
-      self._cache_size_time = 0.0  # refresh the size as soon as clearing finishes
-    elif (now := time.monotonic()) - self._cache_size_time > 0.5:
-      self._cache_size_time = now
-      self.clear_cache_btn.set_value(f"{model_cache_size_mb():.1f} MB")
-
-    # manager is offroad-only, so a refresh queued onroad would never be serviced
-    refreshing = refresh_in_progress(self._refresh_start)
-    self.refresh_btn.set_enabled(ui_state.is_offroad() and not is_downloading and not refreshing)
-    self.refresh_btn.set_value(tr("fetching...") if refreshing else "")
-
     self.current_model_info.current_model_header.set_text(tr("active model"))
     active_text, info_header, info_text = _model_info()
     self.current_model_info.current_model_text.set_text(active_text)

openpilot/selfdrive/ui/sunnypilot/model_info.py

@@ -4,28 +4,12 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
 This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
-import contextlib
-import os
-import time
-
-from openpilot.common.hardware.hw import Paths
 from openpilot.selfdrive.ui.ui_state import ui_state, ChestnutState
 from openpilot.sunnypilot.models.fetcher import get_cached_bundles
 from openpilot.sunnypilot.models.helpers import get_active_source, get_selected_bundle, resolve_bundle_by_ref
 from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL, DEFAULT_MODEL
 
 
-def model_cache_size_mb() -> float:
-  """Bytes on disk under the model cache directory, in MB."""
-  model_root = Paths.model_root()
-  total = 0
-  if os.path.isdir(model_root):
-    for name in os.listdir(model_root):
-      with contextlib.suppress(OSError):
-        total += os.path.getsize(os.path.join(model_root, name))
-  return total / (1024 ** 2)
-
-
 def active_source() -> str:
   return get_active_source(chestnut=ui_state.chestnut_present,
                            chestnut_active=ui_state.chestnut_active, chestnut_loading=ui_state.chestnut_loading,
@@ -99,22 +83,3 @@ def model_info() -> tuple[str, str, str]:
   active_name = active_bundle.displayName if active_bundle else default_model_name(source)
   other_name = other_bundle.displayName if other_bundle else default_model_name(other)
   return source, active_name, other_name
-
-
-# mirrors the manager's ModelCache keys; the manager restamps them on a successful fetch
-MODEL_SYNC_KEYS = ("ModelManager_LastSyncTime", "ModelManager_LastSyncTime_Chestnut")
-MODEL_SYNC_TIMEOUT = 20.0
-
-
-def refresh_model_list() -> None:
-  # zeroing the sync keys makes the manager refetch each manifest on its next tick
-  for key in MODEL_SYNC_KEYS:
-    ui_state.params.put(key, 0)
-
-
-def refresh_in_progress(started_at: float | None) -> bool:
-  """Whether a user refresh is still outstanding. A failed fetch never restamps the
-  sync keys, so the spinner is bounded by MODEL_SYNC_TIMEOUT rather than sticking."""
-  if started_at is None or time.monotonic() - started_at > MODEL_SYNC_TIMEOUT:
-    return False
-  return not all(ui_state.params.get(key) for key in MODEL_SYNC_KEYS)

openpilot/selfdrive/ui/ui_state.py

@@ -225,6 +225,7 @@ class UIState(UIStateSP):
                              ChestnutState.UNCOMPILED if detected else ChestnutState.DISCONNECTED)
       return
 
+    self.chestnut_present = self.chestnut_present or detected
     model_seen = self.sm.recv_frame["modelV2"] > self.started_frame
     if not self.chestnut_present:
       self.chestnut_state = ChestnutState.DISCONNECTED

openpilot/sunnypilot/modeld_v2/camera_offset_helper.py

@@ -15,19 +15,11 @@ class CameraOffsetHelper:
     self.actual_camera_offset = 0.0
 
   @staticmethod
-  def get_v_horizon(intrinsics, rpy_calib):
+  def apply_camera_offset(model_transform, intrinsics, height, offset_param):
     cy = intrinsics[1, 2]
-    if len(rpy_calib) == 3 and np.isfinite(rpy_calib).all():
-      fy = intrinsics[1, 1]
-      pitch = rpy_calib[1]
-      return float(cy - fy * np.tan(pitch))
-    return float(cy)
-
-  @staticmethod
-  def apply_camera_offset(model_transform, height, offset_param, v_horizon):
     shear = np.eye(3, dtype=np.float32)
     shear[0, 1] = offset_param / height
-    shear[0, 2] = -offset_param / height * v_horizon
+    shear[0, 2] = -offset_param / height * cy
     model_transform = (shear @ model_transform).astype(np.float32)
     return model_transform
 
@@ -38,13 +30,10 @@ class CameraOffsetHelper:
     self.actual_camera_offset = (0.9 * self.actual_camera_offset) + (0.1 * self.camera_offset)
     dc = DEVICE_CAMERAS[(str(sm['deviceState'].deviceType), str(sm['narrowRoadCameraState'].sensor))]
     height = sm["extrinsicsCalibration"].height[0] if sm['extrinsicsCalibration'].height else 1.22
-    rpy_calib = sm['extrinsicsCalibration'].rpyCalib
 
     intrinsics_main = dc.wide_road.intrinsics if main_wide_camera else dc.narrow_road.intrinsics
-    v_horizon_main = self.get_v_horizon(intrinsics_main, rpy_calib)
-    model_transform_main = self.apply_camera_offset(model_transform_main, height, self.actual_camera_offset, v_horizon_main)
+    model_transform_main = self.apply_camera_offset(model_transform_main, intrinsics_main, height, self.actual_camera_offset)
 
     intrinsics_extra = dc.wide_road.intrinsics
-    v_horizon_extra = self.get_v_horizon(intrinsics_extra, rpy_calib)
-    model_transform_extra = self.apply_camera_offset(model_transform_extra, height, self.actual_camera_offset, v_horizon_extra)
+    model_transform_extra = self.apply_camera_offset(model_transform_extra, intrinsics_extra, height, self.actual_camera_offset)
     return model_transform_main, model_transform_extra

openpilot/sunnypilot/modeld_v2/compile_modeld.py

@@ -33,7 +33,6 @@ def _patch_tinygrad_fetch_fw():
 _patch_tinygrad_fetch_fw()
 
 import openpilot.selfdrive.modeld.compile_modeld as stock
-import openpilot.sunnypilot.modeld_v2.stock_dependencies as legacy
 from tinygrad import dtypes
 from tinygrad.device import Device
 from tinygrad.engine.jit import TinyJit
@@ -42,7 +41,7 @@ from tinygrad.tensor import Tensor
 MODEL_TYPES = ('vision_policy', 'supercombo', 'vision_multi_policy')
 WARP_INPUTS = ['tfm', 'big_tfm']
 POLICY_INPUTS = ['img_q', 'big_img_q', 'feat_q', 'desire_q', 'packed_npy_inputs']
-
+nv12_copy_size = stock.nv12_copy_size
 
 def _detect_desire_key(shapes: dict) -> str | None:
   return next((key for key in shapes if key.startswith('desire')), None)
@@ -153,8 +152,8 @@ def make_warp_queues(device=Device.DEFAULT):
 
 
 def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, frame_skip: int, input_shapes: dict):
-  sample_skip_fn = partial(legacy.sample_skip, frame_skip=frame_skip)
-  sample_desire_fn = partial(legacy.sample_desire, frame_skip=frame_skip)
+  sample_skip_fn = partial(stock.sample_skip, frame_skip=frame_skip)
+  sample_desire_fn = partial(stock.sample_desire, frame_skip=frame_skip)
 
   desire_key = _detect_desire_key(input_shapes)
   road_key, wide_key = _detect_vision_keys(input_shapes)
@@ -171,14 +170,14 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
     warped_dev = warped.to(Device.DEFAULT)
     Tensor.realize(packed_npy_inputs_dev, warped_dev)
 
-    img = legacy.shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn)
-    big_img = legacy.shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn)
+    img = stock.shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn)
+    big_img = stock.shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn)
 
     unpacked_tensors = [tensor.reshape(shape) for tensor, shape in zip(packed_npy_inputs_dev.split(npy_sizes), npy_shapes.values(), strict=True)]
     unpacked_dict = dict(zip(npy_shapes.keys(), unpacked_tensors, strict=True))
 
     desire_dev = unpacked_dict['desire']
-    desire_buf = legacy.shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn)
+    desire_buf = stock.shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn)
 
     inputs = {desire_key: desire_buf}
     for key, tensor_val in unpacked_dict.items():
@@ -187,13 +186,13 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
 
     if 'prev_feat' in unpacked_dict:
       prev_feat_dev = unpacked_dict['prev_feat']
-      inputs['features_buffer'] = legacy.shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).reshape(input_shapes['features_buffer'])
+      inputs['features_buffer'] = stock.shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).reshape(input_shapes['features_buffer'])
 
     if vision_runner:
       vision_out_cast = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32').realize()
       if 'features_buffer' not in inputs:
         new_feat = vision_out_cast[:, features_slice].reshape(1, -1).unsqueeze(0)
-        inputs['features_buffer'] = legacy.shift_and_sample(feat_q, new_feat, sample_skip_fn).realize()
+        inputs['features_buffer'] = stock.shift_and_sample(feat_q, new_feat, sample_skip_fn).realize()
       policy_outs = [next(iter(pol_runner(inputs).values())).cast('float32').realize() for pol_runner in policy_runners]
       return (vision_out_cast, *policy_outs) if len(policy_outs) > 1 else (vision_out_cast, policy_outs[0])
 
@@ -204,7 +203,7 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
     policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32').realize()
     if 'features_buffer' not in inputs and features_slice is not None:
       new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0)
-      legacy.shift_and_sample(feat_q, new_feat, sample_skip_fn).realize()
+      stock.shift_and_sample(feat_q, new_feat, sample_skip_fn).realize()
     return policy_out
 
   return run_policy
@@ -331,32 +330,16 @@ if __name__ == "__main__":
     output_data['run_model'] = {}
     derived_frame_skip = args.frame_skip or derive_frame_skip({}, model_metadata['input_shapes'])
     model_runner = OnnxRunner(args.supercombo_onnx)
-    new_img_model = 'new_img' in model_runner.graph_inputs
-
-    if new_img_model:
-      input_shapes = {name: (spec.shape, spec.dtype) for name, spec in model_runner.graph_inputs.items()}
-      state_pairs = {name: f'next_{name}' for name in input_shapes if f'next_{name}' in model_runner.graph_outputs}
-      output_data['metadata'] = {'model': model_metadata, **model_metadata, 'input_shapes': input_shapes, 'state_pairs': state_pairs}
-      for cam_w, cam_h in args.camera_resolutions:
-        print(f"Compiling unified run_model JIT for {cam_w}x{cam_h} (new architecture)...")
-        nv12 = stock.NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h))
-        frame_copy_size = stock.nv12_copy_size(nv12.stride, nv12.y_height, nv12.uv_height)
-        make_model_queues = partial(stock.make_input_queues, input_shapes, state_pairs, frame_copy_size=frame_copy_size)
-        warp = stock.make_warp(nv12, model_w, model_h)
-        run_model_jit = TinyJit(stock.make_run_model(warp, model_runner, input_shapes, state_pairs, frame_copy_size), prune=True)
-        output_data['run_model'][(cam_w, cam_h)] = compile_jit(run_model_jit, list(state_pairs.keys()) + ['packed_npy_inputs'], make_model_queues,
-                                                               benchmark_runs=args.benchmark_runs)
-    else:
-      run_policy = legacy.make_legacy_run_policy(model_runner, model_metadata, derived_frame_skip)
-      for cam_w, cam_h in args.camera_resolutions:
-        print(f"Compiling unified run_model JIT for {cam_w}x{cam_h}...")
-        nv12 = stock.NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h))
-        frame_copy_size = stock.nv12_copy_size(nv12.stride, nv12.y_height, nv12.uv_height)
-        make_model_queues = partial(stock.make_input_queues, model_metadata['input_shapes'], derived_frame_skip,
-                                    frame_copy_size=frame_copy_size)
-        warp = stock.make_warp(nv12, model_w, model_h)
-        run_model_jit = TinyJit(legacy.make_legacy_run_model(warp, run_policy, model_metadata, frame_copy_size), prune=True)
-        output_data['run_model'][(cam_w, cam_h)] = compile_jit(run_model_jit, POLICY_INPUTS, make_model_queues, benchmark_runs=args.benchmark_runs)
+    run_policy = stock.make_run_policy(model_runner, model_metadata, derived_frame_skip)
+    for cam_w, cam_h in args.camera_resolutions:
+      print(f"Compiling unified run_model JIT for {cam_w}x{cam_h}...")
+      nv12 = stock.NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h))
+      frame_copy_size = stock.nv12_copy_size(nv12.stride, nv12.y_height, nv12.uv_height)
+      make_model_queues = partial(stock.make_input_queues, model_metadata['input_shapes'], derived_frame_skip,
+                                  frame_copy_size=frame_copy_size)
+      warp = stock.make_warp(nv12, model_w, model_h)
+      run_model_jit = TinyJit(stock.make_run_model(warp, run_policy, model_metadata, frame_copy_size), prune=True)
+      output_data['run_model'][(cam_w, cam_h)] = compile_jit(run_model_jit, stock.MODELD_INPUTS, make_model_queues, benchmark_runs=args.benchmark_runs)
   else:
     vision_runner = OnnxRunner(args.vision_onnx) if args.vision_onnx else None
     if args.model_type == 'vision_policy':
@@ -372,12 +355,13 @@ if __name__ == "__main__":
         output_data['metadata'][name] = make_metadata_dict(runner_arg)
 
     policy_keys = [key for key in output_data['metadata'].keys() if key != 'vision']
-    first_policy_meta: dict = output_data['metadata'][policy_keys[0]] if policy_keys else {}
-    vision_meta: dict = output_data['metadata'].get('vision', {})
+    first_policy_meta = output_data['metadata'][policy_keys[0]] if policy_keys else {}
+    vision_meta = output_data['metadata'].get('vision', {})
 
     derived_frame_skip = args.frame_skip or derive_frame_skip(vision_meta.get('input_shapes', {}), first_policy_meta.get('input_shapes', {}))
     all_shapes = {key: value for meta in output_data['metadata'].values() for key, value in meta['input_shapes'].items()}
-    feat_meta: dict = vision_meta or first_policy_meta
+    feat_meta = output_data['metadata'].get('vision') or output_data['metadata'].get('policy')
+    assert feat_meta is not None
     features_slice = feat_meta['output_slices']['hidden_state']
 
     print(f"Compiling run_policy JIT (model_size={model_w}x{model_h}, frame_skip={derived_frame_skip})...")

openpilot/sunnypilot/modeld_v2/helpers.py

@@ -1,100 +0,0 @@
-"""
-Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
-
-This file is part of sunnypilot and is licensed under the MIT License.
-See the LICENSE.md file in the root directory for more details.
-"""
-
-import io
-import struct
-import pickle
-import inspect
-import importlib
-import enum
-
-
-def _pad_args(func, args, kwargs):
-  try:
-    sig = inspect.signature(func)
-  except Exception:
-    return args, kwargs
-  params = list(sig.parameters.values())
-  if inspect.isfunction(func) and params and params[0].name in ('cls', 'self'):
-    params = params[1:]
-
-  new_args = list(args)
-  has_varargs = any(p.kind == inspect.Parameter.VAR_POSITIONAL for p in params)
-  if len(new_args) > len(params) and not has_varargs:
-    new_args = new_args[:len(params)]
-
-  for i in range(len(new_args), len(params)):
-    param = params[i]
-    if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
-      continue
-    val = param.default if param.default is not inspect.Parameter.empty else None
-    new_args.append(val)
-  return new_args, kwargs
-
-
-def _enum_factory(enum_class):
-  def factory(*args, **kwargs):
-    try:
-      return enum_class(*args, **kwargs)
-    # OptOps and UOp objects in the .pkl are left over from the compilation phase,
-    # reassignment does nothing because they aren't tied to the execution graph
-    # It never executes or evaluates the UOp nodes again.
-    except ValueError:
-      return list(enum_class)[0]
-  factory.__name__ = enum_class.__name__
-  factory.__module__ = enum_class.__module__
-  return factory
-
-
-def _dynamic_factory(real_class):
-  if isinstance(real_class, type) and issubclass(real_class, enum.Enum):
-    return _enum_factory(real_class)
-
-  def factory(*args, **kwargs):
-    try:
-      return real_class(*args, **kwargs)
-    except TypeError:
-      new_args, new_kwargs = _pad_args(real_class, args, kwargs)
-      return real_class(*new_args, **new_kwargs)
-
-  class DynamicMeta(type(real_class)):
-    def __call__(cls, *args, **kwargs):
-      return factory(*args, **kwargs)
-
-  class DynamicProxy(real_class, metaclass=DynamicMeta):
-    __slots__ = ()
-
-    def __new__(cls, *args, **kwargs):
-      return factory(*args, **kwargs)
-
-  DynamicProxy.__name__ = real_class.__name__
-  DynamicProxy.__module__ = real_class.__module__
-  return DynamicProxy
-
-
-class DynamicTinygradUnpickler(pickle.Unpickler):
-  def find_class(self, module, name):
-    if module == "tinygrad.ops":
-      try:
-        importlib.import_module("tinygrad.uops")
-        module = "tinygrad.uops"
-      except ImportError:
-        pass
-    real_class = getattr(importlib.import_module(module), name)
-    if module.startswith("tinygrad"):
-      return _dynamic_factory(real_class)
-    return real_class
-
-
-def load_oob(f):
-  opcodes = f.read(struct.unpack('<q', f.read(8))[0])
-  def buffers():
-    while (h := f.read(8)):
-      pb = pickle.PickleBuffer(bytearray(struct.unpack('<q', h)[0]))
-      f.readinto(pb)
-      yield pb
-  return DynamicTinygradUnpickler(io.BytesIO(opcodes), buffers=buffers()).load()

openpilot/sunnypilot/modeld_v2/modeld.py

@@ -17,7 +17,7 @@ from tinygrad.tensor import Tensor
 
 import openpilot.cereal.messaging as messaging
 from openpilot.common.hardware import COMMA_HARDWARE
-from openpilot.selfdrive.modeld.helpers import chestnut_present
+from openpilot.selfdrive.modeld.helpers import chestnut_present, load_oob
 from openpilot.cereal import log
 from opendbc.car.structs import car
 from openpilot.cereal.services import SERVICE_LIST
@@ -36,9 +36,12 @@ from openpilot.system import sentry
 from openpilot.system.camerad.cameras.nv12_info import get_nv12_info
 from openpilot.selfdrive.controls.lib.desire_helper import DesireHelper
 from openpilot.selfdrive.controls.lib.drive_helpers import get_accel_from_plan, smooth_value
-from openpilot.selfdrive.modeld.modeld import ChestnutGpuState
+from openpilot.selfdrive.modeld.modeld import ChestnutState
 
-from openpilot.selfdrive.modeld.compile_modeld import make_input_queues
+from openpilot.selfdrive.modeld.compile_modeld import (
+  MODELD_INPUTS,
+  make_input_queues as make_stock_input_queues,
+)
 from openpilot.sunnypilot.modeld_v2.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState, get_curvature_from_output
 from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser
 from openpilot.sunnypilot.modeld_v2.constants import ModelConstants, Plan
@@ -47,10 +50,8 @@ from openpilot.sunnypilot.modeld_v2.camera_offset_helper import CameraOffsetHelp
 from openpilot.sunnypilot.modeld_v2.compile_modeld import (derive_frame_skip, make_split_input_queues,
                                                            make_supercombo_input_queues, nv12_copy_size,
                                                            WARP_INPUTS, POLICY_INPUTS)
-from openpilot.sunnypilot.modeld_v2.stock_dependencies import make_legacy_stock_input_queues
 from openpilot.sunnypilot.livedelay.helpers import get_lat_delay
 from openpilot.sunnypilot.modeld_v2.modeld_base import ModelStateBase
-from openpilot.sunnypilot.modeld_v2.helpers import load_oob
 from openpilot.sunnypilot.models.helpers import get_active_bundle
 from openpilot.sunnypilot.selfdrive.controls.lib.relc import RoadEdgeLaneChangeController
 
@@ -139,14 +140,8 @@ class ModelState(ModelStateBase):
       self._vision_input_names = [key for key in self.input_shapes if 'img' in key]
       self.frame_skip = derive_frame_skip({}, self.input_shapes)
       if self.is_run_model:
-        self.state_pairs = model_metadata.get('state_pairs', {})
-        self.is_new_model = len(self.state_pairs) > 0
-        if self.is_new_model:
-          self.input_queues, self.numpy_inputs, self.frame_buffers = make_input_queues(self.input_shapes, self.state_pairs,
-                                                                                       device=self.DEV, frame_copy_size=self.frame_copy_size)
-        else:
-          self.input_queues, self.numpy_inputs, self.frame_buffers = make_legacy_stock_input_queues(self.input_shapes, self.frame_skip, device=self.DEV,
-                                                                                                    frame_copy_size=self.frame_copy_size)
+        self.input_queues, self.numpy_inputs, self.frame_buffers = make_stock_input_queues(
+          self.input_shapes, self.frame_skip, device=self.DEV, frame_copy_size=self.frame_copy_size)
         self.frame_views, self.npy = self.frame_buffers, self.numpy_inputs
         self.run_model, self.run_policy, self.warp = jits['run_model'][(cam_w, cam_h)], None, None
       else:
@@ -195,12 +190,8 @@ class ModelState(ModelStateBase):
     dummy_inputs = {k: np.zeros(v.shape, dtype=v.dtype) for k, v in self.numpy_inputs.items() if k not in ['tfm', 'big_tfm', 'prev_feat']}
     self.run(dummy_frames, transforms, dummy_inputs)
     if self.is_run_model:
-      if self.is_new_model:
-        self.input_queues, self.numpy_inputs, self.frame_buffers = make_input_queues(self.input_shapes, self.state_pairs, device=self.DEV,
-                                                                                     frame_copy_size=self.frame_copy_size)
-      else:
-        self.input_queues, self.numpy_inputs, self.frame_buffers = make_legacy_stock_input_queues(self.input_shapes, self.frame_skip, device=self.DEV,
-                                                                                                  frame_copy_size=self.frame_copy_size)
+      self.input_queues, self.numpy_inputs, self.frame_buffers = make_stock_input_queues(
+        self.input_shapes, self.frame_skip, device=self.DEV, frame_copy_size=self.frame_copy_size)
       self.frame_views = self.frame_buffers
       self.npy = self.numpy_inputs
     else:
@@ -250,10 +241,7 @@ class ModelState(ModelStateBase):
     self.numpy_inputs['big_tfm'][:, :] = transforms[self._wide_key].reshape(3, 3)
 
     if self.run_model is not None:
-      if self.is_new_model:
-        outs, = self.run_model(**self.input_queues)
-      else:
-        outs, = self.run_model(**{k: self.input_queues[k] for k in POLICY_INPUTS})
+      outs, = self.run_model(**{k: self.input_queues[k] for k in MODELD_INPUTS})
       raw_outputs = outs
     else:
       assert self.warp is not None and self.run_policy is not None
@@ -384,7 +372,11 @@ def main(demo=False):
     loader.start()
     loader.join(BIG_MODEL_TIMEOUT)
     model = big_model
+    if model is None:
+      params.put_bool("ChestnutModelError", True)
     params.put_bool("ChestnutActive", model is not None)
+    if model is not None:
+      params.remove("ChestnutModelError")
 
   small_model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, chestnut=False) if model is None or CHESTNUT else None
   if model is None:
@@ -394,12 +386,12 @@ def main(demo=False):
   cloudlog.warning(f"models loaded in {time.monotonic() - st:.1f}s, modeld starting")
 
   # messaging
-  pub_socks = ["modelV2", "drivingModelData", "cameraOdometry", "modelDataV2SP"] + (["chestnutGpuState"] if CHESTNUT else [])
+  pub_socks = ["modelV2", "drivingModelData", "cameraOdometry", "modelDataV2SP"] + (["chestnutState"] if CHESTNUT else [])
   pm = PubMaster(pub_socks)
   sm = SubMaster(["deviceState", "carState", "narrowRoadCameraState", "extrinsicsCalibration", "driverMonitoringState", "carControl", "lateralDelay"])
 
   publish_state = PublishState()
-  chestnut_state = ChestnutGpuState(pm, model.chestnut) if CHESTNUT else None
+  chestnut_state = ChestnutState(pm, model.chestnut) if CHESTNUT else None
 
   # setup filter to track dropped frames
   frame_dropped_filter = FirstOrderFilter(0., 10., 1. / model.constants.MODEL_FREQ)
@@ -521,12 +513,13 @@ def main(demo=False):
     mt1 = time.perf_counter()
     try:
       send_chestnut = (chestnut_state is not None and
-                       run_count % round(model.constants.MODEL_FREQ / SERVICE_LIST['chestnutGpuState'].frequency) == 0)
+                       run_count % round(model.constants.MODEL_FREQ / SERVICE_LIST['chestnutState'].frequency) == 0)
       model_output = model.run(bufs, transforms, inputs, chestnut_state.send if send_chestnut else None)
     except Exception:
       if not params.get_bool("ChestnutActive"):
         raise
       cloudlog.exception("chestnut failed, falling back to small")
+      params.put_bool("ChestnutModelError", True)
       params.put_bool("ChestnutActive", False)
       assert small_model is not None
       model = small_model
@@ -558,7 +551,6 @@ def main(demo=False):
       DH.update(sm['carState'], sm['carControl'].latActive, lane_change_prob, left_edge, right_edge)
       modelv2_send.modelV2.meta.laneChangeState = DH.lane_change_state
       modelv2_send.modelV2.meta.laneChangeDirection = DH.lane_change_direction
-      mdv2sp_send.valid = modelv2_send.valid
       mdv2sp_send.modelDataV2SP.laneTurnDirection = DH.lane_turn_direction
       drivingdata_send.drivingModelData.meta.laneChangeState = DH.lane_change_state
       drivingdata_send.drivingModelData.meta.laneChangeDirection = DH.lane_change_direction

openpilot/sunnypilot/modeld_v2/stock_dependencies.py

@@ -1,119 +0,0 @@
-"""
-Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
-
-This file is part of sunnypilot and is licensed under the MIT License.
-See the LICENSE.md file in the root directory for more details.
-"""
-
-import math
-import numpy as np
-from functools import partial
-from tinygrad import dtypes
-from tinygrad.device import Device
-from tinygrad.tensor import Tensor
-
-# The old openpilot/selfdrive/modeld/compile_modeld.py functions needed for legacy models
-# We freeze them here so they aren't lost.
-
-def shift_and_sample(buf, new_val, sample_fn):
-  buf.assign(buf[1:].cat(new_val, dim=0).contiguous())
-  return sample_fn(buf)
-
-def sample_skip(buf, frame_skip):
-  return buf[::frame_skip].contiguous().flatten(0, 1).unsqueeze(0)
-
-def sample_desire(buf, frame_skip):
-  return buf.reshape(-1, frame_skip, *buf.shape[1:]).max(1).flatten(0, 1).unsqueeze(0)
-
-def _detect_desire_key(shapes: dict) -> str | None:
-  return next((key for key in shapes if key.startswith('desire')), None)
-
-def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tuple[dict, list[int]]:
-  desire_key = _detect_desire_key(input_shapes)
-  shapes = {}
-  if desire_key:
-    shapes['desire'] = (input_shapes[desire_key][2],)
-
-  for key, shape in input_shapes.items():
-    if key not in (desire_key, 'features_buffer') and 'img' not in key:
-      shapes[key] = tuple(shape)
-
-  if is_supercombo and 'features_buffer' in input_shapes:
-    fb = input_shapes['features_buffer']
-    feat_dim = math.prod(fb[2:])
-    shapes['prev_feat'] = (fb[0], feat_dim)
-
-  sizes = [int(np.prod(size)) for size in shapes.values()]
-  return shapes, sizes
-
-def make_legacy_run_policy(model_runner, model_metadata, frame_skip):
-  sample_desire_fn = partial(sample_desire, frame_skip=frame_skip)
-  sample_skip_fn = partial(sample_skip, frame_skip=frame_skip)
-  npy_shapes, npy_sizes = get_policy_npy_shapes(model_metadata['input_shapes'], is_supercombo=True)
-  model_input_dtypes = {name: spec.dtype for name, spec in model_runner.graph_inputs.items()}
-
-  def run_policy(warped, img_q, big_img_q, feat_q, desire_q, packed_npy_inputs):
-    packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT)
-    Tensor.realize(packed_npy_inputs, warped)
-
-    img = shift_and_sample(img_q, warped[0:1], sample_skip_fn)
-    big_img = shift_and_sample(big_img_q, warped[1:2], sample_skip_fn)
-
-    desire, traffic_convention, action_t, prev_feat = (t.reshape(s) for t, s in zip(packed_npy_inputs.split(npy_sizes), npy_shapes.values(), strict=True))
-    desire_buf = shift_and_sample(desire_q, desire.reshape(1, 1, -1), sample_desire_fn)
-    feat_buf = shift_and_sample(feat_q, prev_feat.reshape(1, 1, -1), sample_skip_fn)
-
-    inputs = {
-      'img': img,
-      'big_img': big_img,
-      'features_buffer': feat_buf.reshape(model_metadata['input_shapes']['features_buffer']),
-      'desire_pulse': desire_buf,
-      'traffic_convention': traffic_convention,
-      'action_t': action_t,
-    }
-    inputs = {name: value.cast(model_input_dtypes.get(name, dtypes.float32)) for name, value in inputs.items()}
-    out = next(iter(model_runner(inputs).values())).cast('float32')
-    return out,
-  return run_policy
-
-def make_legacy_run_model(warp, run_policy, model_metadata, frame_copy_size):
-  _, policy_sizes = get_policy_npy_shapes(model_metadata['input_shapes'], is_supercombo=True)
-  packed_npy_size = (18 + sum(policy_sizes)) * np.dtype(np.float32).itemsize
-
-  def run_model(img_q, big_img_q, feat_q, desire_q, packed_npy_inputs):
-    packed_input = packed_npy_inputs.to(Device.DEFAULT)
-    Tensor.realize(packed_input)
-    packed_npy_inputs = packed_input[:packed_npy_size].bitcast('float32')
-    frame = packed_input[packed_npy_size:packed_npy_size + frame_copy_size]
-    big_frame = packed_input[packed_npy_size + frame_copy_size:]
-    tfm, big_tfm, policy_inputs = packed_npy_inputs.split([9, 9, sum(policy_sizes)])
-    warped = warp(tfm.reshape(3, 3), big_tfm.reshape(3, 3), frame, big_frame)
-    return run_policy(warped, img_q, big_img_q, feat_q, desire_q, policy_inputs)
-  return run_model
-
-def make_legacy_stock_input_queues(input_shapes, frame_skip, device, frame_copy_size):
-  img = input_shapes['img']  # (1, 12, 128, 256)
-  fb = input_shapes['features_buffer']  # (1, T-1, ...), past features only; the model appends the current frame's feature
-  feat_dim = math.prod(fb[2:])
-  dp = input_shapes['desire_pulse']  # (1, 25, 8)
-  n_frames = img[1] // 6
-  img_buf_shape = (frame_skip * (n_frames - 1) + 1, 6, img[2], img[3])
-
-  policy_shapes, _ = get_policy_npy_shapes(input_shapes, is_supercombo=True)
-  shapes = {'tfm': (3, 3), 'big_tfm': (3, 3)} | policy_shapes
-  sizes = [math.prod(s) for s in shapes.values()]
-  packed_npy_size = sum(sizes) * np.dtype(np.float32).itemsize
-  packed_input = np.zeros(packed_npy_size + 2 * frame_copy_size, dtype=np.uint8)
-  packed_npy_inputs = packed_input[:packed_npy_size].view(np.float32)
-  frames = packed_input[packed_npy_size:]
-  frame_views = {'img': frames[:frame_copy_size], 'big_img': frames[frame_copy_size:]}
-  # views into the packed inputs, to be refilled at runtime
-  npy = {k: v.reshape(s) for (k, s), v in zip(shapes.items(), np.split(packed_npy_inputs, np.cumsum(sizes[:-1])), strict=True)}
-  input_queues = {
-    'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(),
-    'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(),
-    'feat_q': Tensor(np.zeros((frame_skip * fb[1], fb[0], feat_dim), dtype=np.float32), device=device).contiguous().realize(),
-    'desire_q': Tensor(np.zeros((frame_skip * dp[1], dp[0], dp[2]), dtype=np.float32), device=device).contiguous().realize(),
-    'packed_npy_inputs': Tensor(packed_input, device='NPY').realize(),
-  }
-  return input_queues, npy, frame_views

openpilot/sunnypilot/modeld_v2/tests/test_camera_offset_helper.py

@@ -6,9 +6,8 @@ See the LICENSE.md file in the root directory for more details.
 """
 import numpy as np
 
-from openpilot.common.transformations.camera import DEVICE_CAMERAS, view_frame_from_device_frame
+from openpilot.common.transformations.camera import DEVICE_CAMERAS
 from openpilot.common.transformations.model import get_warp_matrix
-from openpilot.common.transformations.orientation import rot_from_euler
 from openpilot.sunnypilot.modeld_v2.camera_offset_helper import CameraOffsetHelper
 from openpilot.common.test import OpenpilotTestCase
 
@@ -47,50 +46,29 @@ class TestCameraOffset(OpenpilotTestCase):
     self.camera_offset.update(main_transform, extra_transform, sm, False)
     np.testing.assert_almost_equal(self.camera_offset.actual_camera_offset, 0.038)
 
-  def test_apply_camera_offset(self):
+  def test_camera_offset_(self):
     intrinsics = self.dc.narrow_road.intrinsics
-    v_horizon = CameraOffsetHelper.get_v_horizon(intrinsics, [])  # pitch = 0 fallback: v_horizon == cy
     transform = np.eye(3, dtype=np.float32)
     height = 1.22
     offset = 0.1
 
+    cy = intrinsics[1, 2]
     expected_shear = np.eye(3, dtype=np.float32)
     expected_shear[0, 1] = offset / height
-    expected_shear[0, 2] = -offset / height * v_horizon
+    expected_shear[0, 2] = -offset / height * cy
 
-    result = CameraOffsetHelper.apply_camera_offset(transform, height, offset, v_horizon)
+    result = CameraOffsetHelper.apply_camera_offset(transform, intrinsics, height, offset)
     np.testing.assert_array_almost_equal(result, expected_shear)
 
-  def test_v_horizon_empty_rpy(self):
-    intrinsics = self.dc.narrow_road.intrinsics
-    v_horizon = CameraOffsetHelper.get_v_horizon(intrinsics, [])
-    np.testing.assert_almost_equal(v_horizon, intrinsics[1, 2])
-
-  def test_v_horizon_projection(self):
-    intrinsics = self.dc.narrow_road.intrinsics
-    f, cy = intrinsics[1, 1], intrinsics[1, 2]
-
-    for pitch_deg in [6.0, -6.0, 0.0]:
-      rpy = [0.0, np.radians(pitch_deg), 0.0]
-      d_dev = rot_from_euler(rpy) @ np.array([1.0, 0.0, 0.0])
-      view = view_frame_from_device_frame @ d_dev
-      expected = cy + f * view[1] / view[2]
-
-      v_horizon = CameraOffsetHelper.get_v_horizon(intrinsics, rpy)
-      np.testing.assert_almost_equal(v_horizon, expected, decimal=4)
-
   def test_update(self):
-    height = 1.2
-    pitch = np.radians(-8.0)
-
     sm = MockStruct(
       deviceState=MockStruct(deviceType='mici'),
       narrowRoadCameraState=MockStruct(sensor='os04c10'),
-      extrinsicsCalibration=MockStruct(rpyCalib=[0.0, pitch, 0.0], height=[height])
+      extrinsicsCalibration=MockStruct(rpyCalib=[0.0, 0.0, 0.0], height=[1.22])
     )
     intrinsics_main = self.dc.narrow_road.intrinsics
     intrinsics_extra = self.dc.wide_road.intrinsics
-    device_from_calib_euler = np.array(sm['extrinsicsCalibration'].rpyCalib, dtype=np.float32)
+    device_from_calib_euler = np.array([0.0, 0.0, 0.0], dtype=np.float32)
     main_transform = get_warp_matrix(device_from_calib_euler, intrinsics_main, False).astype(np.float32)
     extra_transform = get_warp_matrix(device_from_calib_euler, intrinsics_extra, True).astype(np.float32)
 
@@ -103,13 +81,5 @@ class TestCameraOffset(OpenpilotTestCase):
     main_out, extra_out = self.camera_offset.update(main_transform, extra_transform, sm, False)
     assert not np.array_equal(main_out, main_transform)
     assert not np.array_equal(extra_out, extra_transform)
-
-    # settle the low-pass filter
-    for _ in range(100):
-      main_out, extra_out = self.camera_offset.update(main_transform, extra_transform, sm, False)
-
-    # undo main_transform dot product to get shear matrix
-    shear = main_out @ np.linalg.inv(main_transform)
-    expected_v_horizon = intrinsics_main[1, 2] - intrinsics_main[1, 1] * np.tan(pitch)
-    np.testing.assert_almost_equal(shear[0, 1], self.camera_offset.actual_camera_offset / height, decimal=4)
-    np.testing.assert_almost_equal(shear[0, 2], -self.camera_offset.actual_camera_offset / height * expected_v_horizon, decimal=4)
+    assert main_out[0, 1] != 0.0
+    assert main_out[0, 2] != 0.0

openpilot/sunnypilot/modeld_v2/tests/test_models.py

@@ -1,43 +0,0 @@
-"""
-Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
-
-This file is part of sunnypilot and is licensed under the MIT License.
-See the LICENSE.md file in the root directory for more details.
-"""
-
-import os
-import unittest
-from unittest.mock import patch
-from openpilot.common.file_chunker import open_file_chunked
-from openpilot.sunnypilot.modeld_v2.helpers import load_oob
-from tinygrad.device import Device
-
-
-class TestLegacyModels(unittest.TestCase):
-  def test_legacy_model_load(self):
-    base_name = os.environ.get("MODEL_BASE_NAME")
-    if not base_name:
-      raise unittest.SkipTest("MODEL_BASE_NAME env var not set, skipping integration test.")
-    chunk_dir = os.environ.get("MODEL_CHUNK_DIR", "/tmp/model_chunks")
-    base_path = os.path.join(chunk_dir, base_name)
-
-    try:
-      f = open_file_chunked(base_path)
-    except Exception as error:
-      self.fail(f"Failed to open chunked file {base_path}: {error}")
-    self.addCleanup(f.close)
-
-    real_getitem = Device.__class__.__getitem__
-
-    def safe_getitem(device_self, ix):
-      if ix == "QCOM" and not os.path.exists("/dev/kgsl-3d0"):
-        return real_getitem(device_self, "CPU")
-      if ix == "AMD" and not os.path.exists("/dev/kfd"):
-        return real_getitem(device_self, "CPU")
-      return real_getitem(device_self, ix)
-
-    with patch.object(Device.__class__, "__getitem__", safe_getitem):
-      obj = load_oob(f)
-
-    assert isinstance(obj, dict), "Parsed object is not a dictionary"
-    assert "metadata" in obj, "Metadata key is missing"

openpilot/sunnypilot/models/tests/test_tinygrad_ref.py

@@ -0,0 +1,24 @@
+import requests
+
+from openpilot.sunnypilot.models.tinygrad_ref import get_tinygrad_ref
+from openpilot.sunnypilot.models.fetcher import ModelFetcher
+from openpilot.common.test import OpenpilotTestCase
+
+def fetch_tinygrad_ref():
+  response = requests.get(ModelFetcher.MODEL_URL, timeout=10)
+  response.raise_for_status()
+  json_data = response.json()
+  return json_data.get("tinygrad_ref")
+
+
+class TestTinygradRef(OpenpilotTestCase):
+  def test_tinygrad_ref(self):
+    current_ref = get_tinygrad_ref()
+    remote_ref = fetch_tinygrad_ref()
+    assert remote_ref == current_ref, (
+      f"""tinygrad_repo ref does not match remote tinygrad_ref of current compiled driving models json.
+    Current: {current_ref}
+    Remote: {remote_ref}
+    Please run build-all workflow to update models."""
+    )
+    print("tinygrad_repo ref matches current compiled driving models json ref.")

openpilot/sunnypilot/sunnylink/settings_ui.json

@@ -1675,6 +1675,12 @@
               "widget": "toggle",
               "title": "Onroad Uploads"
             },
+            {
+              "key": "AuxPowerSave",
+              "widget": "toggle",
+              "title": "Disable Aux Port When Offroad",
+              "description": "Power off the aux USB-C port while offroad to save power. It powers back on automatically when you go onroad."
+            },
             {
               "key": "MaxTimeOffroad",
               "widget": "option",

openpilot/sunnypilot/sunnylink/settings_ui_src/pages/device.yaml

@@ -30,6 +30,10 @@ sections:
   - key: OnroadUploads
     widget: toggle
     title: Onroad Uploads
+  - key: AuxPowerSave
+    widget: toggle
+    title: Disable Aux Port When Offroad
+    description: Power off the aux USB-C port while offroad to save power. It powers back on automatically when you go onroad.
   - key: MaxTimeOffroad
     widget: option
     title: Max Time Offroad

openpilot/system/hardware/hardwared.py

@@ -21,6 +21,7 @@ from openpilot.common.hardware import HARDWARE, COMMA_HARDWARE
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.git import get_short_branch
 from openpilot.common.hardware.usb import CHESTNUT_FW_VERSION, CHESTNUT_USB_PRODUCT, get_usb_state, get_usb_topology, is_chestnut_usb_id, set_usb_state
+from openpilot.system.hardware.chestnut.flash import VBUS_PATH
 from openpilot.common.linux import LinuxSystemStats
 from openpilot.system.loggerd.config import get_available_percent
 from openpilot.common.swaglog import cloudlog
@@ -50,6 +51,10 @@ class Chestnut:
     self.last_attempt = 0.
     self.flashed = False
     self.mismatch = False
+    self.vbus_on = None
+    self.params = Params()
+    self.powersave = False
+    self.last_offroad = None
 
   @property
   def failed(self) -> bool:
@@ -61,9 +66,19 @@ class Chestnut:
     cloudlog.event("chestnut flash done", returncode=ret.returncode, output=ret.stdout[-1000:], error=ret.returncode != 0)
     self.flashed = ret.returncode == 0
 
+  def set_vbus(self, on: bool) -> None:
+    if on == self.vbus_on:
+      return
+    subprocess.run(["sudo", "tee", VBUS_PATH], input=b"1" if on else b"0", stdout=subprocess.DEVNULL, check=False)
+    self.vbus_on = on
+
   def update(self, offroad: bool, usb_state: list[dict]) -> None:
     self.mismatch = any(is_chestnut_usb_id(d["vendorId"], d["productId"], include_bootloader=True) and
                         d["product"] != CHESTNUT_USB_PRODUCT for d in usb_state)
+    if offroad != self.last_offroad:
+      self.powersave = self.params.get_bool("AuxPowerSave")
+      self.last_offroad = offroad
+    self.set_vbus((not offroad or self.mismatch) or not self.powersave)
     if not self.mismatch:
       self.flashed = False
       return