ci: unify default model build into single workflow (#1951)

* ci: unify default model build into single workflow

* ci: consolidate upload jobs and add tinygrad ref validation

.github/workflows/build-default-big-model.yaml

@@ -1,83 +0,0 @@
-name: Build default big model
-
-on:
-  workflow_dispatch:
-
-env:
-  HF_REPO: sunnypilot/sunnypilot_models_v1
-  HF_DEFAULTS_PATH: models/defaults/big
-
-jobs:
-  resolve_name:
-    runs-on: ubuntu-24.04
-    outputs:
-      model_name: ${{ steps.name.outputs.model_name }}
-      onnx_ref: ${{ steps.name.outputs.onnx_ref }}
-    steps:
-      - uses: actions/checkout@v4
-      - id: name
-        run: |
-          NAME=$(PYTHONPATH=${{ github.workspace }} python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)")
-          ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx)
-          echo "model_name=${NAME}" >> $GITHUB_OUTPUT
-          echo "onnx_ref=$ONNX_REF" >> $GITHUB_OUTPUT
-
-  build_model:
-    needs: resolve_name
-    uses: ./.github/workflows/sunnypilot-build-model.yaml
-    with:
-      upstream_branch: ${{ needs.resolve_name.outputs.onnx_ref }}
-      custom_name: ${{ needs.resolve_name.outputs.model_name }}
-      target_hardware: usbgpu
-    secrets: inherit
-
-  upload_defaults:
-    needs: [ resolve_name, build_model ]
-    runs-on: ubuntu-24.04
-    permissions:
-      id-token: write
-      contents: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
-
-      - name: Install huggingface_hub
-        run: pip install --upgrade "huggingface_hub>=0.22.0"
-
-      - name: Download artifact name
-        uses: actions/download-artifact@v4
-        with:
-          name: artifact-name-${{ needs.resolve_name.outputs.model_name }}
-          path: artifact_name
-
-      - name: Read artifact name
-        id: artifact
-        run: |
-          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
-          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
-
-      - name: Download model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.artifact.outputs.artifact_name }}
-          path: output
-
-      - name: Upload to HF and update default_models.json
-        env:
-          HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
-          ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
-        run: |
-          rm -f output/artifact_name.txt
-          export PYTHONPATH=$(pwd)
-          python3 release/ci/upload_default_model.py \
-            --hf-repo "${{ env.HF_REPO }}" \
-            --hf-defaults-path "${{ env.HF_DEFAULTS_PATH }}" \
-            --artifact-name "$ARTIFACT_NAME" \
-            --model-dir output \
-            --onnx-path "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" \
-            --onnx-ref "${{ needs.resolve_name.outputs.onnx_ref }}" \
-            --model-name "${{ needs.resolve_name.outputs.model_name }}" \
-            --tinygrad-ref "$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)" \
-            --run-number "${{ github.run_number }}"

.github/workflows/build-default-models.yaml

@@ -0,0 +1,279 @@
+name: Build default models
+
+on:
+  workflow_dispatch:
+    inputs:
+      target:
+        description: 'Model target to build'
+        required: true
+        type: choice
+        options:
+          - small
+          - big
+  workflow_call:
+    inputs:
+      target:
+        description: 'Model target to build (small or big)'
+        required: true
+        type: string
+
+env:
+  HF_REPO: sunnypilot/sunnypilot_models_v1
+
+jobs:
+  resolve:
+    runs-on: ubuntu-24.04
+    outputs:
+      model_name: ${{ steps.resolve.outputs.model_name }}
+      onnx_ref: ${{ steps.resolve.outputs.onnx_ref }}
+      onnx_path: ${{ steps.resolve.outputs.onnx_path }}
+      hf_defaults_path: ${{ steps.resolve.outputs.hf_defaults_path }}
+      target_hardware: ${{ steps.resolve.outputs.target_hardware }}
+      tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }}
+      dm_onnx_ref: ${{ steps.resolve.outputs.dm_onnx_ref }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - id: resolve
+        run: |
+          export PYTHONPATH=${{ github.workspace }}
+
+          if [ "${{ inputs.target }}" = "big" ]; then
+            NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)")
+            ONNX_PATH="openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
+            HF_DEFAULTS_PATH="models/defaults/big"
+            TARGET_HW="usbgpu"
+          else
+            NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL; print(DEFAULT_MODEL)")
+            ONNX_PATH="openpilot/selfdrive/modeld/models/driving_supercombo.onnx"
+            HF_DEFAULTS_PATH="models/defaults/small"
+            TARGET_HW="qcom"
+          fi
+
+          ONNX_REF=$(git log -1 --format='%H' -- "$ONNX_PATH")
+          TINYGRAD_REF=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)
+          if [ -z "$TINYGRAD_REF" ]; then
+            echo "::error::Failed to resolve tinygrad ref"
+            exit 1
+          fi
+
+          DM_ONNX_REF=""
+          if [ "${{ inputs.target }}" = "small" ]; then
+            DM_ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx)
+          fi
+
+          echo "model_name=${NAME}" >> $GITHUB_OUTPUT
+          echo "onnx_ref=${ONNX_REF}" >> $GITHUB_OUTPUT
+          echo "onnx_path=${ONNX_PATH}" >> $GITHUB_OUTPUT
+          echo "hf_defaults_path=${HF_DEFAULTS_PATH}" >> $GITHUB_OUTPUT
+          echo "target_hardware=${TARGET_HW}" >> $GITHUB_OUTPUT
+          echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT
+          echo "dm_onnx_ref=${DM_ONNX_REF}" >> $GITHUB_OUTPUT
+
+  build_driving_model:
+    needs: resolve
+    uses: ./.github/workflows/sunnypilot-build-model.yaml
+    with:
+      upstream_branch: ${{ needs.resolve.outputs.onnx_ref }}
+      custom_name: ${{ needs.resolve.outputs.model_name }}
+      target_hardware: ${{ needs.resolve.outputs.target_hardware }}
+    secrets: inherit
+
+  upload_defaults:
+    needs: [ resolve, build_driving_model, build_dm_model ]
+    if: ${{ !cancelled() && needs.build_driving_model.result == 'success' && (inputs.target != 'small' || needs.build_dm_model.result == 'success') }}
+    runs-on: ubuntu-24.04
+    permissions:
+      id-token: write
+      contents: write
+    env:
+      DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Pull ONNX via LFS
+        run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}${{ inputs.target == 'small' && ',openpilot/selfdrive/modeld/models/dmonitoring_model.onnx' || '' }}"
+
+      - name: Install huggingface_hub
+        run: pip install --upgrade "huggingface_hub>=0.22.0"
+
+      - name: Download driving artifact name
+        uses: actions/download-artifact@v4
+        with:
+          name: artifact-name-${{ needs.resolve.outputs.model_name }}
+          path: artifact_name
+
+      - name: Read driving artifact name
+        id: artifact
+        run: |
+          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
+          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
+
+      - name: Download driving model artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ steps.artifact.outputs.artifact_name }}
+          path: output
+
+      - name: Upload driving model to HF
+        env:
+          HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
+          ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
+        run: |
+          rm -f output/artifact_name.txt
+          export PYTHONPATH=$(pwd)
+          python3 release/ci/upload_default_model.py \
+            --hf-repo "${{ env.HF_REPO }}" \
+            --hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \
+            --artifact-name "$ARTIFACT_NAME" \
+            --model-dir output \
+            --onnx-path "${{ needs.resolve.outputs.onnx_path }}" \
+            --onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \
+            --model-name "${{ needs.resolve.outputs.model_name }}" \
+            --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \
+            --run-number "${{ github.run_number }}"
+
+      - name: Download DM artifact
+        if: ${{ inputs.target == 'small' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: dm-model-${{ github.run_number }}
+          path: dm_output
+
+      - name: Generate DM metadata and upload to HF
+        if: ${{ inputs.target == 'small' }}
+        env:
+          HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
+        run: |
+          export PYTHONPATH=$(pwd)
+          python3 -c "
+          import json, hashlib
+          from pathlib import Path
+          from datetime import datetime, UTC
+
+          dm_dir = Path('dm_output')
+          manifest = list(dm_dir.glob('*.chunkmanifest'))
+          assert manifest, 'No chunkmanifest found'
+          pkl_name = manifest[0].name.removesuffix('.chunkmanifest')
+          num_chunks = int(manifest[0].read_text().strip())
+
+          chunks = []
+          for i in range(num_chunks):
+              chunk = dm_dir / f'{pkl_name}.chunk{i+1:02d}of{num_chunks:02d}'
+              chunks.append({
+                  'file_name': chunk.name,
+                  'sha256': hashlib.sha256(chunk.read_bytes()).hexdigest()
+              })
+
+          digest = hashlib.sha256()
+          for c in chunks:
+              with open(dm_dir / c['file_name'], 'rb') as f:
+                  while block := f.read(1024*1024):
+                      digest.update(block)
+
+          metadata = {
+              'bundles': [{
+                  'short_name': 'DMMODEL',
+                  'display_name': 'dmonitoring_model',
+                  'ref': '${{ needs.resolve.outputs.dm_onnx_ref }}',
+                  'runner': 'tinygrad',
+                  'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'),
+                  'models': [{
+                      'type': 'chunked',
+                      'artifact': {
+                          'file_name': pkl_name,
+                          'download_uri': {'url': '', 'sha256': digest.hexdigest()},
+                          'chunks': chunks
+                      }
+                  }]
+              }]
+          }
+          with open(dm_dir / 'metadata.json', 'w') as f:
+              json.dump(metadata, f, indent=2)
+          print('Generated DM metadata.json')
+          "
+
+          python3 release/ci/upload_default_model.py \
+            --hf-repo "${{ env.HF_REPO }}" \
+            --hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \
+            --artifact-name "dm-model-${{ github.run_number }}" \
+            --model-dir dm_output \
+            --onnx-path "${{ env.DM_ONNX }}" \
+            --onnx-ref "${{ needs.resolve.outputs.dm_onnx_ref }}" \
+            --model-name "dmonitoring_model" \
+            --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \
+            --run-number "${{ github.run_number }}"
+
+  build_dm_model:
+    needs: resolve
+    if: ${{ inputs.target == 'small' }}
+    runs-on: [self-hosted, tici]
+    env:
+      DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx
+      DM_PKL: openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set environment variables
+        run: |
+          source /etc/profile
+          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
+          export UV_PYTHON_PREFERENCE=managed
+          export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
+          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
+          uv sync --frozen
+          printenv >> $GITHUB_ENV
+
+      - name: Disable powersave
+        run: |
+          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
+
+      - name: Compile DM model
+        run: |
+          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+          export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"
+
+          TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
+
+          taskset -c 7 env ${TG_FLAGS} python3 \
+            ${{ github.workspace }}/tinygrad_repo/examples/openpilot/compile3.py \
+            ${{ github.workspace }}/${{ env.DM_ONNX }} \
+            ${{ github.workspace }}/${{ env.DM_PKL }}
+
+      - name: Chunk DM pkl
+        run: |
+          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+          export PYTHONPATH=${{ github.workspace }}
+          python3 -c "
+          from openpilot.common.file_chunker import chunk_file, get_chunk_targets
+          import os
+          pkl = '${{ github.workspace }}/${{ env.DM_PKL }}'
+          size = os.path.getsize(pkl)
+          targets = get_chunk_targets(pkl, size)
+          chunk_file(pkl, targets)
+          print(f'Chunked {pkl} into {len(targets)} chunks')
+          "
+
+      - name: Prepare DM output
+        run: |
+          mkdir -p dm_output
+          cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunk* dm_output/
+          cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunkmanifest dm_output/
+
+      - name: Upload DM artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dm-model-${{ github.run_number }}
+          path: dm_output/
+
+      - name: Re-enable powersave
+        if: always()
+        run: |
+          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
+

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -242,14 +242,14 @@ jobs:
             echo "HF defaults match repo ONNX"
           else
             echo "No matching model on HF — triggering build"
-            gh workflow run build-default-big-model.yaml --ref "${{ github.head_ref || github.ref_name }}"
+            gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=big
 
             echo "Waiting for build to start..."
             sleep 120
 
-            RUN_ID=$(gh run list --workflow=build-default-big-model.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId')
+            RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId')
             if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
-              echo "::error::Failed to find build-default-big-model run"
+              echo "::error::Failed to find build-default-models run"
               exit 1
             fi
 
@@ -258,7 +258,7 @@ jobs:
 
             CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
             if [ "$CONCLUSION" != "success" ]; then
-              echo "::error::build-default-big-model failed: $CONCLUSION"
+              echo "::error::build-default-models failed: $CONCLUSION"
               exit 1
             fi
 

openpilot/cereal/custom.capnp

@@ -352,7 +352,6 @@ struct OnroadEventSP @0xda96579883444c35 {
     speedLimitPending @22;
     e2eChime @23;
     laneChangeRoadEdge @24;
-    bigModelReady @25;
   }
 }
 

openpilot/selfdrive/selfdrived/selfdrived.py

@@ -198,7 +198,6 @@ class SelfdriveD(CruiseHelper):
     loading = self.params.get_bool("UsbGpuLoading")
     if self.big_model_loading and not loading:
       self.big_model_ready_t = time.monotonic()
-      self.events_sp.add(custom.OnroadEventSP.EventName.bigModelReady)
     self.big_model_loading = loading
     if self.big_model_loading:
       self.events.add(EventName.bigModelLoading)

openpilot/sunnypilot/models/fetcher.py

@@ -13,8 +13,6 @@ from openpilot.common.params import Params
 from openpilot.common.swaglog import cloudlog
 from openpilot.common.hardware.hw import Paths
 from openpilot.sunnypilot.models.helpers import is_bundle_version_compatible
-from openpilot.selfdrive.modeld.helpers import usbgpu_present
-
 from openpilot.cereal import custom
 
 
@@ -149,11 +147,10 @@ class ModelFetcher:
     self._is_usbgpu: bool | None = None
     self.model_cache = ModelCache(params)
     self.model_url = self.MODEL_URL
-    self._update_model_source()
 
-  def _update_model_source(self) -> None:
-    """Updates what json to use based on usbgpu availability"""
-    is_usbgpu = usbgpu_present()
+  def _update_model_source(self, chestnut_present: bool) -> None:
+    """Updates what json to use based on chestnut hardware presence via deviceState"""
+    is_usbgpu = chestnut_present
     if is_usbgpu != self._is_usbgpu:
       self._is_usbgpu = is_usbgpu
       self.model_cache = ModelCache(self.params, suffix="_USBGPU" if is_usbgpu else "")
@@ -191,9 +188,9 @@ class ModelFetcher:
 
     return None
 
-  def get_available_bundles(self) -> list[custom.ModelManagerSP.ModelBundle]:
+  def get_available_bundles(self, chestnut_present: bool = False) -> list[custom.ModelManagerSP.ModelBundle]:
     """Gets the list of available models, with smart cache handling"""
-    self._update_model_source()
+    self._update_model_source(chestnut_present)
     cached_data, is_expired = self.model_cache.get()
 
     if cached_data and not is_expired:
@@ -210,10 +207,12 @@ class ModelFetcher:
     cloudlog.warning("Failed to fetch fresh data. Using expired cache as fallback")
     return self.model_parser.parse_models(cached_data)
 
+
 if __name__ == "__main__":
+  from openpilot.selfdrive.modeld.helpers import usbgpu_present
   params = Params()
   model_fetcher = ModelFetcher(params)
-  bundles = model_fetcher.get_available_bundles()
+  bundles = model_fetcher.get_available_bundles(chestnut_present=usbgpu_present())
   for bundle in bundles:
     for model in bundle.models:
       model_overrides = {override.key: override.value for override in bundle.overrides}

openpilot/sunnypilot/models/manager.py

@@ -30,6 +30,7 @@ class ModelManagerSP:
     self.params = Params()
     self.model_fetcher = ModelFetcher(self.params)
     self.pm = messaging.PubMaster(["modelManagerSP"])
+    self.sm = messaging.SubMaster(["deviceState"])
     self.available_models: list[custom.ModelManagerSP.ModelBundle] = []
     self.selected_bundle: custom.ModelManagerSP.ModelBundle = None
     self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params)
@@ -262,7 +263,8 @@ class ModelManagerSP:
 
     while True:
       try:
-        self.available_models = self.model_fetcher.get_available_bundles()
+        self.sm.update(0)
+        self.available_models = self.model_fetcher.get_available_bundles(self.sm['deviceState'].chestnutPresent)
         validate_active_bundle(self.params, self.available_models)
         self.active_bundle = get_active_bundle(self.params)
 

openpilot/sunnypilot/selfdrive/selfdrived/events.py

@@ -252,12 +252,4 @@ EVENTS_SP: dict[int, dict[str, Alert | AlertCallbackType]] = {
       AlertStatus.userPrompt, AlertSize.small,
       Priority.LOW, VisualAlert.none, AudibleAlert.prompt, 0.1),
   },
-
-  EventNameSP.bigModelReady: {
-    ET.PERMANENT: Alert(
-      "Big Model Ready",
-      "",
-      AlertStatus.normal, AlertSize.small,
-      Priority.LOW, VisualAlert.none, AudibleAlert.prompt, 2.),
-  },
 }

openpilot/sunnypilot/sunnylink/athena/sunnylinkd.py

@@ -28,7 +28,6 @@ from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutExce
                        create_connection, WebSocketConnectionClosedException)
 
 import openpilot.cereal.messaging as messaging
-from openpilot.selfdrive.modeld.helpers import usbgpu_present
 from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL, DEFAULT_BIG_MODEL
 from openpilot.sunnypilot.selfdrive.car.sync_sunnylink_params import update_car_list_param
 from openpilot.sunnypilot.sunnylink.api import SunnylinkApi
@@ -182,10 +181,8 @@ def getParamsMetadata() -> str:
     schema = generate_schema()
     schema["capabilities"] = generate_capabilities()
     schema["capability_labels"] = CAPABILITY_LABELS
-    # mirrors get_default_model() — ui_state unavailable in sunnylinkd process
-    show_big = (usbgpu_present()
-                and (params.get_bool("UsbGpuActive") or params.get_bool("UsbGpuLoading") or params.get_bool("IsOffroad")))
-    schema["default_model"] = DEFAULT_BIG_MODEL if show_big else DEFAULT_MODEL
+    schema["default_model"] = DEFAULT_MODEL
+    schema["default_big_model"] = DEFAULT_BIG_MODEL
     schema["usbgpu_active"] = params.get_bool("UsbGpuActive")
     raw = json.dumps(schema, separators=(",", ":")).encode("utf-8")
     return base64.b64encode(gzip.compress(raw)).decode("utf-8")