ci: offload small model compilation (#1952)

* ci: compile default big model with stock modeld * Revert "Revert big RL model (#38627)" This reverts commit 516ec1e682. * ci: compile default small model with stock modeld compiler * ci: offload small model compilation * Reapply "Revert big RL model (#38627)" This reverts commit d06cfabb625e16bfd8f74984b359162be4b71417. * Reapply "Revert big RL model (#38627)" This reverts commit d06cfabb625e16bfd8f74984b359162be4b71417.
2026-09-11 02:33:41 +08:00 · 2026-08-24 16:24:19 -04:00
parent 2bcfed5c71
commit 8e16c9babb
3 changed files with 168 additions and 47 deletions
@@ -31,6 +31,7 @@ jobs:
      target_hardware: ${{ steps.resolve.outputs.target_hardware }}
      tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }}
      dm_onnx_ref: ${{ steps.resolve.outputs.dm_onnx_ref }}
+      dm_onnx_date: ${{ steps.resolve.outputs.dm_onnx_date }}
    steps:
      - uses: actions/checkout@v4
        with:
@@ -60,8 +61,10 @@ jobs:
          fi

          DM_ONNX_REF=""
+          DM_ONNX_DATE=""
          if [ "${{ inputs.target }}" = "small" ]; then
            DM_ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx)
+            DM_ONNX_DATE=$(git log -1 --format=%cd --date=format:'%B %d, %Y' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx)
          fi

          echo "model_name=${NAME}" >> $GITHUB_OUTPUT
@@ -71,6 +74,7 @@ jobs:
          echo "target_hardware=${TARGET_HW}" >> $GITHUB_OUTPUT
          echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT
          echo "dm_onnx_ref=${DM_ONNX_REF}" >> $GITHUB_OUTPUT
+          echo "dm_onnx_date=${DM_ONNX_DATE}" >> $GITHUB_OUTPUT

  build_small_model:
    needs: resolve
@@ -395,7 +399,7 @@ jobs:
          metadata = {
              'bundles': [{
                  'short_name': 'DMMODEL',
-                  'display_name': 'dmonitoring_model',
+                  'display_name': 'dmonitoring_model (${{ needs.resolve.outputs.dm_onnx_date }})',
                  'ref': '${{ needs.resolve.outputs.dm_onnx_ref }}',
                  'runner': 'tinygrad',
                  'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'),
@@ -421,7 +425,7 @@ jobs:
            --model-dir dm_output \
            --onnx-path "${{ env.DM_ONNX }}" \
            --onnx-ref "${{ needs.resolve.outputs.dm_onnx_ref }}" \
-            --model-name "dmonitoring_model" \
+            --model-name "dmonitoring_model (${{ needs.resolve.outputs.dm_onnx_date }})" \
            --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \
            --run-number "${{ github.run_number }}"

@@ -165,7 +165,7 @@ jobs:
          scons -j1 cache_dir="$SCONS_CACHE" --minimal \
            openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd
          echo "Building rest of sunnypilot"
-          /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
+          SKIP_TINYGRAD_COMPILE=1 /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
          touch ${BUILD_DIR}/prebuilt
          if [[ "${{ runner.debug }}" == "1" ]]; then
            ls -la ${BUILD_DIR}
@@ -242,12 +242,13 @@ jobs:
            echo "HF defaults match repo ONNX"
          else
            echo "No matching model on HF — triggering build"
+            TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
            gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=big

            echo "Waiting for build to start..."
            sleep 120

-            RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId')
+            RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --created=">$TRIGGER_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
            if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
              echo "::error::Failed to find build-default-models run"
              exit 1
@@ -276,21 +277,99 @@ jobs:
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

+  prepare_small_models:
+    needs: [ prepare_strategy ]
+    runs-on: ubuntu-24.04
+    outputs:
+      driving_onnx_sha256: ${{ steps.resolve.outputs.driving_onnx_sha256 }}
+      dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }}
+    env:
+      HF_REPO: sunnypilot/sunnypilot_models_v1
+      HF_DEFAULTS_PATH: models/defaults/small
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+          submodules: recursive
+
+      - run: git lfs pull -I "openpilot/selfdrive/modeld/models/driving_supercombo.onnx,openpilot/selfdrive/modeld/models/dmonitoring_model.onnx"
+
+      - name: Check HF defaults and build if needed
+        id: resolve
+        run: |
+          DRIVING_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/driving_supercombo.onnx" | cut -d' ' -f1)
+          DM_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/dmonitoring_model.onnx" | cut -d' ' -f1)
+          TINYGRAD_REF=$(PYTHONPATH=${{ github.workspace }} python3 openpilot/sunnypilot/models/tinygrad_ref.py)
+          echo "driving_onnx_sha256=$DRIVING_HASH" >> $GITHUB_OUTPUT
+          echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT
+          echo "Driving ONNX hash: $DRIVING_HASH"
+          echo "DM ONNX hash: $DM_HASH"
+          echo "tinygrad ref: $TINYGRAD_REF"
+
+          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
+
+          check_defaults() {
+            DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
+            TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null)
+            [ "$TINYGRAD_MATCH" = "true" ] || return 1
+            DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
+            [ -n "$DRIVING" ] && [ "$DRIVING" != "null" ] || return 1
+            DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
+            [ -n "$DM" ] && [ "$DM" != "null" ] || return 1
+          }
+
+          if check_defaults; then
+            echo "HF defaults match repo ONNX hashes and tinygrad ref"
+          else
+            echo "No matching models on HF — triggering build"
+            TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+            gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=small
+
+            echo "Waiting for build to start..."
+            sleep 120
+
+            RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --created=">$TRIGGER_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
+            if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
+              echo "::error::Failed to find build-default-models run"
+              exit 1
+            fi
+
+            echo "Waiting for run $RUN_ID..."
+            gh run watch "$RUN_ID"
+
+            CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
+            if [ "$CONCLUSION" != "success" ]; then
+              echo "::error::build-default-models failed: $CONCLUSION"
+              exit 1
+            fi
+
+            if ! check_defaults; then
+              echo "::error::HF defaults still don't match after build"
+              exit 1
+            fi
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Cancel run on failure
+        if: failure()
+        run: gh run cancel ${{ github.run_id }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
  publish:
    concurrency:
-      # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
-      # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
-      # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
      group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
      cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
    if: ${{
      always() && !cancelled() &&
      needs.build.result == 'success' &&
      needs.prepare_strategy.result == 'success' &&
+      needs.prepare_small_models.result == 'success' &&
      (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
      (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
      }}
-    needs: [ build, prepare_strategy, prepare_chestnut ]
+    needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_models ]
    runs-on: ubuntu-24.04
    environment: ${{ needs.prepare_strategy.outputs.environment }}
    steps:
@@ -306,6 +385,41 @@ jobs:
          mkdir -p ${{ env.OUTPUT_DIR }}
          tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}

+      - name: Download small model chunks from HF
+        env:
+          HF_REPO: sunnypilot/sunnypilot_models_v1
+          HF_DEFAULTS_PATH: models/defaults/small
+        run: |
+          set -o pipefail
+          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
+          DEFAULTS=$(curl -fsSL "$JSON_URL")
+          MODELS_DIR="${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models"
+
+          download_model_chunks() {
+            local ONNX_HASH="$1"
+            local CANONICAL="$2"
+            BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
+            ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
+            BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
+            NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
+
+            echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
+              CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true)
+              if [ -z "$CHUNK_IDX" ]; then
+                echo "::error::Failed to parse chunk index from: $CHUNK_NAME"
+                exit 1
+              fi
+              CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
+              ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
+              echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
+              curl -fsSL -o "${MODELS_DIR}/${CANONICAL_CHUNK}" "$ENCODED_URL"
+            done
+            echo "$NUM_CHUNKS" > "${MODELS_DIR}/${CANONICAL}.chunkmanifest"
+          }
+
+          download_model_chunks "${{ needs.prepare_small_models.outputs.driving_onnx_sha256 }}" "driving_tinygrad.pkl"
+          download_model_chunks "${{ needs.prepare_small_models.outputs.dm_onnx_sha256 }}" "dmonitoring_model_tinygrad.pkl"
+
      - name: Prepare chestnut output
        if: ${{ needs.prepare_chestnut.result == 'success' }}
        run: |
@@ -393,6 +507,7 @@ jobs:
    - build
    - publish
    - prepare_chestnut
+    - prepare_small_models
    runs-on: ubuntu-24.04
    if: ${{ (always() && !cancelled() && !failure())
      && needs.publish.result == 'success'