modeld_v2: chestnut support (#1894)

* modeld_v2: Support eGpu * bump tg * egpu * no pkls please * god no onnx either * fix test * done in build model now * lint * rip * egpu ready build all split * manual seed , reuse memory buffers across runs * dont download big when we dont have big lol * whoops * no i and x * cd * who put those there. ?? * reduce flakiness by using artifact-name from build-model to regex, speed up pub b y checking the name before trying to clone and publish again * try hf as a trusted publisher :) * mf its a dataaset. i knew that * fucking validation wants raw to fetch and full to push. grr * smh * dude i am missing so much * pkl name * move build all to hf * tests: migrate sunnypilot tests to unittest and remove pytest * red diff mf * im scared , this may be a bad idea lol * fetch latest commit. * transition to requests * models: use requests instead of aiohttp * tici * fix * gpu fixes from upstream * lint * bump * needed to say * old * how?? * epgu flag reduce * this made me cry * support monolith still * precache warp in legacy * Move jsons to param for sunnylink --------- Co-authored-by: Jason Wen <haibin.wen3@gmail.com>
2026-08-19 22:23:44 +08:00 · 2026-08-16 18:40:16 -07:00
parent 91d0f3309c
commit 94a32493e3
15 changed files with 411 additions and 554 deletions
@@ -7,6 +7,19 @@ on:
        description: 'Minimum selector version required for the models (see helpers.py or readme.md)'
        required: true
        type: string
+      target_hardware:
+        description: 'Hardware target to compile for (qcom or usbgpu)'
+        required: true
+        type: choice
+        default: 'qcom'
+        options:
+          - qcom
+          - usbgpu
+      hf_repo:
+        description: 'Hugging Face dataset repository'
+        required: false
+        type: string
+        default: 'sunnypilot/sunnypilot_models_v1'

 jobs:
  setup:
@@ -46,13 +59,14 @@ jobs:
        id: get-json
        run: |
          cd docs/docs
-          latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1)
+          PREFIX="driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_' || '' }}v"
+          latest=$(ls ${PREFIX}*.json | sed -E "s/${PREFIX}([0-9]+)\.json/\1/" | sort -n | tail -1)
          next=$((latest+1))
-          json_file="driving_models_v${next}.json"
-          cp "driving_models_v${latest}.json" "$json_file"
+          json_file="${PREFIX}${next}.json"
+          cp "${PREFIX}${latest}.json" "$json_file"
          echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
          echo "json_version=$((next+0))" >> $GITHUB_OUTPUT
-          echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
+          echo "SRC_JSON_FILE=docs/docs/${PREFIX}${latest}.json" >> $GITHUB_ENV

      - name: Extract tinygrad models
        id: set-matrix
@@ -61,45 +75,23 @@ jobs:
          jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
          echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT

-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-      - run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Clone GitLab docs repo and create new recompiled dir
+      - name: Get next recompiled dir number
        id: create-recompiled-dir
        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
+          HF_REPO: ${{ github.event.inputs.hf_repo }}
        run: |
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          git checkout main
-          git sparse-checkout set --no-cone models/
-          cd models
-          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
-          if [[ -z "$latest_dir" ]]; then
-            next_dir=1
-          else
-            next_dir=$((latest_dir+1))
-          fi
-          recompiled_dir="${next_dir}"
-          mkdir -p "recompiled${recompiled_dir}"
-          touch "recompiled${recompiled_dir}/.gitkeep"
-          cd ../..
+          pip install huggingface_hub
+          recompiled_dir=$(python3 -c "
+          from huggingface_hub import HfApi
+          import re, sys
+          api = HfApi()
+          files = api.list_repo_files(repo_id=sys.argv[1], repo_type='dataset')
+          dirs = [re.search(r'models/recompiled([0-9]+)', f) for f in files]
+          nums = [int(m.group(1)) for m in dirs if m]
+          print(max(nums) + 1)
+          " "$HF_REPO")
          echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT

-      - name: Push empty recompiled dir to GitLab
-        run: |
-          cd gitlab_docs
-          git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit"
-          git push origin main
-
      - name: Push new JSON to GitHub docs repo
        run: |
          cd docs
@@ -123,25 +115,30 @@ jobs:
      is_20hz: ${{ matrix.model.is_20hz }}
      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
      json_version: ${{ needs.setup.outputs.json_version }}
+      target_hardware: ${{ github.event.inputs.target_hardware }}
+      hf_repo: ${{ github.event.inputs.hf_repo }}
+      set_min_version: ${{ github.event.inputs.set_min_version }}
+      tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
    secrets: inherit

  retry_failed_models:
    needs: [setup, get_and_build]
    runs-on: ubuntu-latest
-    if: ${{ needs.setup.result != 'failure' && !cancelled() }}
+    if: ${{ !cancelled() && needs.setup.result == 'success' && (needs.get_and_build.result == 'success' || needs.get_and_build.result == 'failure') }}
    outputs:
      retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }}
    steps:
      - uses: actions/download-artifact@v4
        with:
-          pattern: model-*
+          pattern: artifact-name-*
          path: output
+        continue-on-error: true

      - id: set-retry-matrix
        run: |
          echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
-          built=(); while IFS= read -r line; do built+=("$line"); done < <(
-            find output -maxdepth 1 -name 'model-*' -printf "%f\n" | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
+          built=(); while IFS= read -r line; do [ -n "$line" ] && built+=("$line"); done < <(
+            find output -maxdepth 1 -name 'artifact-name-*' -printf "%f\n" 2>/dev/null | sed -E 's/^artifact-name-//' | awk '{gsub(/^ +| +$/, ""); print}'
          )
          jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
            'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
@@ -149,7 +146,7 @@ jobs:

  retry_get_and_build:
    needs: [setup, get_and_build, retry_failed_models]
-    if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }}
+    if: ${{ !cancelled() && needs.retry_failed_models.result == 'success' && needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '' }}
    strategy:
      matrix:
        model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }}
@@ -161,146 +158,9 @@ jobs:
      is_20hz: ${{ matrix.model.is_20hz }}
      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
      json_version: ${{ needs.setup.outputs.json_version }}
+      target_hardware: ${{ github.event.inputs.target_hardware }}
      artifact_suffix: -retry
+      hf_repo: ${{ github.event.inputs.hf_repo }}
+      set_min_version: ${{ github.event.inputs.set_min_version }}
+      tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
    secrets: inherit
-
-  publish_models:
-    name: Publish models sequentially
-    needs: [setup, get_and_build, retry_failed_models, retry_get_and_build]
-    if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      max-parallel: 1
-      matrix:
-        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
-    env:
-      RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }}
-      JSON_FILE: ${{ needs.setup.outputs.json_file }}
-      ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }}
-    steps:
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-
-      - name: Add GitLab.com SSH key to known_hosts
-        run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Clone GitLab docs repo
-        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
-        run: |
-          echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          echo "checkout models/${RECOMPILED_DIR}"
-          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
-          git checkout main
-          cd ..
-
-      - name: Checkout docs repo
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-models
-          ref: gh-pages
-          path: docs
-          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
-      - name: Validate recompiled dir and JSON version
-        run: |
-          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
-            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
-            exit 1
-          fi
-          if [ ! -f "$JSON_FILE" ]; then
-            echo "JSON file $JSON_FILE does not exist!"
-            exit 1
-          fi
-
-      - name: Download artifact name file
-        uses: actions/download-artifact@v4
-        with:
-          name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }}
-          path: artifact_name
-
-      - name: Read artifact name
-        id: read-artifact-name
-        run: |
-          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
-          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
-
-      - name: Download model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
-          path: output
-
-      - name: Remove onnx files bc not needed for recompiled dir since they already exist from single build
-        run: |
-          find output -type f -name '*.onnx' -delete
-          find output -type f -name 'big_*.pkl' -delete
-          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
-
-      - name: Copy model artifacts to gitlab
-        env:
-          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
-        run: |
-          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          mkdir -p "$ARTIFACT_DIR"
-          for path in output/*; do
-            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
-              continue
-            fi
-            name="$(basename "$path")"
-            if [ -d "$path" ]; then
-              mkdir -p "$ARTIFACT_DIR/$name"
-              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
-              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
-            else
-              cp "$path" "$ARTIFACT_DIR/"
-              echo "Copied file $name -> $ARTIFACT_DIR/"
-            fi
-          done
-
-      - name: Push recompiled dir to GitLab
-        env:
-          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-        run: |
-          cd gitlab_docs
-          git checkout main
-          git pull origin main
-          for d in models/"$RECOMPILED_DIR"/*/; do
-            git sparse-checkout add "$d"
-          done
-          git add models/"$RECOMPILED_DIR"
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit"
-          git push origin main
-      - run: |
-          cd docs
-          git pull origin gh-pages
-
-      - name: update json
-        run: |
-          ARGS=""
-          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
-          ARGS="$ARGS --sort-by-date"
-          ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\""
-          eval python3 docs/json_parser.py \
-            --json-path "$JSON_FILE" \
-            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
-            $ARGS
-
-      - name: Push updated json to GitHub
-        run: |
-          cd docs
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git checkout gh-pages
-          git add docs/"$(basename $JSON_FILE)"
-          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
-          git push origin gh-pages
@@ -29,11 +29,24 @@ on:
        required: false
        type: boolean
        default: true
-      bypass_push:
-        description: 'Bypass pushing to GitLab for build-all'
+      target_hardware:
+        description: 'Hardware target to compile for (qcom or usbgpu)'
        required: false
-        default: true
-        type: boolean
+        type: string
+        default: 'qcom'
+      hf_repo:
+        description: 'Hugging Face dataset repository (e.g. sunnypilot/sunnypilot_models_v1)'
+        required: false
+        type: string
+        default: 'sunnypilot/sunnypilot_models_v1'
+      set_min_version:
+        description: 'Minimum selector version'
+        required: false
+        type: string
+      tinygrad_ref:
+        description: 'Tinygrad reference'
+        required: false
+        type: string
  workflow_dispatch:
    inputs:
      upstream_branch:
@@ -65,8 +78,8 @@ on:
          - None
          - Master Models
          - Release Models
-          - 2025 World Models
          - 2026 World Models
+          - 2026 Deep RL Models
          - Custom Merge Models
          - Other
      custom_model_folder:
@@ -81,9 +94,22 @@ on:
        description: 'Minimum selector version'
        required: false
        type: string
+      target_hardware:
+        description: 'Hardware target to compile for'
+        required: false
+        type: choice
+        default: 'qcom'
+        options:
+          - qcom
+          - usbgpu
+      hf_repo:
+        description: 'Hugging Face dataset repository'
+        required: false
+        type: string
+        default: 'sunnypilot/sunnypilot_models_v1'
 env:
  RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
-  JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json
+  JSON_FILE: docs/docs/driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_v' || 'v' }}${{ inputs.json_version }}.json

 jobs:
  build_model:
@@ -93,38 +119,20 @@ jobs:
      custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
      is_20hz: ${{ inputs.is_20hz }}
      artifact_suffix: ${{ inputs.artifact_suffix }}
+      target_hardware: ${{ inputs.target_hardware }}
    secrets: inherit

  publish_model:
-    if: ${{ !inputs.bypass_push && !cancelled() }}
+    if: ${{ !cancelled() && needs.build_model.result == 'success' }}
    concurrency:
-      group: gitlab-push-${{ inputs.recompiled_dir }}
+      group: hf-push-${{ inputs.recompiled_dir }}
      cancel-in-progress: false
    needs: build_model
    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
    steps:
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-
-      - name: Add GitLab.com SSH key to known_hosts
-        run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Clone GitLab docs repo
-        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
-        run: |
-          echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          echo "checkout models/${RECOMPILED_DIR}"
-          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
-          git checkout main
-          cd ..
-
      - name: Checkout docs repo
        uses: actions/checkout@v4
        with:
@@ -133,16 +141,28 @@ jobs:
          path: docs
          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}

-      - name: Validate recompiled dir and JSON version
+      - name: Install huggingface_hub
+        run: pip install --upgrade "huggingface_hub>=0.22.0"
+
+      - name: Validate hf_repo and JSON version
+        env:
+          HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
        run: |
-          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
-            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
-            exit 1
-          fi
          if [ ! -f "$JSON_FILE" ]; then
            echo "JSON file $JSON_FILE does not exist!"
            exit 1
          fi
+          python3 -c "
+          import sys
+          from huggingface_hub import HfApi
+          try:
+              api = HfApi()
+              api.repo_info(repo_id=sys.argv[1], repo_type='dataset')
+              print(f'Success: Repo {sys.argv[1]} exists.')
+          except Exception as e:
+              print('HF validation failed:', e)
+              sys.exit(1)
+          " "${{ inputs.hf_repo }}"

      - name: Download artifact name file
        uses: actions/download-artifact@v4
@@ -162,49 +182,26 @@ jobs:
          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
          path: output

-      - name: Remove unwanted files
-        run: |
-          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
-          find output -type f -name 'dmonitoring_model.onnx' -delete
-
-      - name: Copy model artifact(s) to GitLab recompiled dir
+      - name: Create models folder
        env:
          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
        run: |
-          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          mkdir -p "$ARTIFACT_DIR"
-          for path in output/*; do
-            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
-              continue
-            fi
-            name="$(basename "$path")"
-            if [ -d "$path" ]; then
-              mkdir -p "$ARTIFACT_DIR/$name"
-              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
-              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
-            else
-              cp "$path" "$ARTIFACT_DIR/"
-              echo "Copied file $name -> $ARTIFACT_DIR/"
-            fi
-          done
+          mkdir -p "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
+          cp -r output/* "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/"
+          rm -f "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/artifact_name.txt"

-      - name: Push recompiled dir to GitLab
+      - name: Upload to Hugging Face
        env:
-          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+          HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
+          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
        run: |
-          cd gitlab_docs
-          git checkout main
-          git pull origin main
-          for d in models/"$RECOMPILED_DIR"/*/; do
-            git sparse-checkout add "$d"
-          done
-          git add models/"$RECOMPILED_DIR"
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
-          git push origin main
+          hf upload ${{ inputs.hf_repo }} \
+            output/ \
+            "models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/" \
+            --repo-type=dataset

-      - run: |
+      - name: Pull gh-pages
+        run: |
          cd docs
          git pull origin gh-pages

@@ -220,9 +217,11 @@ jobs:
          fi
          [ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\""
          [ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
+          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
+          [ -n "${{ inputs.tinygrad_ref }}" ] && ARGS="$ARGS --tinygrad-ref \"${{ inputs.tinygrad_ref }}\""
          eval python3 docs/json_parser.py \
            --json-path "$JSON_FILE" \
-            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
+            --recompiled-dir "local_models/$RECOMPILED_DIR" \
            --sort-by-date \
            $ARGS

@@ -80,6 +80,7 @@ jobs:
        with:
          repository: commaai/openpilot
          ref: ${{ inputs.upstream_branch }}
+          fetch-depth: 1
          submodules: recursive
          path: openpilot

@@ -89,18 +90,25 @@ jobs:
        with:
          repository: sunnypilot/sunnypilot
          ref: ${{ inputs.upstream_branch }}
+          fetch-depth: 1
          submodules: recursive
          path: openpilot
      - name: Get commit date
        id: commit-date
        run: |
-          cd ${{ github.workspace }}/openpilot
+          cd ${{ github.workspace }}/openpilot/openpilot
          commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
          echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
          cat $GITHUB_OUTPUT
      - run: |
-          cd ${{ github.workspace }}/openpilot
-          git lfs pull
+          cd ${{ github.workspace }}/openpilot/openpilot
+          if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then
+            git lfs pull -X "selfdrive/modeld/models/big_*.onnx" -X "selfdrive/modeld/models/dmonitoring_*.onnx"
+            rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx
+          else
+            git lfs pull -I "selfdrive/modeld/models/big_*.onnx"
+            find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete
+          fi
      - name: 'Upload Artifact'
        uses: actions/upload-artifact@v4
        with:
@@ -116,24 +124,10 @@ jobs:
    steps:
      - uses: actions/checkout@v4
        with:
+          fetch-depth: 1
          submodules: recursive

      - run: git lfs pull
-      - name: Cache SCons
-        uses: actions/cache@v4
-        with:
-          path: ${{env.SCONS_CACHE_DIR}}
-          key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
-          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
-          restore-keys: |
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}-model
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}-model
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}
-            scons-${{ runner.os }}-${{ runner.arch }}

      - name: Set environment variables
        id: set-env
@@ -144,7 +138,7 @@ jobs:
          export UV_PYTHON_PREFERENCE=managed
          export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          uv sync
+          uv sync --frozen
          printenv >> $GITHUB_ENV
          if [[ "${{ runner.debug }}" == "1" ]]; then
            cat $GITHUB_OUTPUT
@@ -173,8 +167,6 @@ jobs:
        with:
          name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
          path: ${{ env.MODELS_DIR }}
-      - run: |
-          rm -f ${{ env.MODELS_DIR }}/{dmonitoring_model,big_driving_policy,big_driving_vision,big_driving_supercombo}.onnx

      - name: Build Model
        run: |
@@ -191,7 +183,7 @@ jobs:
          if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
            echo "USBGPU build"
            export USBGPU=1
-            TG_FLAGS="DEV=AMD USBGPU=1 IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
+            TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2"
            OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
          else
            echo "QCOM build"
@@ -254,10 +246,8 @@ jobs:
          # Copy the model files
          rsync -avm \
            --include='*.dlc' \
-            --include='*.pkl' \
            --include='*.chunk*' \
            --include='*.chunkmanifest' \
-            --include='*.onnx' \
            --exclude='*' \
            --delete-excluded \
            --chown=comma:comma \