Merge remote-tracking branch 'sunnypilot/sunnypilot/master' into hkg-angle-steering-2025

2026-10-28 02:13:47 +08:00 · 2026-10-25 09:53:27 -04:00
parent d021f6ca41 8e740e0ac6
commit 4e371fb488
517 changed files with 24049 additions and 14748 deletions
@@ -41,6 +41,7 @@ jobs:
        with:
          path: sunnypilot
          submodules: recursive
+          fetch-depth: 1

      - name: Get tinygrad_repo ref
        id: get-tinygrad-ref
@@ -58,6 +59,7 @@ jobs:
          ref: gh-pages
          path: docs
          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
+          fetch-depth: 1

      - name: Get next JSON version to use (from GitHub docs repo)
        id: get-json
@@ -46,8 +46,8 @@ jobs:

          if [ "${{ inputs.target }}" = "big" ]; then
            NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)")
-            ONNX_PATH="openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
            HF_DEFAULTS_PATH="models/defaults/big"
+            ONNX_PATH="openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl"
          elif [ "${{ inputs.target }}" = "dm" ]; then
            ONNX_PATH="openpilot/selfdrive/modeld/models/dmonitoring_model.onnx"
            HF_DEFAULTS_PATH="models/defaults/dm"
@@ -57,8 +57,9 @@ jobs:
            ONNX_PATH="openpilot/selfdrive/modeld/models/driving_supercombo.onnx"
            HF_DEFAULTS_PATH="models/defaults/small"
          fi
+          ONNX_REF=""
+          [ -n "$ONNX_PATH" ] && ONNX_REF=$(git log -1 --format='%H' -- "$ONNX_PATH")

-          ONNX_REF=$(git log -1 --format='%H' -- "$ONNX_PATH")
          TINYGRAD_REF=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)
          if [ -z "$TINYGRAD_REF" ]; then
            echo "::error::Failed to resolve tinygrad ref"
@@ -78,6 +79,8 @@ jobs:
    if: ${{ inputs.target == 'small' }}
    runs-on: [self-hosted, tici]
    env:
+      MODELS_DIR: openpilot/selfdrive/modeld/models
+      COMPILER: tinygrad_repo/examples/openpilot
      SMALL_ONNX: openpilot/selfdrive/modeld/models/driving_supercombo.onnx
      SMALL_PKL: openpilot/selfdrive/modeld/models/driving_tinygrad.pkl
    steps:
@@ -85,9 +88,10 @@ jobs:
        with:
          submodules: recursive

-      - name: Pull ONNX via LFS
+      - name: Pull small ONNX via LFS
        run: git lfs pull -I "${{ env.SMALL_ONNX }}"

+
      - name: Set environment variables
        run: |
          source /etc/profile
@@ -103,38 +107,15 @@ jobs:
          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable

-      - name: Compile small model with stock compiler
+      - name: Compile small model from ONNX
        run: |
          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
          export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"
-
-          MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
-          CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
-          FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)")
-
          TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
-
-          env ${TG_FLAGS} python3 \
-            ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \
-            --onnx ${{ github.workspace }}/${{ env.SMALL_ONNX }} \
-            --model-size $MODEL_SIZE \
-            --camera-resolutions $CAMERA_RES \
-            --frame-skip $FRAME_SKIP \
-            --output ${{ github.workspace }}/${{ env.SMALL_PKL }}
-
-      - name: Chunk small pkl
-        run: |
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
-          export PYTHONPATH=${{ github.workspace }}
-          python3 -c "
-          from openpilot.common.file_chunker import chunk_file, get_chunk_targets
-          import os
-          pkl = '${{ github.workspace }}/${{ env.SMALL_PKL }}'
-          size = os.path.getsize(pkl)
-          targets = get_chunk_targets(pkl, size)
-          chunk_file(pkl, targets)
-          print(f'Chunked into {len(targets)} files')
-          "
+          env ${TG_FLAGS} python3 ${{ github.workspace }}/${{ env.COMPILER }}/compile_onnx.py \
+            ${{ github.workspace }}/${{ env.SMALL_ONNX }} \
+            ${{ github.workspace }}/${{ env.SMALL_PKL }} \
+            --out-of-band --benchmark-runs 1

      - name: Prepare output
        env:
@@ -147,14 +128,16 @@ jobs:
          PKL_BASE="driving_tinygrad.pkl"
          mkdir -p "$OUTPUT_DIR"

-          cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/"
-          cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/"
+          cp "$MODELS_DIR/${PKL_BASE}" "$OUTPUT_DIR/"
+
+          ONNX_SHA256=$(git cat-file blob "$(git ls-files -s "${{ github.workspace }}/${{ env.SMALL_ONNX }}" | awk '{print $2}')" | grep '^oid sha256:' | cut -d: -f2)

          python3 "${{ github.workspace }}/release/ci/model_generator.py" \
            --model-dir "$MODELS_DIR" \
            --output-dir "$OUTPUT_DIR" \
            --custom-name "$MODEL_NAME" \
-            --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}"
+            --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" \
+            --onnx-sha256 "$ONNX_SHA256"

          echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt"

@@ -181,15 +164,16 @@ jobs:
    if: ${{ inputs.target == 'big' }}
    runs-on: [self-hosted, chestnut]
    env:
-      BIG_ONNX: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
+      MODELS_DIR: openpilot/selfdrive/modeld/models
+      COMPILER: tinygrad_repo/examples/openpilot
      BIG_PKL: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive

-      - name: Pull big ONNX via LFS
-        run: git lfs pull -I "${{ env.BIG_ONNX }}"
+      - name: Pull big PKL via LFS
+        run: git lfs pull -I "${{ env.BIG_PKL }}"

      - name: Set environment variables
        run: |
@@ -222,38 +206,33 @@ jobs:
              raise RuntimeError('Chestnut PCIe link not ready after 10 attempts')
          "

-      - name: Compile big model with stock compiler
+      - name: Compile big warp
        run: |
          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
          export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"

+          CHESTNUT_TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
+          COMPILER="${{ github.workspace }}/tinygrad_repo/examples/openpilot"
+          MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models"
          MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
-          CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
-          FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)")

-          TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2"
-
-          env ${TG_FLAGS} python3 \
-            ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \
-            --onnx ${{ github.workspace }}/${{ env.BIG_ONNX }} \
-            --model-size $MODEL_SIZE \
-            --camera-resolutions $CAMERA_RES \
-            --frame-skip $FRAME_SKIP \
-            --output ${{ github.workspace }}/${{ env.BIG_PKL }}
-
-      - name: Chunk big pkl
-        run: |
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
-          export PYTHONPATH=${{ github.workspace }}
-          python3 -c "
-          from openpilot.common.file_chunker import chunk_file, get_chunk_targets
-          import os
-          pkl = '${{ github.workspace }}/${{ env.BIG_PKL }}'
-          size = os.path.getsize(pkl)
-          targets = get_chunk_targets(pkl, size)
-          chunk_file(pkl, targets)
-          print(f'Chunked into {len(targets)} files')
-          "
+          for res in $(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')"); do
+            CAM_W=$(echo $res | cut -d x -f1)
+            CAM_H=$(echo $res | cut -d x -f2)
+            STRIDE_INFO=$(python3 -c "
+          import sys
+          from openpilot.system.camerad.cameras.nv12_info import get_nv12_info
+          s,y,u,sz = get_nv12_info(int(sys.argv[1]), int(sys.argv[2]))
+          print(f'{s},{y},{u},{sz}')
+          " $CAM_W $CAM_H)
+            WARP_PKL="${MODELS_DIR}/big_driving_warp_${res}_tinygrad.pkl"
+            env ${CHESTNUT_TG_FLAGS} python3 "${COMPILER}/compile_warp.py" \
+              --frame ${CAM_W},${CAM_H},${STRIDE_INFO} \
+              --warp-to ${MODEL_SIZE} \
+              --layout yuv420 \
+              --frames 2 \
+              --output "${WARP_PKL}"
+          done

      - name: Prepare output
        env:
@@ -266,14 +245,18 @@ jobs:
          PKL_BASE="big_driving_tinygrad.pkl"
          mkdir -p "$OUTPUT_DIR"

-          cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/"
-          cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/"
+          cp "$MODELS_DIR/${PKL_BASE}" "$OUTPUT_DIR/"
+
+          PKL_LFS_SHA256=$(git cat-file blob "$(git ls-files -s "${{ github.workspace }}/${{ env.BIG_PKL }}" | awk '{print $2}')" | grep '^oid sha256:' | cut -d: -f2)

          python3 "${{ github.workspace }}/release/ci/model_generator.py" \
            --model-dir "$MODELS_DIR" \
            --output-dir "$OUTPUT_DIR" \
            --custom-name "$MODEL_NAME" \
-            --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}"
+            --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" \
+            --onnx-sha256 "$PKL_LFS_SHA256"
+
+          cp "$MODELS_DIR"/big_driving_warp_* "$OUTPUT_DIR/" || true

          echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt"

@@ -311,7 +294,8 @@ jobs:
    steps:
      - uses: actions/checkout@v4

-      - name: Pull ONNX via LFS
+      - name: Pull LFS for resolved ONNX files
+        if: ${{ needs.resolve.outputs.onnx_path != '' }}
        run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}"

      - name: Install huggingface_hub
@@ -376,38 +360,28 @@ jobs:
          from datetime import datetime, UTC

          dm_dir = Path('dm_output')
-          manifest = list(dm_dir.glob('*.chunkmanifest'))
-          assert manifest, 'No chunkmanifest found'
-          pkl_name = manifest[0].name.removesuffix('.chunkmanifest')
-          num_chunks = int(manifest[0].read_text().strip())
-
-          chunks = []
-          for i in range(num_chunks):
-              chunk = dm_dir / f'{pkl_name}.chunk{i+1:02d}of{num_chunks:02d}'
-              chunks.append({
-                  'file_name': chunk.name,
-                  'sha256': hashlib.sha256(chunk.read_bytes()).hexdigest()
-              })
+          matches = sorted(dm_dir.glob('dmonitoring_*_tinygrad.pkl'))
+          assert len(matches) == 1, f'expected one DM pkl in {dm_dir}, found {matches}'
+          pkl_name = matches[0].name

          digest = hashlib.sha256()
-          for c in chunks:
-              with open(dm_dir / c['file_name'], 'rb') as f:
-                  while block := f.read(1024*1024):
-                      digest.update(block)
+          with open(dm_dir / pkl_name, 'rb') as f:
+              while block := f.read(1024*1024):
+                  digest.update(block)

          metadata = {
              'bundles': [{
                  'short_name': 'DMMODEL',
                  'display_name': '${{ needs.resolve.outputs.model_name }}',
                  'ref': '${{ needs.resolve.outputs.onnx_ref }}',
+                  'tinygrad_ref': '${{ needs.resolve.outputs.tinygrad_ref }}',
                  'runner': 'tinygrad',
                  'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'),
                  'models': [{
                      'type': 'chunked',
                      'artifact': {
                          'file_name': pkl_name,
-                          'download_uri': {'url': '', 'sha256': digest.hexdigest()},
-                          'chunks': chunks
+                          'download_uri': {'url': '', 'sha256': digest.hexdigest()}
                      }
                  }]
              }]
@@ -466,23 +440,10 @@ jobs:
          TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"

          taskset -c 7 env ${TG_FLAGS} python3 \
-            ${{ github.workspace }}/tinygrad_repo/examples/openpilot/compile3.py \
+            ${{ github.workspace }}/tinygrad_repo/examples/openpilot/compile_onnx.py \
            ${{ github.workspace }}/${{ env.DM_ONNX }} \
-            ${{ github.workspace }}/${{ env.DM_PKL }}
-
-      - name: Chunk DM pkl
-        run: |
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
-          export PYTHONPATH=${{ github.workspace }}
-          python3 -c "
-          from openpilot.common.file_chunker import chunk_file, get_chunk_targets
-          import os
-          pkl = '${{ github.workspace }}/${{ env.DM_PKL }}'
-          size = os.path.getsize(pkl)
-          targets = get_chunk_targets(pkl, size)
-          chunk_file(pkl, targets)
-          print(f'Chunked {pkl} into {len(targets)} chunks')
-          "
+            ${{ github.workspace }}/${{ env.DM_PKL }} \
+            --out-of-band --benchmark-runs 1

      - name: Compile DM warp
        run: |
@@ -490,22 +451,33 @@ jobs:
          export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"

          TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
-          MODEL_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld"
+          COMPILER="${{ github.workspace }}/tinygrad_repo/examples/openpilot"
+          MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models"
          DM_SIZE=$(python3 -c "from openpilot.common.transformations.model import DM_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")

          for res in $(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')"); do
-            WARP_PKL="${MODEL_DIR}/models/dm_warp_${res}_tinygrad.pkl"
-            taskset -c 7 env ${TG_FLAGS} python3 ${MODEL_DIR}/compile_dm_warp.py \
-              --camera-resolution ${res} \
+            CAM_W=$(echo $res | cut -d x -f1)
+            CAM_H=$(echo $res | cut -d x -f2)
+            STRIDE_INFO=$(python3 -c "
+          import sys
+          from openpilot.system.camerad.cameras.nv12_info import get_nv12_info
+          s,y,u,sz = get_nv12_info(int(sys.argv[1]), int(sys.argv[2]))
+          print(f'{s},{y},{u},{sz}')
+          " $CAM_W $CAM_H)
+            WARP_PKL="${MODELS_DIR}/dm_warp_${res}_tinygrad.pkl"
+            taskset -c 7 env ${TG_FLAGS} python3 "${COMPILER}/compile_warp.py" \
+              --frame ${CAM_W},${CAM_H},${STRIDE_INFO} \
              --warp-to ${DM_SIZE} \
-              --output ${WARP_PKL}
+              --layout luma \
+              --border-fill 16 \
+              --transform-device NPY \
+              --output "${WARP_PKL}"
          done

      - name: Prepare DM output
        run: |
          mkdir -p dm_output
-          cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunk* dm_output/
-          cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunkmanifest dm_output/
+          cp ${{ github.workspace }}/${{ env.DM_PKL }} dm_output/
          cp ${{ github.workspace }}/openpilot/selfdrive/modeld/models/dm_warp_* dm_output/

      - name: Upload DM artifact
@@ -519,4 +491,3 @@ jobs:
        run: |
          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
-
@@ -134,12 +134,10 @@ jobs:

  publish_model:
    if: ${{ !cancelled() && needs.build_model.result == 'success' }}
-    concurrency:
-      group: hf-push-${{ inputs.recompiled_dir }}
-      cancel-in-progress: false
    needs: build_model
    runs-on: ubuntu-latest
    permissions:
+      actions: read
      id-token: write
      contents: write
    steps:
@@ -182,18 +180,21 @@ jobs:
          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT

      - name: Download and extract model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
-          path: output
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
+        run: |
+          id=$(gh api -X GET "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts" -f name="$ARTIFACT_NAME" --jq '.artifacts[0].id')
+          url=$(curl -sf -o /dev/null -w '%{redirect_url}' -H "Authorization: Bearer $GH_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/artifacts/$id/zip")
+          sudo apt-get install -y -qq aria2 > /dev/null
+          aria2c -q -x16 -s16 -k8M --max-tries=5 --file-allocation=none -o model.zip "$url" && unzip -q model.zip -d output && rm model.zip

      - name: Create models folder
        env:
          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
        run: |
          mkdir -p "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          cp -r output/* "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/"
-          rm -f "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/artifact_name.txt"
+          cp output/metadata.json "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/"

      - name: Upload to Hugging Face
        env:
@@ -205,13 +206,12 @@ jobs:
            "models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/" \
            --repo-type=dataset

-      - name: Pull gh-pages
+      - name: Update and push JSON to docs repo
        run: |
          cd docs
-          git pull origin gh-pages
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"

-      - name: Run json_parser.py to update JSON
-        run: |
          FOLDER="${{ inputs.model_folder }}"
          if [ "$FOLDER" = "Other" ]; then
            FOLDER="${{ inputs.custom_model_folder }}"
@@ -224,18 +224,27 @@ jobs:
          [ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
          [ -n "${{ inputs.tinygrad_ref }}" ] && ARGS="$ARGS --tinygrad-ref \"${{ inputs.tinygrad_ref }}\""
-          eval python3 docs/json_parser.py \
-            --json-path "$JSON_FILE" \
-            --recompiled-dir "local_models/$RECOMPILED_DIR" \
-            --sort-by-date \
-            $ARGS

-      - name: Push updated JSON to GitHub docs repo
-        run: |
-          cd docs
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git checkout gh-pages
-          git add docs/"$(basename $JSON_FILE)"
-          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
-          git push origin gh-pages
+          for attempt in $(seq 1 10); do
+            git fetch origin gh-pages
+            git reset --hard origin/gh-pages
+
+            eval python3 json_parser.py \
+              --json-path "docs/$(basename $JSON_FILE)" \
+              --recompiled-dir "../local_models/$RECOMPILED_DIR" \
+              --sort-by-date \
+              $ARGS
+
+            git add docs/"$(basename $JSON_FILE)"
+            git commit -m "Update $(basename $JSON_FILE) after recompiling model" || { echo "No changes to commit"; exit 0; }
+
+            if git push origin gh-pages; then
+              echo "Push succeeded on attempt $attempt"
+              exit 0
+            fi
+
+            echo "Push failed (attempt $attempt/10), retrying after backoff..."
+            sleep $((RANDOM % 10 + 3))
+          done
+          echo "All 10 push attempts failed"
+          exit 1
@@ -0,0 +1,80 @@
+name: Compile warp for sunnypilot modeld
+
+on:
+  workflow_dispatch:
+
+jobs:
+  compile_warps:
+    runs-on: [self-hosted, chestnut]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set environment variables
+        run: |
+          source /etc/profile
+          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
+          export UV_PYTHON_PREFERENCE=managed
+          export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
+          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
+          uv sync --frozen
+          printenv >> $GITHUB_ENV
+
+      - name: Disable powersave
+        run: |
+          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
+
+      - name: Compile Warp Kernels
+        run: |
+          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+          export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"
+          mkdir -p openpilot/sunnypilot/modeld_v2/models/
+          MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
+          CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
+          echo "Model size: $MODEL_SIZE"
+          echo "Camera resolutions: $CAMERA_RES"
+
+          for res in $CAMERA_RES; do
+            FRAME_PARAM=$(python3 -c "from openpilot.system.camerad.cameras.nv12_info import get_nv12_info; w, h = map(int, '${res}'.split('x')); stride, y_h, uv_h, _ = get_nv12_info(w, h); print(f'{w},{h},{stride},{y_h},{uv_h},{stride * (y_h + uv_h)}')")
+
+            WARP_PKL="openpilot/sunnypilot/modeld_v2/models/driving_warp_${res}_tinygrad.pkl"
+            echo "Compiling $WARP_PKL on QCOM"
+            taskset -c 7 env DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1 python3 openpilot/sunnypilot/modeld_v2/compile_warp.py \
+              --frame ${FRAME_PARAM} --warp-to ${MODEL_SIZE} --output "${WARP_PKL}"
+
+            BIG_WARP_PKL="openpilot/sunnypilot/modeld_v2/models/big_driving_warp_${res}_tinygrad.pkl"
+            echo "Compiling $BIG_WARP_PKL on AMD"
+            taskset -c 7 env DEV=USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32 python3 openpilot/sunnypilot/modeld_v2/compile_warp.py \
+              --frame ${FRAME_PARAM} --warp-to ${MODEL_SIZE} --output "${BIG_WARP_PKL}"
+          done
+
+      - name: Re-enable powersave
+        if: always()
+        run: |
+          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          GIT_CONFIG_COUNT: 1
+          GIT_CONFIG_KEY_0: credential.https://huggingface.co.helper
+          GIT_CONFIG_VALUE_0: '!f() { test "$1" = get && echo username=user && echo "password=$HF_TOKEN"; }; f'
+        with:
+          author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "[bot] Update warp pkl for modeld_v2"
+          title: "[bot] Update modeld_v2 Warp"
+          branch: "auto/compile-warp-kernels"
+          base: "master"
+          delete-branch: true
+          labels: bot
+          add-paths: |
+            openpilot/sunnypilot/modeld_v2/models/*.pkl
+
+      - name: Reset lfs access
+        if: always()
+        run: git config --local --unset-all "lfs.$(git config -f .lfsconfig lfs.url).access" || true
@@ -18,24 +18,25 @@ concurrency:
 env:
  GIT_CONFIG_COUNT: 1
  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
+  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl

 jobs:
  docs:
    name: build docs
-    runs-on: ubuntu-24.04
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
    steps:
    - uses: commaai/timeout@v1
-
    - uses: actions/checkout@v7
-      with:
-        submodules: true
+    - run: ./tools/op.sh setup

    # Build
    - name: Build docs
-      run: |
-        git lfs pull
-        python docs/serve.py --build
+      run: ./tools/op.sh docs --build

    # Push to docs.comma.ai
    - uses: actions/checkout@v7
@@ -1,5 +1,5 @@
-name: Download HF model chunks
-description: Resolve and download model chunks from HuggingFace in parallel
+name: Download HF model
+description: Resolve and download models from HuggingFace in parallel

 inputs:
  hf_repo:
@@ -11,16 +11,21 @@ inputs:
  dest_dir:
    description: Destination directory for downloaded chunks
    required: true
+  tinygrad_ref:
+    description: 'Tinygrad ref to match bundle against'
+    required: false
+    default: ''

 runs:
  using: composite
  steps:
-    - name: Download model chunks
+    - name: Download models
      shell: bash
      env:
        HF_REPO: ${{ inputs.hf_repo }}
        MODELS_JSON: ${{ inputs.models }}
        DEST_DIR: ${{ inputs.dest_dir }}
+        TINYGRAD_REF: ${{ inputs.tinygrad_ref }}
      run: |
        set -eo pipefail
        DOWNLOAD_LIST=$(mktemp)
@@ -28,30 +33,41 @@ runs:
        resolve_chunks() {
          local HF_PATH="$1" ONNX_HASH="$2" CANONICAL="$3" DEST_DIR="$4"
          local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_PATH}/default_models.json"
-          local DEFAULTS BUNDLE ARTIFACT BASE_URL NUM_CHUNKS
+          local DEFAULTS BUNDLE ARTIFACT FILE_URL BASE_URL
          DEFAULTS=$(curl -fsSL "$JSON_URL")
-          BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
+          if [ -n "$TINYGRAD_REF" ]; then
+            BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" --arg tg "$TINYGRAD_REF" '
+              . as $root | [.bundles[] | select(.onnx_sha256 == $hash and (.tinygrad_ref == $tg or (.tinygrad_ref == null and $root.tinygrad_ref == $tg)))] | .[0] // empty
+            ')
+          else
+            BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '
+              [.bundles[] | select(.onnx_sha256 == $hash)] | .[0] // empty
+            ')
+          fi
+
+          if [ -z "$BUNDLE" ] || [ "$BUNDLE" = "null" ]; then
+            echo "::error::No model bundle found matching ONNX hash $ONNX_HASH (tinygrad ref: $TINYGRAD_REF) in $HF_PATH"
+            return 1
+          fi
+
          ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
-          BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
-          NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
+          FILE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url')
+          BASE_URL=$(echo "$FILE_URL" | sed 's|/[^/]*$||')

          mkdir -p "$DEST_DIR"
-          while IFS= read -r CHUNK_NAME; do
-            CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true)
-            if [ -z "$CHUNK_IDX" ]; then
-              echo "::error::Failed to parse chunk index from: $CHUNK_NAME"
-              return 1
-            fi
-            ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
-            printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}.chunk${CHUNK_IDX}" >> "$DOWNLOAD_LIST"
-          done < <(echo "$ARTIFACT" | jq -r '.chunks[].file_name')
-          echo "$NUM_CHUNKS" > "${DEST_DIR}/${CANONICAL}.chunkmanifest"
+          ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${FILE_URL}', safe=':/'))")
+          printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}" >> "$DOWNLOAD_LIST"

          if [ "$CANONICAL" = "dmonitoring_model_tinygrad.pkl" ]; then
            for warp in dm_warp_1928x1208_tinygrad.pkl dm_warp_1344x760_tinygrad.pkl; do
              ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${warp}', safe=':/'))")
              printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${warp}" >> "$DOWNLOAD_LIST"
            done
+          elif [ "$CANONICAL" = "big_driving_tinygrad.pkl" ]; then
+            for warp in big_driving_warp_1928x1208_tinygrad.pkl big_driving_warp_1344x760_tinygrad.pkl; do
+              ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${warp}', safe=':/'))")
+              printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${warp}" >> "$DOWNLOAD_LIST"
+            done
          fi
        }

@@ -63,7 +79,7 @@ runs:
        done

        TOTAL=$(wc -l < "$DOWNLOAD_LIST")
-        echo "Downloading $TOTAL chunks with 8 parallel connections..."
+        echo "Downloading $TOTAL files with 8 parallel connections..."
        xargs -P8 -d'\n' -I{} bash -c '
          URL="${1%%	*}"
          DEST="${1#*	}"
@@ -1,8 +1,8 @@
 name: Sync comma's LFS

 env:
-  LFS_URL: 'https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs'
-  LFS_PUSH_URL: 'ssh://git@gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git'
+  LFS_URL: 'https://huggingface.co/sunnypilot/sunnypilot-lfs.git/info/lfs'
+  LFS_PUSH_URL: 'https://huggingface.co/sunnypilot/sunnypilot-lfs.git/info/lfs'

 on:
  schedule:
@@ -33,21 +33,19 @@ jobs:

      - name: LFS Fetch
        run: |
-          git lfs fetch
+          git lfs fetch || echo "::warning::some objects are missing on commaai/openpilot-lfs; mirroring the rest"

      - name: Set up Git
        run: |
          git config --global user.name 'GitHub Action'
          git config --global user.email 'action@github.com'

-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Add GitLab public keys
+      - name: Setup hf
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+          git config --global credential.helper store
+          echo "https://user:${HF_TOKEN}@huggingface.co" > ~/.git-credentials

      - name: Ensure branch
        run: |
@@ -70,6 +68,7 @@ jobs:
          git config lfs.activitytimeout 300
          git config lfs.transfer.maxretries 5
          git config lfs.concurrenttransfers 4
+          git config lfs.allowincompletepush true

      - name: Push LFS
        id: sync-and-commit
@@ -83,3 +82,80 @@ jobs:
          done
          echo "All push attempts failed"
          exit 1
+
+      - name: Checkout sunnypilot
+        uses: actions/checkout@v4
+        with:
+          repository: 'sunnypilot/sunnypilot'
+          path: sunnypilot
+          fetch-depth: 0
+          filter: blob:limit=1k
+          lfs: false
+
+      - name: Reference objects on sunnypilot-lfs
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          set -euo pipefail
+          GIT_LFS_SKIP_SMUDGE=1 git clone --quiet https://huggingface.co/sunnypilot/sunnypilot-lfs hf-lfs
+          mkdir -p hf-lfs/all
+          cat > scan_pointers.py <<'EOF'
+          import re, subprocess, sys, threading
+          repo = sys.argv[1]
+          chk = subprocess.run(["git", "-C", repo, "cat-file", "--batch-all-objects",
+                                "--batch-check=%(objectname) %(objecttype) %(objectsize)"],
+                               capture_output=True, text=True, check=True).stdout
+          small = [f[0] for f in (l.split() for l in chk.splitlines()) if f[1] == "blob" and int(f[2]) < 1024]
+          p = subprocess.Popen(["git", "-C", repo, "cat-file", "--batch=%(objectsize)"],
+                               stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+          def feed():
+            for sha in small:
+              p.stdin.write(sha.encode() + b"\n")
+            p.stdin.close()
+          threading.Thread(target=feed, daemon=True).start()
+          rx = re.compile(rb"^version https://git-lfs\.github\.com/spec/v1\noid sha256:([0-9a-f]{64})\nsize ([0-9]+)\n")
+          seen = {}
+          for _ in small:
+            n = int(p.stdout.readline()); body = p.stdout.read(n); p.stdout.read(1)
+            m = rx.match(body)
+            if m:
+              seen.setdefault(m.group(1).decode(), m.group(2).decode())
+          for oid, size in sorted(seen.items()):
+            print(oid, size)
+          EOF
+          for repo in . sunnypilot; do
+            python3 scan_pointers.py "$repo"
+          done | sort -u | while read -r oid size; do
+            [ -e "hf-lfs/all/$oid" ] && continue
+            printf 'version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %s\n' "$oid" "$size" > "hf-lfs/all/$oid"
+            echo "$oid"
+          done > new_oids.txt
+          echo "new pointer files: $(wc -l < new_oids.txt)"
+
+          dropped=0
+          while mapfile -t -n 100 chunk && ((${#chunk[@]})); do
+            body=$(for oid in "${chunk[@]}"; do
+                     size=$(awk '/^size /{print $2}' "hf-lfs/all/$oid")
+                     printf '{"oid":"%s","size":%s}\n' "$oid" "$size"
+                   done | jq -s '{operation:"download",transfers:["basic"],objects:.}')
+            for attempt in 1 2 3 4 5; do
+              resp=$(curl -sS -u "user:${HF_TOKEN}" -w '\n%{http_code}' \
+                       -H 'Accept: application/vnd.git-lfs+json' -H 'Content-Type: application/vnd.git-lfs+json' \
+                       --data "$body" "${LFS_URL}/objects/batch") && [ "$(tail -n1 <<<"$resp")" = 200 ] && break
+              echo "batch check attempt $attempt failed ($(tail -n1 <<<"$resp")), retrying in 60s"; sleep 60
+            done
+            [ "$(tail -n1 <<<"$resp")" = 200 ] || { echo "batch check failed"; exit 1; }
+            for oid in $(sed '$d' <<<"$resp" | jq -r '.objects[] | select(.error) | .oid'); do
+              rm -f "hf-lfs/all/$oid"; dropped=$((dropped+1))
+            done
+          done < new_oids.txt
+          echo "dropped (not on HF): $dropped"
+
+          cd hf-lfs
+          git add -A all
+          if git diff --cached --quiet; then
+            echo "nothing new to reference"
+            exit 0
+          fi
+          git commit -q -m "Reference $(git diff --cached --name-only | wc -l | tr -d ' ') new LFS objects"
+          git push origin HEAD:main
@@ -7,7 +7,7 @@ on:
 env:
  GIT_CONFIG_COUNT: 1
  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
+  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl

 jobs:
  build___nightly:
@@ -34,3 +34,7 @@ jobs:
    - run: ./tools/op.sh setup
    - name: Push __nightly
      run: BRANCH=__nightly tools/release/build_stripped.sh
+    - name: Push chestnut nightly
+      run: |
+        git lfs pull --exclude=''
+        INCLUDE_BIG_MODEL=1 BRANCH=__nightly-chestnut tools/release/build_stripped.sh
@@ -11,7 +11,7 @@ env:
  PYTHONPATH: ${{ github.workspace }}
  GIT_CONFIG_COUNT: 1
  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
+  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl

 jobs:
  package_updates:
@@ -100,24 +100,56 @@ jobs:
          commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
          echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
          cat $GITHUB_OUTPUT
-      - run: |
+
+      - name: Fetch Models
+        run: |
          cd ${{ github.workspace }}/openpilot/openpilot
+          fetch_lfs() {
+            sudo apt-get install -y -qq aria2 > /dev/null # Parallel ranges because gh is flaky
+            LFS_URL=$(git config -f "$(git rev-parse --show-toplevel)/.lfsconfig" lfs.url)
+            for f in "$@"; do
+              grep -qs "^version https://git-lfs" "$f" || continue
+              OID=$(awk -F'sha256:' '/^oid/{print $2}' "$f"); SIZE=$(awk '/^size/{print $2}' "$f")
+              HREF=$(curl -fsSL --retry 3 -X POST -H "Accept: application/vnd.git-lfs+json" -H "Content-Type: application/vnd.git-lfs+json" "$LFS_URL/objects/batch" \
+                -d "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"$OID\",\"size\":$SIZE}]}" | jq -r '.objects[0].actions.download.href')
+              aria2c -x 16 -s 16 -k 16M --max-tries=5 --retry-wait=5 --file-allocation=none --summary-interval=30 --allow-overwrite=true \
+                -d "$(dirname "$f")" -o "$(basename "$f")" "$HREF"
+              echo "$OID  $f" | sha256sum -c -
+            done
+          }
+          UPSTREAM_PKL=selfdrive/modeld/models/big_driving_tinygrad.pkl
          if [ "${{ inputs.target_hardware }}" != "chestnut" ]; then
-            git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx,**/selfdrive/modeld/models/dmonitoring_*.onnx"
-            rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx
+            git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx,**/selfdrive/modeld/models/dmonitoring_*.onnx,**/selfdrive/modeld/models/big_*.pkl,**/selfdrive/modeld/models/dmonitoring_*.pkl"
+            rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx selfdrive/modeld/models/big_*.pkl selfdrive/modeld/models/dmonitoring_*.pkl
+          elif compgen -G "selfdrive/modeld/models/big_*.onnx" > /dev/null; then
+            fetch_lfs selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/big_*.pkl
+            find selfdrive/modeld/models -type f \( -name "*.onnx" -o -name "*.pkl" \) ! -name "big_*.onnx" ! -name "big_*.pkl" -delete
+          elif [ -f "$UPSTREAM_PKL" ]; then
+            echo "No big model ONNX tracked: converting commaai's precompiled $UPSTREAM_PKL, with the tinygrad it was built with"
+            fetch_lfs "$UPSTREAM_PKL"
+            find selfdrive/modeld/models -type f \( -name "*.onnx" -o -name "*.pkl" \) ! -path "$UPSTREAM_PKL" -delete
+            GITMODULES="$(git rev-parse --show-toplevel)/.gitmodules"
+            TG_NAME=$(git config -f "$GITMODULES" --get-regexp '^submodule\..*\.path$' | awk '$2 == "tinygrad_repo" {sub(/^submodule\./, "", $1); sub(/\.path$/, "", $1); print $1}')
+            TG_URL=$(git config -f "$GITMODULES" "submodule.$TG_NAME.url" | sed 's|^\.\./\.\./|https://github.com/|')
+            echo "$TG_URL $(git rev-parse HEAD:tinygrad_repo) $(sha256sum "$UPSTREAM_PKL" | cut -d' ' -f1)" > selfdrive/modeld/models/upstream_pkl.txt
          else
-            git lfs pull -I "**/selfdrive/modeld/models/big_*.onnx" -X ""
-            find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete
+            echo "::error::no big model ONNX or precompiled big_driving_tinygrad.pkl at this ref"
+            exit 1
          fi
-          if grep -lIF "version https://git-lfs.github.com/spec/v1" selfdrive/modeld/models/*.onnx; then
-            echo "::error::the ONNX files above are still LFS pointers, not real models"
+
+          if grep -lIF "version https://git-lfs.github.com/spec/v1" selfdrive/modeld/models/*.onnx selfdrive/modeld/models/*.pkl 2>/dev/null; then
+            echo "::error::the ONNX or PKL files above are still LFS pointers, not real models"
            exit 1
          fi
      - name: 'Upload Artifact'
        uses: actions/upload-artifact@v4
        with:
          name: models-${{ env.REF }}${{ inputs.artifact_suffix  }}
-          path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
+          compression-level: 0
+          path: |
+            ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
+            ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.pkl
+            ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/upstream_pkl.txt
          if-no-files-found: error

  build_model:
@@ -132,7 +164,6 @@ jobs:
          fetch-depth: 1
          submodules: recursive

-
      - name: Set environment variables
        id: set-env
        run: |
@@ -144,9 +175,6 @@ jobs:
          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
          uv sync --frozen
          printenv >> $GITHUB_ENV
-          if [[ "${{ runner.debug }}" == "1" ]]; then
-            cat $GITHUB_OUTPUT
-          fi

      - name: Setup build environment
        run: |
@@ -158,10 +186,6 @@ jobs:
          echo "VERSION: ${{ steps.set-env.outputs.version }}"
          echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
          echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
-          echo "-------"
-          if [[ "${{ runner.debug }}" == "1" ]]; then
-            printenv
-          fi
          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
          rm -rf ${{ env.MODELS_DIR }}/*.onnx*
@@ -184,11 +208,11 @@ jobs:
          MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
          CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")

-          TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
+          TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 OPENPILOT_HACKS=1"
          if [ "${{ inputs.target_hardware }}" == "chestnut" ]; then
            echo "CHESTNUT build"
            export CHESTNUT=1
-            TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2 TC_OCCUPANCY_OPT=1"
+            TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
            OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
          else
            echo "QCOM build"
@@ -196,64 +220,77 @@ jobs:
            OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
          fi

-          # Generate metadata for all ONNX files
-          find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
-            echo "Generating metadata: $onnx_file"
-            env ${TG_FLAGS_QCOM} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
-          done
+          NATIVE_PKL=$(find "${{ env.MODELS_DIR }}" -maxdepth 1 -name "*.pkl" -print -quit)

-          # Detect model type and build compile args
-          VISION_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do
-            [ -f "$f" ] && VISION_ONNX="$f" && break
-          done
+          if [ -f "${{ env.MODELS_DIR }}/upstream_pkl.txt" ]; then
+            read -r TG_URL TG_SHA _ < "${{ env.MODELS_DIR }}/upstream_pkl.txt"
+            echo "Converting commaai's precompiled model with tinygrad $TG_SHA"
+            UPSTREAM_TG="${{ runner.temp }}/upstream_tinygrad"
+            rm -rf "$UPSTREAM_TG" && git init -q "$UPSTREAM_TG" && git -C "$UPSTREAM_TG" fetch -q --depth 1 "$TG_URL" "$TG_SHA" && git -C "$UPSTREAM_TG" checkout -q FETCH_HEAD
+            CONVERTED="${{ runner.temp }}/big_driving_tinygrad_plain.pkl"
+            env DEV=USB+AMD PYTHONPATH="$UPSTREAM_TG:${{ github.workspace }}" taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py \
+              "${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" "$CONVERTED"
+            mv "$CONVERTED" "$OUTPUT_PKL"
+          elif NEW_ARCH_ONNX=$(find "${{ env.MODELS_DIR }}" -maxdepth 1 -name "${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}*driving*.onnx" -print -quit) &&
+               [ -n "$NEW_ARCH_ONNX" ] && grep -qa new_img "$NEW_ARCH_ONNX"; then # the new architecture: one ONNX taking the warped frames as new_img
+            echo "Compiling new architecture ONNX: $NEW_ARCH_ONNX"
+            env ${TG_FLAGS} taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/compile_modeld_v2.py \
+              "$NEW_ARCH_ONNX" "$OUTPUT_PKL" --device-input "*" --benchmark-runs 1
+          else
+            echo "Detected onnx architecture. Routing to native compile_modeld.py."
+            # Detect model type and build compile args
+            VISION_ONNX=""
+            for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do
+              [ -f "$f" ] && VISION_ONNX="$f" && break
+            done

-          POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do
-            [ -f "$f" ] && POLICY_ONNX="$f" && break
-          done
+            POLICY_ONNX=""
+            for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do
+              [ -f "$f" ] && POLICY_ONNX="$f" && break
+            done

-          OFF_POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do
-            [ -f "$f" ] && OFF_POLICY_ONNX="$f" && break
-          done
+            OFF_POLICY_ONNX=""
+            for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do
+              [ -f "$f" ] && OFF_POLICY_ONNX="$f" && break
+            done

-          ON_POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do
-            [ -f "$f" ] && ON_POLICY_ONNX="$f" && break
-          done
+            ON_POLICY_ONNX=""
+            for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do
+              [ -f "$f" ] && ON_POLICY_ONNX="$f" && break
+            done

-          SUPERCOMBO_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do
-            [ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break
-          done
+            SUPERCOMBO_ONNX=""
+            for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do
+              [ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break
+            done

-          MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
-          if [ -f "$VISION_ONNX" ]; then
-            ONNX_ARGS="--vision-onnx $VISION_ONNX"
-            if [ -f "$ON_POLICY_ONNX" ] && [ -f "$OFF_POLICY_ONNX" ]; then
-              MODEL_TYPE=vision_multi_policy
-              ONNX_ARGS="$ONNX_ARGS --off-policy-onnx $OFF_POLICY_ONNX --on-policy-onnx $ON_POLICY_ONNX"
-            elif [ -f "$OFF_POLICY_ONNX" ] && [ -f "$POLICY_ONNX" ]; then
-              MODEL_TYPE=vision_multi_policy
-              ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX --off-policy-onnx $OFF_POLICY_ONNX"
-            elif [ -f "$POLICY_ONNX" ]; then
-              MODEL_TYPE=vision_policy
-              ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX"
+            MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
+            if [ -f "$VISION_ONNX" ]; then
+              ONNX_ARGS="--vision-onnx $VISION_ONNX"
+              if [ -f "$ON_POLICY_ONNX" ] && [ -f "$OFF_POLICY_ONNX" ]; then
+                MODEL_TYPE=vision_multi_policy
+                ONNX_ARGS="$ONNX_ARGS --off-policy-onnx $OFF_POLICY_ONNX --on-policy-onnx $ON_POLICY_ONNX"
+              elif [ -f "$OFF_POLICY_ONNX" ] && [ -f "$POLICY_ONNX" ]; then
+                MODEL_TYPE=vision_multi_policy
+                ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX --off-policy-onnx $OFF_POLICY_ONNX"
+              elif [ -f "$POLICY_ONNX" ]; then
+                MODEL_TYPE=vision_policy
+                ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX"
+              fi
+            elif [ -f "$SUPERCOMBO_ONNX" ]; then
+              MODEL_TYPE=supercombo
+              ONNX_ARGS="--supercombo-onnx $SUPERCOMBO_ONNX"
            fi
-          elif [ -f "$SUPERCOMBO_ONNX" ]; then
-            MODEL_TYPE=supercombo
-            ONNX_ARGS="--supercombo-onnx $SUPERCOMBO_ONNX"
-          fi

-          if [ -n "$MODEL_TYPE" ]; then
-            echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL"
-            env ${TG_FLAGS} python3 "$COMPILE_MODELD" \
-              --model-type $MODEL_TYPE \
-              --model-size $MODEL_SIZE \
-              --camera-resolutions $CAMERA_RES \
-              $ONNX_ARGS \
-              --output "$OUTPUT_PKL"
+            if [ -n "$MODEL_TYPE" ]; then
+              echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL"
+              env ${TG_FLAGS} taskset -c 7 python3 "$COMPILE_MODELD" \
+                --model-type $MODEL_TYPE \
+                --model-size $MODEL_SIZE \
+                --camera-resolutions $CAMERA_RES \
+                $ONNX_ARGS \
+                --output "$OUTPUT_PKL"
+            fi
          fi

      - name: Prepare Output
@@ -264,8 +301,7 @@ jobs:
          # Copy the model files
          rsync -avm \
            --include='*.dlc' \
-            --include='*.chunk*' \
-            --include='*.chunkmanifest' \
+            --include='/${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}driving_tinygrad.pkl' \
            --exclude='*' \
            --delete-excluded \
            --chown=comma:comma \
@@ -276,6 +312,7 @@ jobs:
            --output-dir "${{ env.OUTPUT_DIR }}" \
            --custom-name "${{ env.MODEL_NAME }}" \
            --upstream-branch "${{ inputs.upstream_branch }}" \
+            $([ -f "${{ env.MODELS_DIR }}/upstream_pkl.txt" ] && echo "--onnx-sha256 $(cut -d' ' -f3 "${{ env.MODELS_DIR }}/upstream_pkl.txt")") \
            ${{ inputs.is_20hz && '--is-20hz' || '' }}

      - name: Write artifact name to file
@@ -287,6 +324,7 @@ jobs:
        with:
          name: model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}
          path: ${{ env.OUTPUT_DIR }}
+          compression-level: 0

      - name: Upload artifact name file
        uses: actions/upload-artifact@v4
@@ -170,6 +170,9 @@ jobs:
          scons -j1 cache_dir="$SCONS_CACHE" --minimal \
            openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd
          echo "Building rest of sunnypilot"
+          sudo rm -rf /tmp/* || true
+          mkdir -p "$BUILD_DIR/tmp"
+          export TMPDIR="$BUILD_DIR/tmp"
          SKIP_TINYGRAD_COMPILE=1 /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
          touch ${BUILD_DIR}/prebuilt
          if [[ "${{ runner.debug }}" == "1" ]]; then
@@ -221,37 +224,40 @@ jobs:
      cancel-in-progress: false
    outputs:
      onnx_sha256: ${{ steps.resolve.outputs.onnx_sha256 }}
+      tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }}
    env:
      GH_REPO: ${{ github.repository }}
      HF_REPO: sunnypilot/sunnypilot_models_v1
      HF_DEFAULTS_PATH: models/defaults/big
    steps:
-      - name: Resolve ONNX hash and tinygrad ref via API
+      - name: Resolve tinygrad ref via API
        id: resolve
        run: |
          REF="${{ github.head_ref || github.ref_name }}"

-          BLOB_SHA=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx?ref=${REF}" --jq '.sha')
+          BLOB_SHA=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl?ref=${REF}" --jq '.sha')
+          [ -n "$BLOB_SHA" ] || { echo "::error::Failed to resolve big_driving_tinygrad.pkl blob SHA"; exit 1; }
          ONNX_HASH=$(gh api "repos/${GH_REPO}/git/blobs/${BLOB_SHA}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2)
-          echo "ONNX hash: $ONNX_HASH"
-          [ -n "$ONNX_HASH" ] || { echo "::error::Failed to extract ONNX hash"; exit 1; }
-          echo "onnx_sha256=$ONNX_HASH" >> $GITHUB_OUTPUT
+          echo "big PKL hash: $ONNX_HASH"
+          [ -n "$ONNX_HASH" ] || { echo "::error::Failed to extract big PKL hash"; exit 1; }
+          echo "onnx_sha256=$ONNX_HASH" >> "$GITHUB_OUTPUT"

          TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha')
          echo "tinygrad ref: $TINYGRAD_REF"
+          echo "tinygrad_ref=$TINYGRAD_REF" >> "$GITHUB_OUTPUT"

          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"

          check_defaults() {
            DEFAULTS=$(curl -fsSL "${JSON_URL}?t=$(date +%s)" 2>/dev/null) || return 1
-            TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null)
-            [ "$TINYGRAD_MATCH" = "true" ] || return 1
-            BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
+            BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" --arg ref "$TINYGRAD_REF" '
+              . as $root | [.bundles[] | select(.onnx_sha256 == $hash and (.tinygrad_ref == $ref or (.tinygrad_ref == null and $root.tinygrad_ref == $ref)))] | .[0] // empty
+            ' 2>/dev/null)
            [ -n "$BUNDLE" ] && [ "$BUNDLE" != "null" ]
          }

          if check_defaults; then
-            echo "HF defaults match repo ONNX hash and tinygrad ref"
+            echo "HF defaults match repo tinygrad ref"
            exit 0
          fi

@@ -304,12 +310,13 @@ jobs:
      cancel-in-progress: false
    outputs:
      driving_onnx_sha256: ${{ steps.resolve.outputs.driving_onnx_sha256 }}
+      tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }}
    env:
      GH_REPO: ${{ github.repository }}
      HF_REPO: sunnypilot/sunnypilot_models_v1
      HF_DEFAULTS_PATH: models/defaults/small
    steps:
-      - name: Resolve ONNX hash and tinygrad ref via API
+      - name: Resolve tinygrad ref via API
        id: resolve
        run: |
          REF="${{ github.head_ref || github.ref_name }}"
@@ -318,23 +325,24 @@ jobs:
          DRIVING_HASH=$(gh api "repos/${GH_REPO}/git/blobs/${BLOB_SHA}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2)
          echo "Driving ONNX hash: $DRIVING_HASH"
          [ -n "$DRIVING_HASH" ] || { echo "::error::Failed to extract driving ONNX hash"; exit 1; }
-          echo "driving_onnx_sha256=$DRIVING_HASH" >> $GITHUB_OUTPUT
+          echo "driving_onnx_sha256=$DRIVING_HASH" >> "$GITHUB_OUTPUT"

          TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha')
          echo "tinygrad ref: $TINYGRAD_REF"
+          echo "tinygrad_ref=$TINYGRAD_REF" >> "$GITHUB_OUTPUT"

          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"

          check_defaults() {
            DEFAULTS=$(curl -fsSL "${JSON_URL}?t=$(date +%s)" 2>/dev/null) || return 1
-            TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null)
-            [ "$TINYGRAD_MATCH" = "true" ] || return 1
-            DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
-            [ -n "$DRIVING" ] && [ "$DRIVING" != "null" ] || return 1
+            DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" --arg ref "$TINYGRAD_REF" '
+              . as $root | [.bundles[] | select(.onnx_sha256 == $hash and (.tinygrad_ref == $ref or (.tinygrad_ref == null and $root.tinygrad_ref == $ref)))] | .[0] // empty
+            ' 2>/dev/null)
+            [ -n "$DRIVING" ] && [ "$DRIVING" != "null" ]
          }

          if check_defaults; then
-            echo "HF defaults match repo ONNX hash and tinygrad ref"
+            echo "HF defaults match repo tinygrad ref"
            exit 0
          fi

@@ -387,12 +395,13 @@ jobs:
      cancel-in-progress: false
    outputs:
      dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }}
+      tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }}
    env:
      GH_REPO: ${{ github.repository }}
      HF_REPO: sunnypilot/sunnypilot_models_v1
      HF_DEFAULTS_PATH: models/defaults/dm
    steps:
-      - name: Resolve ONNX hash and tinygrad ref via API
+      - name: Resolve tinygrad ref via API
        id: resolve
        run: |
          REF="${{ github.head_ref || github.ref_name }}"
@@ -401,23 +410,24 @@ jobs:
          DM_HASH=$(gh api "repos/${GH_REPO}/git/blobs/${BLOB_SHA}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2)
          echo "DM ONNX hash: $DM_HASH"
          [ -n "$DM_HASH" ] || { echo "::error::Failed to extract DM ONNX hash"; exit 1; }
-          echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT
+          echo "dm_onnx_sha256=$DM_HASH" >> "$GITHUB_OUTPUT"

          TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha')
          echo "tinygrad ref: $TINYGRAD_REF"
+          echo "tinygrad_ref=$TINYGRAD_REF" >> "$GITHUB_OUTPUT"

          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"

          check_defaults() {
            DEFAULTS=$(curl -fsSL "${JSON_URL}?t=$(date +%s)" 2>/dev/null) || return 1
-            TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null)
-            [ "$TINYGRAD_MATCH" = "true" ] || return 1
-            DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
-            [ -n "$DM" ] && [ "$DM" != "null" ] || return 1
+            DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" --arg ref "$TINYGRAD_REF" '
+              . as $root | [.bundles[] | select(.onnx_sha256 == $hash and (.tinygrad_ref == $ref or (.tinygrad_ref == null and $root.tinygrad_ref == $ref)))] | .[0] // empty
+            ' 2>/dev/null)
+            [ -n "$DM" ] && [ "$DM" != "null" ]
          }

          if check_defaults; then
-            echo "HF defaults match DM ONNX hash and tinygrad ref"
+            echo "HF defaults match DM tinygrad ref"
            exit 0
          fi

@@ -493,11 +503,12 @@ jobs:
          mkdir -p ${{ env.OUTPUT_DIR }}
          tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}

-      - name: Download model chunks from HF
-        uses: ./.github/workflows/download-hf-model-chunks
+      - name: Download models from HF
+        uses: ./.github/workflows/download-hf-model
        with:
          hf_repo: sunnypilot/sunnypilot_models_v1
          dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models
+          tinygrad_ref: ${{ needs.prepare_dm_model.outputs.tinygrad_ref }}
          models: |
            [
              {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"},
@@ -562,11 +573,12 @@ jobs:
          mkdir -p ${{ env.OUTPUT_DIR }}
          tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}

-      - name: Download model chunks from HF
-        uses: ./.github/workflows/download-hf-model-chunks
+      - name: Download models from HF
+        uses: ./.github/workflows/download-hf-model
        with:
          hf_repo: sunnypilot/sunnypilot_models_v1
          dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models
+          tinygrad_ref: ${{ needs.prepare_dm_model.outputs.tinygrad_ref }}
          models: |
            [
              {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"},
@@ -582,8 +594,14 @@ jobs:
      - name: Publish chestnut branch
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
+          cp .lfsconfig "${{ env.OUTPUT_DIR }}/"
+          echo 'openpilot/selfdrive/modeld/models/big_*_tinygrad.pkl filter=lfs diff=lfs merge=lfs -text' > "${{ env.OUTPUT_DIR }}/.gitattributes"
+          git lfs install
+          git config --global credential.helper store
+          echo "https://user:${HF_TOKEN}@huggingface.co" > ~/.git-credentials

          ${{ env.CI_DIR }}/publish.sh \
            "${{ github.workspace }}" \
@@ -3,9 +3,62 @@ on:
  pull_request:
    paths:
      - 'tinygrad_repo'
+      - 'openpilot/sunnypilot/modeld_v2/**'
  workflow_dispatch:

+env:
+  CI: 1
+  GIT_CONFIG_COUNT: 1
+  GIT_CONFIG_KEY_0: lfs.fetchexclude
+  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl
+
 jobs:
+  plain-artifact:
+    name: Plain artifact round trip and pickle converter
+    runs-on: ubuntu-24.04
+    env:
+      PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
+      TINYGRAD_TESTS: 1
+    steps:
+      - uses: actions/checkout@v7
+      - run: ./tools/op.sh setup
+      - name: Build openpilot
+        run: scons -j$(nproc)
+      - name: Compile, load, cache and fuse a model with the pinned tinygrad
+        env:
+          DEV: CPU
+        run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_plain_artifact.py
+      - name: Install LLVM 20 and ROCm comgr for the amd mock, as tinygrad_repo/.github/actions/setup-tinygrad does
+        run: |
+          . /etc/os-release
+          curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
+          echo "deb http://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+          echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.1 $VERSION_CODENAME main" | sudo tee /etc/apt/sources.list.d/rocm.list
+          printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+          sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends libllvm20 clang-20 lld-20 comgr
+          printf '/opt/rocm/lib\n/opt/rocm/lib64\n' | sudo tee /etc/ld.so.conf.d/rocm.conf && sudo ldconfig
+          echo "/usr/lib/llvm-20/bin" >> "$GITHUB_PATH"
+      - name: Convert a pickled USB+AMD jit and replay it under tinygrad's amd mock
+        env:
+          DEV: MOCKUSB+AMD
+          CHECK_OOB: 0
+          GMMU: 0
+        run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py
+      - name: Where tinygrad moved, for the helper or failure point named above
+        if: failure() && github.event_name == 'pull_request'
+        run: |
+          git fetch -q --depth=1 origin "${{ github.base_ref }}"
+          old=$(git rev-parse "origin/${{ github.base_ref }}:tinygrad_repo") new=$(git rev-parse HEAD:tinygrad_repo)
+          git -C tinygrad_repo fetch -q origin "$old" || true
+          echo "tinygrad_repo $old -> $new, in what helpers.py and plain_artifact_compiler.py read:"
+          git -C tinygrad_repo diff "$old" "$new" -- tinygrad/uop/ops.py tinygrad/engine/jit.py tinygrad/engine/realize.py tinygrad/device.py \
+            tinygrad/helpers.py tinygrad/dtype.py tinygrad/renderer/__init__.py
+          echo "and in what convert_upstream_pkl.py decodes:"
+          git -C tinygrad_repo diff --stat "$old" "$new" -- tinygrad/runtime/autogen/am/pm4_nv.py tinygrad/runtime/autogen/amd_gpu.py
+          git -C tinygrad_repo diff "$old" "$new" -- tinygrad/runtime/support/hcq2.py tinygrad/runtime/support/usb.py tinygrad/runtime/ops_amd.py
+
  generate-matrix:
    runs-on: ubuntu-latest
    outputs:
@@ -47,33 +100,23 @@ jobs:
        with:
          submodules: true

-      - name: Download Model Chunks in Parallel
+      - name: Download model
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          MODEL_URL: ${{ matrix.artifact.download_uri.url }}
+          MODEL_FILE: ${{ matrix.artifact.file_name }}
+        shell: bash
        run: |
-          mkdir -p /tmp/model_chunks
-          echo '${{ toJson(matrix.artifact.chunks) }}' > chunks.json
-
-          BASE_URL="${{ matrix.artifact.download_uri.url }}"
-          export BASE_DIR=$(dirname "$BASE_URL")
-
-          python3 -c '
-          import json, os
-          with open("chunks.json") as f:
-            chunks = json.load(f)
-          manifest_path = f"/tmp/model_chunks/${{ matrix.artifact.file_name }}.chunkmanifest"
-          with open(manifest_path, "w") as f:
-            f.write(str(len(chunks)))
-          base_dir = os.environ["BASE_DIR"]
-          with open("/tmp/curl_config.txt", "w") as f:
-            for c in chunks:
-              fn = c["file_name"]
-              f.write(f"url = \"{base_dir}/{fn}\"\noutput = \"/tmp/model_chunks/{fn}\"\n")
-          '
-          curl -Z --parallel-immediate --parallel-max 16 -s -S -f -L -K /tmp/curl_config.txt
+          mkdir -p /tmp/models
+          AUTH=()
+          if [ -n "$HF_TOKEN" ]; then AUTH=(-H "Authorization: Bearer $HF_TOKEN"); fi
+          curl --retry 3 --retry-all-errors -s -S -f -L "${AUTH[@]}" -o "/tmp/models/$MODEL_FILE" "$MODEL_URL"

      - name: Run Model Compatibility Test
        env:
          MODEL_BASE_NAME: ${{ matrix.artifact.file_name }}
-          MODEL_CHUNK_DIR: "/tmp/model_chunks"
-          PYTHONPATH: ".:./tinygrad_repo"
+          MODEL_DIR: "/tmp/models"
        run: |
+          export PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo:$GITHUB_WORKSPACE:$PYTHONPATH"
+          python3 -m pip uninstall -y tinygrad || uv pip uninstall -y tinygrad --system || true
          python3 -m pytest openpilot/sunnypilot/modeld_v2/tests/test_models.py
@@ -22,7 +22,7 @@ env:
  PYTHONPATH: ${{ github.workspace }}
  GIT_CONFIG_COUNT: 1
  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
+  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl

 jobs:
  build_release: