change cache to hf

.github/workflows/build-single-tinygrad-model.yaml

@@ -34,6 +34,11 @@ on:
         required: false
         type: string
         default: 'qcom'
+      on_device:
+        description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
+        required: false
+        type: boolean
+        default: false
       hf_repo:
         description: 'Hugging Face dataset repository (e.g. sunnypilot/sunnypilot_models_v1)'
         required: false
@@ -107,6 +112,11 @@ on:
         options:
           - qcom
           - chestnut
+      on_device:
+        description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
+        required: false
+        type: boolean
+        default: false
       hf_repo:
         description: 'Hugging Face dataset repository'
         required: false
@@ -130,6 +140,7 @@ jobs:
       is_20hz: ${{ inputs.is_20hz }}
       artifact_suffix: ${{ inputs.artifact_suffix }}
       target_hardware: ${{ inputs.target_hardware }}
+      on_device: ${{ inputs.on_device || false }}
     secrets: inherit
 
   publish_model:

.github/workflows/chestnut-build-parity/action.yml

@@ -0,0 +1,61 @@
+name: Chestnut mock build parity
+description: >-
+  Convert one commaai precompiled chestnut model (LS2) on this runner under tinygrad's amd mock, as a hosted chestnut build does, and require the
+  exact bytes the chestnut runner built (the published artifact). Needs setup-amd-mock, numpy and zstandard; tinygrad_repo checked out.
+runs:
+  using: composite
+  steps:
+    - name: Key this check by everything its result depends on
+      id: key
+      shell: bash
+      env:
+        SOURCES: ${{ hashFiles('tinygrad_repo/tinygrad/**/*.py', 'tinygrad_repo/test/mockgpu/**', 'openpilot/sunnypilot/modeld_v2/*.py', '.github/workflows/chestnut-build-parity/action.yml') }}
+      run: echo "key=chestnut-build-parity-$SOURCES-$(sha256sum "$LLVM_PATH" | cut -c1-16)-$(python3 -c 'import platform; print(platform.python_version())')" >> "$GITHUB_OUTPUT"
+
+    - name: Passed before with all of it identical?
+      id: passed
+      uses: actions/cache/restore@v4
+      with:
+        path: ~/.chestnut_build_parity.passed
+        key: ${{ steps.key.outputs.key }}
+
+    - name: Rebuild LS2 on the mock and require the chestnut runner's artifact byte for byte
+      if: steps.passed.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        REF: b92fc4828664f8767bce5e92f1630ed319cbe509  # commaai/openpilot: precompiled big_driving_tinygrad.pkl, pins tinygrad ec91b74b5b
+        PKL_OID: b4dead3ee091d66ab002a6b8e716c26f4f79e3c931e4be5f48e017ba92f6e960
+        PKL_SIZE: "767228907"
+        TG_SHA: ec91b74b5b6b0fc924b72d3f27689679ca2438b9
+        WANT: 680a8d629785f2b1d1a2b0e40d0dfbcd0f7291c21e20c51259f1d768a22ec217  # driving_light_speed_2_november_04_2026_tinygrad.pkl, recompiled46
+      run: |
+        set -euo pipefail
+        work="$RUNNER_TEMP/chestnut_parity" && rm -rf "$work" && mkdir -p "$work"
+        href=$(curl -fsSL --retry 3 -X POST -H "Accept: application/vnd.git-lfs+json" -H "Content-Type: application/vnd.git-lfs+json" \
+          https://huggingface.co/commaai/openpilot-lfs.git/info/lfs/objects/batch \
+          -d "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"$PKL_OID\",\"size\":$PKL_SIZE}]}" | jq -r '.objects[0].actions.download.href')
+        curl -fsSL --retry 3 -o "$work/big_driving_tinygrad.pkl" "$href"
+        echo "$PKL_OID  $work/big_driving_tinygrad.pkl" | sha256sum -c -
+        tg="$work/tinygrad" && git init -q "$tg" && git -C "$tg" fetch -q --depth 1 https://github.com/tinygrad/tinygrad "$TG_SHA" && git -C "$tg" checkout -q FETCH_HEAD
+        CHECK_OOB=0 GMMU=0 PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo" python3 tinygrad_repo/test/mockgpu/usb_server.py \
+          env DEV=USB+AMD PYTHONPATH="$tg:$GITHUB_WORKSPACE" python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py "$work/big_driving_tinygrad.pkl" "$work/LS2.pkl"
+        got=$(sha256sum "$work/LS2.pkl" | cut -d' ' -f1)
+        if [ "$got" != "$WANT" ]; then
+          echo "::error::the mock's LS2 build ($got) is not the chestnut runner's ($WANT): this runner's LLVM or the mock no longer builds what the chestnut does"
+          exit 1
+        fi
+        echo "LS2 rebuilt on the mock: byte-identical to the chestnut runner's build"
+        touch ~/.chestnut_build_parity.passed
+
+    - name: Remember the pass
+      if: steps.passed.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: ~/.chestnut_build_parity.passed
+        key: ${{ steps.key.outputs.key }}
+
+    - name: Already passed
+      if: steps.passed.outputs.cache-hit == 'true'
+      shell: bash
+      run: |
+        echo "LS2 rebuild skipped: it already matched the chestnut runner's build with this exact mock, converter, LLVM and python"

.github/workflows/setup-amd-mock/action.yml

@@ -0,0 +1,60 @@
+name: Set up tinygrad's amd mock
+description: LLVM 20 and ROCm comgr, as tinygrad_repo/.github/actions/setup-tinygrad installs them for the mock gpu
+inputs:
+  emu_cache:
+    description: >-
+      Keep tinygrad's cache db (the emulator's compiled kernels and gate results) between runs under this name, e.g. the model's file name.
+      Empty: no cache.
+    default: ''
+runs:
+  using: composite
+  steps:
+    - name: Install LLVM 20 and ROCm comgr
+      shell: bash
+      run: |
+        . /etc/os-release
+        curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
+        echo "deb http://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
+        sudo mkdir -p /etc/apt/keyrings
+        curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.1 $VERSION_CODENAME main" | sudo tee /etc/apt/sources.list.d/rocm.list
+        printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+        sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends libllvm20 clang-20 lld-20 comgr
+        printf '/opt/rocm/lib\n/opt/rocm/lib64\n' | sudo tee /etc/ld.so.conf.d/rocm.conf && sudo ldconfig
+        echo "/usr/lib/llvm-20/bin" >> "$GITHUB_PATH"
+        llvm=$(ls /usr/lib/llvm-20/lib/libLLVM*.so* /usr/lib/*-linux-gnu/libLLVM-20.so* 2>/dev/null | head -1 || true)
+        [ -n "$llvm" ] || { echo "::error::libLLVM 20 not found"; exit 1; }
+        echo "LLVM_PATH=$llvm" >> "$GITHUB_ENV"
+
+    - name: Cache the chestnut's firmware
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/tinygrad/downloads/fw
+        key: chestnut-fw-${{ hashFiles('tinygrad_repo/tinygrad/runtime/autogen/am/fw.py') }}
+
+    - name: Fetch the chestnut's firmware (gitlab times out now and then; the mock boots from it)
+      shell: bash
+      run: |
+        for i in 1 2 3 4 5; do
+          PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo" python3 -c "
+        from tinygrad.helpers import fetch_fw
+        from tinygrad.runtime.autogen.am import fw
+        for n, h in fw.hashes.items():
+          if n.startswith(('psp_14_0_2_', 'smu_14_0_2.', 'sdma_7_0_0.', 'gc_12_0_0_')): fetch_fw('amdgpu', n, h)" && exit 0
+          sleep $((i * 10))
+        done
+        echo "::error::could not fetch the chestnut's firmware from gitlab"; exit 1
+
+    - name: runner cpu
+      if: inputs.emu_cache != ''
+      id: cpu
+      shell: bash
+      run: echo "cpu=$( (grep -m1 '^model name' /proc/cpuinfo; grep -m1 '^flags' /proc/cpuinfo) | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
+
+    - name: Keep the emulator's compiled kernels between runs
+      if: inputs.emu_cache != ''
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/tinygrad/cache.db
+        key: emu-${{ steps.cpu.outputs.cpu }}-${{ hashFiles('tinygrad_repo/tinygrad/**/*.py', 'tinygrad_repo/test/mockgpu/**') }}-${{ inputs.emu_cache }}
+        restore-keys: emu-${{ steps.cpu.outputs.cpu }}-${{ hashFiles('tinygrad_repo/tinygrad/**/*.py', 'tinygrad_repo/test/mockgpu/**') }}-

.github/workflows/sunnypilot-build-model.yaml

@@ -35,6 +35,11 @@ on:
         required: false
         type: string
         default: 'qcom'
+      on_device:
+        description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
+        required: false
+        type: boolean
+        default: false
   workflow_dispatch:
     inputs:
       upstream_branch:
@@ -59,6 +64,11 @@ on:
           - qcom
           - chestnut
         default: 'qcom'
+      on_device:
+        description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
+        required: false
+        type: boolean
+        default: false
 
 
 run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
@@ -153,18 +163,74 @@ jobs:
           if-no-files-found: error
 
   build_model:
-    runs-on: [self-hosted, "${{ inputs.target_hardware == 'chestnut' && 'chestnut' || 'tici' }}"]
+    runs-on: ${{ inputs.on_device && fromJSON(inputs.target_hardware == 'chestnut' && '["self-hosted","chestnut"]' || '["self-hosted","tici"]') || 'ubuntu-24.04' }}
     needs: get_model
     env:
       MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
       REF: ${{ inputs.upstream_branch }}
+      MOCK_BUILD: ${{ !inputs.on_device }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
           submodules: recursive
 
+      - name: Set up the hosted mock build
+        if: env.MOCK_BUILD == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install LLVM 20 and ROCm comgr for the amd mock
+        if: env.MOCK_BUILD == 'true' && inputs.target_hardware == 'chestnut'
+        uses: ./.github/workflows/setup-amd-mock
+        with:
+          emu_cache: build
+
+      - name: Install the amd mock build dependencies
+        if: env.MOCK_BUILD == 'true' && inputs.target_hardware == 'chestnut'
+        run: |
+          python3 -m pip install -q numpy zstandard
+
+      - name: Install the qcom mock build dependencies
+        if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut'
+        id: qcom_deps
+        run: |
+          sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends qemu-user-static aria2
+          python3 -m pip install -q numpy zstandard tinymesa==25.2.7.2
+          echo "qemu=$(dpkg-query -W -f='${Version}' qemu-user-static)" >> "$GITHUB_OUTPUT"
+
+      - name: Compiler check passed before with all of it identical?
+        if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut'
+        id: qcom_compiler
+        env:
+          KEY: qcom-compiler-parity-${{ steps.qcom_deps.outputs.qemu }}-${{ hashFiles('tinygrad_repo/test/device/qcom/compiler_parity.py', 'tinygrad_repo/test/mockgpu/qcom/a630_compiled.pkl.xz', 'tinygrad_repo/tinygrad/runtime/support/compiler_qcom.py') }}
+        run: |
+          MARKER="ci/qcom-compiler-parity/$(printf '%s' "$KEY" | tr -c 'A-Za-z0-9._-' '_').txt"
+          CODE=$(curl -s -o /dev/null -w '%{http_code}' -I -L "https://huggingface.co/datasets/sunnypilot/sunnypilot_models_v1/resolve/main/$MARKER")
+          echo "$MARKER: HTTP $CODE"
+          { echo "key=$KEY"; echo "marker=$MARKER"; echo "passed=$([ "$CODE" = 200 ] && echo true || echo false)"; } >> "$GITHUB_OUTPUT"
+
+      - name: Check the off-device qcom compiler builds the a630's bytes
+        if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut' && steps.qcom_compiler.outputs.passed != 'true'
+        run: |
+          # every kernel an a630 has compiled must compile to the same bytes here before a model is built here
+          PYTHONPATH="$TINYGRAD_PATH" python3 "$TINYGRAD_PATH/test/device/qcom/compiler_parity.py" check "$TINYGRAD_PATH/test/mockgpu/qcom/a630_compiled.pkl.xz"
+
+      - name: Remember the compiler check passed
+        if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut' && steps.qcom_compiler.outputs.passed != 'true'
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          KEY: ${{ steps.qcom_compiler.outputs.key }}
+          MARKER: ${{ steps.qcom_compiler.outputs.marker }}
+        run: |
+          python3 -m pip install -q --upgrade "huggingface_hub>=0.34"
+          printf '%s\n%s\n' "$KEY" "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > "$RUNNER_TEMP/marker.txt"
+          hf upload --repo-type dataset sunnypilot/sunnypilot_models_v1 "$RUNNER_TEMP/marker.txt" "$MARKER" \
+            --commit-message "qcom compiler parity passed: $KEY" || echo "::warning::could not record the pass, the next build checks again"
+
       - name: Set environment variables
+        if: env.MOCK_BUILD != 'true'
         id: set-env
         run: |
           # Set up common environment
@@ -177,6 +243,7 @@ jobs:
           printenv >> $GITHUB_ENV
 
       - name: Setup build environment
+        if: env.MOCK_BUILD != 'true'
         run: |
           mkdir -p "${BUILD_DIR}/"
           sudo find $BUILD_DIR/ -mindepth 1 -delete
@@ -198,21 +265,28 @@ jobs:
 
       - name: Build Model
         run: |
-          source /etc/profile
+          if [ "$MOCK_BUILD" = "true" ]; then
-          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
+            TASKSET=""
-          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
+            export CHECK_OOB=0 GMMU=0  # as test/mockgpu runs the amd mock
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+          else
+            source /etc/profile
+            export UV_PROJECT_ENVIRONMENT=${HOME}/venv
+            export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
+            source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+            TASKSET="taskset -c 7"
+          fi
           export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
 
           COMPILE_MODELD="${{ github.workspace }}/openpilot/sunnypilot/modeld_v2/compile_modeld.py"
           MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
           CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
 
-          TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 OPENPILOT_HACKS=1"
+          MOCK=$([ "$MOCK_BUILD" = "true" ] && echo MOCK || true)
+          TG_FLAGS_QCOM="DEV=${MOCK:+MOCK+}QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 OPENPILOT_HACKS=1"
           if [ "${{ inputs.target_hardware }}" == "chestnut" ]; then
             echo "CHESTNUT build"
             export CHESTNUT=1
-            TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
+            TG_FLAGS="DEBUG=1 DEV=${MOCK}USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
             OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
           else
             echo "QCOM build"
@@ -227,14 +301,16 @@ jobs:
             echo "Converting commaai's precompiled model with tinygrad $TG_SHA"
             UPSTREAM_TG="${{ runner.temp }}/upstream_tinygrad"
             rm -rf "$UPSTREAM_TG" && git init -q "$UPSTREAM_TG" && git -C "$UPSTREAM_TG" fetch -q --depth 1 "$TG_URL" "$TG_SHA" && git -C "$UPSTREAM_TG" checkout -q FETCH_HEAD
+            ON_MOCK=""
+            if [ "$MOCK_BUILD" = "true" ]; then ON_MOCK="env PYTHONPATH=$TINYGRAD_PATH python3 $TINYGRAD_PATH/test/mockgpu/usb_server.py"; fi
             CONVERTED="${{ runner.temp }}/big_driving_tinygrad_plain.pkl"
-            env DEV=USB+AMD PYTHONPATH="$UPSTREAM_TG:${{ github.workspace }}" taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py \
+            $ON_MOCK env DEV=USB+AMD PYTHONPATH="$UPSTREAM_TG:${{ github.workspace }}" $TASKSET python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py \
               "${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" "$CONVERTED"
             mv "$CONVERTED" "$OUTPUT_PKL"
           elif NEW_ARCH_ONNX=$(find "${{ env.MODELS_DIR }}" -maxdepth 1 -name "${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}*driving*.onnx" -print -quit) &&
                [ -n "$NEW_ARCH_ONNX" ] && grep -qa new_img "$NEW_ARCH_ONNX"; then # the new architecture: one ONNX taking the warped frames as new_img
             echo "Compiling new architecture ONNX: $NEW_ARCH_ONNX"
-            env ${TG_FLAGS} taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/compile_modeld_v2.py \
+            env ${TG_FLAGS} $TASKSET python3 openpilot/sunnypilot/modeld_v2/compile_modeld_v2.py \
               "$NEW_ARCH_ONNX" "$OUTPUT_PKL" --device-input "*" --benchmark-runs 1
           else
             echo "Detected onnx architecture. Routing to native compile_modeld.py."
@@ -284,7 +360,7 @@ jobs:
 
             if [ -n "$MODEL_TYPE" ]; then
               echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL"
-              env ${TG_FLAGS} taskset -c 7 python3 "$COMPILE_MODELD" \
+              env ${TG_FLAGS} $TASKSET python3 "$COMPILE_MODELD" \
                 --model-type $MODEL_TYPE \
                 --model-size $MODEL_SIZE \
                 --camera-resolutions $CAMERA_RES \
@@ -293,6 +369,13 @@ jobs:
             fi
           fi
 
+      - name: Replay its check frame on the mock and require the result bit for bit
+        if: env.MOCK_BUILD == 'true'
+        env:
+          DEV: ${{ inputs.target_hardware == 'chestnut' && 'MOCKUSB+AMD' || 'MOCK+QCOM' }}
+          PYTHONPATH: ${{ env.TINYGRAD_PATH }}:${{ github.workspace }}
+        run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "${{ env.MODELS_DIR }}/${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}driving_tinygrad.pkl"
+
       - name: Prepare Output
         run: |
           sudo rm -rf ${{ env.OUTPUT_DIR }}
@@ -304,7 +387,7 @@ jobs:
             --include='/${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}driving_tinygrad.pkl' \
             --exclude='*' \
             --delete-excluded \
-            --chown=comma:comma \
+            $([ "$MOCK_BUILD" = "true" ] || echo --chown=comma:comma) \
             ${{ env.MODELS_DIR }}/ ${{ env.OUTPUT_DIR }}/
 
           python3 "${{ github.workspace }}/release/ci/model_generator.py" \
@@ -333,7 +416,7 @@ jobs:
           path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
 
       - name: Re-enable powersave
-        if: always()
+        if: always() && env.MOCK_BUILD != 'true'
         run: |
           source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable

.github/workflows/test_models.yml

@@ -28,18 +28,7 @@ jobs:
         env:
           DEV: CPU
         run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_plain_artifact.py
-      - name: Install LLVM 20 and ROCm comgr for the amd mock, as tinygrad_repo/.github/actions/setup-tinygrad does
+      - uses: ./.github/workflows/setup-amd-mock
-        run: |
-          . /etc/os-release
-          curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
-          echo "deb http://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
-          sudo mkdir -p /etc/apt/keyrings
-          curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
-          echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.1 $VERSION_CODENAME main" | sudo tee /etc/apt/sources.list.d/rocm.list
-          printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/rocm-pin-600
-          sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends libllvm20 clang-20 lld-20 comgr
-          printf '/opt/rocm/lib\n/opt/rocm/lib64\n' | sudo tee /etc/ld.so.conf.d/rocm.conf && sudo ldconfig
-          echo "/usr/lib/llvm-20/bin" >> "$GITHUB_PATH"
       - name: Convert a pickled USB+AMD jit and replay it under tinygrad's amd mock
         env:
           DEV: MOCKUSB+AMD
@@ -80,6 +69,8 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       models: ${{ steps.set-matrix.outputs.models }}
+      chestnut: ${{ steps.set-matrix.outputs.chestnut }}
+      qcom: ${{ steps.set-matrix.outputs.qcom }}
     steps:
       - uses: actions/checkout@v4
       - name: Fetch and Parse json
@@ -91,16 +82,22 @@ jobs:
           with open("openpilot/sunnypilot/models/fetcher.py", "r") as f:
             urls = re.findall(r"MODEL_URL(?:_CHESTNUT)?\s*=\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", f.read())
 
-          artifacts = []
+          artifacts, chestnut, qcom = [], [], []
           for url in urls:
             data = json.loads(urllib.request.urlopen(url).read())
             for bundle in data.get("bundles", []):
               for model in bundle.get("models", []):
                 if "artifact" in model:
                   artifacts.append(model["artifact"])
+                  if "chestnut" in url:
+                    chestnut.append(model["artifact"])
+                  elif "tinygrad" in model["artifact"]["file_name"]:
+                    qcom.append(model["artifact"])
 
           with open(os.environ["GITHUB_OUTPUT"], "a") as f:
             f.write(f"models={json.dumps(artifacts)}\n")
+            f.write(f"chestnut={json.dumps(chestnut)}\n")
+            f.write(f"qcom={json.dumps(qcom)}\n")
           '
 
   test-model:
@@ -137,3 +134,81 @@ jobs:
           export PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo:$GITHUB_WORKSPACE:$PYTHONPATH"
           python3 -m pip uninstall -y tinygrad || uv pip uninstall -y tinygrad --system || true
           python3 -m pytest openpilot/sunnypilot/modeld_v2/tests/test_models.py
+
+  chestnut-mock-parity:
+    name: Mock parity ${{ matrix.artifact.file_name }}
+    needs: generate-matrix
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        artifact: ${{ fromJson(needs.generate-matrix.outputs.chestnut) }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-amd-mock
+        with:
+          emu_cache: ${{ matrix.artifact.file_name }}
+      - run: python3 -m pip install -q numpy
+      - name: Download model
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          MODEL_URL: ${{ matrix.artifact.download_uri.url }}
+          MODEL_SHA: ${{ matrix.artifact.download_uri.sha256 }}
+        shell: bash
+        run: |
+          AUTH=()
+          if [ -n "$HF_TOKEN" ]; then AUTH=(-H "Authorization: Bearer $HF_TOKEN"); fi
+          curl --retry 3 --retry-all-errors -s -S -f -L "${AUTH[@]}" -o "$RUNNER_TEMP/model.pkl" "$MODEL_URL"
+          if [ -n "$MODEL_SHA" ]; then echo "$MODEL_SHA  $RUNNER_TEMP/model.pkl" | sha256sum -c -; fi
+      - name: Replay its check frame on the amd mock and require the chestnut's result bit for bit
+        env:
+          PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
+        run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "$RUNNER_TEMP/model.pkl"
+
+  qcom-mock-parity:
+    name: QCOM mock parity ${{ matrix.artifact.file_name }}
+    needs: generate-matrix
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        artifact: ${{ fromJson(needs.generate-matrix.outputs.qcom) }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python3 -m pip install -q numpy tinymesa==25.2.7.2
+      - name: Download model
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          MODEL_URL: ${{ matrix.artifact.download_uri.url }}
+          MODEL_SHA: ${{ matrix.artifact.download_uri.sha256 }}
+        shell: bash
+        run: |
+          AUTH=()
+          if [ -n "$HF_TOKEN" ]; then AUTH=(-H "Authorization: Bearer $HF_TOKEN"); fi
+          curl --retry 3 --retry-all-errors -s -S -f -L "${AUTH[@]}" -o "$RUNNER_TEMP/model.pkl" "$MODEL_URL"
+          if [ -n "$MODEL_SHA" ]; then echo "$MODEL_SHA  $RUNNER_TEMP/model.pkl" | sha256sum -c -; fi
+      - name: Replay its check frame on the qcom mock and require the comma device's result bit for bit
+        env:
+          DEV: MOCK+QCOM
+          PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
+        run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "$RUNNER_TEMP/model.pkl"
+
+  chestnut-mock-build-parity:
+    name: Chestnut build on the amd mock = chestnut runner build
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-amd-mock
+        with:
+          emu_cache: chestnut-build-parity
+      - run: python3 -m pip install -q numpy zstandard
+      - uses: ./.github/workflows/chestnut-build-parity

openpilot/sunnypilot/modeld_v2/compile_modeld.py

@@ -371,7 +371,7 @@ def _compile_warp_resolution_worker(cam_w, cam_h, model_w, model_h, benchmark_ru
 
 
 if __name__ == "__main__":
-  if 'USB' in os.getenv('DEV', '') or os.getenv('CHESTNUT'):
+  if ('USB' in os.getenv('DEV', '') or os.getenv('CHESTNUT')) and not os.getenv('DEV', '').startswith('MOCK'):
     from openpilot.system.hardware.chestnut.flash import link_up
     for _ in range(10):
       if link_up():

openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py

@@ -10,6 +10,7 @@ import dataclasses
 import inspect
 import itertools
 import mmap
+import os
 import re
 import struct
 import tempfile
@@ -194,7 +195,8 @@ def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
         base, at, nbytes = _address(argument_words[slot])
         args.append({"buffer": buffers.setdefault(base, len(buffers)), "offset": at, "nbytes": nbytes or base.max_numel() * base.dtype.itemsize, "dtype": u8})
     outs, ins = kernel[6]
-    calls.append({"kind": "program", "name": re.sub(r"\x1b\[[0-9;]*m", "", kernel[1]), "lib": programs[image_alloc], "target": dataclasses.asdict(target),
+    device_target = dataclasses.asdict(dataclasses.replace(target, interface=target.interface.removeprefix("MOCK")))
+    calls.append({"kind": "program", "name": re.sub(r"\x1b\[[0-9;]*m", "", kernel[1]), "lib": programs[image_alloc], "target": device_target,
                   "signature": [[f"data{i}", i, u8, [a.get("nbytes", 1)]] for i, a in enumerate(args)], "global_size": grid, "local_size": local,
                   "globals": list(range(len(args))), "outs": list(outs), "ins": list(ins), "args": args})
     needs_init |= {args[i]["buffer"] for i in ins if "buffer" in args[i] and args[i]["buffer"] not in written}
@@ -235,14 +237,16 @@ def main():
   frame, _ = capture_frame(artifact["run"], kwargs, "output_buffers")
   part = convert(artifact["run"], kwargs, frame, Device[Device.DEFAULT].renderer.target)
 
-  plain = PlainJit(*part)  # replays the frame, within tolerance
+  if not os.getenv("MOCKUSB_SOCKET"):
-  replay = {path[0]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 1}
+    plain = PlainJit(*part)
-  replay["output_buffers"] = {path[1]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 2}
+    replay = {path[0]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 1}
-  flatten(plain(**replay))
+    replay["output_buffers"] = {path[1]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 2}
-  for index, (got, expected) in enumerate(zip([t.numpy() for t in replay["output_buffers"].values()], frame["outputs"], strict=True)):
+    flatten(plain(**replay))
-    assert np.array_equal(got, expected), f"output {index} of the plain artifact differs from the pickle"
+    for index, (got, expected) in enumerate(zip([t.numpy() for t in replay["output_buffers"].values()], frame["outputs"], strict=True)):
+      assert np.array_equal(got, expected), f"output {index} of the plain artifact differs from the pickle"
   write_bundle(args.output, {"run": part}, {key: artifact[key] for key in ("metadata", "input_specs", "output_specs")})
-  print(f"converted {len(part[0]['calls'])} kernels, bit-exact with the pickle: {args.output}")
+  checked = "left to mock_parity" if os.getenv("MOCKUSB_SOCKET") else "bit-exact with the pickle"
+  print(f"converted {len(part[0]['calls'])} kernels, {checked}: {args.output}")
 
 
 if __name__ == "__main__":

openpilot/sunnypilot/modeld_v2/mock_parity.py

@@ -0,0 +1,153 @@
+import json
+import os
+import shutil
+import struct
+import sys
+import tempfile
+import time
+
+os.environ.setdefault("DEV", "MOCKUSB+AMD")
+QCOM = "QCOM" in os.environ["DEV"]
+ON_MOCK = os.environ["DEV"].startswith("MOCK")
+ARCH, HARDWARE = ("a630", "comma 3x/4 (a630)") if QCOM else ("gfx1200", "chestnut")
+os.environ.setdefault("CHECK_OOB", "0")
+os.environ.setdefault("GMMU", "0")
+if sys.platform == "darwin" and os.environ["DEV"].startswith("MOCKUSB"):  # the mock's VRAM is a memfd, which macOS lacks: an unlinked file is the same
+  from tinygrad.runtime.autogen import libc
+
+  def _memfd_create(name, flags):
+    fd, path = tempfile.mkstemp()
+    os.unlink(path)
+    return fd
+  libc.memfd_create = _memfd_create
+
+import numpy as np
+
+from openpilot.sunnypilot.modeld_v2.helpers import ALIGNMENT, MAGIC, PlainJit, _device, read_parts
+
+
+class ParityError(RuntimeError):
+  pass
+
+
+def _emulator():
+  try:
+    if QCOM:
+      from test.mockgpu.qcom import emu
+    else:
+      from test.mockgpu.amd import emu
+  except ImportError as e:
+    raise ParityError(f"tinygrad's {'qcom' if QCOM else 'amd'} mock is not importable: put tinygrad_repo on PYTHONPATH") from e
+  if not hasattr(emu, "hw_unverified"):
+    raise ParityError(f"this tinygrad's emulator has no {ARCH} rounding model, its check frames would not match a {HARDWARE}")
+  return emu
+
+
+def unverified(header: dict, blob: memoryview) -> dict[str, int]:
+  emu, out = _emulator(), {}
+  for call in header["calls"]:
+    if call["kind"] != "program":
+      continue
+    if (arch := call["target"].get("arch", "")).split(",")[0] != ARCH:  # IMAGE builds append ",IMAGE_PITCH_ALIGNMENT=64"
+      out[f"arch={arch}"] = out.get(f"arch={arch}", 0) + 1
+      continue
+    for sig, n in emu.hw_unverified(bytes(blob[slice(*call["lib"])])).items():
+      out[sig] = out.get(sig, 0) + n
+  return out
+
+
+def _array(blob: memoryview, stored: dict) -> np.ndarray:
+  return np.frombuffer(blob[slice(*stored["data"])], dtype=np.dtype(stored["dtype"])).reshape(stored["shape"]).copy()
+
+
+def replay(header: dict, blob: memoryview, shared: dict) -> list[np.ndarray]:
+  from tinygrad import Tensor
+  check = header["check"]
+  jit = PlainJit(header, blob, None, verify=False, shared=shared)
+  kwargs: dict = {}
+  for path, stored, device in check["inputs"]:
+    tensor = Tensor(_array(blob, stored), device=_device(device, None)).realize()
+    kwargs[path[0]] = tensor if len(path) == 1 else {**kwargs.get(path[0], {}), path[1]: tensor}
+  result = jit(**kwargs)
+  outputs = kwargs[check["output_container"]].values() if check["output_container"] else result
+  return [t.numpy() for t in (outputs if isinstance(outputs, (list, tuple)) or check["output_container"] else [outputs])]
+
+
+def _parts_with_offsets(path: str) -> tuple[dict, dict]:
+  parts, _ = read_parts(path)
+  with open(path, "rb") as f:
+    head = f.read(len(MAGIC) + 8)
+    start = len(MAGIC) + 8 + struct.unpack_from("<Q", head, len(MAGIC))[0]
+    f.seek(len(MAGIC) + 8)
+    entries = json.loads(f.read(start - len(MAGIC) - 8))["parts"]
+  base = start + -start % ALIGNMENT
+  offsets = {tuple(e["key"]) if isinstance(e["key"], list) else e["key"]: base + e["blob"][0] for e in entries}
+  return parts, offsets
+
+
+def _gate(parts: dict) -> None:
+  missing: dict[str, int] = {}
+  for header, blob in parts.values():
+    if "check" not in header:
+      raise ParityError("artifact has no check frame")
+    for sig, n in unverified(header, blob).items():
+      missing[sig] = missing.get(sig, 0) + n
+  if missing:
+    raise ParityError(f"kernels use instructions the emulator is not verified against a {HARDWARE} for: " +
+                      ", ".join(f"{s} ({n}x)" for s, n in sorted(missing.items(), key=lambda kv: -kv[1])))
+
+
+def check(path: str) -> None:
+  parts, _ = read_parts(path)
+  if ON_MOCK:
+    _gate(parts)
+  shared: dict = {}
+  for key, (header, blob) in parts.items():
+    got = replay(header, blob, shared)
+    for i, (g, stored) in enumerate(zip(got, header["check"]["outputs"], strict=True)):
+      want = _array(blob, stored)
+      if g.shape != want.shape or not np.array_equal(g.view(np.uint8), want.view(np.uint8)):
+        n = int(np.count_nonzero(g != want)) if g.shape == want.shape else -1
+        where = "mock" if ON_MOCK else "device"
+        raise ParityError(f"part {key} output {i}: the {where}'s check frame differs from the stored one in {n} of {want.size} values")
+
+
+def record(path: str, out: str) -> None:
+  parts, offsets = _parts_with_offsets(path)
+  _gate(parts)
+  shared, writes = {}, []
+  for key, (header, blob) in parts.items():
+    for g, stored in zip(replay(header, blob, shared), header["check"]["outputs"], strict=True):
+      want = np.dtype(stored["dtype"])
+      if tuple(g.shape) != tuple(stored["shape"]) or g.dtype != want:
+        raise ParityError(f"part {key}: replayed output {g.dtype}{g.shape} does not fit the stored {want}{tuple(stored['shape'])}")
+      writes.append((offsets[key] + stored["data"][0], np.ascontiguousarray(g).tobytes()))
+  shutil.copyfile(path, out)
+  with open(out, "r+b") as f:
+    for offset, data in writes:
+      f.seek(offset)
+      f.write(data)
+
+
+def main(argv: list[str]) -> int:
+  if len(argv) < 2 or argv[0] not in ("check", "record") or (argv[0] == "record" and len(argv) != 3):
+    print("usage: mock_parity check ARTIFACT... | record ARTIFACT OUT", file=sys.stderr)
+    return 2
+  try:
+    if argv[0] == "record":
+      st = time.perf_counter()
+      record(argv[1], argv[2])
+      print(f"{argv[2]}: check frame recorded on the mock in {time.perf_counter() - st:.0f}s")
+      return 0
+    for path in argv[1:]:
+      st = time.perf_counter()
+      check(path)
+      print(f"{os.path.basename(path)}: bit-identical to its {HARDWARE} check frame on the {'mock' if ON_MOCK else 'device'} ({time.perf_counter() - st:.0f}s)")
+    return 0
+  except ParityError as e:
+    print(f"PARITY FAIL: {e}", file=sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+  sys.exit(main(sys.argv[1:]))

openpilot/sunnypilot/modeld_v2/plain_artifact_compiler.py

@@ -80,7 +80,8 @@ def serialize_jit(jit, kernel_linear, check: dict | None = None) -> tuple[dict,
     if ast.op is Ops.PROGRAM:
       elf, info = ast.to_elf(), ast.arg
       global_size, local_size = info.launch_dims({})
-      calls.append({"kind": "program", "name": elf.name, "lib": blob.add(bytes(elf.lib)), "target": dataclasses.asdict(elf.target),
+      device_target = dataclasses.asdict(dataclasses.replace(elf.target, interface=elf.target.interface.removeprefix("MOCK")))
+      calls.append({"kind": "program", "name": elf.name, "lib": blob.add(bytes(elf.lib)), "target": device_target,
                     "signature": kernel_signature(ast, elf),
                     "global_size": [int(x) for x in global_size], "local_size": [int(x) for x in local_size],
                     "globals": list(info.globals), "outs": list(info.outs), "ins": list(info.ins), "args": arguments})

openpilot/sunnypilot/modeld_v2/tests/test_models.py

@@ -17,4 +17,4 @@ class TestRegistryModels(unittest.TestCase):
       self.skipTest("MODEL_BASE_NAME env var not set, skipping integration test.")
     bundle = load_model(os.path.join(os.environ.get("MODEL_DIR", "/tmp/models"), base_name), {"QCOM": "CPU", "AMD": "CPU"}, verify=False, cache=False)
     jits = [value for value in bundle.values() if isinstance(value, PlainJit)]
-    assert "metadata" in bundle and jits and all(len(jit.captured.linear.src) > 0 for jit in jits)
+    assert "metadata" in bundle and jits and all(len(jit.captured._linear.src) > 0 for jit in jits)