ci: offload small model compilation (#1952)

* ci: compile default big model with stock modeld * Revert "Revert big RL model (#38627)" This reverts commit 516ec1e682. * ci: compile default small model with stock modeld compiler * ci: offload small model compilation * Reapply "Revert big RL model (#38627)" This reverts commit d06cfabb625e16bfd8f74984b359162be4b71417. * Reapply "Revert big RL model (#38627)" This reverts commit d06cfabb625e16bfd8f74984b359162be4b71417.
2026-09-11 02:33:41 +08:00 · 2026-08-24 16:24:19 -04:00
parent 2bcfed5c71
commit 8e16c9babb
3 changed files with 168 additions and 47 deletions
@@ -31,6 +31,7 @@ jobs:
      target_hardware: ${{ steps.resolve.outputs.target_hardware }}
      tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }}
      dm_onnx_ref: ${{ steps.resolve.outputs.dm_onnx_ref }}
+      dm_onnx_date: ${{ steps.resolve.outputs.dm_onnx_date }}
    steps:
      - uses: actions/checkout@v4
        with:
@@ -60,8 +61,10 @@ jobs:
          fi

          DM_ONNX_REF=""
+          DM_ONNX_DATE=""
          if [ "${{ inputs.target }}" = "small" ]; then
            DM_ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx)
+            DM_ONNX_DATE=$(git log -1 --format=%cd --date=format:'%B %d, %Y' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx)
          fi

          echo "model_name=${NAME}" >> $GITHUB_OUTPUT
@@ -71,6 +74,7 @@ jobs:
          echo "target_hardware=${TARGET_HW}" >> $GITHUB_OUTPUT
          echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT
          echo "dm_onnx_ref=${DM_ONNX_REF}" >> $GITHUB_OUTPUT
+          echo "dm_onnx_date=${DM_ONNX_DATE}" >> $GITHUB_OUTPUT

  build_small_model:
    needs: resolve
@@ -395,7 +399,7 @@ jobs:
          metadata = {
              'bundles': [{
                  'short_name': 'DMMODEL',
-                  'display_name': 'dmonitoring_model',
+                  'display_name': 'dmonitoring_model (${{ needs.resolve.outputs.dm_onnx_date }})',
                  'ref': '${{ needs.resolve.outputs.dm_onnx_ref }}',
                  'runner': 'tinygrad',
                  'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'),
@@ -421,7 +425,7 @@ jobs:
            --model-dir dm_output \
            --onnx-path "${{ env.DM_ONNX }}" \
            --onnx-ref "${{ needs.resolve.outputs.dm_onnx_ref }}" \
-            --model-name "dmonitoring_model" \
+            --model-name "dmonitoring_model (${{ needs.resolve.outputs.dm_onnx_date }})" \
            --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \
            --run-number "${{ github.run_number }}"

@@ -165,7 +165,7 @@ jobs:
          scons -j1 cache_dir="$SCONS_CACHE" --minimal \
            openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd
          echo "Building rest of sunnypilot"
-          /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
+          SKIP_TINYGRAD_COMPILE=1 /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
          touch ${BUILD_DIR}/prebuilt
          if [[ "${{ runner.debug }}" == "1" ]]; then
            ls -la ${BUILD_DIR}
@@ -242,12 +242,13 @@ jobs:
            echo "HF defaults match repo ONNX"
          else
            echo "No matching model on HF — triggering build"
+            TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
            gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=big

            echo "Waiting for build to start..."
            sleep 120

-            RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId')
+            RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --created=">$TRIGGER_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
            if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
              echo "::error::Failed to find build-default-models run"
              exit 1
@@ -276,21 +277,99 @@ jobs:
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

+  prepare_small_models:
+    needs: [ prepare_strategy ]
+    runs-on: ubuntu-24.04
+    outputs:
+      driving_onnx_sha256: ${{ steps.resolve.outputs.driving_onnx_sha256 }}
+      dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }}
+    env:
+      HF_REPO: sunnypilot/sunnypilot_models_v1
+      HF_DEFAULTS_PATH: models/defaults/small
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+          submodules: recursive
+
+      - run: git lfs pull -I "openpilot/selfdrive/modeld/models/driving_supercombo.onnx,openpilot/selfdrive/modeld/models/dmonitoring_model.onnx"
+
+      - name: Check HF defaults and build if needed
+        id: resolve
+        run: |
+          DRIVING_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/driving_supercombo.onnx" | cut -d' ' -f1)
+          DM_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/dmonitoring_model.onnx" | cut -d' ' -f1)
+          TINYGRAD_REF=$(PYTHONPATH=${{ github.workspace }} python3 openpilot/sunnypilot/models/tinygrad_ref.py)
+          echo "driving_onnx_sha256=$DRIVING_HASH" >> $GITHUB_OUTPUT
+          echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT
+          echo "Driving ONNX hash: $DRIVING_HASH"
+          echo "DM ONNX hash: $DM_HASH"
+          echo "tinygrad ref: $TINYGRAD_REF"
+
+          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
+
+          check_defaults() {
+            DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
+            TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null)
+            [ "$TINYGRAD_MATCH" = "true" ] || return 1
+            DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
+            [ -n "$DRIVING" ] && [ "$DRIVING" != "null" ] || return 1
+            DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
+            [ -n "$DM" ] && [ "$DM" != "null" ] || return 1
+          }
+
+          if check_defaults; then
+            echo "HF defaults match repo ONNX hashes and tinygrad ref"
+          else
+            echo "No matching models on HF — triggering build"
+            TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+            gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=small
+
+            echo "Waiting for build to start..."
+            sleep 120
+
+            RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --created=">$TRIGGER_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
+            if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
+              echo "::error::Failed to find build-default-models run"
+              exit 1
+            fi
+
+            echo "Waiting for run $RUN_ID..."
+            gh run watch "$RUN_ID"
+
+            CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
+            if [ "$CONCLUSION" != "success" ]; then
+              echo "::error::build-default-models failed: $CONCLUSION"
+              exit 1
+            fi
+
+            if ! check_defaults; then
+              echo "::error::HF defaults still don't match after build"
+              exit 1
+            fi
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Cancel run on failure
+        if: failure()
+        run: gh run cancel ${{ github.run_id }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
  publish:
    concurrency:
-      # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
-      # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
-      # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
      group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
      cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
    if: ${{
      always() && !cancelled() &&
      needs.build.result == 'success' &&
      needs.prepare_strategy.result == 'success' &&
+      needs.prepare_small_models.result == 'success' &&
      (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
      (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
      }}
-    needs: [ build, prepare_strategy, prepare_chestnut ]
+    needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_models ]
    runs-on: ubuntu-24.04
    environment: ${{ needs.prepare_strategy.outputs.environment }}
    steps:
@@ -306,6 +385,41 @@ jobs:
          mkdir -p ${{ env.OUTPUT_DIR }}
          tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}

+      - name: Download small model chunks from HF
+        env:
+          HF_REPO: sunnypilot/sunnypilot_models_v1
+          HF_DEFAULTS_PATH: models/defaults/small
+        run: |
+          set -o pipefail
+          JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
+          DEFAULTS=$(curl -fsSL "$JSON_URL")
+          MODELS_DIR="${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models"
+
+          download_model_chunks() {
+            local ONNX_HASH="$1"
+            local CANONICAL="$2"
+            BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
+            ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
+            BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
+            NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
+
+            echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
+              CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true)
+              if [ -z "$CHUNK_IDX" ]; then
+                echo "::error::Failed to parse chunk index from: $CHUNK_NAME"
+                exit 1
+              fi
+              CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
+              ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
+              echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
+              curl -fsSL -o "${MODELS_DIR}/${CANONICAL_CHUNK}" "$ENCODED_URL"
+            done
+            echo "$NUM_CHUNKS" > "${MODELS_DIR}/${CANONICAL}.chunkmanifest"
+          }
+
+          download_model_chunks "${{ needs.prepare_small_models.outputs.driving_onnx_sha256 }}" "driving_tinygrad.pkl"
+          download_model_chunks "${{ needs.prepare_small_models.outputs.dm_onnx_sha256 }}" "dmonitoring_model_tinygrad.pkl"
+
      - name: Prepare chestnut output
        if: ${{ needs.prepare_chestnut.result == 'success' }}
        run: |
@@ -393,6 +507,7 @@ jobs:
    - build
    - publish
    - prepare_chestnut
+    - prepare_small_models
    runs-on: ubuntu-24.04
    if: ${{ (always() && !cancelled() && !failure())
      && needs.publish.result == 'success'
@@ -73,44 +73,45 @@ compile_modeld_script = [
 model_w, model_h = MEDMODEL_INPUT_SIZE
 frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ

-for usbgpu in [False, True] if USBGPU else [False]:
-  target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath
-  # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU
-  file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags)
-  driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath)
-  camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS)
-  # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it.
-  taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else ''
-  cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py '
-        f'--model-size {model_w}x{model_h} '
-        f'--camera-resolutions {camera_res_args} '
-        f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} '
-        f'--output {target_pkl_path} --frame-skip {frame_skip}')
-  onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps)
-  chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum))
-  def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets):
-    from openpilot.system.hardware.chestnut.flash import link_up
-    # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars
-    for _ in range(10):
-      if link_up():
-        break
-      time.sleep(1)
-    else:
-      print("Chestnut not ready, skipping big model build")
-      return
-    if ret := env.Execute(command):
-      return ret
-    chunk_file(pkl, chunks)
-  def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets):
-    chunk_file(pkl, chunks)
-  actions = Action(do_compile, "  [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, "  [CHUNK] $TARGET")]
-  node = lenv.Command(
-    chunk_targets,
-    tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file],
-    actions,
-  )
-  if usbgpu:
-    lenv.SideEffect(usbgpu_lock, node)
+if not os.getenv('SKIP_TINYGRAD_COMPILE'):
+  for usbgpu in [False, True] if USBGPU else [False]:
+    target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath
+    # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU
+    file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags)
+    driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath)
+    camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS)
+    # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it.
+    taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else ''
+    cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py '
+          f'--model-size {model_w}x{model_h} '
+          f'--camera-resolutions {camera_res_args} '
+          f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} '
+          f'--output {target_pkl_path} --frame-skip {frame_skip}')
+    onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps)
+    chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum))
+    def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets):
+      from openpilot.system.hardware.chestnut.flash import link_up
+      # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars
+      for _ in range(10):
+        if link_up():
+          break
+        time.sleep(1)
+      else:
+        print("Chestnut not ready, skipping big model build")
+        return
+      if ret := env.Execute(command):
+        return ret
+      chunk_file(pkl, chunks)
+    def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets):
+      chunk_file(pkl, chunks)
+    actions = Action(do_compile, "  [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, "  [CHUNK] $TARGET")]
+    node = lenv.Command(
+      chunk_targets,
+      tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file],
+      actions,
+    )
+    if usbgpu:
+      lenv.SideEffect(usbgpu_lock, node)

 # get model metadata
 fn = File(f"models/dmonitoring_model").abspath
@@ -142,4 +143,5 @@ def tg_compile(flags, model_name):
     Action(do_chunk, "  [CHUNK] $TARGET")],
  )

-tg_compile(tg_flags, 'dmonitoring_model')
+if not os.getenv('SKIP_TINYGRAD_COMPILE'):
+  tg_compile(tg_flags, 'dmonitoring_model')