Merge branch 'master' into ccnc-port

This reverts commit 15e5d2efb9.

.github/workflows/sunnypilot-build-model.yaml

@@ -103,10 +103,10 @@ jobs:
       - run: |
           cd ${{ github.workspace }}/openpilot/openpilot
           if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then
-            git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx" -X "**/selfdrive/modeld/models/dmonitoring_*.onnx"
+            git lfs pull -X "selfdrive/modeld/models/big_*.onnx" -X "selfdrive/modeld/models/dmonitoring_*.onnx"
             rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx
           else
-            git lfs pull -I "**/selfdrive/modeld/models/big_*.onnx"
+            git lfs pull -I "selfdrive/modeld/models/big_*.onnx"
             find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete
           fi
       - name: 'Upload Artifact'
@@ -116,7 +116,7 @@ jobs:
           path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
 
   build_model:
-    runs-on: [self-hosted, usbgpu]
+    runs-on: [self-hosted, tici]
     needs: get_model
     env:
       MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
@@ -127,6 +127,7 @@ jobs:
           fetch-depth: 1
           submodules: recursive
 
+      - run: git lfs pull
 
       - name: Set environment variables
         id: set-env
@@ -179,7 +180,6 @@ jobs:
           MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
           CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
 
-          TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
           if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
             echo "USBGPU build"
             export USBGPU=1
@@ -187,40 +187,27 @@ jobs:
             OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
           else
             echo "QCOM build"
-            TG_FLAGS="$TG_FLAGS_QCOM"
+            TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
             OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
           fi
 
           # Generate metadata for all ONNX files
           find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
             echo "Generating metadata: $onnx_file"
-            env ${TG_FLAGS_QCOM} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
+            env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
           done
 
           # Detect model type and build compile args
-          VISION_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do
-            [ -f "$f" ] && VISION_ONNX="$f" && break
-          done
-
-          POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do
-            [ -f "$f" ] && POLICY_ONNX="$f" && break
-          done
-
-          OFF_POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do
-            [ -f "$f" ] && OFF_POLICY_ONNX="$f" && break
-          done
-
-          ON_POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do
-            [ -f "$f" ] && ON_POLICY_ONNX="$f" && break
-          done
-
+          VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
+          POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
+          OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
+          ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
           SUPERCOMBO_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do
-            [ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break
+          for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do
+            if [ -f "$f" ]; then
+              SUPERCOMBO_ONNX="$f"
+              break
+            fi
           done
 
           MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""

openpilot/sunnypilot/modeld_v2/compile_modeld.py

@@ -296,14 +296,14 @@ def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]:
 
 
 if __name__ == "__main__":
-  # if 'USB' in os.getenv('DEV', '') or os.getenv('USBGPU'):
-  #   from openpilot.system.hardware.chestnut.flash import link_up
-  #   for _ in range(10):
-  #     if link_up():
-  #       break
-  #     time.sleep(1)
-  #   else:
-  #     raise RuntimeError("Chestnut not ready, skipping big model build")
+  if 'USB' in os.getenv('DEV', '') or os.getenv('USBGPU'):
+    from openpilot.system.hardware.chestnut.flash import link_up
+    for _ in range(10):
+      if link_up():
+        break
+      time.sleep(1)
+    else:
+      raise RuntimeError("Chestnut not ready, skipping big model build")
 
   from openpilot.common.file_chunker import chunk_file, get_chunk_targets
   from openpilot.selfdrive.modeld.get_model_metadata import make_metadata_dict

release/ci/model_generator.py

@@ -66,11 +66,11 @@ def _read_pkl_bytes(pkl_path: Path) -> bytes:
 
 
 def _find_driving_pkl(output_path: Path) -> Path | None:
-  for pattern in ('*driving_tinygrad.pkl', '*driving_*_tinygrad.pkl'):
+  for pattern in ('driving_tinygrad.pkl', 'driving_*_tinygrad.pkl'):
     matches = sorted(output_path.glob(pattern))
     if matches:
       return matches[0]
-  for pattern in ('*driving_tinygrad.pkl.chunkmanifest', '*driving_*_tinygrad.pkl.chunkmanifest'):
+  for pattern in ('driving_tinygrad.pkl.chunkmanifest', 'driving_*_tinygrad.pkl.chunkmanifest'):
     matches = sorted(output_path.glob(pattern))
     if matches:
       return Path(str(matches[0]).removesuffix('.chunkmanifest'))