From 049d225d5aec12ff44200b12ca700a504d1ddb52 Mon Sep 17 00:00:00 2001 From: James Vecellio-Grant <159560811+Discountchubbs@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:18:17 -0700 Subject: [PATCH] ci: Dedicated Model Runner (#1922) * ci: Dedicated Model Runner * recurse * not needed * fix wrapper * whoops * bypass * modeld_v2: restore chestnut link check before big model build * modeld_v2: stage onnx to disk instead of shared memory * ci: clear unchunked onnx temps before model build * ci: stream the pkl hash instead of loading it into memory --------- Co-authored-by: Jason Wen --- .github/workflows/sunnypilot-build-model.yaml | 45 ++++++++++++------- .../sunnypilot/modeld_v2/compile_modeld.py | 23 +++++----- .../modeld_v2/tests/test_compile_modeld.py | 37 ++++++++++++++- release/ci/model_generator.py | 24 +++++----- 4 files changed, 90 insertions(+), 39 deletions(-) diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index 17981d68e5..50d216be31 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -103,10 +103,10 @@ jobs: - run: | cd ${{ github.workspace }}/openpilot/openpilot if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then - git lfs pull -X "selfdrive/modeld/models/big_*.onnx" -X "selfdrive/modeld/models/dmonitoring_*.onnx" + git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx" -X "**/selfdrive/modeld/models/dmonitoring_*.onnx" rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx else - git lfs pull -I "selfdrive/modeld/models/big_*.onnx" + git lfs pull -I "**/selfdrive/modeld/models/big_*.onnx" find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete fi - name: 'Upload Artifact' @@ -116,7 +116,7 @@ jobs: path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx build_model: - runs-on: [self-hosted, tici] + runs-on: [self-hosted, usbgpu] needs: get_model env: MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }}) @@ -127,7 +127,6 @@ jobs: fetch-depth: 1 submodules: recursive - - run: git lfs pull - name: Set environment variables id: set-env @@ -160,7 +159,7 @@ jobs: fi source ${UV_PROJECT_ENVIRONMENT}/bin/activate PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable - rm -rf ${{ env.MODELS_DIR }}/*.onnx + rm -rf ${{ env.MODELS_DIR }}/*.onnx* - name: Download model artifacts uses: actions/download-artifact@v4 @@ -180,6 +179,7 @@ jobs: MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then echo "USBGPU build" export USBGPU=1 @@ -187,27 +187,40 @@ jobs: OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" else echo "QCOM build" - TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + TG_FLAGS="$TG_FLAGS_QCOM" OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl" fi # Generate metadata for all ONNX files find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do echo "Generating metadata: $onnx_file" - env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true + env ${TG_FLAGS_QCOM} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true done # Detect model type and build compile args - VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx" - POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx" - OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx" - ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx" + VISION_ONNX="" + for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do + [ -f "$f" ] && VISION_ONNX="$f" && break + done + + POLICY_ONNX="" + for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do + [ -f "$f" ] && POLICY_ONNX="$f" && break + done + + OFF_POLICY_ONNX="" + for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do + [ -f "$f" ] && OFF_POLICY_ONNX="$f" && break + done + + ON_POLICY_ONNX="" + for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do + [ -f "$f" ] && ON_POLICY_ONNX="$f" && break + done + SUPERCOMBO_ONNX="" - for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do - if [ -f "$f" ]; then - SUPERCOMBO_ONNX="$f" - break - fi + for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do + [ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break done MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME="" diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 927e5f3392..85ae57c078 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -272,18 +272,17 @@ def _parse_size(size_str: str) -> tuple[int, int]: return int(width), int(height) -def read_file_chunked_to_shm(path): +def read_file_chunked_to_disk(path): if not path: return None import atexit import shutil from openpilot.common.file_chunker import open_file_chunked - from openpilot.common.hardware.hw import Paths - shm_path = os.path.join(Paths.shm_path(), os.path.basename(path)) - atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path)) - with open(shm_path, 'wb') as dst, open_file_chunked(path) as src: - shutil.copyfileobj(src, dst) - return shm_path + tmp_path = f'{path}.unchunked' + with open(tmp_path, 'wb') as f, open_file_chunked(path) as src: + shutil.copyfileobj(src, f) + atexit.register(lambda: os.path.exists(tmp_path) and os.remove(tmp_path)) + return tmp_path def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]: @@ -327,11 +326,11 @@ if __name__ == "__main__": model_w, model_h = args.model_size output_data = {} - args.vision_onnx = read_file_chunked_to_shm(args.vision_onnx) - args.policy_onnx = read_file_chunked_to_shm(args.policy_onnx) - args.off_policy_onnx = read_file_chunked_to_shm(args.off_policy_onnx) - args.on_policy_onnx = read_file_chunked_to_shm(args.on_policy_onnx) - args.supercombo_onnx = read_file_chunked_to_shm(args.supercombo_onnx) + args.vision_onnx = read_file_chunked_to_disk(args.vision_onnx) + args.policy_onnx = read_file_chunked_to_disk(args.policy_onnx) + args.off_policy_onnx = read_file_chunked_to_disk(args.off_policy_onnx) + args.on_policy_onnx = read_file_chunked_to_disk(args.on_policy_onnx) + args.supercombo_onnx = read_file_chunked_to_disk(args.supercombo_onnx) vision_runner = OnnxRunner(args.vision_onnx) if args.vision_onnx else None diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py b/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py index 885696b853..96bfb42638 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py @@ -5,10 +5,15 @@ This file is part of sunnypilot and is licensed under the MIT License. See the LICENSE.md file in the root directory for more details. """ +import os +import tempfile +from pathlib import Path + import numpy as np from openpilot.common.parameterized import parameterized -from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key +from openpilot.common.file_chunker import chunk_file, get_chunk_targets +from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key, read_file_chunked_to_disk from openpilot.common.test import OpenpilotTestCase @@ -160,3 +165,33 @@ class TestOutputSlicePreservation(OpenpilotTestCase): policy_slices = {'plan': slice(0, 495), 'meta': slice(495, 550)} assert set(vision_slices.keys()) & set(policy_slices.keys()) == set(), \ "vision and policy slices should not overlap in keys" + + +class TestReadFileChunkedToDisk(OpenpilotTestCase): + def test_none_passthrough(self): + assert read_file_chunked_to_disk(None) is None + + def test_unchunked_source_staged_on_disk(self): + with tempfile.TemporaryDirectory() as d: + src = Path(d) / "driving_supercombo.onnx" + payload = os.urandom(1024) + src.write_bytes(payload) + + out = Path(read_file_chunked_to_disk(str(src))) + + assert out.parent == Path(d) + assert out.name == "driving_supercombo.onnx.unchunked" + assert out.read_bytes() == payload + + def test_chunked_source_reassembled_on_disk(self): + with tempfile.TemporaryDirectory() as d: + src = Path(d) / "driving_supercombo.onnx" + payload = os.urandom(4096) + src.write_bytes(payload) + chunk_file(str(src), get_chunk_targets(str(src), len(payload))) + assert not src.exists() + + out = Path(read_file_chunked_to_disk(str(src))) + + assert out.parent == Path(d) + assert out.read_bytes() == payload diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py index 607260f145..80b102a6f1 100755 --- a/release/ci/model_generator.py +++ b/release/ci/model_generator.py @@ -53,24 +53,28 @@ def create_pkl_name(full_name: str) -> str: return pkl -def _read_pkl_bytes(pkl_path: Path) -> bytes: +def _hash_pkl(pkl_path: Path) -> str: manifest = Path(f"{pkl_path}.chunkmanifest") if manifest.exists(): num_chunks = int(manifest.read_text().strip()) - parts = [] - for i in range(num_chunks): - chunk = Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}") - parts.append(chunk.read_bytes()) - return b''.join(parts) - return pkl_path.read_bytes() + paths = [Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}") for i in range(num_chunks)] + else: + paths = [pkl_path] + + digest = hashlib.sha256() + for path in paths: + with path.open('rb') as f: + while block := f.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() def _find_driving_pkl(output_path: Path) -> Path | None: - for pattern in ('driving_tinygrad.pkl', 'driving_*_tinygrad.pkl'): + for pattern in ('*driving_tinygrad.pkl', '*driving_*_tinygrad.pkl'): matches = sorted(output_path.glob(pattern)) if matches: return matches[0] - for pattern in ('driving_tinygrad.pkl.chunkmanifest', 'driving_*_tinygrad.pkl.chunkmanifest'): + for pattern in ('*driving_tinygrad.pkl.chunkmanifest', '*driving_*_tinygrad.pkl.chunkmanifest'): matches = sorted(output_path.glob(pattern)) if matches: return Path(str(matches[0]).removesuffix('.chunkmanifest')) @@ -87,7 +91,7 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path: def generate_chunked_model(driving_pkl: Path) -> dict: - tinygrad_hash = hashlib.sha256(_read_pkl_bytes(driving_pkl)).hexdigest() + tinygrad_hash = _hash_pkl(driving_pkl) chunks_config = [] manifest_file = Path(f"{driving_pkl}.chunkmanifest")