diff --git a/.github/workflows/build-default-models.yaml b/.github/workflows/build-default-models.yaml index 55efda4e4d..996e89fc3e 100644 --- a/.github/workflows/build-default-models.yaml +++ b/.github/workflows/build-default-models.yaml @@ -31,6 +31,7 @@ jobs: target_hardware: ${{ steps.resolve.outputs.target_hardware }} tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }} dm_onnx_ref: ${{ steps.resolve.outputs.dm_onnx_ref }} + dm_onnx_date: ${{ steps.resolve.outputs.dm_onnx_date }} steps: - uses: actions/checkout@v4 with: @@ -60,8 +61,10 @@ jobs: fi DM_ONNX_REF="" + DM_ONNX_DATE="" if [ "${{ inputs.target }}" = "small" ]; then DM_ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx) + DM_ONNX_DATE=$(git log -1 --format=%cd --date=format:'%B %d, %Y' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx) fi echo "model_name=${NAME}" >> $GITHUB_OUTPUT @@ -71,19 +74,239 @@ jobs: echo "target_hardware=${TARGET_HW}" >> $GITHUB_OUTPUT echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT echo "dm_onnx_ref=${DM_ONNX_REF}" >> $GITHUB_OUTPUT + echo "dm_onnx_date=${DM_ONNX_DATE}" >> $GITHUB_OUTPUT - build_driving_model: + build_small_model: needs: resolve - uses: ./.github/workflows/sunnypilot-build-model.yaml - with: - upstream_branch: ${{ needs.resolve.outputs.onnx_ref }} - custom_name: ${{ needs.resolve.outputs.model_name }} - target_hardware: ${{ needs.resolve.outputs.target_hardware }} - secrets: inherit + if: ${{ inputs.target == 'small' }} + runs-on: [self-hosted, tici] + env: + SMALL_ONNX: openpilot/selfdrive/modeld/models/driving_supercombo.onnx + SMALL_PKL: openpilot/selfdrive/modeld/models/driving_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull ONNX via LFS + run: git lfs pull -I "${{ env.SMALL_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Compile small model with stock compiler + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") + CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)") + + TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + + env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \ + --onnx ${{ github.workspace }}/${{ env.SMALL_ONNX }} \ + --model-size $MODEL_SIZE \ + --camera-resolutions $CAMERA_RES \ + --frame-skip $FRAME_SKIP \ + --output ${{ github.workspace }}/${{ env.SMALL_PKL }} + + - name: Chunk small pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.SMALL_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked into {len(targets)} files') + " + + - name: Prepare output + env: + MODEL_NAME: ${{ needs.resolve.outputs.model_name }} + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models" + OUTPUT_DIR="${{ github.workspace }}/small_output" + PKL_BASE="driving_tinygrad.pkl" + mkdir -p "$OUTPUT_DIR" + + cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/" + cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/" + + python3 "${{ github.workspace }}/release/ci/model_generator.py" \ + --model-dir "$MODELS_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --custom-name "$MODEL_NAME" \ + --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" + + echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt" + + - name: Upload small model artifact + uses: actions/upload-artifact@v4 + with: + name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }} + path: ${{ github.workspace }}/small_output/ + + - name: Upload artifact name file + uses: actions/upload-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: ${{ github.workspace }}/small_output/artifact_name.txt + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + + build_big_model: + needs: resolve + if: ${{ inputs.target == 'big' }} + runs-on: [self-hosted, usbgpu] + env: + BIG_ONNX: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx + BIG_PKL: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull big ONNX via LFS + run: git lfs pull -I "${{ env.BIG_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Wait for chestnut PCIe link + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + python3 -c " + import time + from openpilot.system.hardware.chestnut.flash import link_up + for i in range(10): + if link_up(): + print(f'PCIe link up after {i+1} attempt(s)') + break + time.sleep(1) + else: + raise RuntimeError('Chestnut PCIe link not ready after 10 attempts') + " + + - name: Compile big model with stock compiler + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") + CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)") + + TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" + + env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \ + --onnx ${{ github.workspace }}/${{ env.BIG_ONNX }} \ + --model-size $MODEL_SIZE \ + --camera-resolutions $CAMERA_RES \ + --frame-skip $FRAME_SKIP \ + --output ${{ github.workspace }}/${{ env.BIG_PKL }} + + - name: Chunk big pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.BIG_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked into {len(targets)} files') + " + + - name: Prepare output + env: + MODEL_NAME: ${{ needs.resolve.outputs.model_name }} + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models" + OUTPUT_DIR="${{ github.workspace }}/big_output" + PKL_BASE="big_driving_tinygrad.pkl" + mkdir -p "$OUTPUT_DIR" + + cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/" + cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/" + + python3 "${{ github.workspace }}/release/ci/model_generator.py" \ + --model-dir "$MODELS_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --custom-name "$MODEL_NAME" \ + --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" + + echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt" + + - name: Upload big model artifact + uses: actions/upload-artifact@v4 + with: + name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }} + path: ${{ github.workspace }}/big_output/ + + - name: Upload artifact name file + uses: actions/upload-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: ${{ github.workspace }}/big_output/artifact_name.txt + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable upload_defaults: - needs: [ resolve, build_driving_model, build_dm_model ] - if: ${{ !cancelled() && needs.build_driving_model.result == 'success' && (inputs.target != 'small' || needs.build_dm_model.result == 'success') }} + needs: [ resolve, build_small_model, build_big_model, build_dm_model ] + if: | + ${{ + !cancelled() && + (inputs.target == 'big' && needs.build_big_model.result == 'success' || + inputs.target == 'small' && needs.build_small_model.result == 'success') && + (inputs.target != 'small' || needs.build_dm_model.result == 'success') + }} runs-on: ubuntu-24.04 permissions: id-token: write @@ -176,7 +399,7 @@ jobs: metadata = { 'bundles': [{ 'short_name': 'DMMODEL', - 'display_name': 'dmonitoring_model', + 'display_name': 'dmonitoring_model (${{ needs.resolve.outputs.dm_onnx_date }})', 'ref': '${{ needs.resolve.outputs.dm_onnx_ref }}', 'runner': 'tinygrad', 'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'), @@ -202,7 +425,7 @@ jobs: --model-dir dm_output \ --onnx-path "${{ env.DM_ONNX }}" \ --onnx-ref "${{ needs.resolve.outputs.dm_onnx_ref }}" \ - --model-name "dmonitoring_model" \ + --model-name "dmonitoring_model (${{ needs.resolve.outputs.dm_onnx_date }})" \ --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \ --run-number "${{ github.run_number }}" diff --git a/.github/workflows/sunnypilot-build-prebuilt.yaml b/.github/workflows/sunnypilot-build-prebuilt.yaml index c7232505ee..2f824be964 100644 --- a/.github/workflows/sunnypilot-build-prebuilt.yaml +++ b/.github/workflows/sunnypilot-build-prebuilt.yaml @@ -165,7 +165,7 @@ jobs: scons -j1 cache_dir="$SCONS_CACHE" --minimal \ openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd echo "Building rest of sunnypilot" - /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal + SKIP_TINYGRAD_COMPILE=1 /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal touch ${BUILD_DIR}/prebuilt if [[ "${{ runner.debug }}" == "1" ]]; then ls -la ${BUILD_DIR} @@ -242,12 +242,13 @@ jobs: echo "HF defaults match repo ONNX" else echo "No matching model on HF — triggering build" + TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=big echo "Waiting for build to start..." sleep 120 - RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId') + RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --created=">$TRIGGER_TIME" --limit=1 --json databaseId --jq '.[0].databaseId') if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then echo "::error::Failed to find build-default-models run" exit 1 @@ -276,21 +277,99 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + prepare_small_models: + needs: [ prepare_strategy ] + runs-on: ubuntu-24.04 + outputs: + driving_onnx_sha256: ${{ steps.resolve.outputs.driving_onnx_sha256 }} + dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }} + env: + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/small + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref || github.ref_name }} + submodules: recursive + + - run: git lfs pull -I "openpilot/selfdrive/modeld/models/driving_supercombo.onnx,openpilot/selfdrive/modeld/models/dmonitoring_model.onnx" + + - name: Check HF defaults and build if needed + id: resolve + run: | + DRIVING_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/driving_supercombo.onnx" | cut -d' ' -f1) + DM_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/dmonitoring_model.onnx" | cut -d' ' -f1) + TINYGRAD_REF=$(PYTHONPATH=${{ github.workspace }} python3 openpilot/sunnypilot/models/tinygrad_ref.py) + echo "driving_onnx_sha256=$DRIVING_HASH" >> $GITHUB_OUTPUT + echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT + echo "Driving ONNX hash: $DRIVING_HASH" + echo "DM ONNX hash: $DM_HASH" + echo "tinygrad ref: $TINYGRAD_REF" + + JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" + + check_defaults() { + DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + [ -n "$DRIVING" ] && [ "$DRIVING" != "null" ] || return 1 + DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + [ -n "$DM" ] && [ "$DM" != "null" ] || return 1 + } + + if check_defaults; then + echo "HF defaults match repo ONNX hashes and tinygrad ref" + else + echo "No matching models on HF — triggering build" + TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) + gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=small + + echo "Waiting for build to start..." + sleep 120 + + RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --created=">$TRIGGER_TIME" --limit=1 --json databaseId --jq '.[0].databaseId') + if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then + echo "::error::Failed to find build-default-models run" + exit 1 + fi + + echo "Waiting for run $RUN_ID..." + gh run watch "$RUN_ID" + + CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion') + if [ "$CONCLUSION" != "success" ]; then + echo "::error::build-default-models failed: $CONCLUSION" + exit 1 + fi + + if ! check_defaults; then + echo "::error::HF defaults still don't match after build" + exit 1 + fi + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cancel run on failure + if: failure() + run: gh run cancel ${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + publish: concurrency: - # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name. - # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other. - # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time. group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }} cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }} if: ${{ always() && !cancelled() && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && + needs.prepare_small_models.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) && (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success') }} - needs: [ build, prepare_strategy, prepare_chestnut ] + needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_models ] runs-on: ubuntu-24.04 environment: ${{ needs.prepare_strategy.outputs.environment }} steps: @@ -306,6 +385,41 @@ jobs: mkdir -p ${{ env.OUTPUT_DIR }} tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} + - name: Download small model chunks from HF + env: + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/small + run: | + set -o pipefail + JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" + DEFAULTS=$(curl -fsSL "$JSON_URL") + MODELS_DIR="${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models" + + download_model_chunks() { + local ONNX_HASH="$1" + local CANONICAL="$2" + BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)') + ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact') + BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||') + NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length') + + echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do + CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true) + if [ -z "$CHUNK_IDX" ]; then + echo "::error::Failed to parse chunk index from: $CHUNK_NAME" + exit 1 + fi + CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}" + ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))") + echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK" + curl -fsSL -o "${MODELS_DIR}/${CANONICAL_CHUNK}" "$ENCODED_URL" + done + echo "$NUM_CHUNKS" > "${MODELS_DIR}/${CANONICAL}.chunkmanifest" + } + + download_model_chunks "${{ needs.prepare_small_models.outputs.driving_onnx_sha256 }}" "driving_tinygrad.pkl" + download_model_chunks "${{ needs.prepare_small_models.outputs.dm_onnx_sha256 }}" "dmonitoring_model_tinygrad.pkl" + - name: Prepare chestnut output if: ${{ needs.prepare_chestnut.result == 'success' }} run: | @@ -393,6 +507,7 @@ jobs: - build - publish - prepare_chestnut + - prepare_small_models runs-on: ubuntu-24.04 if: ${{ (always() && !cancelled() && !failure()) && needs.publish.result == 'success' diff --git a/openpilot/selfdrive/modeld/SConscript b/openpilot/selfdrive/modeld/SConscript index 30a31aae27..19ce7d5e00 100644 --- a/openpilot/selfdrive/modeld/SConscript +++ b/openpilot/selfdrive/modeld/SConscript @@ -73,44 +73,45 @@ compile_modeld_script = [ model_w, model_h = MEDMODEL_INPUT_SIZE frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ -for usbgpu in [False, True] if USBGPU else [False]: - target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath - # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU - file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags) - driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath) - camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS) - # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it. - taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else '' - cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py ' - f'--model-size {model_w}x{model_h} ' - f'--camera-resolutions {camera_res_args} ' - f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} ' - f'--output {target_pkl_path} --frame-skip {frame_skip}') - onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps) - chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum)) - def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets): - from openpilot.system.hardware.chestnut.flash import link_up - # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars - for _ in range(10): - if link_up(): - break - time.sleep(1) - else: - print("Chestnut not ready, skipping big model build") - return - if ret := env.Execute(command): - return ret - chunk_file(pkl, chunks) - def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets): - chunk_file(pkl, chunks) - actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")] - node = lenv.Command( - chunk_targets, - tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file], - actions, - ) - if usbgpu: - lenv.SideEffect(usbgpu_lock, node) +if not os.getenv('SKIP_TINYGRAD_COMPILE'): + for usbgpu in [False, True] if USBGPU else [False]: + target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath + # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU + file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags) + driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath) + camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS) + # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it. + taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else '' + cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py ' + f'--model-size {model_w}x{model_h} ' + f'--camera-resolutions {camera_res_args} ' + f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} ' + f'--output {target_pkl_path} --frame-skip {frame_skip}') + onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps) + chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum)) + def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets): + from openpilot.system.hardware.chestnut.flash import link_up + # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars + for _ in range(10): + if link_up(): + break + time.sleep(1) + else: + print("Chestnut not ready, skipping big model build") + return + if ret := env.Execute(command): + return ret + chunk_file(pkl, chunks) + def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets): + chunk_file(pkl, chunks) + actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")] + node = lenv.Command( + chunk_targets, + tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file], + actions, + ) + if usbgpu: + lenv.SideEffect(usbgpu_lock, node) # get model metadata fn = File(f"models/dmonitoring_model").abspath @@ -142,4 +143,5 @@ def tg_compile(flags, model_name): Action(do_chunk, " [CHUNK] $TARGET")], ) -tg_compile(tg_flags, 'dmonitoring_model') +if not os.getenv('SKIP_TINYGRAD_COMPILE'): + tg_compile(tg_flags, 'dmonitoring_model')