diff --git a/.github/workflows/build-default-big-model.yaml b/.github/workflows/build-default-big-model.yaml deleted file mode 100644 index 4b05a7977d..0000000000 --- a/.github/workflows/build-default-big-model.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: Build default big model - -on: - workflow_dispatch: - -env: - HF_REPO: sunnypilot/sunnypilot_models_v1 - HF_DEFAULTS_PATH: models/defaults/big - -jobs: - resolve_name: - runs-on: ubuntu-24.04 - outputs: - model_name: ${{ steps.name.outputs.model_name }} - onnx_ref: ${{ steps.name.outputs.onnx_ref }} - steps: - - uses: actions/checkout@v4 - - id: name - run: | - NAME=$(PYTHONPATH=${{ github.workspace }} python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)") - ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx) - echo "model_name=${NAME}" >> $GITHUB_OUTPUT - echo "onnx_ref=$ONNX_REF" >> $GITHUB_OUTPUT - - build_model: - needs: resolve_name - uses: ./.github/workflows/sunnypilot-build-model.yaml - with: - upstream_branch: ${{ needs.resolve_name.outputs.onnx_ref }} - custom_name: ${{ needs.resolve_name.outputs.model_name }} - target_hardware: usbgpu - secrets: inherit - - upload_defaults: - needs: [ resolve_name, build_model ] - runs-on: ubuntu-24.04 - permissions: - id-token: write - contents: write - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" - - - name: Install huggingface_hub - run: pip install --upgrade "huggingface_hub>=0.22.0" - - - name: Download artifact name - uses: actions/download-artifact@v4 - with: - name: artifact-name-${{ needs.resolve_name.outputs.model_name }} - path: artifact_name - - - name: Read artifact name - id: artifact - run: | - ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt) - echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT - - - name: Download model artifact - uses: actions/download-artifact@v4 - with: - name: ${{ steps.artifact.outputs.artifact_name }} - path: output - - - name: Upload to HF and update default_models.json - env: - HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} - ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }} - run: | - rm -f output/artifact_name.txt - export PYTHONPATH=$(pwd) - python3 release/ci/upload_default_model.py \ - --hf-repo "${{ env.HF_REPO }}" \ - --hf-defaults-path "${{ env.HF_DEFAULTS_PATH }}" \ - --artifact-name "$ARTIFACT_NAME" \ - --model-dir output \ - --onnx-path "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" \ - --onnx-ref "${{ needs.resolve_name.outputs.onnx_ref }}" \ - --model-name "${{ needs.resolve_name.outputs.model_name }}" \ - --tinygrad-ref "$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)" \ - --run-number "${{ github.run_number }}" diff --git a/.github/workflows/build-default-models.yaml b/.github/workflows/build-default-models.yaml new file mode 100644 index 0000000000..bf00845e27 --- /dev/null +++ b/.github/workflows/build-default-models.yaml @@ -0,0 +1,501 @@ +name: Build default models + +on: + workflow_dispatch: + inputs: + target: + description: 'Model target to build' + required: true + type: choice + options: + - small + - big + - dm + workflow_call: + inputs: + target: + description: 'Model target to build (small, big, or dm)' + required: true + type: string + +concurrency: + group: build-default-models-${{ inputs.target }} + cancel-in-progress: false + +env: + HF_REPO: sunnypilot/sunnypilot_models_v1 + +jobs: + resolve: + runs-on: ubuntu-24.04 + outputs: + model_name: ${{ steps.resolve.outputs.model_name }} + onnx_ref: ${{ steps.resolve.outputs.onnx_ref }} + onnx_path: ${{ steps.resolve.outputs.onnx_path }} + hf_defaults_path: ${{ steps.resolve.outputs.hf_defaults_path }} + tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - id: resolve + run: | + export PYTHONPATH=${{ github.workspace }} + + if [ "${{ inputs.target }}" = "big" ]; then + NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)") + ONNX_PATH="openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" + HF_DEFAULTS_PATH="models/defaults/big" + elif [ "${{ inputs.target }}" = "dm" ]; then + ONNX_PATH="openpilot/selfdrive/modeld/models/dmonitoring_model.onnx" + HF_DEFAULTS_PATH="models/defaults/dm" + NAME="dmonitoring_model ($(git log -1 --format=%cd --date=format:'%B %d, %Y' -- "$ONNX_PATH"))" + else + NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL; print(DEFAULT_MODEL)") + ONNX_PATH="openpilot/selfdrive/modeld/models/driving_supercombo.onnx" + HF_DEFAULTS_PATH="models/defaults/small" + fi + + ONNX_REF=$(git log -1 --format='%H' -- "$ONNX_PATH") + TINYGRAD_REF=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py) + if [ -z "$TINYGRAD_REF" ]; then + echo "::error::Failed to resolve tinygrad ref" + exit 1 + fi + + echo "model_name=${NAME}" >> $GITHUB_OUTPUT + echo "onnx_ref=${ONNX_REF}" >> $GITHUB_OUTPUT + echo "onnx_path=${ONNX_PATH}" >> $GITHUB_OUTPUT + echo "hf_defaults_path=${HF_DEFAULTS_PATH}" >> $GITHUB_OUTPUT + echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT + + build_small_model: + needs: resolve + if: ${{ inputs.target == 'small' }} + runs-on: [self-hosted, tici] + env: + SMALL_ONNX: openpilot/selfdrive/modeld/models/driving_supercombo.onnx + SMALL_PKL: openpilot/selfdrive/modeld/models/driving_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull ONNX via LFS + run: git lfs pull -I "${{ env.SMALL_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Compile small model with stock compiler + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") + CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)") + + TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + + env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \ + --onnx ${{ github.workspace }}/${{ env.SMALL_ONNX }} \ + --model-size $MODEL_SIZE \ + --camera-resolutions $CAMERA_RES \ + --frame-skip $FRAME_SKIP \ + --output ${{ github.workspace }}/${{ env.SMALL_PKL }} + + - name: Chunk small pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.SMALL_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked into {len(targets)} files') + " + + - name: Prepare output + env: + MODEL_NAME: ${{ needs.resolve.outputs.model_name }} + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models" + OUTPUT_DIR="${{ github.workspace }}/small_output" + PKL_BASE="driving_tinygrad.pkl" + mkdir -p "$OUTPUT_DIR" + + cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/" + cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/" + + python3 "${{ github.workspace }}/release/ci/model_generator.py" \ + --model-dir "$MODELS_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --custom-name "$MODEL_NAME" \ + --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" + + echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt" + + - name: Upload small model artifact + uses: actions/upload-artifact@v4 + with: + name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }} + path: ${{ github.workspace }}/small_output/ + + - name: Upload artifact name file + uses: actions/upload-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: ${{ github.workspace }}/small_output/artifact_name.txt + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + + build_big_model: + needs: resolve + if: ${{ inputs.target == 'big' }} + runs-on: [self-hosted, usbgpu] + env: + BIG_ONNX: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx + BIG_PKL: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull big ONNX via LFS + run: git lfs pull -I "${{ env.BIG_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Wait for chestnut PCIe link + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + python3 -c " + import time + from openpilot.system.hardware.chestnut.flash import link_up + for i in range(10): + if link_up(): + print(f'PCIe link up after {i+1} attempt(s)') + break + time.sleep(1) + else: + raise RuntimeError('Chestnut PCIe link not ready after 10 attempts') + " + + - name: Compile big model with stock compiler + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") + CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)") + + TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" + + env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \ + --onnx ${{ github.workspace }}/${{ env.BIG_ONNX }} \ + --model-size $MODEL_SIZE \ + --camera-resolutions $CAMERA_RES \ + --frame-skip $FRAME_SKIP \ + --output ${{ github.workspace }}/${{ env.BIG_PKL }} + + - name: Chunk big pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.BIG_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked into {len(targets)} files') + " + + - name: Prepare output + env: + MODEL_NAME: ${{ needs.resolve.outputs.model_name }} + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models" + OUTPUT_DIR="${{ github.workspace }}/big_output" + PKL_BASE="big_driving_tinygrad.pkl" + mkdir -p "$OUTPUT_DIR" + + cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/" + cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/" + + python3 "${{ github.workspace }}/release/ci/model_generator.py" \ + --model-dir "$MODELS_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --custom-name "$MODEL_NAME" \ + --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" + + echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt" + + - name: Upload big model artifact + uses: actions/upload-artifact@v4 + with: + name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }} + path: ${{ github.workspace }}/big_output/ + + - name: Upload artifact name file + uses: actions/upload-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: ${{ github.workspace }}/big_output/artifact_name.txt + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + + upload_defaults: + needs: [ resolve, build_small_model, build_big_model, build_dm_model ] + if: | + ${{ + !cancelled() && + (inputs.target == 'big' && needs.build_big_model.result == 'success' || + inputs.target == 'small' && needs.build_small_model.result == 'success' || + inputs.target == 'dm' && needs.build_dm_model.result == 'success') + }} + runs-on: ubuntu-24.04 + permissions: + id-token: write + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Pull ONNX via LFS + run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}" + + - name: Install huggingface_hub + run: pip install --upgrade "huggingface_hub>=0.22.0" + + - name: Download artifact name + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} + uses: actions/download-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: artifact_name + + - name: Read artifact name + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} + id: artifact + run: | + ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt) + echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT + + - name: Download model artifact + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} + uses: actions/download-artifact@v4 + with: + name: ${{ steps.artifact.outputs.artifact_name }} + path: output + + - name: Upload model to HF + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} + env: + HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} + ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }} + run: | + rm -f output/artifact_name.txt + export PYTHONPATH=$(pwd) + python3 release/ci/upload_default_model.py \ + --hf-repo "${{ env.HF_REPO }}" \ + --hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \ + --artifact-name "$ARTIFACT_NAME" \ + --model-dir output \ + --onnx-path "${{ needs.resolve.outputs.onnx_path }}" \ + --onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \ + --model-name "${{ needs.resolve.outputs.model_name }}" \ + --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \ + --run-number "${{ github.run_number }}" + + - name: Download DM artifact + if: ${{ inputs.target == 'dm' }} + uses: actions/download-artifact@v4 + with: + name: dm-model-${{ github.run_number }} + path: dm_output + + - name: Generate DM metadata and upload to HF + if: ${{ inputs.target == 'dm' }} + env: + HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} + run: | + export PYTHONPATH=$(pwd) + python3 -c " + import json, hashlib + from pathlib import Path + from datetime import datetime, UTC + + dm_dir = Path('dm_output') + manifest = list(dm_dir.glob('*.chunkmanifest')) + assert manifest, 'No chunkmanifest found' + pkl_name = manifest[0].name.removesuffix('.chunkmanifest') + num_chunks = int(manifest[0].read_text().strip()) + + chunks = [] + for i in range(num_chunks): + chunk = dm_dir / f'{pkl_name}.chunk{i+1:02d}of{num_chunks:02d}' + chunks.append({ + 'file_name': chunk.name, + 'sha256': hashlib.sha256(chunk.read_bytes()).hexdigest() + }) + + digest = hashlib.sha256() + for c in chunks: + with open(dm_dir / c['file_name'], 'rb') as f: + while block := f.read(1024*1024): + digest.update(block) + + metadata = { + 'bundles': [{ + 'short_name': 'DMMODEL', + 'display_name': '${{ needs.resolve.outputs.model_name }}', + 'ref': '${{ needs.resolve.outputs.onnx_ref }}', + 'runner': 'tinygrad', + 'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'models': [{ + 'type': 'chunked', + 'artifact': { + 'file_name': pkl_name, + 'download_uri': {'url': '', 'sha256': digest.hexdigest()}, + 'chunks': chunks + } + }] + }] + } + with open(dm_dir / 'metadata.json', 'w') as f: + json.dump(metadata, f, indent=2) + print('Generated DM metadata.json') + " + + python3 release/ci/upload_default_model.py \ + --hf-repo "${{ env.HF_REPO }}" \ + --hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \ + --artifact-name "dm-model-${{ github.run_number }}" \ + --model-dir dm_output \ + --onnx-path "${{ needs.resolve.outputs.onnx_path }}" \ + --onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \ + --model-name "${{ needs.resolve.outputs.model_name }}" \ + --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \ + --run-number "${{ github.run_number }}" + + build_dm_model: + needs: resolve + if: ${{ inputs.target == 'dm' }} + runs-on: [self-hosted, tici] + env: + DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx + DM_PKL: openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull DM ONNX via LFS + run: git lfs pull -I "${{ env.DM_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Compile DM model + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + + taskset -c 7 env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/tinygrad_repo/examples/openpilot/compile3.py \ + ${{ github.workspace }}/${{ env.DM_ONNX }} \ + ${{ github.workspace }}/${{ env.DM_PKL }} + + - name: Chunk DM pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.DM_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked {pkl} into {len(targets)} chunks') + " + + - name: Prepare DM output + run: | + mkdir -p dm_output + cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunk* dm_output/ + cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunkmanifest dm_output/ + + - name: Upload DM artifact + uses: actions/upload-artifact@v4 + with: + name: dm-model-${{ github.run_number }} + path: dm_output/ + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + diff --git a/.github/workflows/download-hf-model-chunks/action.yml b/.github/workflows/download-hf-model-chunks/action.yml new file mode 100644 index 0000000000..01ab5385da --- /dev/null +++ b/.github/workflows/download-hf-model-chunks/action.yml @@ -0,0 +1,66 @@ +name: Download HF model chunks +description: Resolve and download model chunks from HuggingFace in parallel + +inputs: + hf_repo: + description: HuggingFace dataset repo + required: true + models: + description: 'JSON array of {hf_path, onnx_hash, canonical} objects' + required: true + dest_dir: + description: Destination directory for downloaded chunks + required: true + +runs: + using: composite + steps: + - name: Download model chunks + shell: bash + env: + HF_REPO: ${{ inputs.hf_repo }} + MODELS_JSON: ${{ inputs.models }} + DEST_DIR: ${{ inputs.dest_dir }} + run: | + set -eo pipefail + DOWNLOAD_LIST=$(mktemp) + + resolve_chunks() { + local HF_PATH="$1" ONNX_HASH="$2" CANONICAL="$3" DEST_DIR="$4" + local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_PATH}/default_models.json" + local DEFAULTS BUNDLE ARTIFACT BASE_URL NUM_CHUNKS + DEFAULTS=$(curl -fsSL "$JSON_URL") + BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)') + ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact') + BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||') + NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length') + + mkdir -p "$DEST_DIR" + while IFS= read -r CHUNK_NAME; do + CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true) + if [ -z "$CHUNK_IDX" ]; then + echo "::error::Failed to parse chunk index from: $CHUNK_NAME" + return 1 + fi + ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))") + printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}.chunk${CHUNK_IDX}" >> "$DOWNLOAD_LIST" + done < <(echo "$ARTIFACT" | jq -r '.chunks[].file_name') + echo "$NUM_CHUNKS" > "${DEST_DIR}/${CANONICAL}.chunkmanifest" + } + + echo "$MODELS_JSON" | jq -c '.[]' | while IFS= read -r model; do + HF_PATH=$(echo "$model" | jq -r '.hf_path') + ONNX_HASH=$(echo "$model" | jq -r '.onnx_hash') + CANONICAL=$(echo "$model" | jq -r '.canonical') + resolve_chunks "$HF_PATH" "$ONNX_HASH" "$CANONICAL" "$DEST_DIR" + done + + TOTAL=$(wc -l < "$DOWNLOAD_LIST") + echo "Downloading $TOTAL chunks with 8 parallel connections..." + xargs -P8 -d'\n' -I{} bash -c ' + URL="${1%% *}" + DEST="${1#* }" + echo "Downloading $(basename "$DEST")" + curl -fsSL --retry 3 --retry-delay 5 -o "$DEST" "$URL" + ' _ {} < "$DOWNLOAD_LIST" + rm -f "$DOWNLOAD_LIST" diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index bc132ae1bf..5c2e3bf204 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -188,7 +188,7 @@ jobs: if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then echo "USBGPU build" export USBGPU=1 - TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" + TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" else echo "QCOM build" diff --git a/.github/workflows/sunnypilot-build-prebuilt.yaml b/.github/workflows/sunnypilot-build-prebuilt.yaml index 84fe6b3cc1..c93dbe02c1 100644 --- a/.github/workflows/sunnypilot-build-prebuilt.yaml +++ b/.github/workflows/sunnypilot-build-prebuilt.yaml @@ -39,6 +39,8 @@ jobs: include_big_model: ${{ steps.strategy.outputs.include_big_model }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Extract deploy strategy id: strategy run: | @@ -96,6 +98,8 @@ jobs: }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Wait for Tests uses: ./.github/workflows/wait-for-action # Path to where you place the action with: @@ -119,6 +123,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + fetch-depth: 1 submodules: recursive ref: ${{ env.SOURCE_BRANCH }} repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }} @@ -165,7 +170,7 @@ jobs: scons -j1 cache_dir="$SCONS_CACHE" --minimal \ openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd echo "Building rest of sunnypilot" - /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal + SKIP_TINYGRAD_COMPILE=1 /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal touch ${BUILD_DIR}/prebuilt if [[ "${{ runner.debug }}" == "1" ]]; then ls -la ${BUILD_DIR} @@ -214,59 +219,174 @@ jobs: outputs: onnx_sha256: ${{ steps.resolve.outputs.onnx_sha256 }} env: + GH_REPO: ${{ github.repository }} HF_REPO: sunnypilot/sunnypilot_models_v1 HF_DEFAULTS_PATH: models/defaults/big steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref || github.ref_name }} - - - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" - - - name: Check HF defaults and build if needed + - name: Resolve ONNX hash and tinygrad ref via API id: resolve run: | - ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1) - echo "Repo ONNX hash: $ACTUAL_ONNX_HASH" - echo "onnx_sha256=$ACTUAL_ONNX_HASH" >> $GITHUB_OUTPUT + REF="${{ github.head_ref || github.ref_name }}" + + ONNX_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2) + echo "ONNX hash: $ONNX_HASH" + echo "onnx_sha256=$ONNX_HASH" >> $GITHUB_OUTPUT + + TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha') + echo "tinygrad ref: $TINYGRAD_REF" JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" - check_hash() { + check_defaults() { DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 - BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) [ -n "$BUNDLE" ] && [ "$BUNDLE" != "null" ] } - if check_hash; then - echo "HF defaults match repo ONNX" - else - echo "No matching model on HF — triggering build" - gh workflow run build-default-big-model.yaml --ref "${{ github.head_ref || github.ref_name }}" - - echo "Waiting for build to start..." - sleep 120 - - RUN_ID=$(gh run list --workflow=build-default-big-model.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId') - if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then - echo "::error::Failed to find build-default-big-model run" - exit 1 - fi - - echo "Waiting for run $RUN_ID..." - gh run watch "$RUN_ID" - - CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion') - if [ "$CONCLUSION" != "success" ]; then - echo "::error::build-default-big-model failed: $CONCLUSION" - exit 1 - fi - - if ! check_hash; then - echo "::error::HF defaults still don't match after build" - exit 1 - fi + if check_defaults; then + echo "HF defaults match repo ONNX hash and tinygrad ref" + exit 0 fi + + echo "No matching model on HF — dispatching build" + gh workflow run build-default-models.yaml --ref "$REF" -f target=big + + echo "Polling HF for big model availability..." + for i in $(seq 1 90); do + sleep 30 + if check_defaults; then + echo "Big model available on HF after $((i * 30))s" + exit 0 + fi + echo "Poll $i/90: not yet available" + done + + echo "::error::Big model not available on HF after 45 minutes" + exit 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cancel run on failure + if: failure() + run: gh run cancel ${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + prepare_small_model: + needs: [ prepare_strategy ] + runs-on: ubuntu-24.04 + outputs: + driving_onnx_sha256: ${{ steps.resolve.outputs.driving_onnx_sha256 }} + env: + GH_REPO: ${{ github.repository }} + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/small + steps: + - name: Resolve ONNX hash and tinygrad ref via API + id: resolve + run: | + REF="${{ github.head_ref || github.ref_name }}" + + DRIVING_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/driving_supercombo.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2) + echo "Driving ONNX hash: $DRIVING_HASH" + echo "driving_onnx_sha256=$DRIVING_HASH" >> $GITHUB_OUTPUT + + TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha') + echo "tinygrad ref: $TINYGRAD_REF" + + JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" + + check_defaults() { + DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + [ -n "$DRIVING" ] && [ "$DRIVING" != "null" ] || return 1 + } + + if check_defaults; then + echo "HF defaults match repo ONNX hash and tinygrad ref" + exit 0 + fi + + echo "No matching model on HF — dispatching build" + gh workflow run build-default-models.yaml --ref "$REF" -f target=small + + echo "Polling HF for model availability..." + for i in $(seq 1 60); do + sleep 30 + if check_defaults; then + echo "Model available on HF after $((i * 30))s" + exit 0 + fi + echo "Poll $i/60: not yet available" + done + + echo "::error::Small driving model not available on HF after 30 minutes" + exit 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cancel run on failure + if: failure() + run: gh run cancel ${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + prepare_dm_model: + needs: [ prepare_strategy ] + runs-on: ubuntu-24.04 + outputs: + dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }} + env: + GH_REPO: ${{ github.repository }} + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/dm + steps: + - name: Resolve ONNX hash and tinygrad ref via API + id: resolve + run: | + REF="${{ github.head_ref || github.ref_name }}" + + DM_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/dmonitoring_model.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2) + echo "DM ONNX hash: $DM_HASH" + echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT + + TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha') + echo "tinygrad ref: $TINYGRAD_REF" + + JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" + + check_defaults() { + DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + [ -n "$DM" ] && [ "$DM" != "null" ] || return 1 + } + + if check_defaults; then + echo "HF defaults match DM ONNX hash and tinygrad ref" + exit 0 + fi + + echo "No matching DM model on HF — dispatching build" + gh workflow run build-default-models.yaml --ref "$REF" -f target=dm + + echo "Polling HF for DM model availability..." + for i in $(seq 1 60); do + sleep 30 + if check_defaults; then + echo "DM model available on HF after $((i * 30))s" + exit 0 + fi + echo "Poll $i/60: not yet available" + done + + echo "::error::DM model not available on HF after 30 minutes" + exit 1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -278,23 +398,24 @@ jobs: publish: concurrency: - # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name. - # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other. - # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time. group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }} cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }} if: ${{ always() && !cancelled() && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && + needs.prepare_small_model.result == 'success' && + needs.prepare_dm_model.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) && (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success') }} - needs: [ build, prepare_strategy, prepare_chestnut ] + needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ] runs-on: ubuntu-24.04 environment: ${{ needs.prepare_strategy.outputs.environment }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Download prebuilt artifact uses: actions/download-artifact@v4 @@ -306,43 +427,16 @@ jobs: mkdir -p ${{ env.OUTPUT_DIR }} tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} - - name: Prepare chestnut output - if: ${{ needs.prepare_chestnut.result == 'success' }} - run: | - mkdir -p "${{ github.workspace }}/chestnut_output" - tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output" - - - name: Download big model chunks from HF - if: ${{ needs.prepare_chestnut.result == 'success' }} - env: - HF_REPO: sunnypilot/sunnypilot_models_v1 - HF_DEFAULTS_PATH: models/defaults/big - run: | - ONNX_HASH="${{ needs.prepare_chestnut.outputs.onnx_sha256 }}" - JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" - DEFAULTS=$(curl -fsSL "$JSON_URL") - BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)') - - mkdir -p big_model_chunks - ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact') - BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||') - NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length') - - CANONICAL="big_driving_tinygrad.pkl" - echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do - CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+') - CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}" - ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))") - echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK" - curl -fsSL -o "big_model_chunks/${CANONICAL_CHUNK}" "$ENCODED_URL" - done - - echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest" - - - name: Inject big model into chestnut - if: ${{ needs.prepare_chestnut.result == 'success' }} - run: | - cp big_model_chunks/* "${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models/" + - name: Download model chunks from HF + uses: ./.github/workflows/download-hf-model-chunks + with: + hf_repo: sunnypilot/sunnypilot_models_v1 + dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models + models: | + [ + {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"}, + {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"} + ] - name: Configure Git run: | @@ -364,22 +458,6 @@ jobs: "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" - - name: Publish chestnut branch - if: ${{ needs.prepare_chestnut.result == 'success' }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut" - CHESTNUT_DIR="${{ github.workspace }}/chestnut_output" - - ${{ env.CI_DIR }}/publish.sh \ - "${{ github.workspace }}" \ - "$CHESTNUT_DIR" \ - "$CHESTNUT_BRANCH" \ - "${{ needs.prepare_strategy.outputs.version }}" \ - "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ - "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" - - name: Tag ${{ needs.prepare_strategy.outputs.environment }} if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }} run: | @@ -387,12 +465,77 @@ jobs: git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}." git push -f origin ${TAG} + publish_chestnut: + concurrency: + group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}-chestnut + cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }} + if: ${{ + always() && !cancelled() && + needs.build.result == 'success' && + needs.prepare_strategy.result == 'success' && + needs.prepare_small_model.result == 'success' && + needs.prepare_dm_model.result == 'success' && + needs.prepare_chestnut.result == 'success' && + (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) + }} + needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ] + runs-on: ubuntu-24.04 + environment: ${{ needs.prepare_strategy.outputs.environment }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Download prebuilt artifact + uses: actions/download-artifact@v4 + with: + name: prebuilt + + - name: Untar prebuilt + run: | + mkdir -p ${{ env.OUTPUT_DIR }} + tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} + + - name: Download model chunks from HF + uses: ./.github/workflows/download-hf-model-chunks + with: + hf_repo: sunnypilot/sunnypilot_models_v1 + dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models + models: | + [ + {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"}, + {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"}, + {"hf_path": "models/defaults/big", "onnx_hash": "${{ needs.prepare_chestnut.outputs.onnx_sha256 }}", "canonical": "big_driving_tinygrad.pkl"} + ] + + - name: Configure Git + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + + - name: Publish chestnut branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut" + + ${{ env.CI_DIR }}/publish.sh \ + "${{ github.workspace }}" \ + "${{ env.OUTPUT_DIR }}" \ + "$CHESTNUT_BRANCH" \ + "${{ needs.prepare_strategy.outputs.version }}" \ + "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ + "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" + notify: needs: - prepare_strategy - build - publish + - publish_chestnut - prepare_chestnut + - prepare_small_model + - prepare_dm_model runs-on: ubuntu-24.04 if: ${{ (always() && !cancelled() && !failure()) && needs.publish.result == 'success' @@ -400,6 +543,8 @@ jobs: && (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Prepare notification message id: message diff --git a/openpilot/selfdrive/modeld/SConscript b/openpilot/selfdrive/modeld/SConscript index 30a31aae27..19ce7d5e00 100644 --- a/openpilot/selfdrive/modeld/SConscript +++ b/openpilot/selfdrive/modeld/SConscript @@ -73,44 +73,45 @@ compile_modeld_script = [ model_w, model_h = MEDMODEL_INPUT_SIZE frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ -for usbgpu in [False, True] if USBGPU else [False]: - target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath - # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU - file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags) - driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath) - camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS) - # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it. - taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else '' - cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py ' - f'--model-size {model_w}x{model_h} ' - f'--camera-resolutions {camera_res_args} ' - f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} ' - f'--output {target_pkl_path} --frame-skip {frame_skip}') - onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps) - chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum)) - def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets): - from openpilot.system.hardware.chestnut.flash import link_up - # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars - for _ in range(10): - if link_up(): - break - time.sleep(1) - else: - print("Chestnut not ready, skipping big model build") - return - if ret := env.Execute(command): - return ret - chunk_file(pkl, chunks) - def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets): - chunk_file(pkl, chunks) - actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")] - node = lenv.Command( - chunk_targets, - tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file], - actions, - ) - if usbgpu: - lenv.SideEffect(usbgpu_lock, node) +if not os.getenv('SKIP_TINYGRAD_COMPILE'): + for usbgpu in [False, True] if USBGPU else [False]: + target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath + # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU + file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags) + driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath) + camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS) + # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it. + taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else '' + cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py ' + f'--model-size {model_w}x{model_h} ' + f'--camera-resolutions {camera_res_args} ' + f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} ' + f'--output {target_pkl_path} --frame-skip {frame_skip}') + onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps) + chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum)) + def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets): + from openpilot.system.hardware.chestnut.flash import link_up + # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars + for _ in range(10): + if link_up(): + break + time.sleep(1) + else: + print("Chestnut not ready, skipping big model build") + return + if ret := env.Execute(command): + return ret + chunk_file(pkl, chunks) + def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets): + chunk_file(pkl, chunks) + actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")] + node = lenv.Command( + chunk_targets, + tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file], + actions, + ) + if usbgpu: + lenv.SideEffect(usbgpu_lock, node) # get model metadata fn = File(f"models/dmonitoring_model").abspath @@ -142,4 +143,5 @@ def tg_compile(flags, model_name): Action(do_chunk, " [CHUNK] $TARGET")], ) -tg_compile(tg_flags, 'dmonitoring_model') +if not os.getenv('SKIP_TINYGRAD_COMPILE'): + tg_compile(tg_flags, 'dmonitoring_model') diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 85ae57c078..17687908c8 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -7,6 +7,7 @@ See the LICENSE.md file in the root directory for more details. """ import argparse +import math import os import tempfile import time @@ -66,14 +67,15 @@ def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tu if desire_key: shapes['desire'] = (input_shapes[desire_key][2],) - if is_supercombo and 'features_buffer' in input_shapes: - fb = input_shapes['features_buffer'] - shapes['prev_feat'] = (fb[0], fb[2]) - for key, shape in input_shapes.items(): if key not in (desire_key, 'features_buffer') and 'img' not in key: shapes[key] = tuple(shape) + if is_supercombo and 'features_buffer' in input_shapes: + fb = input_shapes['features_buffer'] + feat_dim = math.prod(fb[2:]) + shapes['prev_feat'] = (fb[0], feat_dim) + sizes = [int(np.prod(size)) for size in shapes.values()] return shapes, sizes @@ -117,8 +119,9 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D } if features_buffer: + feat_dim = math.prod(features_buffer[2:]) feat_q_len = frame_skip * features_buffer[1] if is_supercombo else frame_skip * (features_buffer[1] - 1) + 1 - queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], features_buffer[2]), + queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], feat_dim), dtype=np.float32), device=device).contiguous().realize() queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')}) @@ -183,14 +186,14 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, warped_dev = warped.to(Device.DEFAULT) Tensor.realize(packed_npy_inputs_dev, warped_dev) - img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn).realize() - big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn).realize() + img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn) + big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn) unpacked_tensors = [tensor.reshape(shape) for tensor, shape in zip(packed_npy_inputs_dev.split(npy_sizes), npy_shapes.values(), strict=True)] unpacked_dict = dict(zip(npy_shapes.keys(), unpacked_tensors, strict=True)) desire_dev = unpacked_dict['desire'] - desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn).realize() + desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn) inputs = {desire_key: desire_buf} for key, tensor_val in unpacked_dict.items(): @@ -199,7 +202,7 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, if 'prev_feat' in unpacked_dict: prev_feat_dev = unpacked_dict['prev_feat'] - inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).realize() + inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).reshape(input_shapes['features_buffer']) if vision_runner: vision_out_cast = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32').realize() @@ -211,7 +214,7 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, inputs.update({road_key: img, wide_key: big_img}) if 'features_buffer' not in inputs: - inputs['features_buffer'] = sample_skip_fn(feat_q) + inputs['features_buffer'] = sample_skip_fn(feat_q).reshape(input_shapes['features_buffer']) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32').realize() if 'features_buffer' not in inputs and features_slice is not None: diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py b/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py index 96bfb42638..86974b14f1 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py @@ -195,3 +195,85 @@ class TestReadFileChunkedToDisk(OpenpilotTestCase): assert out.parent == Path(d) assert out.read_bytes() == payload + + +class Test4DFeaturesBuffer(OpenpilotTestCase): + def test_get_policy_npy_shapes_4d(self): + from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes + input_shapes = { + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 32, 512), # compare 4d to 3d for regression + 'traffic_convention': (1, 2), + 'action_t': (1, 2) + } + shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True) + assert shapes['prev_feat'] == (1, 16384) + assert sizes == [8, 2, 2, 16384] + + def test_get_policy_npy_shapes_3d(self): + from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes + input_shapes = { + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 512), + 'traffic_convention': (1, 2), + 'action_t': (1, 2) + } + shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True) + assert shapes['prev_feat'] == (1, 512) + assert sizes == [8, 2, 2, 512] + + +class TestStockCompileModeldEquivalence(OpenpilotTestCase): + def test_get_policy_npy_shapes_matches_stock(self): + from openpilot.selfdrive.modeld.compile_modeld import get_policy_npy_shapes as stock_get_policy_npy_shapes + from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes as sunny_get_policy_npy_shapes + + stock_input_shapes = { + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 512), # see below comment + 'traffic_convention': (1, 2), + 'action_t': (1, 2), + } + + stock_shapes, stock_sizes = stock_get_policy_npy_shapes(stock_input_shapes) + sunny_shapes, sunny_sizes = sunny_get_policy_npy_shapes(stock_input_shapes, is_supercombo=True) + + assert sunny_shapes == stock_shapes + assert sunny_sizes == stock_sizes + assert sunny_shapes['prev_feat'] == (1, 512) + + def test_make_input_queues_full_stock_equivalence(self): + from openpilot.selfdrive.modeld.compile_modeld import make_input_queues as stock_make_input_queues + from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues as sunny_make_supercombo_input_queues + input_shapes = { + 'img': (1, 12, 128, 256), + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 512), # when https://github.com/commaai/openpilot/pull/38681 merges, update to 1,24,32,512 + 'traffic_convention': (1, 2), + 'action_t': (1, 2), + } + frame_skip = 4 + + stock_queues, stock_npy = stock_make_input_queues(input_shapes, frame_skip, device='NPY') + sunny_queues, sunny_npy = sunny_make_supercombo_input_queues(input_shapes, frame_skip, device='NPY') + assert set(sunny_queues.keys()) == set(stock_queues.keys()) + for key in stock_queues: + assert sunny_queues[key].shape == stock_queues[key].shape, \ + f"Queue shape mismatch for {key}: sunny {sunny_queues[key].shape} != stock {stock_queues[key].shape}" + assert set(sunny_npy.keys()) == set(stock_npy.keys()) + for key in stock_npy: + assert sunny_npy[key].shape == stock_npy[key].shape, \ + f"Numpy array shape mismatch for {key}: sunny {sunny_npy[key].shape} != stock {stock_npy[key].shape}" + + def test_make_warp_queues_stock_equivalence(self): + from openpilot.selfdrive.modeld.compile_modeld import make_warp_input_queues as stock_make_warp_queues + from openpilot.sunnypilot.modeld_v2.compile_modeld import make_warp_queues as sunny_make_warp_queues + stock_vision_shapes = {'img': (1, 12, 128, 256)} # for now? + stock_queues, stock_npy = stock_make_warp_queues(stock_vision_shapes, frame_skip=4, device='NPY') + sunny_queues, sunny_npy = sunny_make_warp_queues(device='NPY') + + assert set(sunny_npy.keys()) == set(stock_npy.keys()) == {'tfm', 'big_tfm'} + for key in sunny_npy: + assert sunny_npy[key].shape == stock_npy[key].shape == (3, 3) + + diff --git a/openpilot/sunnypilot/models/fetcher.py b/openpilot/sunnypilot/models/fetcher.py index 773eb5b95c..c9e86edd0c 100644 --- a/openpilot/sunnypilot/models/fetcher.py +++ b/openpilot/sunnypilot/models/fetcher.py @@ -138,8 +138,8 @@ class ModelCache: class ModelFetcher: """Handles fetching and caching of model data from remote source""" - MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v20.json" - MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v21.json" + MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v21.json" + MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v22.json" def __init__(self, params: Params): self.params = params diff --git a/openpilot/sunnypilot/models/helpers.py b/openpilot/sunnypilot/models/helpers.py index b5c97467d3..d0fb2e37ec 100644 --- a/openpilot/sunnypilot/models/helpers.py +++ b/openpilot/sunnypilot/models/helpers.py @@ -18,7 +18,7 @@ from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRai from openpilot.common.hardware.hw import Paths # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO -REQUIRED_JSON_VERSION = 17 +REQUIRED_JSON_VERSION = 18 CUSTOM_MODEL_PATH = Paths.model_root() METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl' diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py index ff9be64783..2d35d319c2 100755 --- a/release/ci/model_generator.py +++ b/release/ci/model_generator.py @@ -136,7 +136,7 @@ def generate_chunked_model(driving_pkl: Path) -> dict: def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown", - onnx_sha256=None) -> None: + onnx_sha256=None, is_big=False) -> None: bundle_json = { "short_name": short_name, "display_name": custom_name or upstream_branch, @@ -149,6 +149,7 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short "generation": "-1", "build_time": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "overrides": {}, + "is_big": is_big, "models": models, } @@ -186,6 +187,8 @@ if __name__ == "__main__": print(f"No driving_tinygrad.pkl found in {_output_dir}", file=sys.stderr) sys.exit(1) + is_big = _driving_pkl.name.startswith('big_') + if _pkl: new_pkl = _output_dir / f"driving_{_pkl}_tinygrad.pkl" if not new_pkl.exists(): @@ -196,4 +199,4 @@ if __name__ == "__main__": _model_metadata = generate_chunked_model(_driving_pkl) _onnx_sha256 = _hash_onnx_files(Path(args.model_dir)) create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch, - onnx_sha256=_onnx_sha256) + onnx_sha256=_onnx_sha256, is_big=is_big) diff --git a/release/ci/publish.sh b/release/ci/publish.sh index fd1a61a87c..4b328a035c 100755 --- a/release/ci/publish.sh +++ b/release/ci/publish.sh @@ -47,7 +47,7 @@ git rm -rf $OUTPUT_DIR/.git || true # Doing cleanup, but it might fail if the .g git remote remove origin || true # ensure cleanup git remote add origin $GIT_ORIGIN #git push origin -d $DEV_BRANCH || true # Ensuring we delete the remote branch if it exists as we are wiping it out -git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH) +git fetch --depth 1 origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH) echo "[-] committing version $VERSION T=$SECONDS" git add -f .