diff --git a/.github/workflows/build-default-models.yaml b/.github/workflows/build-default-models.yaml index 55efda4e4d..bf00845e27 100644 --- a/.github/workflows/build-default-models.yaml +++ b/.github/workflows/build-default-models.yaml @@ -10,13 +10,18 @@ on: options: - small - big + - dm workflow_call: inputs: target: - description: 'Model target to build (small or big)' + description: 'Model target to build (small, big, or dm)' required: true type: string +concurrency: + group: build-default-models-${{ inputs.target }} + cancel-in-progress: false + env: HF_REPO: sunnypilot/sunnypilot_models_v1 @@ -28,9 +33,7 @@ jobs: onnx_ref: ${{ steps.resolve.outputs.onnx_ref }} onnx_path: ${{ steps.resolve.outputs.onnx_path }} hf_defaults_path: ${{ steps.resolve.outputs.hf_defaults_path }} - target_hardware: ${{ steps.resolve.outputs.target_hardware }} tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }} - dm_onnx_ref: ${{ steps.resolve.outputs.dm_onnx_ref }} steps: - uses: actions/checkout@v4 with: @@ -44,12 +47,14 @@ jobs: NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)") ONNX_PATH="openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" HF_DEFAULTS_PATH="models/defaults/big" - TARGET_HW="usbgpu" + elif [ "${{ inputs.target }}" = "dm" ]; then + ONNX_PATH="openpilot/selfdrive/modeld/models/dmonitoring_model.onnx" + HF_DEFAULTS_PATH="models/defaults/dm" + NAME="dmonitoring_model ($(git log -1 --format=%cd --date=format:'%B %d, %Y' -- "$ONNX_PATH"))" else NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL; print(DEFAULT_MODEL)") ONNX_PATH="openpilot/selfdrive/modeld/models/driving_supercombo.onnx" HF_DEFAULTS_PATH="models/defaults/small" - TARGET_HW="qcom" fi ONNX_REF=$(git log -1 --format='%H' -- "$ONNX_PATH") @@ -59,65 +64,279 @@ jobs: exit 1 fi - DM_ONNX_REF="" - if [ "${{ inputs.target }}" = "small" ]; then - DM_ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx) - fi - echo "model_name=${NAME}" >> $GITHUB_OUTPUT echo "onnx_ref=${ONNX_REF}" >> $GITHUB_OUTPUT echo "onnx_path=${ONNX_PATH}" >> $GITHUB_OUTPUT echo "hf_defaults_path=${HF_DEFAULTS_PATH}" >> $GITHUB_OUTPUT - echo "target_hardware=${TARGET_HW}" >> $GITHUB_OUTPUT echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT - echo "dm_onnx_ref=${DM_ONNX_REF}" >> $GITHUB_OUTPUT - build_driving_model: + build_small_model: needs: resolve - uses: ./.github/workflows/sunnypilot-build-model.yaml - with: - upstream_branch: ${{ needs.resolve.outputs.onnx_ref }} - custom_name: ${{ needs.resolve.outputs.model_name }} - target_hardware: ${{ needs.resolve.outputs.target_hardware }} - secrets: inherit + if: ${{ inputs.target == 'small' }} + runs-on: [self-hosted, tici] + env: + SMALL_ONNX: openpilot/selfdrive/modeld/models/driving_supercombo.onnx + SMALL_PKL: openpilot/selfdrive/modeld/models/driving_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull ONNX via LFS + run: git lfs pull -I "${{ env.SMALL_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Compile small model with stock compiler + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") + CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)") + + TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + + env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \ + --onnx ${{ github.workspace }}/${{ env.SMALL_ONNX }} \ + --model-size $MODEL_SIZE \ + --camera-resolutions $CAMERA_RES \ + --frame-skip $FRAME_SKIP \ + --output ${{ github.workspace }}/${{ env.SMALL_PKL }} + + - name: Chunk small pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.SMALL_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked into {len(targets)} files') + " + + - name: Prepare output + env: + MODEL_NAME: ${{ needs.resolve.outputs.model_name }} + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models" + OUTPUT_DIR="${{ github.workspace }}/small_output" + PKL_BASE="driving_tinygrad.pkl" + mkdir -p "$OUTPUT_DIR" + + cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/" + cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/" + + python3 "${{ github.workspace }}/release/ci/model_generator.py" \ + --model-dir "$MODELS_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --custom-name "$MODEL_NAME" \ + --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" + + echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt" + + - name: Upload small model artifact + uses: actions/upload-artifact@v4 + with: + name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }} + path: ${{ github.workspace }}/small_output/ + + - name: Upload artifact name file + uses: actions/upload-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: ${{ github.workspace }}/small_output/artifact_name.txt + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + + build_big_model: + needs: resolve + if: ${{ inputs.target == 'big' }} + runs-on: [self-hosted, usbgpu] + env: + BIG_ONNX: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx + BIG_PKL: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull big ONNX via LFS + run: git lfs pull -I "${{ env.BIG_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Wait for chestnut PCIe link + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + python3 -c " + import time + from openpilot.system.hardware.chestnut.flash import link_up + for i in range(10): + if link_up(): + print(f'PCIe link up after {i+1} attempt(s)') + break + time.sleep(1) + else: + raise RuntimeError('Chestnut PCIe link not ready after 10 attempts') + " + + - name: Compile big model with stock compiler + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") + CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)") + + TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" + + env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \ + --onnx ${{ github.workspace }}/${{ env.BIG_ONNX }} \ + --model-size $MODEL_SIZE \ + --camera-resolutions $CAMERA_RES \ + --frame-skip $FRAME_SKIP \ + --output ${{ github.workspace }}/${{ env.BIG_PKL }} + + - name: Chunk big pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.BIG_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked into {len(targets)} files') + " + + - name: Prepare output + env: + MODEL_NAME: ${{ needs.resolve.outputs.model_name }} + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models" + OUTPUT_DIR="${{ github.workspace }}/big_output" + PKL_BASE="big_driving_tinygrad.pkl" + mkdir -p "$OUTPUT_DIR" + + cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/" + cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/" + + python3 "${{ github.workspace }}/release/ci/model_generator.py" \ + --model-dir "$MODELS_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --custom-name "$MODEL_NAME" \ + --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" + + echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt" + + - name: Upload big model artifact + uses: actions/upload-artifact@v4 + with: + name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }} + path: ${{ github.workspace }}/big_output/ + + - name: Upload artifact name file + uses: actions/upload-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: ${{ github.workspace }}/big_output/artifact_name.txt + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable upload_defaults: - needs: [ resolve, build_driving_model, build_dm_model ] - if: ${{ !cancelled() && needs.build_driving_model.result == 'success' && (inputs.target != 'small' || needs.build_dm_model.result == 'success') }} + needs: [ resolve, build_small_model, build_big_model, build_dm_model ] + if: | + ${{ + !cancelled() && + (inputs.target == 'big' && needs.build_big_model.result == 'success' || + inputs.target == 'small' && needs.build_small_model.result == 'success' || + inputs.target == 'dm' && needs.build_dm_model.result == 'success') + }} runs-on: ubuntu-24.04 permissions: id-token: write contents: write - env: - DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx steps: - uses: actions/checkout@v4 - name: Pull ONNX via LFS - run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}${{ inputs.target == 'small' && ',openpilot/selfdrive/modeld/models/dmonitoring_model.onnx' || '' }}" + run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}" - name: Install huggingface_hub run: pip install --upgrade "huggingface_hub>=0.22.0" - - name: Download driving artifact name + - name: Download artifact name + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} uses: actions/download-artifact@v4 with: name: artifact-name-${{ needs.resolve.outputs.model_name }} path: artifact_name - - name: Read driving artifact name + - name: Read artifact name + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} id: artifact run: | ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt) echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT - - name: Download driving model artifact + - name: Download model artifact + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} uses: actions/download-artifact@v4 with: name: ${{ steps.artifact.outputs.artifact_name }} path: output - - name: Upload driving model to HF + - name: Upload model to HF + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} env: HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }} @@ -136,14 +355,14 @@ jobs: --run-number "${{ github.run_number }}" - name: Download DM artifact - if: ${{ inputs.target == 'small' }} + if: ${{ inputs.target == 'dm' }} uses: actions/download-artifact@v4 with: name: dm-model-${{ github.run_number }} path: dm_output - name: Generate DM metadata and upload to HF - if: ${{ inputs.target == 'small' }} + if: ${{ inputs.target == 'dm' }} env: HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} run: | @@ -176,8 +395,8 @@ jobs: metadata = { 'bundles': [{ 'short_name': 'DMMODEL', - 'display_name': 'dmonitoring_model', - 'ref': '${{ needs.resolve.outputs.dm_onnx_ref }}', + 'display_name': '${{ needs.resolve.outputs.model_name }}', + 'ref': '${{ needs.resolve.outputs.onnx_ref }}', 'runner': 'tinygrad', 'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'), 'models': [{ @@ -200,15 +419,15 @@ jobs: --hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \ --artifact-name "dm-model-${{ github.run_number }}" \ --model-dir dm_output \ - --onnx-path "${{ env.DM_ONNX }}" \ - --onnx-ref "${{ needs.resolve.outputs.dm_onnx_ref }}" \ - --model-name "dmonitoring_model" \ + --onnx-path "${{ needs.resolve.outputs.onnx_path }}" \ + --onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \ + --model-name "${{ needs.resolve.outputs.model_name }}" \ --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \ --run-number "${{ github.run_number }}" build_dm_model: needs: resolve - if: ${{ inputs.target == 'small' }} + if: ${{ inputs.target == 'dm' }} runs-on: [self-hosted, tici] env: DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx @@ -218,6 +437,9 @@ jobs: with: submodules: recursive + - name: Pull DM ONNX via LFS + run: git lfs pull -I "${{ env.DM_ONNX }}" + - name: Set environment variables run: | source /etc/profile diff --git a/.github/workflows/download-hf-model-chunks/action.yml b/.github/workflows/download-hf-model-chunks/action.yml new file mode 100644 index 0000000000..01ab5385da --- /dev/null +++ b/.github/workflows/download-hf-model-chunks/action.yml @@ -0,0 +1,66 @@ +name: Download HF model chunks +description: Resolve and download model chunks from HuggingFace in parallel + +inputs: + hf_repo: + description: HuggingFace dataset repo + required: true + models: + description: 'JSON array of {hf_path, onnx_hash, canonical} objects' + required: true + dest_dir: + description: Destination directory for downloaded chunks + required: true + +runs: + using: composite + steps: + - name: Download model chunks + shell: bash + env: + HF_REPO: ${{ inputs.hf_repo }} + MODELS_JSON: ${{ inputs.models }} + DEST_DIR: ${{ inputs.dest_dir }} + run: | + set -eo pipefail + DOWNLOAD_LIST=$(mktemp) + + resolve_chunks() { + local HF_PATH="$1" ONNX_HASH="$2" CANONICAL="$3" DEST_DIR="$4" + local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_PATH}/default_models.json" + local DEFAULTS BUNDLE ARTIFACT BASE_URL NUM_CHUNKS + DEFAULTS=$(curl -fsSL "$JSON_URL") + BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)') + ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact') + BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||') + NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length') + + mkdir -p "$DEST_DIR" + while IFS= read -r CHUNK_NAME; do + CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true) + if [ -z "$CHUNK_IDX" ]; then + echo "::error::Failed to parse chunk index from: $CHUNK_NAME" + return 1 + fi + ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))") + printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}.chunk${CHUNK_IDX}" >> "$DOWNLOAD_LIST" + done < <(echo "$ARTIFACT" | jq -r '.chunks[].file_name') + echo "$NUM_CHUNKS" > "${DEST_DIR}/${CANONICAL}.chunkmanifest" + } + + echo "$MODELS_JSON" | jq -c '.[]' | while IFS= read -r model; do + HF_PATH=$(echo "$model" | jq -r '.hf_path') + ONNX_HASH=$(echo "$model" | jq -r '.onnx_hash') + CANONICAL=$(echo "$model" | jq -r '.canonical') + resolve_chunks "$HF_PATH" "$ONNX_HASH" "$CANONICAL" "$DEST_DIR" + done + + TOTAL=$(wc -l < "$DOWNLOAD_LIST") + echo "Downloading $TOTAL chunks with 8 parallel connections..." + xargs -P8 -d'\n' -I{} bash -c ' + URL="${1%% *}" + DEST="${1#* }" + echo "Downloading $(basename "$DEST")" + curl -fsSL --retry 3 --retry-delay 5 -o "$DEST" "$URL" + ' _ {} < "$DOWNLOAD_LIST" + rm -f "$DOWNLOAD_LIST" diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index bc132ae1bf..5c2e3bf204 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -188,7 +188,7 @@ jobs: if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then echo "USBGPU build" export USBGPU=1 - TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" + TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" else echo "QCOM build" diff --git a/.github/workflows/sunnypilot-build-prebuilt.yaml b/.github/workflows/sunnypilot-build-prebuilt.yaml index c7232505ee..c93dbe02c1 100644 --- a/.github/workflows/sunnypilot-build-prebuilt.yaml +++ b/.github/workflows/sunnypilot-build-prebuilt.yaml @@ -39,6 +39,8 @@ jobs: include_big_model: ${{ steps.strategy.outputs.include_big_model }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Extract deploy strategy id: strategy run: | @@ -96,6 +98,8 @@ jobs: }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Wait for Tests uses: ./.github/workflows/wait-for-action # Path to where you place the action with: @@ -119,6 +123,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + fetch-depth: 1 submodules: recursive ref: ${{ env.SOURCE_BRANCH }} repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }} @@ -165,7 +170,7 @@ jobs: scons -j1 cache_dir="$SCONS_CACHE" --minimal \ openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd echo "Building rest of sunnypilot" - /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal + SKIP_TINYGRAD_COMPILE=1 /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal touch ${BUILD_DIR}/prebuilt if [[ "${{ runner.debug }}" == "1" ]]; then ls -la ${BUILD_DIR} @@ -214,59 +219,174 @@ jobs: outputs: onnx_sha256: ${{ steps.resolve.outputs.onnx_sha256 }} env: + GH_REPO: ${{ github.repository }} HF_REPO: sunnypilot/sunnypilot_models_v1 HF_DEFAULTS_PATH: models/defaults/big steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref || github.ref_name }} - - - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" - - - name: Check HF defaults and build if needed + - name: Resolve ONNX hash and tinygrad ref via API id: resolve run: | - ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1) - echo "Repo ONNX hash: $ACTUAL_ONNX_HASH" - echo "onnx_sha256=$ACTUAL_ONNX_HASH" >> $GITHUB_OUTPUT + REF="${{ github.head_ref || github.ref_name }}" + + ONNX_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2) + echo "ONNX hash: $ONNX_HASH" + echo "onnx_sha256=$ONNX_HASH" >> $GITHUB_OUTPUT + + TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha') + echo "tinygrad ref: $TINYGRAD_REF" JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" - check_hash() { + check_defaults() { DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 - BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) [ -n "$BUNDLE" ] && [ "$BUNDLE" != "null" ] } - if check_hash; then - echo "HF defaults match repo ONNX" - else - echo "No matching model on HF — triggering build" - gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=big - - echo "Waiting for build to start..." - sleep 120 - - RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId') - if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then - echo "::error::Failed to find build-default-models run" - exit 1 - fi - - echo "Waiting for run $RUN_ID..." - gh run watch "$RUN_ID" - - CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion') - if [ "$CONCLUSION" != "success" ]; then - echo "::error::build-default-models failed: $CONCLUSION" - exit 1 - fi - - if ! check_hash; then - echo "::error::HF defaults still don't match after build" - exit 1 - fi + if check_defaults; then + echo "HF defaults match repo ONNX hash and tinygrad ref" + exit 0 fi + + echo "No matching model on HF — dispatching build" + gh workflow run build-default-models.yaml --ref "$REF" -f target=big + + echo "Polling HF for big model availability..." + for i in $(seq 1 90); do + sleep 30 + if check_defaults; then + echo "Big model available on HF after $((i * 30))s" + exit 0 + fi + echo "Poll $i/90: not yet available" + done + + echo "::error::Big model not available on HF after 45 minutes" + exit 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cancel run on failure + if: failure() + run: gh run cancel ${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + prepare_small_model: + needs: [ prepare_strategy ] + runs-on: ubuntu-24.04 + outputs: + driving_onnx_sha256: ${{ steps.resolve.outputs.driving_onnx_sha256 }} + env: + GH_REPO: ${{ github.repository }} + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/small + steps: + - name: Resolve ONNX hash and tinygrad ref via API + id: resolve + run: | + REF="${{ github.head_ref || github.ref_name }}" + + DRIVING_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/driving_supercombo.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2) + echo "Driving ONNX hash: $DRIVING_HASH" + echo "driving_onnx_sha256=$DRIVING_HASH" >> $GITHUB_OUTPUT + + TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha') + echo "tinygrad ref: $TINYGRAD_REF" + + JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" + + check_defaults() { + DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + [ -n "$DRIVING" ] && [ "$DRIVING" != "null" ] || return 1 + } + + if check_defaults; then + echo "HF defaults match repo ONNX hash and tinygrad ref" + exit 0 + fi + + echo "No matching model on HF — dispatching build" + gh workflow run build-default-models.yaml --ref "$REF" -f target=small + + echo "Polling HF for model availability..." + for i in $(seq 1 60); do + sleep 30 + if check_defaults; then + echo "Model available on HF after $((i * 30))s" + exit 0 + fi + echo "Poll $i/60: not yet available" + done + + echo "::error::Small driving model not available on HF after 30 minutes" + exit 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cancel run on failure + if: failure() + run: gh run cancel ${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + prepare_dm_model: + needs: [ prepare_strategy ] + runs-on: ubuntu-24.04 + outputs: + dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }} + env: + GH_REPO: ${{ github.repository }} + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/dm + steps: + - name: Resolve ONNX hash and tinygrad ref via API + id: resolve + run: | + REF="${{ github.head_ref || github.ref_name }}" + + DM_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/dmonitoring_model.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2) + echo "DM ONNX hash: $DM_HASH" + echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT + + TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha') + echo "tinygrad ref: $TINYGRAD_REF" + + JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" + + check_defaults() { + DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + [ -n "$DM" ] && [ "$DM" != "null" ] || return 1 + } + + if check_defaults; then + echo "HF defaults match DM ONNX hash and tinygrad ref" + exit 0 + fi + + echo "No matching DM model on HF — dispatching build" + gh workflow run build-default-models.yaml --ref "$REF" -f target=dm + + echo "Polling HF for DM model availability..." + for i in $(seq 1 60); do + sleep 30 + if check_defaults; then + echo "DM model available on HF after $((i * 30))s" + exit 0 + fi + echo "Poll $i/60: not yet available" + done + + echo "::error::DM model not available on HF after 30 minutes" + exit 1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -278,23 +398,24 @@ jobs: publish: concurrency: - # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name. - # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other. - # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time. group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }} cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }} if: ${{ always() && !cancelled() && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && + needs.prepare_small_model.result == 'success' && + needs.prepare_dm_model.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) && (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success') }} - needs: [ build, prepare_strategy, prepare_chestnut ] + needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ] runs-on: ubuntu-24.04 environment: ${{ needs.prepare_strategy.outputs.environment }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Download prebuilt artifact uses: actions/download-artifact@v4 @@ -306,43 +427,16 @@ jobs: mkdir -p ${{ env.OUTPUT_DIR }} tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} - - name: Prepare chestnut output - if: ${{ needs.prepare_chestnut.result == 'success' }} - run: | - mkdir -p "${{ github.workspace }}/chestnut_output" - tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output" - - - name: Download big model chunks from HF - if: ${{ needs.prepare_chestnut.result == 'success' }} - env: - HF_REPO: sunnypilot/sunnypilot_models_v1 - HF_DEFAULTS_PATH: models/defaults/big - run: | - ONNX_HASH="${{ needs.prepare_chestnut.outputs.onnx_sha256 }}" - JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" - DEFAULTS=$(curl -fsSL "$JSON_URL") - BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)') - - mkdir -p big_model_chunks - ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact') - BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||') - NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length') - - CANONICAL="big_driving_tinygrad.pkl" - echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do - CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+') - CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}" - ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))") - echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK" - curl -fsSL -o "big_model_chunks/${CANONICAL_CHUNK}" "$ENCODED_URL" - done - - echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest" - - - name: Inject big model into chestnut - if: ${{ needs.prepare_chestnut.result == 'success' }} - run: | - cp big_model_chunks/* "${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models/" + - name: Download model chunks from HF + uses: ./.github/workflows/download-hf-model-chunks + with: + hf_repo: sunnypilot/sunnypilot_models_v1 + dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models + models: | + [ + {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"}, + {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"} + ] - name: Configure Git run: | @@ -364,22 +458,6 @@ jobs: "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" - - name: Publish chestnut branch - if: ${{ needs.prepare_chestnut.result == 'success' }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut" - CHESTNUT_DIR="${{ github.workspace }}/chestnut_output" - - ${{ env.CI_DIR }}/publish.sh \ - "${{ github.workspace }}" \ - "$CHESTNUT_DIR" \ - "$CHESTNUT_BRANCH" \ - "${{ needs.prepare_strategy.outputs.version }}" \ - "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ - "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" - - name: Tag ${{ needs.prepare_strategy.outputs.environment }} if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }} run: | @@ -387,12 +465,77 @@ jobs: git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}." git push -f origin ${TAG} + publish_chestnut: + concurrency: + group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}-chestnut + cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }} + if: ${{ + always() && !cancelled() && + needs.build.result == 'success' && + needs.prepare_strategy.result == 'success' && + needs.prepare_small_model.result == 'success' && + needs.prepare_dm_model.result == 'success' && + needs.prepare_chestnut.result == 'success' && + (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) + }} + needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ] + runs-on: ubuntu-24.04 + environment: ${{ needs.prepare_strategy.outputs.environment }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Download prebuilt artifact + uses: actions/download-artifact@v4 + with: + name: prebuilt + + - name: Untar prebuilt + run: | + mkdir -p ${{ env.OUTPUT_DIR }} + tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} + + - name: Download model chunks from HF + uses: ./.github/workflows/download-hf-model-chunks + with: + hf_repo: sunnypilot/sunnypilot_models_v1 + dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models + models: | + [ + {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"}, + {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"}, + {"hf_path": "models/defaults/big", "onnx_hash": "${{ needs.prepare_chestnut.outputs.onnx_sha256 }}", "canonical": "big_driving_tinygrad.pkl"} + ] + + - name: Configure Git + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + + - name: Publish chestnut branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut" + + ${{ env.CI_DIR }}/publish.sh \ + "${{ github.workspace }}" \ + "${{ env.OUTPUT_DIR }}" \ + "$CHESTNUT_BRANCH" \ + "${{ needs.prepare_strategy.outputs.version }}" \ + "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ + "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" + notify: needs: - prepare_strategy - build - publish + - publish_chestnut - prepare_chestnut + - prepare_small_model + - prepare_dm_model runs-on: ubuntu-24.04 if: ${{ (always() && !cancelled() && !failure()) && needs.publish.result == 'success' @@ -400,6 +543,8 @@ jobs: && (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Prepare notification message id: message diff --git a/openpilot/cereal/custom.capnp b/openpilot/cereal/custom.capnp index c20bf923be..086b10c01c 100644 --- a/openpilot/cereal/custom.capnp +++ b/openpilot/cereal/custom.capnp @@ -131,6 +131,7 @@ struct ModelManagerSP @0xaedffd8f31e7b55d { downloaded @2; cached @3; failed @4; + verifying @5; } struct DownloadProgress { diff --git a/openpilot/selfdrive/modeld/SConscript b/openpilot/selfdrive/modeld/SConscript index 30a31aae27..19ce7d5e00 100644 --- a/openpilot/selfdrive/modeld/SConscript +++ b/openpilot/selfdrive/modeld/SConscript @@ -73,44 +73,45 @@ compile_modeld_script = [ model_w, model_h = MEDMODEL_INPUT_SIZE frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ -for usbgpu in [False, True] if USBGPU else [False]: - target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath - # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU - file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags) - driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath) - camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS) - # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it. - taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else '' - cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py ' - f'--model-size {model_w}x{model_h} ' - f'--camera-resolutions {camera_res_args} ' - f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} ' - f'--output {target_pkl_path} --frame-skip {frame_skip}') - onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps) - chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum)) - def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets): - from openpilot.system.hardware.chestnut.flash import link_up - # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars - for _ in range(10): - if link_up(): - break - time.sleep(1) - else: - print("Chestnut not ready, skipping big model build") - return - if ret := env.Execute(command): - return ret - chunk_file(pkl, chunks) - def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets): - chunk_file(pkl, chunks) - actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")] - node = lenv.Command( - chunk_targets, - tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file], - actions, - ) - if usbgpu: - lenv.SideEffect(usbgpu_lock, node) +if not os.getenv('SKIP_TINYGRAD_COMPILE'): + for usbgpu in [False, True] if USBGPU else [False]: + target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath + # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU + file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags) + driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath) + camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS) + # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it. + taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else '' + cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py ' + f'--model-size {model_w}x{model_h} ' + f'--camera-resolutions {camera_res_args} ' + f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} ' + f'--output {target_pkl_path} --frame-skip {frame_skip}') + onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps) + chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum)) + def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets): + from openpilot.system.hardware.chestnut.flash import link_up + # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars + for _ in range(10): + if link_up(): + break + time.sleep(1) + else: + print("Chestnut not ready, skipping big model build") + return + if ret := env.Execute(command): + return ret + chunk_file(pkl, chunks) + def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets): + chunk_file(pkl, chunks) + actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")] + node = lenv.Command( + chunk_targets, + tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file], + actions, + ) + if usbgpu: + lenv.SideEffect(usbgpu_lock, node) # get model metadata fn = File(f"models/dmonitoring_model").abspath @@ -142,4 +143,5 @@ def tg_compile(flags, model_name): Action(do_chunk, " [CHUNK] $TARGET")], ) -tg_compile(tg_flags, 'dmonitoring_model') +if not os.getenv('SKIP_TINYGRAD_COMPILE'): + tg_compile(tg_flags, 'dmonitoring_model') diff --git a/openpilot/selfdrive/ui/layouts/sidebar.py b/openpilot/selfdrive/ui/layouts/sidebar.py index f950edaa46..5429a35851 100644 --- a/openpilot/selfdrive/ui/layouts/sidebar.py +++ b/openpilot/selfdrive/ui/layouts/sidebar.py @@ -168,9 +168,16 @@ class Sidebar(Widget, SidebarSP): # Home/Flag button flag_pressed = mouse_down and rl.check_collision_point_rec(mouse_pos, HOME_BTN) button_img = self._flag_img if ui_state.started else self._home_img + button_pos = rl.Vector2(HOME_BTN.x, HOME_BTN.y) + icon_opacity = 1.0 + + if gui_app.sunnypilot_ui(): + button_img, button_pos, icon_opacity = SidebarSP._get_home_icon(self, button_img) tint = Colors.BUTTON_PRESSED if (ui_state.started and flag_pressed) else Colors.BUTTON_NORMAL - rl.draw_texture_ex(button_img, rl.Vector2(HOME_BTN.x, HOME_BTN.y), 0.0, 1.0, tint) + if icon_opacity < 1.0: + tint = rl.Color(tint[0], tint[1], tint[2], int(255 * icon_opacity)) + rl.draw_texture_ex(button_img, button_pos, 0.0, 1.0, tint) # Microphone button if self._recording_audio: diff --git a/openpilot/selfdrive/ui/mici/layouts/home.py b/openpilot/selfdrive/ui/mici/layouts/home.py index ff8d350e08..fd74979404 100644 --- a/openpilot/selfdrive/ui/mici/layouts/home.py +++ b/openpilot/selfdrive/ui/mici/layouts/home.py @@ -248,8 +248,11 @@ class MiciHomeLayout(Widget): # ***** Center-aligned bottom section icons ***** self._experimental_icon.set_visible(ui_state.experimental_mode) - self._egpu_icon.set_visible(ui_state.sm["deviceState"].chestnutPresent and ui_state.usbgpu_compiled) - self._egpu_icon_gray.set_visible(ui_state.sm["deviceState"].chestnutPresent and not ui_state.usbgpu_compiled) + if gui_app.sunnypilot_ui(): + self._set_egpu_visibility() + else: + self._egpu_icon.set_visible(ui_state.sm["deviceState"].chestnutPresent and ui_state.usbgpu_compiled) + self._egpu_icon_gray.set_visible(ui_state.sm["deviceState"].chestnutPresent and not ui_state.usbgpu_compiled) self._mic_icon.set_visible(ui_state.recording_audio) self._body_icon.set_visible(bool(ui_state.is_body)) diff --git a/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py b/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py index 3f8ca49cdc..3aa115139f 100644 --- a/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py +++ b/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py @@ -10,11 +10,10 @@ import time import pyray as rl from openpilot.cereal import custom -from openpilot.sunnypilot.models.fetcher import ModelFetcher, get_cached_bundles from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_selected_bundle, resolve_bundle_by_ref from openpilot.common.constants import CV from openpilot.selfdrive.ui.ui_state import device, ui_state -from openpilot.selfdrive.ui.sunnypilot.model_info import model_info +from openpilot.selfdrive.ui.sunnypilot.model_info import big_model_state, bundles_for_source, carrying_model, default_model_name, queued_name from openpilot.system.ui.lib.multilang import tr from openpilot.system.ui.lib.application import gui_app from openpilot.system.ui.widgets import DialogResult, Widget @@ -40,6 +39,8 @@ class ModelsLayout(Widget): self.model_dialog = None self._selection_source = None self._downloading = False + self._verifying = False + self._last_note = None self.last_cache_calc_time = 0 self._initialize_items() @@ -51,17 +52,17 @@ class ModelsLayout(Widget): self._scroller = Scroller(self.items, line_separator=True, spacing=0) def _initialize_items(self): - self.current_model_item = ListItemSP( - title=tr("Active Model"), + self.small_model_item = ListItemSP( + title=tr("Small Model"), description="", action_item=ScrollingButtonAction(tr("SELECT")), - callback=self._handle_current_model_clicked + callback=lambda: self._open_source_dialog("qcom") ) - self.other_model_item = ListItemSP( + self.big_model_item = ListItemSP( title=tr("Big Model"), action_item=ScrollingButtonAction(tr("SELECT")), - callback=self._handle_other_model_clicked + callback=lambda: self._open_source_dialog("usbgpu") ) self.download_item = download_status_item(lambda: tr("Download") if self._downloading else tr("Model Status")) @@ -78,7 +79,8 @@ class ModelsLayout(Widget): callback=self._clear_cache ) - self.cancel_download_item = button_item(tr("Cancel Download"), tr("Cancel"), "", + self.cancel_download_item = button_item(lambda: tr("Cancel Verification") if self._verifying else tr("Cancel Download"), + tr("Cancel"), "", lambda: ui_state.params.remove("ModelManager_DownloadRef")) self.lane_turn_value_control = option_item_sp(tr("Adjust Lane Turn Speed"), "LaneTurnValue", 500, 2000, @@ -104,7 +106,7 @@ class ModelsLayout(Widget): 1, None, True, "", style.BUTTON_ACTION_WIDTH, None, True, lambda v: f"{v / 100:.2f} m") - self.items = [self.current_model_item, self.other_model_item, self.cancel_download_item, self.download_item, self.refresh_item, self.clear_cache_item, + self.items = [self.small_model_item, self.big_model_item, self.cancel_download_item, self.download_item, self.refresh_item, self.clear_cache_item, self.lane_turn_desire_toggle, self.lane_turn_value_control, self.lagd_toggle, self.delay_control, self.camera_offset] def _update_lagd_description(self, lagd_toggle: bool): @@ -118,16 +120,16 @@ class ModelsLayout(Widget): desc += f"
{tr('Actuator Delay:')} {cp:.2f} s + {tr('Software Delay:')} {sw:.2f} s = {tr('Total Delay:')} {cp + sw:.2f} s" self.lagd_toggle.set_description(desc) - def _is_downloading(self): - return (self.model_manager and self.model_manager.selectedBundle and - self.model_manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.downloading) - @staticmethod def calculate_cache_size(): cache_size = 0.0 if os.path.exists(CUSTOM_MODEL_PATH): - cache_size = sum(os.path.getsize(os.path.join(CUSTOM_MODEL_PATH, file)) for file in os.listdir(CUSTOM_MODEL_PATH)) / (1024**2) - return cache_size + for file in os.listdir(CUSTOM_MODEL_PATH): + try: + cache_size += os.path.getsize(os.path.join(CUSTOM_MODEL_PATH, file)) + except OSError: + continue + return cache_size / (1024**2) def _clear_cache(self): def _callback(response): @@ -140,36 +142,90 @@ class ModelsLayout(Widget): gui_app.push_widget(dialog) def _handle_bundle_download_progress(self): - self.download_item.set_visible(False) self.cancel_download_item.set_visible(False) self._downloading = False - - if not self.model_manager or (not self.model_manager.selectedBundle and not self.model_manager.activeBundle): - return - - bundle = self.model_manager.selectedBundle if self._is_downloading() or ( - self.model_manager.selectedBundle and self.model_manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.failed - ) else self.model_manager.activeBundle - if not bundle: - return - - self.cancel_download_item.set_visible(bool(self.model_manager.selectedBundle) and ui_state.params.get("ModelManager_DownloadRef") is not None) + self._verifying = False + self.download_item.set_visible(True) if (current_time := time.monotonic()) - self.last_cache_calc_time > 0.5: self.last_cache_calc_time = current_time self.clear_cache_item.action_item.set_value(f"{self.calculate_cache_size():.2f} MB") + bundle = self.model_manager.selectedBundle if self.model_manager else None + progresses = [model.artifact.downloadProgress for model in bundle.models if model.artifact.fileName] if bundle else [] + if not progresses or bundle.status not in (custom.ModelManagerSP.DownloadStatus.downloading, + custom.ModelManagerSP.DownloadStatus.failed): + self.download_item.action_item.update(name="", segments=self._slot_segments()) + return + + self.cancel_download_item.set_visible(ui_state.params.get("ModelManager_DownloadRef") is not None) if bundle.status == custom.ModelManagerSP.DownloadStatus.downloading: device._reset_interactive_timeout() - # every bundle is a single chunked artifact now - progresses = [model.artifact.downloadProgress for model in bundle.models if model.artifact.fileName] - if not progresses: - return - - self.download_item.set_visible(True) - self.download_item.action_item.update(**self._download_row_state(progresses, bundle.internalName)) + state = self._download_row_state(progresses, bundle.internalName) + if queued := queued_name(bundle.ref): + state["name"] += f" | {queued} {tr('queued')}" + self.download_item.action_item.update(**state) self._downloading = self.download_item.action_item.downloading + ds = custom.ModelManagerSP.DownloadStatus + self._verifying = any(getattr(p.status, 'raw', p.status) == ds.verifying for p in progresses) + + def _slot_segments(self): + """small and big slots side by side; green marks the slot whose pick is actually + driving (runner-matched, so a failed Default big greens neither slot), an empty + slot shows its default.""" + big_state = big_model_state() + carry_source, carry_internal, _ = carrying_model() + segments = [] + for source, label in (("qcom", tr("small")), ("usbgpu", tr("big"))): + if segments: + segments.append(("|", rl.GRAY, None, None)) + bundle = get_selected_bundle(ui_state.params, source) + name = bundle.internalName if bundle else default_model_name(source) + color = ON_COLOR if (source == carry_source and name == carry_internal) else rl.LIGHTGRAY + name = "● " + name + if source == "usbgpu": + if big_state == 'failed': + color = rl.RED + elif big_state == 'loading': + color = rl.GOLD + segments.append((label, rl.GRAY, None, None)) + segments.append((name, color, None, None)) + return segments + + @staticmethod + def _set_item_note(item, text): + # a description renders only while shown; hide before clearing or the + # empty description keeps its visible state + if text: + item.set_description(text) + item.show_description(True) + else: + item.show_description(False) + item.set_description("") + + def _status_note(self) -> str: + """The failover story for the Model Status row. One-way big -> small, and the + fallback is runner-matched: a Default big can only fall back to the Default + small (stock modeld), a custom big has no automatic fallback yet.""" + if not ui_state.usbgpu: + return "" + big_bundle = get_selected_bundle(ui_state.params, "usbgpu") + big_name = big_bundle.internalName if big_bundle else default_model_name("usbgpu") + big_is_default = big_bundle is None + fallback_name = default_model_name("qcom") + state = big_model_state() + if state == 'failed': + if big_is_default: + return tr("Big model unavailable, {} is driving until the next drive.").format(fallback_name) + return tr("Big model unavailable until the next drive.") + if state == 'loading': + if big_is_default: + return tr("{} drives until the big model is ready.").format(fallback_name) + return tr("Getting the big model ready.") + if big_is_default: + return tr("{} will drive. If it fails during a drive, {} takes over until the next drive.").format(big_name, fallback_name) + return tr("{} will drive when the eGPU is ready.").format(big_name) @staticmethod def _download_row_state(progresses, name: str) -> dict: @@ -182,6 +238,8 @@ class ModelsLayout(Widget): if ds.failed in statuses: # close.png is authored black and a tint cannot lift it, hence close2 return {"name": name, "status_text": tr("download failed"), "text_color": rl.RED, "icon": "icons/close2.png"} + if ds.verifying in statuses: + return {"name": name, "downloading": True, "progress": progress, "status_text": tr("verifying")} if ds.downloading in statuses: return {"name": name, "downloading": True, "progress": progress} if statuses <= {ds.downloaded, ds.cached}: @@ -203,12 +261,7 @@ class ModelsLayout(Widget): ui_state.params.put("ModelManager_DownloadRef", selected_bundle.ref) def _resolve_selected_bundle(self, ref): - """Finds the bundle for a ref across both hardware manifests.""" - active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent) - source_bundles = { - source: self.model_manager.availableBundles if source == active else get_cached_bundles(ui_state.params, source) - for source in ("qcom", "usbgpu") - } + source_bundles = {source: bundles_for_source(source) for source in ("qcom", "usbgpu")} resolved = resolve_bundle_by_ref(ref, source_bundles) return resolved[0] if resolved else None @@ -228,18 +281,10 @@ class ModelsLayout(Widget): folders_list.append(TreeFolder(name, [self._bundle_to_node(bundle) for bundle in folder_bundles])) if favorites and (fav_bundles := [bundle for bundle in bundles if bundle.ref in favorites]): - folders_list.insert(1, TreeFolder("Favorites", [self._bundle_to_node(bundle) for bundle in fav_bundles])) + folders_list.insert(0, TreeFolder("Favorites", [self._bundle_to_node(bundle) for bundle in fav_bundles])) return folders_list - def _handle_current_model_clicked(self): - self._open_source_dialog(ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent)) - - def _handle_other_model_clicked(self): - active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent) - self._open_source_dialog("qcom" if active == "usbgpu" else "usbgpu") - def _open_source_dialog(self, source): - """Opens the picker for one hardware: its model folders plus the Default reset entry.""" self._selection_source = source favs = ui_state.params.get("ModelManager_Favs") favorites = set(favs.split(';')) if favs else set() @@ -252,12 +297,10 @@ class ModelsLayout(Widget): gui_app.push_widget(self.model_dialog) def _source_folders(self, favorites, source): - """Default reset entry on top, then the hardware's model folders.""" - active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent) - bundles = self.model_manager.availableBundles if source == active else get_cached_bundles(ui_state.params, source) + bundles = bundles_for_source(source) if not bundles: return [] - folders_list = [TreeFolder("", [TreeNode("Default", {'display_name': "Default"})])] + folders_list = [TreeFolder("", [TreeNode("Default", {'display_name': default_model_name(source)})])] folders_list.extend(self._get_folders(favorites, bundles)) return folders_list @@ -284,20 +327,27 @@ class ModelsLayout(Widget): self._update_lagd_description(live_delay) self.model_manager = ui_state.sm["modelManagerSP"] self._handle_bundle_download_progress() - source, active_name, other_name = model_info() - self.current_model_item.action_item.set_value(active_name) - self.other_model_item.set_title(tr("Big Model") if source == "qcom" else tr("Small Model")) - self.other_model_item.action_item.set_value(other_name) - if not ui_state.is_offroad(): - self.current_model_item.action_item.set_enabled(False) - self.current_model_item.set_description(tr("Only available when vehicle is off, or always offroad mode is on")) - else: - self.current_model_item.action_item.set_enabled(True) - self.current_model_item.set_description("") + carry_source, _, carry_display = carrying_model() + for item, item_source in ((self.small_model_item, "qcom"), (self.big_model_item, "usbgpu")): + bundle = get_selected_bundle(ui_state.params, item_source) + name = bundle.displayName if bundle else default_model_name(item_source) + color = ON_COLOR if (item_source == carry_source and name == carry_display) else style.ITEM_TEXT_VALUE_COLOR + item.action_item.set_value(name, color) + + note = self._status_note() + if note != self._last_note: + self._last_note = note + self._set_item_note(self.download_item, note) + + offroad = ui_state.is_offroad() + self.small_model_item.action_item.set_enabled(offroad) + self.big_model_item.action_item.set_enabled(offroad) + self.small_model_item.set_description("" if offroad else tr("Only available when vehicle is off, or always offroad mode is on")) def _render(self, rect): self._scroller.render(rect) def show_event(self): self._scroller.show_event() + self._last_note = None # re-expand the failover note every time the page opens diff --git a/openpilot/selfdrive/ui/sunnypilot/layouts/sidebar.py b/openpilot/selfdrive/ui/sunnypilot/layouts/sidebar.py index 79bb15dbb8..7c74c48469 100644 --- a/openpilot/selfdrive/ui/sunnypilot/layouts/sidebar.py +++ b/openpilot/selfdrive/ui/sunnypilot/layouts/sidebar.py @@ -4,11 +4,14 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. This file is part of sunnypilot and is licensed under the MIT License. See the LICENSE.md file in the root directory for more details. """ +import math + import pyray as rl import time from dataclasses import dataclass from openpilot.selfdrive.ui.ui_state import ui_state from openpilot.sunnypilot.sunnylink.api import UNREGISTERED_SUNNYLINK_DONGLE_ID +from openpilot.system.ui.lib.application import gui_app from openpilot.system.ui.lib.multilang import tr_noop @@ -18,6 +21,9 @@ METRIC_MARGIN = 30 METRIC_START_Y = 300 HOME_BTN = rl.Rectangle(60, 860, 180, 180) +EGPU_ICON_WIDTH = 180 +EGPU_ICON_HEIGHT = 133 + # Color scheme class Colors: @@ -53,6 +59,10 @@ class MetricData: class SidebarSP: def __init__(self): self._sunnylink_status = MetricData(tr_noop("SUNNYLINK"), tr_noop("OFFLINE"), Colors.WARNING) + self._egpu_green_img = gui_app.texture("icons_mici/egpu_green.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT) + self._egpu_default_img = gui_app.texture("icons_mici/egpu.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT) + self._egpu_orange_img = gui_app.texture("icons_mici/egpu_orange.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT) + self._egpu_gray_img = gui_app.texture("icons_mici/egpu_gray.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT) def _update_sunnylink_status(self): if not ui_state.params.get_bool("SunnylinkEnabled"): @@ -78,6 +88,29 @@ class SidebarSP: self._sunnylink_status.update(tr_noop("SUNNYLINK"), status, color) + def _get_home_icon(self, default_img: rl.Texture) -> tuple[rl.Texture, rl.Vector2, float]: + default_pos = rl.Vector2(HOME_BTN.x, HOME_BTN.y) + if not ui_state.sm["deviceState"].chestnutPresent: + return default_img, default_pos, 1.0 + + big_model_selected = ui_state.usbgpu_compiled or ui_state.model_runner_tinygrad + big_model_failed = ui_state.started and ui_state.big_model_failed + loading = ui_state.usbgpu_loading or (big_model_selected and ui_state.started and ui_state.usbgpu_active is None) + + if loading: + icon = self._egpu_default_img + opacity = 0.35 + 0.65 * (0.5 - 0.5 * math.cos(rl.get_time() * 6.0)) + elif big_model_selected and big_model_failed: + icon, opacity = self._egpu_orange_img, 1.0 + elif big_model_selected: + icon, opacity = self._egpu_green_img, 1.0 + else: + icon, opacity = self._egpu_gray_img, 1.0 + + x = HOME_BTN.x + (HOME_BTN.width - icon.width) / 2 + y = HOME_BTN.y + (HOME_BTN.height - icon.height) / 2 + return icon, rl.Vector2(x, y), opacity + def _draw_metrics_w_sunnylink(self, rect: rl.Rectangle, _temp, _panda, _connect): metrics = [_temp, _panda, _connect, self._sunnylink_status] start_y = int(rect.y) + METRIC_START_Y diff --git a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/home.py b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/home.py index d29e579c52..b261373947 100644 --- a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/home.py +++ b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/home.py @@ -4,8 +4,14 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. This file is part of sunnypilot and is licensed under the MIT License. See the LICENSE.md file in the root directory for more details. """ +import math + +import pyray as rl + from openpilot.selfdrive.ui.mici.layouts.home import MiciHomeLayout +from openpilot.selfdrive.ui.ui_state import ui_state from openpilot.system.ui.lib.application import FontWeight +from openpilot.system.ui.widgets.icon_widget import IconWidget from openpilot.system.ui.widgets.label import UnifiedLabel @@ -13,3 +19,35 @@ class MiciHomeLayoutSP(MiciHomeLayout): def __init__(self): super().__init__() self._openpilot_label = UnifiedLabel("sunnypilot", font_size=88, font_weight=FontWeight.AUDIOWIDE, max_width=480, wrap_text=False) + self._egpu_icon_default = IconWidget("icons_mici/egpu.png", (50, 37)) + self._egpu_icon_default.set_visible(False) + self._egpu_icon_orange = IconWidget("icons_mici/egpu_orange.png", (50, 37)) + self._egpu_icon_orange.set_visible(False) + gray_idx = self._status_bar_layout.widgets.index(self._egpu_icon_gray) + self._status_bar_layout.widgets.insert(gray_idx + 1, self._egpu_icon_default) + self._status_bar_layout.widgets.insert(gray_idx + 2, self._egpu_icon_orange) + + def _set_egpu_visibility(self): + chestnut = ui_state.sm["deviceState"].chestnutPresent + if not chestnut: + self._egpu_icon.set_visible(False) + self._egpu_icon_default.set_visible(False) + self._egpu_icon_orange.set_visible(False) + self._egpu_icon_gray.set_visible(False) + return + + big_model_selected = ui_state.usbgpu_compiled or ui_state.model_runner_tinygrad + big_model_failed = ui_state.started and ui_state.big_model_failed + loading = ui_state.usbgpu_loading or (big_model_selected and ui_state.started and ui_state.usbgpu_active is None) + + if loading: + self._egpu_icon_default._opacity = 0.35 + 0.65 * (0.5 - 0.5 * math.cos(rl.get_time() * 6.0)) + self._egpu_icon_default.set_visible(True) + self._egpu_icon.set_visible(False) + self._egpu_icon_orange.set_visible(False) + self._egpu_icon_gray.set_visible(False) + else: + self._egpu_icon_default.set_visible(False) + self._egpu_icon.set_visible(big_model_selected and not big_model_failed) + self._egpu_icon_orange.set_visible(big_model_selected and big_model_failed) + self._egpu_icon_gray.set_visible(not big_model_selected) diff --git a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py index 4d456bdb76..183b47fa58 100644 --- a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py +++ b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py @@ -8,11 +8,11 @@ import pyray as rl from openpilot.cereal import custom from openpilot.selfdrive.ui.mici.widgets.dialog import BigDialog -from openpilot.sunnypilot.models.fetcher import ModelFetcher, get_cached_bundles -from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS +from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_selected_bundle from openpilot.selfdrive.ui.mici.widgets.button import BigButton from openpilot.selfdrive.ui.ui_state import ui_state, device -from openpilot.selfdrive.ui.sunnypilot.model_info import model_info +from openpilot.selfdrive.ui.sunnypilot.model_info import (active_source, big_model_state, bundles_for_source, carrying_model, + default_model_name, model_info, queued_name) from openpilot.system.ui.lib.application import FontWeight, gui_app from openpilot.system.ui.lib.multilang import tr from openpilot.system.ui.widgets import Widget @@ -20,10 +20,22 @@ from openpilot.system.ui.widgets.label import UnifiedLabel from openpilot.system.ui.widgets.scroller import NavScroller def _model_info() -> tuple[str, str, str]: - """(active model, other-model header, other-model text) for the panel.""" + """(active model, info header, info text) for the panel. Runner-matched: the + active line names what actually drives, and a notable big-model state takes + the info pair.""" source, active_name, other_name = model_info() + state = big_model_state() + _, _, carry_display = carrying_model() + if carry_display is None: + big = get_selected_bundle(ui_state.params, "usbgpu") + carry_display = big.displayName if big else default_model_name("usbgpu") + active_text = (carry_display or active_name).lower() + if state == 'failed': + return active_text, tr("big model"), tr("unavailable") + if state == 'loading': + return active_text, tr("big model"), tr("getting ready") header = tr("small model") if source == "usbgpu" else tr("big model") - return active_name.lower(), header, other_name.lower() + return active_text, header, other_name.lower() class CurrentModelInfo(Widget): @@ -100,8 +112,13 @@ class ModelsLayoutMici(NavScroller): self.focused_widget = self.select_model_btn hardware_btns = [] + active = active_source() for source, label in (("qcom", tr("small models")), ("usbgpu", tr("big models"))): - btn = BigButton(label.lower()) + bundle = get_selected_bundle(ui_state.params, source) + value = (bundle.internalName if bundle else default_model_name(source)).lower() + if source == active: + value += f" ({tr('active')})" + btn = BigButton(label.lower(), value=value) btn.set_click_callback(lambda s=source: self._select_hardware(s)) hardware_btns.append(btn) self._push_selection_view(hardware_btns) @@ -111,20 +128,16 @@ class ModelsLayoutMici(NavScroller): favs = ui_state.params.get("ModelManager_Favs") favorites = set(favs.split(';')) if favs else set() - active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent) - if source != active: - bundles = get_cached_bundles(ui_state.params, source) - if not bundles: - gui_app.push_widget(BigDialog(title=tr("No models available"), - description=tr("No models are available for this hardware yet. Connect to the internet and refresh the model list."))) - return - else: - bundles = self.model_manager.availableBundles + bundles = bundles_for_source(source) + if not bundles: + gui_app.push_widget(BigDialog(title=tr("No models available"), + description=tr("No models are available for this hardware yet. Connect to the internet and refresh the model list."))) + return folders = self._get_grouped_bundles(bundles, favorites) folder_buttons = [] - default_btn = BigButton(tr("default")) + default_btn = BigButton(default_model_name(source).lower()) default_btn.set_click_callback(lambda s=source: self._select_default(s)) folder_buttons.append(default_btn) @@ -155,13 +168,8 @@ class ModelsLayoutMici(NavScroller): return favs = ui_state.params.get("ModelManager_Favs") favorites = set(favs.split(';')) if favs else set() - active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent) - if source != active: - bundles = get_cached_bundles(ui_state.params, source) - else: - bundles = self.model_manager.availableBundles - folders = self._get_grouped_bundles(bundles, favorites) + folders = self._get_grouped_bundles(bundles_for_source(source), favorites) bundles = sorted(folders.get(folder_name, []), key=lambda b: b.index, reverse=True) btns = [] @@ -212,18 +220,25 @@ class ModelsLayoutMici(NavScroller): device.set_override_interactive_timeout(5) progress = 0.0 count = 0 + verifying = False for model in manager.selectedBundle.models: count += 1 p = model.artifact.downloadProgress - if p.status == custom.ModelManagerSP.DownloadStatus.downloading: + if p.status in (custom.ModelManagerSP.DownloadStatus.downloading, + custom.ModelManagerSP.DownloadStatus.verifying): progress += p.progress + verifying = verifying or p.status == custom.ModelManagerSP.DownloadStatus.verifying elif p.status in (custom.ModelManagerSP.DownloadStatus.downloaded, custom.ModelManagerSP.DownloadStatus.cached): progress += 100.0 - self.current_model_info.current_model_header.set_text(tr("downloading")) + self.current_model_info.current_model_header.set_text(tr("verifying") if verifying else tr("downloading")) + self.cancel_download_btn.set_text(tr("cancel verification") if verifying else tr("cancel download")) self.current_model_info.current_model_header._shimmer = True - self.current_model_info.current_model_text.set_text(f"{manager.selectedBundle.internalName.lower()}") + name_text = manager.selectedBundle.internalName.lower() + if queued := queued_name(manager.selectedBundle.ref): + name_text += f" | {queued.lower()} {tr('queued')}" + self.current_model_info.current_model_text.set_text(name_text) self.current_model_info.info_header.set_text(tr("progress") + self._download_progress) self.current_model_info.info_header._shimmer = True self.current_model_info.info_text.set_text(f"{progress/count:.2f}%") diff --git a/openpilot/selfdrive/ui/sunnypilot/mici/onroad/hud_renderer.py b/openpilot/selfdrive/ui/sunnypilot/mici/onroad/hud_renderer.py index 9d39d01727..ad75f7e969 100644 --- a/openpilot/selfdrive/ui/sunnypilot/mici/onroad/hud_renderer.py +++ b/openpilot/selfdrive/ui/sunnypilot/mici/onroad/hud_renderer.py @@ -7,6 +7,7 @@ See the LICENSE.md file in the root directory for more details. import pyray as rl from openpilot.selfdrive.ui.mici.onroad.hud_renderer import HudRenderer +from openpilot.selfdrive.ui.ui_state import ui_state from openpilot.selfdrive.ui.sunnypilot.onroad.blind_spot_indicators import BlindSpotIndicators @@ -21,6 +22,8 @@ class HudRendererSP(HudRenderer): def _render(self, rect: rl.Rectangle) -> None: super()._render(rect) + if ui_state.usbgpu and not ui_state.usbgpu_compiled and ui_state.model_runner_tinygrad: + self._draw_model_source(rect) self.blind_spot_indicators.render(rect) def _has_blind_spot_detected(self) -> bool: diff --git a/openpilot/selfdrive/ui/sunnypilot/model_info.py b/openpilot/selfdrive/ui/sunnypilot/model_info.py index 5222ef70c6..a93a06f187 100644 --- a/openpilot/selfdrive/ui/sunnypilot/model_info.py +++ b/openpilot/selfdrive/ui/sunnypilot/model_info.py @@ -5,21 +5,84 @@ This file is part of sunnypilot and is licensed under the MIT License. See the LICENSE.md file in the root directory for more details. """ from openpilot.selfdrive.ui.ui_state import ui_state -from openpilot.sunnypilot.models.helpers import get_active_source, get_selected_bundle +from openpilot.sunnypilot.models.fetcher import get_cached_bundles +from openpilot.sunnypilot.models.helpers import get_active_source, get_selected_bundle, resolve_bundle_by_ref from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL, DEFAULT_MODEL +def active_source() -> str: + return get_active_source(usbgpu=ui_state.usbgpu, + usbgpu_active=ui_state.usbgpu_active, usbgpu_loading=ui_state.usbgpu_loading, + offroad=ui_state.is_offroad()) + + +def bundles_for_source(source: str): + if source == active_source(): + return ui_state.sm["modelManagerSP"].availableBundles + return get_cached_bundles(ui_state.params, source) + + +def default_model(source: str) -> str: + return DEFAULT_BIG_MODEL if source == 'usbgpu' else DEFAULT_MODEL + + +def default_model_name(source: str) -> str: + return f"{default_model(source)} (Default)" + + +def big_model_state() -> str | None: + """'failed' | 'loading' | None, mirroring the sidebar's detection (#1969).""" + if ui_state.started and ui_state.usbgpu and ui_state.big_model_failed: + return 'failed' + big_selected = ui_state.usbgpu_compiled or ui_state.model_runner_tinygrad + if ui_state.usbgpu_loading or (big_selected and ui_state.started and ui_state.usbgpu_active is None): + return 'loading' + return None + + +def carrying_model() -> tuple[str | None, str | None, str | None]: + """(source, internal name, display name) of what actually drives. Runner-matched: + when a Default big cannot carry, stock modeld runs the Default small, never the + small slot's pick; a custom big has no automatic fallback yet -> (None, None, None).""" + source = active_source() + if source == "usbgpu": + bundle = get_selected_bundle(ui_state.params, "usbgpu") + if bundle: + return "usbgpu", bundle.internalName, bundle.displayName + name = default_model_name("usbgpu") + return "usbgpu", name, name + if ui_state.usbgpu: + if get_selected_bundle(ui_state.params, "usbgpu") is None: + name = default_model_name("qcom") + return "qcom", name, name + return None, None, None + bundle = get_selected_bundle(ui_state.params, "qcom") + if bundle: + return "qcom", bundle.internalName, bundle.displayName + name = default_model_name("qcom") + return "qcom", name, name + + +def queued_name(current_ref) -> str | None: + ref = ui_state.params.get("ModelManager_DownloadRef") + if ref and ref != current_ref: + source_bundles = {source: bundles_for_source(source) for source in ("qcom", "usbgpu")} + if resolved := resolve_bundle_by_ref(ref, source_bundles): + return resolved[0].internalName + return None + + def model_info() -> tuple[str, str, str]: - """returns (active source, active model name, other model name)""" - source = get_active_source(usbgpu=ui_state.usbgpu, - usbgpu_active=ui_state.usbgpu_active, usbgpu_loading=ui_state.usbgpu_loading, - offroad=ui_state.is_offroad()) + """returns (active source, active model name, other model name) + + Names come from the params slots, never modelManagerSP.activeBundle — the + manager republishes a tick after a chestnut change, so the stale bundle + would flash the wrong model.""" + source = active_source() other = "qcom" if source == "usbgpu" else "usbgpu" active_bundle = get_selected_bundle(ui_state.params, source) other_bundle = get_selected_bundle(ui_state.params, other) - active_name = active_bundle.displayName if active_bundle \ - else f"{DEFAULT_BIG_MODEL if source == 'usbgpu' else DEFAULT_MODEL} (Default)" - other_name = other_bundle.displayName if other_bundle \ - else f"{DEFAULT_MODEL if source == 'usbgpu' else DEFAULT_BIG_MODEL} (Default)" + active_name = active_bundle.displayName if active_bundle else default_model_name(source) + other_name = other_bundle.displayName if other_bundle else default_model_name(other) return source, active_name, other_name diff --git a/openpilot/selfdrive/ui/sunnypilot/ui_state.py b/openpilot/selfdrive/ui/sunnypilot/ui_state.py index 1b209e8359..9bed533d3f 100644 --- a/openpilot/selfdrive/ui/sunnypilot/ui_state.py +++ b/openpilot/selfdrive/ui/sunnypilot/ui_state.py @@ -10,7 +10,7 @@ from openpilot.cereal import messaging, log, custom from opendbc.car.structs import car from openpilot.common.params import Params from openpilot.selfdrive.ui.sunnypilot.layouts.settings.display import OnroadBrightness -from openpilot.sunnypilot.models.helpers import get_active_bundle +from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_active_source from openpilot.sunnypilot.sunnylink.sunnylink_state import SunnylinkState from openpilot.system.ui.lib.application import gui_app from openpilot.system.ui.sunnypilot.widgets.screen_saver import ScreenSaverSP @@ -44,6 +44,7 @@ class UIStateSP: self.screensaver_enabled: bool = False self.active_bundle = None + self.model_runner_tinygrad: bool = False self.blindspot: bool = False self.chevron_metrics = None self.custom_interactive_timeout: int = 0 @@ -151,7 +152,10 @@ class UIStateSP: self.has_icbm = self.CP_SP.intelligentCruiseButtonManagementAvailable and self.params.get_bool("IntelligentCruiseButtonManagement") self._enforce_constraints() - self.active_bundle = get_active_bundle(self.params) + source = get_active_source(usbgpu=self.usbgpu, usbgpu_active=self.usbgpu_active, + usbgpu_loading=self.usbgpu_loading, offroad=self.is_offroad()) + self.active_bundle = self.params.get(ACTIVE_BUNDLE_KEYS[source]) + self.model_runner_tinygrad = self.active_bundle is not None and self.active_bundle.get("runner") == "tinygrad" self.blindspot = self.params.get_bool("BlindSpot") self.chevron_metrics = self.params.get("ChevronInfo") self.custom_interactive_timeout = self.params.get("InteractivityTimeout", return_default=True) diff --git a/openpilot/selfdrive/ui/ui_state.py b/openpilot/selfdrive/ui/ui_state.py index e0aca74ff2..7e2a2494ea 100644 --- a/openpilot/selfdrive/ui/ui_state.py +++ b/openpilot/selfdrive/ui/ui_state.py @@ -112,6 +112,15 @@ class UIState(UIStateSP): def add_on_body_changed_callbacks(self, callback: Callable[[], None]): self._on_body_changed_callbacks.append(callback) + @property + def big_model_failed(self) -> bool: + # Mirrors the onroad HUD's four-condition check so sidebar and home icons reflect the same failure states + return (self.usbgpu_active is False or + not self.sm['deviceState'].chestnutPresent or + (self.usbgpu_active is True and self.sm.recv_frame['modelV2'] > self.started_frame and + not self.sm.alive['modelV2']) or + (self.usbgpu_active is None and self.sm.recv_frame['modelV2'] > self.started_frame)) + @property def engaged(self) -> bool: return self.started and (self.sm["selfdriveState"].enabled or self.sm["selfdriveStateSP"].mads.enabled) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 85ae57c078..17687908c8 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -7,6 +7,7 @@ See the LICENSE.md file in the root directory for more details. """ import argparse +import math import os import tempfile import time @@ -66,14 +67,15 @@ def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tu if desire_key: shapes['desire'] = (input_shapes[desire_key][2],) - if is_supercombo and 'features_buffer' in input_shapes: - fb = input_shapes['features_buffer'] - shapes['prev_feat'] = (fb[0], fb[2]) - for key, shape in input_shapes.items(): if key not in (desire_key, 'features_buffer') and 'img' not in key: shapes[key] = tuple(shape) + if is_supercombo and 'features_buffer' in input_shapes: + fb = input_shapes['features_buffer'] + feat_dim = math.prod(fb[2:]) + shapes['prev_feat'] = (fb[0], feat_dim) + sizes = [int(np.prod(size)) for size in shapes.values()] return shapes, sizes @@ -117,8 +119,9 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D } if features_buffer: + feat_dim = math.prod(features_buffer[2:]) feat_q_len = frame_skip * features_buffer[1] if is_supercombo else frame_skip * (features_buffer[1] - 1) + 1 - queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], features_buffer[2]), + queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], feat_dim), dtype=np.float32), device=device).contiguous().realize() queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')}) @@ -183,14 +186,14 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, warped_dev = warped.to(Device.DEFAULT) Tensor.realize(packed_npy_inputs_dev, warped_dev) - img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn).realize() - big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn).realize() + img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn) + big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn) unpacked_tensors = [tensor.reshape(shape) for tensor, shape in zip(packed_npy_inputs_dev.split(npy_sizes), npy_shapes.values(), strict=True)] unpacked_dict = dict(zip(npy_shapes.keys(), unpacked_tensors, strict=True)) desire_dev = unpacked_dict['desire'] - desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn).realize() + desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn) inputs = {desire_key: desire_buf} for key, tensor_val in unpacked_dict.items(): @@ -199,7 +202,7 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, if 'prev_feat' in unpacked_dict: prev_feat_dev = unpacked_dict['prev_feat'] - inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).realize() + inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).reshape(input_shapes['features_buffer']) if vision_runner: vision_out_cast = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32').realize() @@ -211,7 +214,7 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, inputs.update({road_key: img, wide_key: big_img}) if 'features_buffer' not in inputs: - inputs['features_buffer'] = sample_skip_fn(feat_q) + inputs['features_buffer'] = sample_skip_fn(feat_q).reshape(input_shapes['features_buffer']) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32').realize() if 'features_buffer' not in inputs and features_slice is not None: diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 9f3d709537..d180012279 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -91,7 +91,7 @@ class ModelState(ModelStateBase): if env_pkl and os.path.exists(env_pkl): model_bundle = None else: - model_bundle = get_active_bundle() + model_bundle = get_active_bundle(usbgpu=usbgpu) self.generation = model_bundle.generation if model_bundle is not None else None overrides = {override.key: override.value for override in model_bundle.overrides} if model_bundle else {} diff --git a/openpilot/sunnypilot/modeld_v2/tests/helpers.py b/openpilot/sunnypilot/modeld_v2/tests/helpers.py index ee59e82785..6e66bf771a 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/helpers.py +++ b/openpilot/sunnypilot/modeld_v2/tests/helpers.py @@ -190,8 +190,8 @@ def tmp_path(): def patch_modeld(monkeypatch): def _patch(bundle): - monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None: bundle) - monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None: bundle) + monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle) + monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle) return _patch diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py b/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py index 3396649a1d..ccd8cbc7f3 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py @@ -59,8 +59,8 @@ class TestFindDrivingPkl(OpenpilotTestCase): class TestModelStateCombinedInit(OpenpilotTestCase): def test_asserts_when_no_pkl(self, monkeypatch): bundle = DummyBundle(models=[], is_20hz=True) - monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None: bundle) - monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None: bundle) + monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle) + monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle) with self.assertRaisesRegex(AssertionError, "No driving pkl found"): ModelState(cam_w=CAM_W, cam_h=CAM_H) diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py b/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py index 96bfb42638..86974b14f1 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py @@ -195,3 +195,85 @@ class TestReadFileChunkedToDisk(OpenpilotTestCase): assert out.parent == Path(d) assert out.read_bytes() == payload + + +class Test4DFeaturesBuffer(OpenpilotTestCase): + def test_get_policy_npy_shapes_4d(self): + from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes + input_shapes = { + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 32, 512), # compare 4d to 3d for regression + 'traffic_convention': (1, 2), + 'action_t': (1, 2) + } + shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True) + assert shapes['prev_feat'] == (1, 16384) + assert sizes == [8, 2, 2, 16384] + + def test_get_policy_npy_shapes_3d(self): + from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes + input_shapes = { + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 512), + 'traffic_convention': (1, 2), + 'action_t': (1, 2) + } + shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True) + assert shapes['prev_feat'] == (1, 512) + assert sizes == [8, 2, 2, 512] + + +class TestStockCompileModeldEquivalence(OpenpilotTestCase): + def test_get_policy_npy_shapes_matches_stock(self): + from openpilot.selfdrive.modeld.compile_modeld import get_policy_npy_shapes as stock_get_policy_npy_shapes + from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes as sunny_get_policy_npy_shapes + + stock_input_shapes = { + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 512), # see below comment + 'traffic_convention': (1, 2), + 'action_t': (1, 2), + } + + stock_shapes, stock_sizes = stock_get_policy_npy_shapes(stock_input_shapes) + sunny_shapes, sunny_sizes = sunny_get_policy_npy_shapes(stock_input_shapes, is_supercombo=True) + + assert sunny_shapes == stock_shapes + assert sunny_sizes == stock_sizes + assert sunny_shapes['prev_feat'] == (1, 512) + + def test_make_input_queues_full_stock_equivalence(self): + from openpilot.selfdrive.modeld.compile_modeld import make_input_queues as stock_make_input_queues + from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues as sunny_make_supercombo_input_queues + input_shapes = { + 'img': (1, 12, 128, 256), + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 512), # when https://github.com/commaai/openpilot/pull/38681 merges, update to 1,24,32,512 + 'traffic_convention': (1, 2), + 'action_t': (1, 2), + } + frame_skip = 4 + + stock_queues, stock_npy = stock_make_input_queues(input_shapes, frame_skip, device='NPY') + sunny_queues, sunny_npy = sunny_make_supercombo_input_queues(input_shapes, frame_skip, device='NPY') + assert set(sunny_queues.keys()) == set(stock_queues.keys()) + for key in stock_queues: + assert sunny_queues[key].shape == stock_queues[key].shape, \ + f"Queue shape mismatch for {key}: sunny {sunny_queues[key].shape} != stock {stock_queues[key].shape}" + assert set(sunny_npy.keys()) == set(stock_npy.keys()) + for key in stock_npy: + assert sunny_npy[key].shape == stock_npy[key].shape, \ + f"Numpy array shape mismatch for {key}: sunny {sunny_npy[key].shape} != stock {stock_npy[key].shape}" + + def test_make_warp_queues_stock_equivalence(self): + from openpilot.selfdrive.modeld.compile_modeld import make_warp_input_queues as stock_make_warp_queues + from openpilot.sunnypilot.modeld_v2.compile_modeld import make_warp_queues as sunny_make_warp_queues + stock_vision_shapes = {'img': (1, 12, 128, 256)} # for now? + stock_queues, stock_npy = stock_make_warp_queues(stock_vision_shapes, frame_skip=4, device='NPY') + sunny_queues, sunny_npy = sunny_make_warp_queues(device='NPY') + + assert set(sunny_npy.keys()) == set(stock_npy.keys()) == {'tfm', 'big_tfm'} + for key in sunny_npy: + assert sunny_npy[key].shape == stock_npy[key].shape == (3, 3) + + diff --git a/openpilot/sunnypilot/models/fetcher.py b/openpilot/sunnypilot/models/fetcher.py index c70c70710e..1bbfb02f70 100644 --- a/openpilot/sunnypilot/models/fetcher.py +++ b/openpilot/sunnypilot/models/fetcher.py @@ -138,8 +138,8 @@ class ModelCache: class ModelFetcher: """Handles fetching and caching of model data from remote source""" - MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v20.json" - MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v21.json" + MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v21.json" + MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v22.json" MODEL_SOURCES = { "qcom": (MODEL_URL, ""), @@ -149,27 +149,20 @@ class ModelFetcher: def __init__(self, params: Params): self.params = params self.model_parser = ModelParser() - self._active_json_published = False self.model_caches = { source: ModelCache(params, suffix=suffix) for source, (_, suffix) in self.MODEL_SOURCES.items() } - self.model_url = self.MODEL_URL - self._update_model_source() + self._refetched: set[str] = set() + self.params.put("ModelManager_ActiveJson", { + "qcom": self.MODEL_URL, + "usbgpu": self.MODEL_URL_USBGPU, + }, block=True) @staticmethod def active_source(chestnut_present: bool) -> str: return "usbgpu" if chestnut_present else "qcom" - def _update_model_source(self) -> None: - """Publishes the manifest URLs for both sources""" - if not self._active_json_published: - self._active_json_published = True - self.params.put("ModelManager_ActiveJson", { - "qcom": self.MODEL_URL, - "usbgpu": self.MODEL_URL_USBGPU, - }, block=True) - def _fetch_and_cache_models(self, source: str) -> list[custom.ModelManagerSP.ModelBundle] | None: """Fetches fresh model data from remote and updates cache. Returns None on transport errors. Raises on 404 and other fatal HTTP errors. @@ -206,17 +199,22 @@ class ModelFetcher: @staticmethod def _cache_matches_source(source: str, cached_data: dict) -> bool: - """Confirms a cached manifest contains requested source's models.""" bundles = cached_data.get("bundles", []) if source == "usbgpu": return any(bundle.get("is_big") is True for bundle in bundles) return not any(bundle.get("is_big") is True for bundle in bundles) - def _get_source_bundles(self, source: str) -> list[custom.ModelManagerSP.ModelBundle]: + def get_bundles_for_source(self, source: str) -> list[custom.ModelManagerSP.ModelBundle]: + if source not in self.MODEL_SOURCES: + cloudlog.warning(f"Unknown model source: {source}") + return [] + cached_data, is_expired = self.model_caches[source].get() if cached_data and not is_expired: - if self._cache_matches_source(source, cached_data): + # a source is refetched over a mismatch at most once per process: if the fresh + # manifest still mismatches, the URL is authoritative and the cache is trusted + if self._cache_matches_source(source, cached_data) or source in self._refetched: try: parsed = self.model_parser.parse_models(cached_data) except Exception: @@ -229,7 +227,8 @@ class ModelFetcher: # manifest version) - do not trust it, refetch so the source is repopulated cloudlog.warning(f"Cached models for {source} have no valid bundles; refetching") else: - cloudlog.warning(f"Cached models for {source} not valid; refetching") + self._refetched.add(source) + cloudlog.warning(f"Cached models for {source} not valid; refetching once") fetched_bundles = self._fetch_and_cache_models(source) if fetched_bundles is not None: @@ -244,16 +243,8 @@ class ModelFetcher: except Exception: return [] - def get_bundles_for_source(self, source: str) -> list[custom.ModelManagerSP.ModelBundle]: - """Gets the list of available models for a specific source, with smart cache handling.""" - if source not in self.MODEL_SOURCES: - cloudlog.warning(f"Unknown model source: {source}") - return [] - return self._get_source_bundles(source) - def get_cached_bundles(params: Params, source: str) -> list[custom.ModelManagerSP.ModelBundle]: - """Reads a source's cached manifest from params and parses it into bundles.""" if source not in ModelFetcher.MODEL_SOURCES: cloudlog.warning(f"Unknown model source: {source}") diff --git a/openpilot/sunnypilot/models/helpers.py b/openpilot/sunnypilot/models/helpers.py index 16cb2cf4a0..707b86f722 100644 --- a/openpilot/sunnypilot/models/helpers.py +++ b/openpilot/sunnypilot/models/helpers.py @@ -19,7 +19,7 @@ from openpilot.common.hardware.hw import Paths from openpilot.selfdrive.modeld.helpers import usbgpu_present # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO -REQUIRED_JSON_VERSION = 17 +REQUIRED_JSON_VERSION = 18 CUSTOM_MODEL_PATH = Paths.model_root() METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl' @@ -29,7 +29,7 @@ ACTIVE_BUNDLE_KEYS = { "qcom": "ModelManager_ActiveBundle", "usbgpu": "ModelManager_ActiveBundleUSBGPU", } -_LAST_VALIDATED_RAW: dict[str, bytes | None] = {} +_LAST_VALIDATED_RAW: dict[str, dict | None] = {} def _compute_hash(file_path: str) -> str | None: @@ -92,11 +92,11 @@ def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, availa if available_bundles is not None: matching_bundle = None for bundle in available_bundles: - if getattr(active_bundle, 'ref', None) and getattr(bundle, 'ref', None): + if active_bundle.ref and bundle.ref: if active_bundle.ref == bundle.ref: matching_bundle = bundle break - elif getattr(active_bundle, 'internalName', None) == getattr(bundle, 'internalName', None): + elif active_bundle.internalName == bundle.internalName: matching_bundle = bundle break @@ -104,12 +104,8 @@ def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, availa return True if active_bundle.minimumSelectorVersion != matching_bundle.minimumSelectorVersion: return True - - active_runner = getattr(active_bundle, 'runner', None) - matching_runner = getattr(matching_bundle, 'runner', None) - if active_runner is not None and matching_runner is not None: - if getattr(active_runner, 'raw', active_runner) != getattr(matching_runner, 'raw', matching_runner): - return True + if active_bundle.runner != matching_bundle.runner: + return True if set(_bundle_artifacts(active_bundle)) != set(_bundle_artifacts(matching_bundle)): return True @@ -127,7 +123,7 @@ def _parse_active_bundle(raw_bundle) -> "custom.ModelManagerSP.ModelBundle | Non def get_selected_bundle(params: Params | None = None, source: str = "qcom") -> "custom.ModelManagerSP.ModelBundle | None": params = params or Params() - return _parse_active_bundle(params.get(ACTIVE_BUNDLE_KEYS.get(source, "ModelManager_ActiveBundle"))) + return _parse_active_bundle(params.get(ACTIVE_BUNDLE_KEYS[source])) def get_active_source(usbgpu: bool | None = None, usbgpu_active: bool | None = None, @@ -140,17 +136,15 @@ def get_active_source(usbgpu: bool | None = None, usbgpu_active: bool | None = N def get_active_bundle(params: Params | None = None, *, usbgpu: bool | None = None) -> "custom.ModelManagerSP.ModelBundle | None": + # no cross-slot fallback: an empty active slot means the hardware default, which + # only stock modeld can run - modeld_v2 requires a real bundle params = params or Params() - if get_active_source(usbgpu=usbgpu) == "usbgpu": - if bundle := get_selected_bundle(params, "usbgpu"): - return bundle - return get_selected_bundle(params, "qcom") + return get_selected_bundle(params, get_active_source(usbgpu=usbgpu)) def resolve_bundle_by_ref( ref: str, source_bundles: dict[str, list[custom.ModelManagerSP.ModelBundle]], ) -> "tuple[custom.ModelManagerSP.ModelBundle, str] | None": - """Finds the bundle matching a ref across all sources.""" for source, bundles in source_bundles.items(): for bundle in bundles: if bundle.ref == ref: @@ -173,15 +167,16 @@ def _validate_active_bundle(params: Params, source: str, available_bundles: list if active_bundle is None or _bundle_needs_reset(active_bundle, available_bundles): cloudlog.warning(f"Active model bundle invalid for {source}; resetting to default") params.remove(key) - params.put("ModelRunnerTypeCache", int(custom.ModelManagerSP.Runner.stock), block=True) _LAST_VALIDATED_RAW[key] = None else: _LAST_VALIDATED_RAW[key] = raw_bundle def validate_active_bundles(params: Params, source_bundles: dict[str, list[custom.ModelManagerSP.ModelBundle]]) -> None: + # an empty list means the fetch failed, not that the catalog dropped the bundle for source, bundles in source_bundles.items(): - _validate_active_bundle(params, source, bundles) + _validate_active_bundle(params, source, bundles or None) + get_active_model_runner(params, force_check=True) def get_active_model_runner(params: Params | None = None, force_check: bool = False) -> int: diff --git a/openpilot/sunnypilot/models/manager.py b/openpilot/sunnypilot/models/manager.py index bdc6c88741..178d6c04e5 100644 --- a/openpilot/sunnypilot/models/manager.py +++ b/openpilot/sunnypilot/models/manager.py @@ -24,6 +24,10 @@ from openpilot.sunnypilot.models.helpers import (ACTIVE_BUNDLE_KEYS, get_active_ DOWNLOAD_TIMEOUT = (30, 30) +class DownloadCancelled(Exception): + pass + + class ModelManagerSP: """Manages model downloads and status reporting""" @@ -39,6 +43,17 @@ class ModelManagerSP: self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params, usbgpu=self.chestnut_present) self._chunk_size = 128 * 1000 # 128 KB chunks self._download_start_times: dict[str, float] = {} # Track start time per model + self._download_ref: bytes | str | None = None + + def _download_interrupted(self) -> bool: + # only removal cancels: a different ref is a queued selection that + # _release_download_ref leaves in place for the next tick + return self.params.get("ModelManager_DownloadRef") is None + + def _release_download_ref(self) -> None: + if self.params.get("ModelManager_DownloadRef") == self._download_ref: + self.params.remove("ModelManager_DownloadRef") + self._download_ref = None def _sync_artifact_progress(self, source_artifact) -> None: """Mirror download progress to all artifacts sharing the same filename in the selected bundle.""" @@ -80,8 +95,8 @@ class ModelManagerSP: f.write(chunk) bytes_downloaded += len(chunk) - if self.params.get("ModelManager_DownloadRef") is None: - raise Exception("Download cancelled") + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") if total_size > 0: progress = (bytes_downloaded / total_size) * 100 @@ -94,7 +109,7 @@ class ModelManagerSP: # Clean up start time after download completes del self._download_start_times[model.fileName] - async def _download_chunked(self, base_url: str, base_path: str, artifact) -> None: + async def _download_chunked(self, base_url: str, base_path: str, artifact, skip: frozenset[int] | set[int] = frozenset()) -> None: from openpilot.common.file_chunker import get_chunk_name, get_manifest_path num_chunks = len(artifact.chunks) @@ -106,8 +121,11 @@ class ModelManagerSP: # Shared connection saves a TCP+TLS handshake per chunk. # Keep sequential: the link saturates on one stream and Session is not thread-safe. + completed = len(skip) with requests.Session() as session: for i, _ in enumerate(artifact.chunks): + if i in skip: + continue chunk_url = get_chunk_name(base_url, i, num_chunks) chunk_path = get_chunk_name(base_path, i, num_chunks) chunk_downloaded = 0 @@ -118,15 +136,16 @@ class ModelManagerSP: for data in response.iter_content(chunk_size=self._chunk_size): f.write(data) chunk_downloaded += len(data) - if self.params.get("ModelManager_DownloadRef") is None: - raise Exception("Download cancelled") + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") intra = chunk_downloaded / max(chunk_size, 1) - progress = min(99.0, ((i + intra) / num_chunks) * 100) + progress = min(99.0, ((completed + intra) / num_chunks) * 100) artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloading artifact.downloadProgress.progress = progress artifact.downloadProgress.eta = self._calculate_eta(artifact.fileName, progress) self._sync_artifact_progress(artifact) self._report_status() + completed += 1 with open(manifest_path, 'w') as f: # noqa: ASYNC230 f.write(str(num_chunks)) @@ -137,6 +156,8 @@ class ModelManagerSP: async def _process_artifact(self, artifact, destination_path: str) -> None: if not artifact.downloadUri.uri: return None + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") url = artifact.downloadUri.uri expected_hash = artifact.downloadUri.sha256 @@ -144,21 +165,23 @@ class ModelManagerSP: full_path = os.path.join(destination_path, filename) try: + # progress counts only valid chunks so a resumed download continues the + # bar from where verification left it, instead of falling back to zero is_cached = False + valid_chunks: set[int] = set() if len(artifact.chunks) > 0: from openpilot.common.file_chunker import get_chunk_name num_chunks = len(artifact.chunks) - chunks_valid = True for i, chunk in enumerate(artifact.chunks): - chunk_path = get_chunk_name(full_path, i, num_chunks) - if not await verify_file(chunk_path, chunk.sha256): - chunks_valid = False - break - artifact.downloadProgress.progress = ((i + 1) / num_chunks) * 100 + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") + if await verify_file(get_chunk_name(full_path, i, num_chunks), chunk.sha256): + valid_chunks.add(i) + artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.verifying + artifact.downloadProgress.progress = (len(valid_chunks) / num_chunks) * 100 self._sync_artifact_progress(artifact) self._report_status() - if chunks_valid and num_chunks > 0: - is_cached = True + is_cached = len(valid_chunks) == num_chunks else: if await verify_file(full_path, expected_hash): is_cached = True @@ -172,7 +195,7 @@ class ModelManagerSP: return if len(artifact.chunks) > 0: - await self._download_chunked(url, full_path, artifact) + await self._download_chunked(url, full_path, artifact, skip=valid_chunks) from openpilot.common.file_chunker import get_chunk_name for i, chunk in enumerate(artifact.chunks): chunk_path = get_chunk_name(full_path, i, len(artifact.chunks)) @@ -189,6 +212,17 @@ class ModelManagerSP: self._sync_artifact_progress(artifact) self._report_status() + except DownloadCancelled: + # a cancel keeps whatever is on disk: complete chunks resume the next attempt + self._download_start_times.pop(artifact.fileName, None) + artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.failed + artifact.downloadProgress.eta = 0 + self._sync_artifact_progress(artifact) + if self.selected_bundle: + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed + self._report_status() + raise + except Exception as e: cloudlog.error(f"Error downloading {filename}: {str(e)}") for f in [full_path] + [p for p in (os.path.join(destination_path, f) for f in os.listdir(destination_path)) if filename in p]: @@ -221,7 +255,6 @@ class ModelManagerSP: self.pm.send('modelManagerSP', msg) async def _download_bundle(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str, source: str) -> None: - """Downloads a bundle and sets it as the active bundle for its source""" self.selected_bundle = model_bundle self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloading for model in self.selected_bundle.models: @@ -243,6 +276,8 @@ class ModelManagerSP: seen_artifacts.add(artifact.fileName) await self._process_artifact(artifact, destination_path) + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded self.params.put(ACTIVE_BUNDLE_KEYS[source], model_bundle.to_dict(), block=True) self.active_bundle = get_active_bundle(self.params, usbgpu=self.chestnut_present) @@ -259,6 +294,27 @@ class ModelManagerSP: """Main entry point for downloading a model bundle""" asyncio.run(self._download_bundle(model_bundle, destination_path, source)) + def _process_download_requests(self) -> None: + # loops so a ref queued during a download starts in the same tick, without + # the bar dropping to idle for a tick between the two transfers + last_ref = None + while (ref_to_download := self.params.get("ModelManager_DownloadRef")) is not None: + if ref_to_download == last_ref: # a repeating ref falls back to the next tick instead of spinning + return + last_ref = ref_to_download + resolved = resolve_bundle_by_ref(ref_to_download, self.source_models) + if not resolved: + return + model_to_download, source = resolved + self._download_ref = ref_to_download + try: + self.download(model_to_download, Paths.model_root(), source) + except Exception as e: + cloudlog.exception(e) + finally: + self._release_download_ref() + self.selected_bundle = None + def main_thread(self) -> None: """Main thread for model management""" rk = Ratekeeper(1, print_delay_threshold=None) @@ -272,16 +328,7 @@ class ModelManagerSP: validate_active_bundles(self.params, self.source_models) self.active_bundle = get_active_bundle(self.params, usbgpu=self.chestnut_present) - if (ref_to_download := self.params.get("ModelManager_DownloadRef")) is not None: - if resolved := resolve_bundle_by_ref(ref_to_download, self.source_models): - model_to_download, source = resolved - try: - self.download(model_to_download, Paths.model_root(), source) - except Exception as e: - cloudlog.exception(e) - finally: - self.params.remove("ModelManager_DownloadRef") - self.selected_bundle = None + self._process_download_requests() if self.params.get("ModelManager_ClearCache"): self.clear_model_cache() @@ -305,7 +352,7 @@ class ModelManagerSP: for source in ACTIVE_BUNDLE_KEYS: if selected_bundle := get_selected_bundle(self.params, source): for model in selected_bundle.models: - if hasattr(model, 'artifact') and model.artifact.fileName: + if model.artifact.fileName: active_files.append(model.artifact.fileName) # Remove all files except active ones (including their chunk files) diff --git a/openpilot/sunnypilot/models/tests/test_manager_download.py b/openpilot/sunnypilot/models/tests/test_manager_download.py index 4346f74e4a..4d3b7989fb 100644 --- a/openpilot/sunnypilot/models/tests/test_manager_download.py +++ b/openpilot/sunnypilot/models/tests/test_manager_download.py @@ -25,7 +25,9 @@ from openpilot.common.file_chunker import get_chunk_name, get_manifest_path from openpilot.selfdrive.test.helpers import http_server_context from openpilot.sunnypilot.models import manager as manager_module from openpilot.sunnypilot.models.fetcher import ModelFetcher, get_cached_bundles -from openpilot.sunnypilot.models.helpers import get_active_bundle, get_active_source, get_selected_bundle, resolve_bundle_by_ref +from openpilot.sunnypilot.models import helpers +from openpilot.sunnypilot.models.helpers import (get_active_bundle, get_active_source, get_selected_bundle, + resolve_bundle_by_ref, validate_active_bundles) from openpilot.sunnypilot.models.manager import ModelManagerSP CHUNK_BODIES = [b'A' * 5000, b'B' * 5000, b'C' * 3000] @@ -101,6 +103,7 @@ class ManagerDownloadTestBase(OpenpilotTestCase): self.manager = ModelManagerSP.__new__(ModelManagerSP) self.manager.params = mock.MagicMock() self.manager.params.get.return_value = b'0' # not cancelled + self.manager._download_ref = b'0' self.manager.pm = mock.MagicMock() self.manager.pm.send.side_effect = self._record_progress self.manager.selected_bundle = None @@ -259,6 +262,7 @@ class TestManagerDownload(ManagerDownloadTestBase): artifact = self.make_artifact(chunked=True) base_path = os.path.join(self.dest, artifact.fileName) self.manager.params.get.side_effect = lambda key: b"ref" if key == "ModelManager_DownloadRef" else None + self.manager._download_ref = b"ref" asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact)) assert os.path.isfile(get_manifest_path(base_path)) self.run_with_server(body) @@ -277,12 +281,92 @@ class TestManagerDownload(ManagerDownloadTestBase): return b"0" self.manager.params.get.side_effect = get + self.manager._download_ref = b"ref" with self.assertRaises(Exception) as ctx: asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact)) assert 'cancelled' in str(ctx.exception).lower() assert not os.path.isfile(get_manifest_path(base_path)) self.run_with_server(body) + def test_replaced_download_ref_queues_instead_of_cancelling(self): + """Selecting another model mid-transfer lets the running download finish.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + self.manager.params.get.side_effect = lambda key: b"other-ref" if key == "ModelManager_DownloadRef" else None + self.manager._download_ref = b"ref" + asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact)) + assert os.path.isfile(get_manifest_path(base_path)) + self.run_with_server(body) + + def test_replaced_download_ref_is_kept(self): + """A selection made during a download must survive that download's cleanup.""" + self.manager.params.get.return_value = b"new-ref" + self.manager._download_ref = b"old-ref" + self.manager._release_download_ref() + self.manager.params.remove.assert_not_called() + + def test_own_download_ref_is_released(self): + self.manager.params.get.return_value = b"ref" + self.manager._download_ref = b"ref" + self.manager._release_download_ref() + self.manager.params.remove.assert_called_once_with("ModelManager_DownloadRef") + + def test_cached_bundle_cancel_skips_slot_write(self): + """A cancel must stop an already-on-disk bundle before it is applied to the slot.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + for i, data in enumerate(CHUNK_BODIES): + with open(get_chunk_name(base_path, i, len(CHUNK_BODIES)), 'wb') as f: + f.write(data) + self._bundle.ref = "test-ref" + params, store = self._make_params_with_store() + store["ModelManager_DownloadRef"] = None # removed -> cancelled + self.manager.params = params + self.manager._download_ref = b"ref" + with self.assertRaises(Exception) as ctx: + asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "qcom")) + assert 'cancelled' in str(ctx.exception).lower() + assert "ModelManager_ActiveBundle" not in store + assert all(os.path.isfile(p) for p in self.chunk_paths(base_path)), "cancel must not delete cached chunks" + self.run_with_server(body) + + def test_resume_skips_valid_chunks(self): + """A chunk already on disk is kept and not re-downloaded; progress starts above its share.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + with open(get_chunk_name(base_path, 0, len(CHUNK_BODIES)), 'wb') as f: + f.write(CHUNK_BODIES[0]) + + asyncio.run(self.manager._process_artifact(artifact, self.dest)) + + chunk0_suffix = get_chunk_name('', 0, len(CHUNK_BODIES)) + assert not any(p.endswith(chunk0_suffix) for p in DownloadHandler.request_paths), "valid chunk was re-downloaded" + for i, expected in enumerate(CHUNK_BODIES): + with open(get_chunk_name(base_path, i, len(CHUNK_BODIES)), 'rb') as f: + assert f.read() == expected + assert os.path.isfile(get_manifest_path(base_path)) + assert min(self.reported) >= (1 / len(CHUNK_BODIES)) * 100 - 1, "progress must not restart below the resumed share" + self.run_with_server(body) + + def test_verify_reports_valid_fraction_then_cached(self): + """A fully cached bundle publishes climbing verify progress and ends cached.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + for i, data in enumerate(CHUNK_BODIES): + with open(get_chunk_name(base_path, i, len(CHUNK_BODIES)), 'wb') as f: + f.write(data) + + asyncio.run(self.manager._process_artifact(artifact, self.dest)) + + assert DownloadHandler.request_paths == [], "cached bundle must not hit the network" + assert [round(p) for p in self.reported[:3]] == [33, 67, 100] + assert artifact.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.cached + self.run_with_server(body) + def _make_params_with_store(self): params = mock.MagicMock() store = {} @@ -302,7 +386,7 @@ class TestManagerDownload(ManagerDownloadTestBase): def body(): artifact = self.make_artifact(chunked=True) self._bundle.ref = "test-ref" - self._bundle.minimumSelectorVersion = 17 + self._bundle.minimumSelectorVersion = 18 params, store = self._make_params_with_store() self.manager.params = params asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "qcom")) @@ -322,7 +406,7 @@ class TestManagerDownload(ManagerDownloadTestBase): def body(): self.make_artifact(chunked=True) self._bundle.ref = "big-ref" - self._bundle.minimumSelectorVersion = 17 + self._bundle.minimumSelectorVersion = 18 params, store = self._make_params_with_store() self.manager.params = params asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "usbgpu")) @@ -385,7 +469,7 @@ def manifest_bundle(short_name: str, ref: str, index: int = 0, is_big: bool = Fa "environment": "release", "runner": "tinygrad", "is_big": is_big, - "minimum_selector_version": "17", + "minimum_selector_version": "18", "ref": ref, "models": [{ "type": "supercombo", @@ -532,6 +616,20 @@ class TestSourceCacheIntegrity(OpenpilotTestCase): fetch.assert_called_once_with("qcom") assert [bundle.ref for bundle in bundles] == ["ddd"] + def test_mismatched_refetch_happens_once(self): + """If the fresh manifest still fails the source check, the URL is authoritative: + trust it instead of refetching at 1 Hz forever.""" + params = self._make_params({"bundles": [manifest_bundle("big", "bbb", is_big=True)]}, + {"bundles": [manifest_bundle("big2", "ccc", is_big=True)]}) + fetcher = ModelFetcher(params) + fetched = self._fetched(manifest_bundle("big", "bbb", is_big=True)) + with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched) as fetch: + first = fetcher.get_bundles_for_source("qcom") + second = fetcher.get_bundles_for_source("qcom") + fetch.assert_called_once_with("qcom") + assert [bundle.ref for bundle in first] == ["bbb"] + assert [bundle.ref for bundle in second] == ["bbb"] + def test_corrupt_cache_is_refetched(self): """A cache that fails to parse (e.g. truncated/foreign JSON) must trigger a refetch instead of raising every loop and never recovering.""" @@ -545,15 +643,61 @@ class TestSourceCacheIntegrity(OpenpilotTestCase): assert [bundle.ref for bundle in bundles] == ["aaa"] +class TestActiveBundleValidation(OpenpilotTestCase): + """Validation is per-slot: a failed fetch (empty bundle list) must not reset a slot, + and resetting one slot must not stomp the runner cache derived from the other.""" + + def setUp(self): + super().setUp() + helpers._LAST_VALIDATED_RAW.clear() + + @staticmethod + def _raw_bundle(ref: str, runner: int | None = None) -> dict: + bundle = custom.ModelManagerSP.ModelBundle.new_message() + bundle.ref = ref + bundle.minimumSelectorVersion = 18 + if runner is not None: + bundle.runner = runner + return bundle.to_dict() + + def _params(self, qcom=None, usbgpu=None): + params = mock.MagicMock() + + def get(key, *args, **kwargs): + return {"ModelManager_ActiveBundle": qcom, "ModelManager_ActiveBundleUSBGPU": usbgpu}.get(key) + + params.get.side_effect = get + return params + + def test_empty_catalog_does_not_reset_slot(self): + params = self._params(qcom=self._raw_bundle("small")) + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False): + validate_active_bundles(params, {"qcom": [], "usbgpu": []}) + params.remove.assert_not_called() + + def test_reset_recomputes_runner_from_surviving_slot(self): + tinygrad = int(custom.ModelManagerSP.Runner.tinygrad) + big_raw = self._raw_bundle("big", runner=tinygrad) + params = self._params(qcom=self._raw_bundle("gone"), usbgpu=big_raw) + catalog = {"qcom": [custom.ModelManagerSP.ModelBundle(**self._raw_bundle("other"))], + "usbgpu": [custom.ModelManagerSP.ModelBundle(**big_raw)]} + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True): + validate_active_bundles(params, catalog) + params.remove.assert_called_once_with("ModelManager_ActiveBundle") + runner_puts = [call for call in params.put.call_args_list if call.args[0] == "ModelRunnerTypeCache"] + assert [call.args[1] for call in runner_puts] == [tinygrad] + + class TestActiveBundleSelection(OpenpilotTestCase): - """The effective active bundle follows the hardware: the usbgpu slot wins when a GPU - is present and compiled, otherwise the qcom slot. Each slot keeps its own selection.""" + """The effective active bundle is the active source's slot: usbgpu when a GPU is + present, qcom otherwise. An empty active slot means the hardware default (stock + runner), never the other slot's pick - modeld_v2 requires a real bundle.""" @staticmethod def _raw_bundle(ref: str) -> dict: bundle = custom.ModelManagerSP.ModelBundle.new_message() bundle.ref = ref - bundle.minimumSelectorVersion = 17 + bundle.minimumSelectorVersion = 18 return bundle.to_dict() def _params(self, qcom=None, usbgpu=None): @@ -584,10 +728,10 @@ class TestActiveBundleSelection(OpenpilotTestCase): with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True): assert get_active_bundle(params).ref == "big" - def test_gpu_without_big_selection_falls_back_to_small(self): + def test_gpu_without_big_selection_is_hardware_default(self): params = self._params(qcom=self._raw_bundle("small"), usbgpu=None) with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True): - assert get_active_bundle(params).ref == "small" + assert get_active_bundle(params) is None class TestEffectiveSource(OpenpilotTestCase): @@ -600,7 +744,7 @@ class TestEffectiveSource(OpenpilotTestCase): def _raw_bundle(ref: str) -> dict: bundle = custom.ModelManagerSP.ModelBundle.new_message() bundle.ref = ref - bundle.minimumSelectorVersion = 17 + bundle.minimumSelectorVersion = 18 return bundle.to_dict() def test_runtime_no_gpu(self): diff --git a/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py b/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py index fd389f93c0..d6d82dfb32 100644 --- a/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py +++ b/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py @@ -1,13 +1,11 @@ import requests -from openpilot.common.params import Params from openpilot.sunnypilot.models.tinygrad_ref import get_tinygrad_ref from openpilot.sunnypilot.models.fetcher import ModelFetcher from openpilot.common.test import OpenpilotTestCase def fetch_tinygrad_ref(): - fetcher = ModelFetcher(Params()) - response = requests.get(fetcher.model_url, timeout=10) + response = requests.get(ModelFetcher.MODEL_URL, timeout=10) response.raise_for_status() json_data = response.json() return json_data.get("tinygrad_ref") diff --git a/openpilot/sunnypilot/system/params_migration.py b/openpilot/sunnypilot/system/params_migration.py index 130fd64310..f0f0d7248a 100644 --- a/openpilot/sunnypilot/system/params_migration.py +++ b/openpilot/sunnypilot/system/params_migration.py @@ -84,6 +84,21 @@ def _migrate_tesla_mads_screen_button(_params): cloudlog.exception(f"Error migrating TeslaMadsScreenButton: {e}") +def _migrate_model_bundle_slots(_params): + # Pre-split, a chestnut user's big-model selection lived in the single + # ActiveBundle. Seed both slots; validation drops whichever does not match + # its own manifest. + try: + if _params.get("ModelManager_ActiveBundleUSBGPU") is not None: + return + if (bundle := _params.get("ModelManager_ActiveBundle")) is None: + return + _params.put("ModelManager_ActiveBundleUSBGPU", bundle, block=True) + cloudlog.info("params_migration: seeded ModelManager_ActiveBundleUSBGPU from ModelManager_ActiveBundle") + except Exception as e: + cloudlog.exception(f"Error migrating model bundle slots: {e}") + + def run_migration(_params): # migrate OnroadScreenOffBrightness if _params.get("OnroadScreenOffBrightnessMigrated") != ONROAD_BRIGHTNESS_MIGRATION_VERSION: @@ -120,3 +135,6 @@ def run_migration(_params): # seed TeslaMadsScreenButton for existing Tesla installs _migrate_tesla_mads_screen_button(_params) + + # seed the usbgpu model slot from the pre-split single slot + _migrate_model_bundle_slots(_params) diff --git a/openpilot/sunnypilot/system/tests/__init__.py b/openpilot/sunnypilot/system/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openpilot/sunnypilot/system/tests/test_params_migration.py b/openpilot/sunnypilot/system/tests/test_params_migration.py new file mode 100644 index 0000000000..328a7a65af --- /dev/null +++ b/openpilot/sunnypilot/system/tests/test_params_migration.py @@ -0,0 +1,36 @@ +""" +Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + +This file is part of sunnypilot and is licensed under the MIT License. +See the LICENSE.md file in the root directory for more details. +""" + +from openpilot.common.params import Params +from openpilot.common.test import OpenpilotTestCase +from openpilot.sunnypilot.system.params_migration import _migrate_model_bundle_slots + + +class TestModelBundleSlotMigration(OpenpilotTestCase): + """Pre-split, a chestnut user's big-model selection lived in the single ActiveBundle. + The migration seeds both slots; per-source validation later drops whichever does not + match its own manifest.""" + + def test_seeds_usbgpu_slot_from_active_bundle(self): + params = Params() + bundle = {"ref": "big", "minimumSelectorVersion": 18} + params.put("ModelManager_ActiveBundle", bundle, block=True) + _migrate_model_bundle_slots(params) + assert params.get("ModelManager_ActiveBundleUSBGPU") == bundle + assert params.get("ModelManager_ActiveBundle") == bundle + + def test_noop_when_usbgpu_slot_already_set(self): + params = Params() + params.put("ModelManager_ActiveBundle", {"ref": "small"}, block=True) + params.put("ModelManager_ActiveBundleUSBGPU", {"ref": "big"}, block=True) + _migrate_model_bundle_slots(params) + assert params.get("ModelManager_ActiveBundleUSBGPU") == {"ref": "big"} + + def test_noop_when_no_selection(self): + params = Params() + _migrate_model_bundle_slots(params) + assert params.get("ModelManager_ActiveBundleUSBGPU") is None diff --git a/openpilot/system/ui/sunnypilot/lib/utils.py b/openpilot/system/ui/sunnypilot/lib/utils.py index b9ed152aff..6ae30d13ae 100644 --- a/openpilot/system/ui/sunnypilot/lib/utils.py +++ b/openpilot/system/ui/sunnypilot/lib/utils.py @@ -8,12 +8,26 @@ from collections.abc import Callable import pyray as rl -from openpilot.system.ui.lib.application import FontWeight +from openpilot.system.ui.lib.application import gui_app, FontWeight from openpilot.system.ui.sunnypilot.lib.styles import style from openpilot.system.ui.sunnypilot.widgets.list_view import ButtonActionSP -from openpilot.system.ui.widgets.label import UnifiedLabel +from openpilot.system.ui.widgets.label import ScrollState, UnifiedLabel from openpilot.system.ui.widgets.list_view import BUTTON_WIDTH, BUTTON_HEIGHT, TEXT_PADDING, _resolve_value +SCROLL_SPEED = 1.2 # stock is 0.8, boosted 50% to compensate for larger font (50 vs 32) +SCROLL_REFERENCE_FPS = 60. + + +class UnifiedLabelSP(UnifiedLabel): + # stock scroll formula (0.8 / 60 * fps) is inverted — pre-correct so speed is constant px/sec + def _render(self, _): + if self._needs_scroll and self._scroll_state == ScrollState.SCROLLING: + fps = gui_app.target_fps + wrong_step = 0.8 / SCROLL_REFERENCE_FPS * fps + correct_step = SCROLL_SPEED * SCROLL_REFERENCE_FPS / fps + self._scroll_offset -= (correct_step - wrong_step) + super()._render(_) + class NoElideButtonAction(ButtonActionSP): def get_width_hint(self): @@ -21,14 +35,12 @@ class NoElideButtonAction(ButtonActionSP): class ScrollingButtonAction(ButtonActionSP): - """ButtonActionSP whose value scrolls instead of eliding when it doesn't fit.""" - def __init__(self, text: str | Callable[[], str], width: int = style.BUTTON_ACTION_WIDTH, enabled: bool | Callable[[], bool] = True): super().__init__(text=text, width=width, enabled=enabled) - self._value_label = UnifiedLabel("", font_size=style.ITEM_TEXT_FONT_SIZE, font_weight=FontWeight.NORMAL, - text_color=self._value_color, scroll=True, - alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) + self._value_label = UnifiedLabelSP("", font_size=style.ITEM_TEXT_FONT_SIZE, font_weight=FontWeight.NORMAL, + text_color=self._value_color, scroll=True, + alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) def set_value(self, value: str | Callable[[], str], color: rl.Color = style.ITEM_TEXT_VALUE_COLOR): if self.value != _resolve_value(value, ""): diff --git a/openpilot/system/ui/sunnypilot/widgets/download_status.py b/openpilot/system/ui/sunnypilot/widgets/download_status.py index b299c464f1..135bd151a5 100644 --- a/openpilot/system/ui/sunnypilot/widgets/download_status.py +++ b/openpilot/system/ui/sunnypilot/widgets/download_status.py @@ -16,6 +16,7 @@ from openpilot.system.ui.lib.text_measure import measure_text_cached from openpilot.system.ui.sunnypilot.lib.styles import style from openpilot.system.ui.sunnypilot.widgets.list_view import ListItemSP from openpilot.system.ui.widgets.label import UnifiedLabel +from openpilot.system.ui.sunnypilot.lib.utils import UnifiedLabelSP from openpilot.system.ui.widgets.list_view import ItemAction FONT_SIZE = style.ITEM_TEXT_FONT_SIZE @@ -24,6 +25,8 @@ ICON_PADDING = 12 BAR_WIDTH = 1100 BAR_HEIGHT = 20 +SEGMENT_GAP = 24 +SEGMENT_NAME_MAX_WIDTH = 380 BAR_GAP = 16 BAR_RADIUS = BAR_HEIGHT / 2 CAPSULE_POINTS = 24 @@ -45,6 +48,8 @@ class DownloadStatusAction(ItemAction): super().__init__(width=BAR_WIDTH) self.name = "" self.status_text = "" + self.segments: list[tuple[str, rl.Color, str | None, rl.Color | None]] | None = None + self._segment_labels: list[UnifiedLabelSP] = [] self.downloading = False self.text_color = rl.GRAY self.icon: str | None = None @@ -62,7 +67,8 @@ class DownloadStatusAction(ItemAction): alignment=rl.GuiTextAlignment.TEXT_ALIGN_RIGHT, alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) - def update(self, name, downloading=False, progress=0.0, status_text="", text_color=rl.GRAY, icon=None, icon_color=None): + def update(self, name, downloading=False, progress=0.0, status_text="", text_color=rl.GRAY, icon=None, icon_color=None, segments=None): + self.segments = segments if downloading and not self.downloading: self._name_label.reset_shimmer() self._progress.x = progress @@ -85,11 +91,22 @@ class DownloadStatusAction(ItemAction): def get_width_hint(self) -> float: if self.downloading: return BAR_WIDTH + if self.segments: + return sum(total for _, _, total in self._measured_segments()) width = measure_text_cached(self._font, self._idle_text, FONT_SIZE).x if self.icon: width += ICON_SIZE + ICON_PADDING return width + def _measured_segments(self): + """[(segment, text width, total width incl. icon and gap)]""" + out = [] + for i, seg in enumerate(self.segments or []): + text_width = min(measure_text_cached(self._font, seg[0], FONT_SIZE).x, SEGMENT_NAME_MAX_WIDTH) + total = text_width + (ICON_PADDING + ICON_SIZE if seg[2] else 0) + (SEGMENT_GAP if i else 0) + out.append((seg, text_width, total)) + return out + def _render(self, rect: rl.Rectangle): if self.downloading: self._render_downloading(rect) @@ -134,6 +151,8 @@ class DownloadStatusAction(ItemAction): def _render_downloading(self, rect: rl.Rectangle): percent = f"{int(self._progress.x)}%" + if self.status_text: + percent = f"{self.status_text} {percent}" text_height = measure_text_cached(self._font, percent, FONT_SIZE).y top = rect.y + (rect.height - (text_height + BAR_GAP + BAR_HEIGHT)) / 2 @@ -148,6 +167,9 @@ class DownloadStatusAction(ItemAction): self._draw_fill(rail, max(0.0, min(rect.width, rect.width * (self._progress.x / 100.0)))) def _render_idle(self, rect: rl.Rectangle): + if self.segments: + self._render_segments(rect) + return text = self._idle_text text_size = measure_text_cached(self._font, text, FONT_SIZE) right = rect.x + rect.width @@ -161,6 +183,29 @@ class DownloadStatusAction(ItemAction): rl.draw_text_ex(self._font, text, rl.Vector2(right - text_size.x, rect.y + (rect.height - text_size.y) / 2), FONT_SIZE, 0, self.text_color) + def _render_segments(self, rect: rl.Rectangle): + measured = self._measured_segments() + while len(self._segment_labels) < len(measured): + self._segment_labels.append(UnifiedLabelSP("", font_size=FONT_SIZE, max_width=SEGMENT_NAME_MAX_WIDTH, + scroll=True, wrap_text=False)) + x = rect.x + rect.width - sum(total for _, _, total in measured) + for i, ((text, color, icon, icon_color), text_width, _) in enumerate(measured): + if i: + x += SEGMENT_GAP + label = self._segment_labels[i] + if label.text != text: + label.set_text(text) + label.set_text_color(color) + text_height = measure_text_cached(self._font, text, FONT_SIZE).y + label.set_position(x, rect.y + (rect.height - text_height) / 2) + label.render() + x += text_width + if icon: + texture = gui_app.texture(icon, ICON_SIZE, ICON_SIZE, keep_aspect_ratio=True) + rl.draw_texture_v(texture, rl.Vector2(x + ICON_PADDING, rect.y + (rect.height - texture.height) / 2), + icon_color or color) + x += ICON_PADDING + ICON_SIZE + def download_status_item(title): return ListItemSP(title=title, action_item=DownloadStatusAction(), title_color=style.ITEM_TEXT_COLOR) diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py index ff9be64783..2d35d319c2 100755 --- a/release/ci/model_generator.py +++ b/release/ci/model_generator.py @@ -136,7 +136,7 @@ def generate_chunked_model(driving_pkl: Path) -> dict: def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown", - onnx_sha256=None) -> None: + onnx_sha256=None, is_big=False) -> None: bundle_json = { "short_name": short_name, "display_name": custom_name or upstream_branch, @@ -149,6 +149,7 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short "generation": "-1", "build_time": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "overrides": {}, + "is_big": is_big, "models": models, } @@ -186,6 +187,8 @@ if __name__ == "__main__": print(f"No driving_tinygrad.pkl found in {_output_dir}", file=sys.stderr) sys.exit(1) + is_big = _driving_pkl.name.startswith('big_') + if _pkl: new_pkl = _output_dir / f"driving_{_pkl}_tinygrad.pkl" if not new_pkl.exists(): @@ -196,4 +199,4 @@ if __name__ == "__main__": _model_metadata = generate_chunked_model(_driving_pkl) _onnx_sha256 = _hash_onnx_files(Path(args.model_dir)) create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch, - onnx_sha256=_onnx_sha256) + onnx_sha256=_onnx_sha256, is_big=is_big) diff --git a/release/ci/publish.sh b/release/ci/publish.sh index fd1a61a87c..4b328a035c 100755 --- a/release/ci/publish.sh +++ b/release/ci/publish.sh @@ -47,7 +47,7 @@ git rm -rf $OUTPUT_DIR/.git || true # Doing cleanup, but it might fail if the .g git remote remove origin || true # ensure cleanup git remote add origin $GIT_ORIGIN #git push origin -d $DEV_BRANCH || true # Ensuring we delete the remote branch if it exists as we are wiping it out -git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH) +git fetch --depth 1 origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH) echo "[-] committing version $VERSION T=$SECONDS" git add -f .