diff --git a/.github/workflows/build-default-models.yaml b/.github/workflows/build-default-models.yaml new file mode 100644 index 0000000000..bf00845e27 --- /dev/null +++ b/.github/workflows/build-default-models.yaml @@ -0,0 +1,501 @@ +name: Build default models + +on: + workflow_dispatch: + inputs: + target: + description: 'Model target to build' + required: true + type: choice + options: + - small + - big + - dm + workflow_call: + inputs: + target: + description: 'Model target to build (small, big, or dm)' + required: true + type: string + +concurrency: + group: build-default-models-${{ inputs.target }} + cancel-in-progress: false + +env: + HF_REPO: sunnypilot/sunnypilot_models_v1 + +jobs: + resolve: + runs-on: ubuntu-24.04 + outputs: + model_name: ${{ steps.resolve.outputs.model_name }} + onnx_ref: ${{ steps.resolve.outputs.onnx_ref }} + onnx_path: ${{ steps.resolve.outputs.onnx_path }} + hf_defaults_path: ${{ steps.resolve.outputs.hf_defaults_path }} + tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - id: resolve + run: | + export PYTHONPATH=${{ github.workspace }} + + if [ "${{ inputs.target }}" = "big" ]; then + NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)") + ONNX_PATH="openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" + HF_DEFAULTS_PATH="models/defaults/big" + elif [ "${{ inputs.target }}" = "dm" ]; then + ONNX_PATH="openpilot/selfdrive/modeld/models/dmonitoring_model.onnx" + HF_DEFAULTS_PATH="models/defaults/dm" + NAME="dmonitoring_model ($(git log -1 --format=%cd --date=format:'%B %d, %Y' -- "$ONNX_PATH"))" + else + NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL; print(DEFAULT_MODEL)") + ONNX_PATH="openpilot/selfdrive/modeld/models/driving_supercombo.onnx" + HF_DEFAULTS_PATH="models/defaults/small" + fi + + ONNX_REF=$(git log -1 --format='%H' -- "$ONNX_PATH") + TINYGRAD_REF=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py) + if [ -z "$TINYGRAD_REF" ]; then + echo "::error::Failed to resolve tinygrad ref" + exit 1 + fi + + echo "model_name=${NAME}" >> $GITHUB_OUTPUT + echo "onnx_ref=${ONNX_REF}" >> $GITHUB_OUTPUT + echo "onnx_path=${ONNX_PATH}" >> $GITHUB_OUTPUT + echo "hf_defaults_path=${HF_DEFAULTS_PATH}" >> $GITHUB_OUTPUT + echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT + + build_small_model: + needs: resolve + if: ${{ inputs.target == 'small' }} + runs-on: [self-hosted, tici] + env: + SMALL_ONNX: openpilot/selfdrive/modeld/models/driving_supercombo.onnx + SMALL_PKL: openpilot/selfdrive/modeld/models/driving_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull ONNX via LFS + run: git lfs pull -I "${{ env.SMALL_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Compile small model with stock compiler + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") + CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)") + + TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + + env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \ + --onnx ${{ github.workspace }}/${{ env.SMALL_ONNX }} \ + --model-size $MODEL_SIZE \ + --camera-resolutions $CAMERA_RES \ + --frame-skip $FRAME_SKIP \ + --output ${{ github.workspace }}/${{ env.SMALL_PKL }} + + - name: Chunk small pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.SMALL_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked into {len(targets)} files') + " + + - name: Prepare output + env: + MODEL_NAME: ${{ needs.resolve.outputs.model_name }} + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models" + OUTPUT_DIR="${{ github.workspace }}/small_output" + PKL_BASE="driving_tinygrad.pkl" + mkdir -p "$OUTPUT_DIR" + + cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/" + cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/" + + python3 "${{ github.workspace }}/release/ci/model_generator.py" \ + --model-dir "$MODELS_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --custom-name "$MODEL_NAME" \ + --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" + + echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt" + + - name: Upload small model artifact + uses: actions/upload-artifact@v4 + with: + name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }} + path: ${{ github.workspace }}/small_output/ + + - name: Upload artifact name file + uses: actions/upload-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: ${{ github.workspace }}/small_output/artifact_name.txt + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + + build_big_model: + needs: resolve + if: ${{ inputs.target == 'big' }} + runs-on: [self-hosted, usbgpu] + env: + BIG_ONNX: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx + BIG_PKL: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull big ONNX via LFS + run: git lfs pull -I "${{ env.BIG_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Wait for chestnut PCIe link + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + python3 -c " + import time + from openpilot.system.hardware.chestnut.flash import link_up + for i in range(10): + if link_up(): + print(f'PCIe link up after {i+1} attempt(s)') + break + time.sleep(1) + else: + raise RuntimeError('Chestnut PCIe link not ready after 10 attempts') + " + + - name: Compile big model with stock compiler + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") + CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)") + + TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" + + env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \ + --onnx ${{ github.workspace }}/${{ env.BIG_ONNX }} \ + --model-size $MODEL_SIZE \ + --camera-resolutions $CAMERA_RES \ + --frame-skip $FRAME_SKIP \ + --output ${{ github.workspace }}/${{ env.BIG_PKL }} + + - name: Chunk big pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.BIG_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked into {len(targets)} files') + " + + - name: Prepare output + env: + MODEL_NAME: ${{ needs.resolve.outputs.model_name }} + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models" + OUTPUT_DIR="${{ github.workspace }}/big_output" + PKL_BASE="big_driving_tinygrad.pkl" + mkdir -p "$OUTPUT_DIR" + + cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/" + cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/" + + python3 "${{ github.workspace }}/release/ci/model_generator.py" \ + --model-dir "$MODELS_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --custom-name "$MODEL_NAME" \ + --upstream-branch "${{ needs.resolve.outputs.onnx_ref }}" + + echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt" + + - name: Upload big model artifact + uses: actions/upload-artifact@v4 + with: + name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }} + path: ${{ github.workspace }}/big_output/ + + - name: Upload artifact name file + uses: actions/upload-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: ${{ github.workspace }}/big_output/artifact_name.txt + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + + upload_defaults: + needs: [ resolve, build_small_model, build_big_model, build_dm_model ] + if: | + ${{ + !cancelled() && + (inputs.target == 'big' && needs.build_big_model.result == 'success' || + inputs.target == 'small' && needs.build_small_model.result == 'success' || + inputs.target == 'dm' && needs.build_dm_model.result == 'success') + }} + runs-on: ubuntu-24.04 + permissions: + id-token: write + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Pull ONNX via LFS + run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}" + + - name: Install huggingface_hub + run: pip install --upgrade "huggingface_hub>=0.22.0" + + - name: Download artifact name + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} + uses: actions/download-artifact@v4 + with: + name: artifact-name-${{ needs.resolve.outputs.model_name }} + path: artifact_name + + - name: Read artifact name + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} + id: artifact + run: | + ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt) + echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT + + - name: Download model artifact + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} + uses: actions/download-artifact@v4 + with: + name: ${{ steps.artifact.outputs.artifact_name }} + path: output + + - name: Upload model to HF + if: ${{ inputs.target == 'small' || inputs.target == 'big' }} + env: + HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} + ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }} + run: | + rm -f output/artifact_name.txt + export PYTHONPATH=$(pwd) + python3 release/ci/upload_default_model.py \ + --hf-repo "${{ env.HF_REPO }}" \ + --hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \ + --artifact-name "$ARTIFACT_NAME" \ + --model-dir output \ + --onnx-path "${{ needs.resolve.outputs.onnx_path }}" \ + --onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \ + --model-name "${{ needs.resolve.outputs.model_name }}" \ + --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \ + --run-number "${{ github.run_number }}" + + - name: Download DM artifact + if: ${{ inputs.target == 'dm' }} + uses: actions/download-artifact@v4 + with: + name: dm-model-${{ github.run_number }} + path: dm_output + + - name: Generate DM metadata and upload to HF + if: ${{ inputs.target == 'dm' }} + env: + HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} + run: | + export PYTHONPATH=$(pwd) + python3 -c " + import json, hashlib + from pathlib import Path + from datetime import datetime, UTC + + dm_dir = Path('dm_output') + manifest = list(dm_dir.glob('*.chunkmanifest')) + assert manifest, 'No chunkmanifest found' + pkl_name = manifest[0].name.removesuffix('.chunkmanifest') + num_chunks = int(manifest[0].read_text().strip()) + + chunks = [] + for i in range(num_chunks): + chunk = dm_dir / f'{pkl_name}.chunk{i+1:02d}of{num_chunks:02d}' + chunks.append({ + 'file_name': chunk.name, + 'sha256': hashlib.sha256(chunk.read_bytes()).hexdigest() + }) + + digest = hashlib.sha256() + for c in chunks: + with open(dm_dir / c['file_name'], 'rb') as f: + while block := f.read(1024*1024): + digest.update(block) + + metadata = { + 'bundles': [{ + 'short_name': 'DMMODEL', + 'display_name': '${{ needs.resolve.outputs.model_name }}', + 'ref': '${{ needs.resolve.outputs.onnx_ref }}', + 'runner': 'tinygrad', + 'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'models': [{ + 'type': 'chunked', + 'artifact': { + 'file_name': pkl_name, + 'download_uri': {'url': '', 'sha256': digest.hexdigest()}, + 'chunks': chunks + } + }] + }] + } + with open(dm_dir / 'metadata.json', 'w') as f: + json.dump(metadata, f, indent=2) + print('Generated DM metadata.json') + " + + python3 release/ci/upload_default_model.py \ + --hf-repo "${{ env.HF_REPO }}" \ + --hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \ + --artifact-name "dm-model-${{ github.run_number }}" \ + --model-dir dm_output \ + --onnx-path "${{ needs.resolve.outputs.onnx_path }}" \ + --onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \ + --model-name "${{ needs.resolve.outputs.model_name }}" \ + --tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \ + --run-number "${{ github.run_number }}" + + build_dm_model: + needs: resolve + if: ${{ inputs.target == 'dm' }} + runs-on: [self-hosted, tici] + env: + DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx + DM_PKL: openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Pull DM ONNX via LFS + run: git lfs pull -I "${{ env.DM_ONNX }}" + + - name: Set environment variables + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export UV_PYTHON_PREFERENCE=managed + export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + uv sync --frozen + printenv >> $GITHUB_ENV + + - name: Disable powersave + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable + + - name: Compile DM model + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}" + + TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + + taskset -c 7 env ${TG_FLAGS} python3 \ + ${{ github.workspace }}/tinygrad_repo/examples/openpilot/compile3.py \ + ${{ github.workspace }}/${{ env.DM_ONNX }} \ + ${{ github.workspace }}/${{ env.DM_PKL }} + + - name: Chunk DM pkl + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + export PYTHONPATH=${{ github.workspace }} + python3 -c " + from openpilot.common.file_chunker import chunk_file, get_chunk_targets + import os + pkl = '${{ github.workspace }}/${{ env.DM_PKL }}' + size = os.path.getsize(pkl) + targets = get_chunk_targets(pkl, size) + chunk_file(pkl, targets) + print(f'Chunked {pkl} into {len(targets)} chunks') + " + + - name: Prepare DM output + run: | + mkdir -p dm_output + cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunk* dm_output/ + cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunkmanifest dm_output/ + + - name: Upload DM artifact + uses: actions/upload-artifact@v4 + with: + name: dm-model-${{ github.run_number }} + path: dm_output/ + + - name: Re-enable powersave + if: always() + run: | + source ${UV_PROJECT_ENVIRONMENT}/bin/activate + PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + diff --git a/.github/workflows/download-hf-model-chunks/action.yml b/.github/workflows/download-hf-model-chunks/action.yml new file mode 100644 index 0000000000..01ab5385da --- /dev/null +++ b/.github/workflows/download-hf-model-chunks/action.yml @@ -0,0 +1,66 @@ +name: Download HF model chunks +description: Resolve and download model chunks from HuggingFace in parallel + +inputs: + hf_repo: + description: HuggingFace dataset repo + required: true + models: + description: 'JSON array of {hf_path, onnx_hash, canonical} objects' + required: true + dest_dir: + description: Destination directory for downloaded chunks + required: true + +runs: + using: composite + steps: + - name: Download model chunks + shell: bash + env: + HF_REPO: ${{ inputs.hf_repo }} + MODELS_JSON: ${{ inputs.models }} + DEST_DIR: ${{ inputs.dest_dir }} + run: | + set -eo pipefail + DOWNLOAD_LIST=$(mktemp) + + resolve_chunks() { + local HF_PATH="$1" ONNX_HASH="$2" CANONICAL="$3" DEST_DIR="$4" + local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_PATH}/default_models.json" + local DEFAULTS BUNDLE ARTIFACT BASE_URL NUM_CHUNKS + DEFAULTS=$(curl -fsSL "$JSON_URL") + BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)') + ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact') + BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||') + NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length') + + mkdir -p "$DEST_DIR" + while IFS= read -r CHUNK_NAME; do + CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true) + if [ -z "$CHUNK_IDX" ]; then + echo "::error::Failed to parse chunk index from: $CHUNK_NAME" + return 1 + fi + ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))") + printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}.chunk${CHUNK_IDX}" >> "$DOWNLOAD_LIST" + done < <(echo "$ARTIFACT" | jq -r '.chunks[].file_name') + echo "$NUM_CHUNKS" > "${DEST_DIR}/${CANONICAL}.chunkmanifest" + } + + echo "$MODELS_JSON" | jq -c '.[]' | while IFS= read -r model; do + HF_PATH=$(echo "$model" | jq -r '.hf_path') + ONNX_HASH=$(echo "$model" | jq -r '.onnx_hash') + CANONICAL=$(echo "$model" | jq -r '.canonical') + resolve_chunks "$HF_PATH" "$ONNX_HASH" "$CANONICAL" "$DEST_DIR" + done + + TOTAL=$(wc -l < "$DOWNLOAD_LIST") + echo "Downloading $TOTAL chunks with 8 parallel connections..." + xargs -P8 -d'\n' -I{} bash -c ' + URL="${1%% *}" + DEST="${1#* }" + echo "Downloading $(basename "$DEST")" + curl -fsSL --retry 3 --retry-delay 5 -o "$DEST" "$URL" + ' _ {} < "$DOWNLOAD_LIST" + rm -f "$DOWNLOAD_LIST" diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index 17981d68e5..5c2e3bf204 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -103,20 +103,25 @@ jobs: - run: | cd ${{ github.workspace }}/openpilot/openpilot if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then - git lfs pull -X "selfdrive/modeld/models/big_*.onnx" -X "selfdrive/modeld/models/dmonitoring_*.onnx" + git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx,**/selfdrive/modeld/models/dmonitoring_*.onnx" rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx else - git lfs pull -I "selfdrive/modeld/models/big_*.onnx" + git lfs pull -I "**/selfdrive/modeld/models/big_*.onnx" -X "" find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete fi + if grep -lIF "version https://git-lfs.github.com/spec/v1" selfdrive/modeld/models/*.onnx; then + echo "::error::the ONNX files above are still LFS pointers, not real models" + exit 1 + fi - name: 'Upload Artifact' uses: actions/upload-artifact@v4 with: name: models-${{ env.REF }}${{ inputs.artifact_suffix }} path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx + if-no-files-found: error build_model: - runs-on: [self-hosted, tici] + runs-on: [self-hosted, usbgpu] needs: get_model env: MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }}) @@ -127,7 +132,6 @@ jobs: fetch-depth: 1 submodules: recursive - - run: git lfs pull - name: Set environment variables id: set-env @@ -160,7 +164,7 @@ jobs: fi source ${UV_PROJECT_ENVIRONMENT}/bin/activate PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable - rm -rf ${{ env.MODELS_DIR }}/*.onnx + rm -rf ${{ env.MODELS_DIR }}/*.onnx* - name: Download model artifacts uses: actions/download-artifact@v4 @@ -180,34 +184,48 @@ jobs: MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") + TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then echo "USBGPU build" export USBGPU=1 - TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" + TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" else echo "QCOM build" - TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + TG_FLAGS="$TG_FLAGS_QCOM" OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl" fi # Generate metadata for all ONNX files find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do echo "Generating metadata: $onnx_file" - env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true + env ${TG_FLAGS_QCOM} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true done # Detect model type and build compile args - VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx" - POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx" - OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx" - ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx" + VISION_ONNX="" + for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do + [ -f "$f" ] && VISION_ONNX="$f" && break + done + + POLICY_ONNX="" + for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do + [ -f "$f" ] && POLICY_ONNX="$f" && break + done + + OFF_POLICY_ONNX="" + for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do + [ -f "$f" ] && OFF_POLICY_ONNX="$f" && break + done + + ON_POLICY_ONNX="" + for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do + [ -f "$f" ] && ON_POLICY_ONNX="$f" && break + done + SUPERCOMBO_ONNX="" - for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do - if [ -f "$f" ]; then - SUPERCOMBO_ONNX="$f" - break - fi + for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do + [ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break done MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME="" diff --git a/.github/workflows/sunnypilot-build-prebuilt.yaml b/.github/workflows/sunnypilot-build-prebuilt.yaml index 12a3a7bce0..c93dbe02c1 100644 --- a/.github/workflows/sunnypilot-build-prebuilt.yaml +++ b/.github/workflows/sunnypilot-build-prebuilt.yaml @@ -4,12 +4,8 @@ env: BUILD_DIR: "/data/openpilot" OUTPUT_DIR: ${{ github.workspace }}/output CI_DIR: ${{ github.workspace }}/release/ci - SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot" - # Branch configurations - STAGING_SOURCE_BRANCH: 'master' - # Runtime configuration SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}" @@ -40,8 +36,11 @@ jobs: publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }} is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }} build: ${{ steps.strategy.outputs.build }} + include_big_model: ${{ steps.strategy.outputs.include_big_model }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Extract deploy strategy id: strategy run: | @@ -82,6 +81,9 @@ jobs: stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g'); echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT + + include_big_model="$(echo "$CONFIG" | jq -r '.include_big_model // false')"; + echo "include_big_model=$include_big_model" >> $GITHUB_OUTPUT fi echo "build=$BUILD" >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT @@ -96,6 +98,8 @@ jobs: }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Wait for Tests uses: ./.github/workflows/wait-for-action # Path to where you place the action with: @@ -109,11 +113,6 @@ jobs: group: build-${{ github.head_ref || github.ref_name }} cancel-in-progress: false runs-on: [self-hosted, tici] - outputs: - new_branch: ${{ needs.prepare_strategy.outputs.new_branch }} - version: ${{ needs.prepare_strategy.outputs.version }} - extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }} - commit_sha: ${{ github.sha }} if: ${{ (always() && !cancelled() && !failure()) && needs.prepare_strategy.result == 'success' && @@ -124,31 +123,14 @@ jobs: steps: - uses: actions/checkout@v4 with: + fetch-depth: 1 submodules: recursive ref: ${{ env.SOURCE_BRANCH }} repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }} - run: git lfs pull - - name: Cache SCons - uses: actions/cache@v4 - with: - path: ${{env.SCONS_CACHE_DIR}} - key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }} - # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) - # for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden. - restore-keys: | - scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }} - scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }} - scons-${{ runner.os }}-${{ runner.arch }} - - name: Set environment variables - id: set-env run: | - echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT - echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT - echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT - echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT - # Set up common environment source /etc/profile; export UV_PROJECT_ENVIRONMENT=${HOME}/venv @@ -157,9 +139,6 @@ jobs: export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT uv sync printenv >> $GITHUB_ENV - if [[ "${{ runner.debug }}" == "1" ]]; then - cat $GITHUB_OUTPUT - fi - name: Setup build environment run: | @@ -168,7 +147,7 @@ jobs: echo "Starting build stage..." echo "BUILD_DIR: ${BUILD_DIR}" echo "CI_DIR: ${CI_DIR}" - echo "VERSION: ${{ steps.set-env.outputs.version }}" + echo "VERSION: ${{ needs.prepare_strategy.outputs.version }}" echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}" echo "VIRTUAL_ENV: ${VIRTUAL_ENV}" echo "-------" @@ -180,61 +159,44 @@ jobs: - name: Build Main Project run: | - export PYTHONPATH="$BUILD_DIR" - ./tools/release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/ + export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo" + ./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" -- + # outside the checkout, which is wiped each run. /data/scons_cache is the device's, not ours. + SCONS_CACHE="$RUNNER_WORKSPACE/scons_cache" + mkdir -p "$SCONS_CACHE" cd $BUILD_DIR - ln -sfn msgq_repo/msgq msgq - ln -sfn opendbc_repo/opendbc opendbc - ln -sfn rednose_repo/rednose rednose - ln -sfn teleoprtc_repo/teleoprtc teleoprtc - ln -sfn tinygrad_repo/tinygrad tinygrad - sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py - echo "Building sunnypilot's modeld_v2..." - scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/sunnypilot/modeld_v2 - echo "Building sunnypilot's locationd..." - scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/sunnypilot/selfdrive/locationd - echo "Building openpilot's locationd..." - scons -j1 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/selfdrive/locationd + echo "Building locationd..." + # -j1: parallel rednose generators OOM the device + scons -j1 cache_dir="$SCONS_CACHE" --minimal \ + openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd echo "Building rest of sunnypilot" - scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal + SKIP_TINYGRAD_COMPILE=1 /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal touch ${BUILD_DIR}/prebuilt if [[ "${{ runner.debug }}" == "1" ]]; then ls -la ${BUILD_DIR} fi - - name: Prepare Output + - name: Strip release tree run: | - sudo rm -rf ${OUTPUT_DIR} - mkdir -p ${OUTPUT_DIR} - rsync -am${RUNNER_DEBUG:+v} \ - --exclude='.sconsign.dblite' \ - --exclude='*.a' \ - --exclude='*.o' \ - --exclude='*.os' \ - --exclude='*.pyc' \ - --exclude='moc_*' \ - --exclude='__pycache__' \ - --exclude='Jenkinsfile' \ - --exclude='**/release/' \ - --exclude='**/.github/' \ - --exclude='**/openpilot/selfdrive/ui/replay/' \ - --exclude='**/__pycache__/' \ - --exclude='${{env.SCONS_CACHE_DIR}}' \ - --exclude='**/.git/' \ - --exclude='**/SConstruct' \ - --exclude='**/SConscript' \ - --exclude='**/.venv/' \ - --exclude='openpilot/selfdrive/modeld/models/*.onnx*' \ - --exclude='openpilot/sunnypilot/modeld*/models/*.onnx*' \ - --exclude='openpilot/third_party/*x86*' \ - --exclude='openpilot/third_party/*Darwin*' \ - --delete-excluded \ - --chown=comma:comma \ - ${BUILD_DIR}/ ${OUTPUT_DIR}/ + cd $BUILD_DIR + find . -name '*.a' -delete + find . -name '*.o' -delete + find . -name '*.os' -delete + find . -name '*.pyc' -delete + find . -name 'moc_*' -delete + find . -name '__pycache__' -type d -exec rm -rf {} + + find . -name 'SConstruct' -delete + find . -name 'SConscript' -delete + rm -rf .sconsign.dblite Jenkinsfile tools/release/ release/ + rm -f openpilot/selfdrive/modeld/models/*.onnx* + rm -f openpilot/sunnypilot/modeld*/models/*.onnx* + find openpilot/third_party/ -name '*x86*' -exec rm -r {} + + find openpilot/third_party/ -name '*Darwin*' -exec rm -r {} + + cd - - name: 'Tar.gz files' run: | - tar czf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} . + tar czf prebuilt.tar.gz -C ${{ env.BUILD_DIR }} . ls -la prebuilt.tar.gz - name: 'Upload Artifact' @@ -242,6 +204,7 @@ jobs: with: name: prebuilt path: prebuilt.tar.gz + compression-level: 0 - name: Re-enable powersave if: always() @@ -249,22 +212,212 @@ jobs: source ${UV_PROJECT_ENVIRONMENT}/bin/activate PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + prepare_chestnut: + needs: [ prepare_strategy ] + runs-on: ubuntu-24.04 + if: ${{ needs.prepare_strategy.outputs.include_big_model == 'true' }} + outputs: + onnx_sha256: ${{ steps.resolve.outputs.onnx_sha256 }} + env: + GH_REPO: ${{ github.repository }} + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/big + steps: + - name: Resolve ONNX hash and tinygrad ref via API + id: resolve + run: | + REF="${{ github.head_ref || github.ref_name }}" + + ONNX_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2) + echo "ONNX hash: $ONNX_HASH" + echo "onnx_sha256=$ONNX_HASH" >> $GITHUB_OUTPUT + + TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha') + echo "tinygrad ref: $TINYGRAD_REF" + + JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" + + check_defaults() { + DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + [ -n "$BUNDLE" ] && [ "$BUNDLE" != "null" ] + } + + if check_defaults; then + echo "HF defaults match repo ONNX hash and tinygrad ref" + exit 0 + fi + + echo "No matching model on HF — dispatching build" + gh workflow run build-default-models.yaml --ref "$REF" -f target=big + + echo "Polling HF for big model availability..." + for i in $(seq 1 90); do + sleep 30 + if check_defaults; then + echo "Big model available on HF after $((i * 30))s" + exit 0 + fi + echo "Poll $i/90: not yet available" + done + + echo "::error::Big model not available on HF after 45 minutes" + exit 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cancel run on failure + if: failure() + run: gh run cancel ${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + prepare_small_model: + needs: [ prepare_strategy ] + runs-on: ubuntu-24.04 + outputs: + driving_onnx_sha256: ${{ steps.resolve.outputs.driving_onnx_sha256 }} + env: + GH_REPO: ${{ github.repository }} + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/small + steps: + - name: Resolve ONNX hash and tinygrad ref via API + id: resolve + run: | + REF="${{ github.head_ref || github.ref_name }}" + + DRIVING_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/driving_supercombo.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2) + echo "Driving ONNX hash: $DRIVING_HASH" + echo "driving_onnx_sha256=$DRIVING_HASH" >> $GITHUB_OUTPUT + + TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha') + echo "tinygrad ref: $TINYGRAD_REF" + + JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" + + check_defaults() { + DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + [ -n "$DRIVING" ] && [ "$DRIVING" != "null" ] || return 1 + } + + if check_defaults; then + echo "HF defaults match repo ONNX hash and tinygrad ref" + exit 0 + fi + + echo "No matching model on HF — dispatching build" + gh workflow run build-default-models.yaml --ref "$REF" -f target=small + + echo "Polling HF for model availability..." + for i in $(seq 1 60); do + sleep 30 + if check_defaults; then + echo "Model available on HF after $((i * 30))s" + exit 0 + fi + echo "Poll $i/60: not yet available" + done + + echo "::error::Small driving model not available on HF after 30 minutes" + exit 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cancel run on failure + if: failure() + run: gh run cancel ${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + prepare_dm_model: + needs: [ prepare_strategy ] + runs-on: ubuntu-24.04 + outputs: + dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }} + env: + GH_REPO: ${{ github.repository }} + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/dm + steps: + - name: Resolve ONNX hash and tinygrad ref via API + id: resolve + run: | + REF="${{ github.head_ref || github.ref_name }}" + + DM_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/dmonitoring_model.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2) + echo "DM ONNX hash: $DM_HASH" + echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT + + TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha') + echo "tinygrad ref: $TINYGRAD_REF" + + JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json" + + check_defaults() { + DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1 + TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null) + [ "$TINYGRAD_MATCH" = "true" ] || return 1 + DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null) + [ -n "$DM" ] && [ "$DM" != "null" ] || return 1 + } + + if check_defaults; then + echo "HF defaults match DM ONNX hash and tinygrad ref" + exit 0 + fi + + echo "No matching DM model on HF — dispatching build" + gh workflow run build-default-models.yaml --ref "$REF" -f target=dm + + echo "Polling HF for DM model availability..." + for i in $(seq 1 60); do + sleep 30 + if check_defaults; then + echo "DM model available on HF after $((i * 30))s" + exit 0 + fi + echo "Poll $i/60: not yet available" + done + + echo "::error::DM model not available on HF after 30 minutes" + exit 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cancel run on failure + if: failure() + run: gh run cancel ${{ github.run_id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} publish: concurrency: - # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name. - # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other. - # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time. group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }} cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }} - if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }} - needs: [ build, prepare_strategy ] + if: ${{ + always() && !cancelled() && + needs.build.result == 'success' && + needs.prepare_strategy.result == 'success' && + needs.prepare_small_model.result == 'success' && + needs.prepare_dm_model.result == 'success' && + (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) && + (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success') + }} + needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ] runs-on: ubuntu-24.04 environment: ${{ needs.prepare_strategy.outputs.environment }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - - name: Download build artifacts + - name: Download prebuilt artifact uses: actions/download-artifact@v4 with: name: prebuilt @@ -274,6 +427,17 @@ jobs: mkdir -p ${{ env.OUTPUT_DIR }} tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} + - name: Download model chunks from HF + uses: ./.github/workflows/download-hf-model-chunks + with: + hf_repo: sunnypilot/sunnypilot_models_v1 + dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models + models: | + [ + {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"}, + {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"} + ] + - name: Configure Git run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" @@ -283,36 +447,95 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - echo '${{ toJSON(needs.build.outputs) }}' + echo '${{ toJSON(needs.prepare_strategy.outputs) }}' ls -la ${{ env.OUTPUT_DIR }} ${{ env.CI_DIR }}/publish.sh \ "${{ github.workspace }}" \ "${{ env.OUTPUT_DIR }}" \ - "${{ needs.build.outputs.new_branch }}" \ - "${{ needs.build.outputs.version }}" \ + "${{ needs.prepare_strategy.outputs.new_branch }}" \ + "${{ needs.prepare_strategy.outputs.version }}" \ "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ - "${{ needs.build.outputs.extra_version_identifier }}" - - echo "" - echo "---- ℹ️ To update the list of branches that auto deploy prebuilts -----" - echo "" - echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES" - echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}" - echo "3. Update as needed (JSON array with no spaces)" + "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" - name: Tag ${{ needs.prepare_strategy.outputs.environment }} if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }} run: | TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}" - git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}." + git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}." git push -f origin ${TAG} + publish_chestnut: + concurrency: + group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}-chestnut + cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }} + if: ${{ + always() && !cancelled() && + needs.build.result == 'success' && + needs.prepare_strategy.result == 'success' && + needs.prepare_small_model.result == 'success' && + needs.prepare_dm_model.result == 'success' && + needs.prepare_chestnut.result == 'success' && + (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) + }} + needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ] + runs-on: ubuntu-24.04 + environment: ${{ needs.prepare_strategy.outputs.environment }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Download prebuilt artifact + uses: actions/download-artifact@v4 + with: + name: prebuilt + + - name: Untar prebuilt + run: | + mkdir -p ${{ env.OUTPUT_DIR }} + tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} + + - name: Download model chunks from HF + uses: ./.github/workflows/download-hf-model-chunks + with: + hf_repo: sunnypilot/sunnypilot_models_v1 + dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models + models: | + [ + {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"}, + {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"}, + {"hf_path": "models/defaults/big", "onnx_hash": "${{ needs.prepare_chestnut.outputs.onnx_sha256 }}", "canonical": "big_driving_tinygrad.pkl"} + ] + + - name: Configure Git + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + + - name: Publish chestnut branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut" + + ${{ env.CI_DIR }}/publish.sh \ + "${{ github.workspace }}" \ + "${{ env.OUTPUT_DIR }}" \ + "$CHESTNUT_BRANCH" \ + "${{ needs.prepare_strategy.outputs.version }}" \ + "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ + "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" + notify: needs: - prepare_strategy - build - publish + - publish_chestnut + - prepare_chestnut + - prepare_small_model + - prepare_dm_model runs-on: ubuntu-24.04 if: ${{ (always() && !cancelled() && !failure()) && needs.publish.result == 'success' @@ -320,11 +543,12 @@ jobs: && (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Prepare notification message id: message run: | - TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}' export VERSION="${{ needs.prepare_strategy.outputs.version }}" export branch_name="${{ env.SOURCE_BRANCH }}" export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}" @@ -333,6 +557,7 @@ jobs: export commit_short_sha="${commit_short_sha:0:7}" export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}" export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}" + export chestnut_branch="${{ needs.prepare_chestnut.result == 'success' && format('{0}-chestnut', needs.prepare_strategy.outputs.new_branch) || '' }}" MESSAGE=$(cat << 'EOF' | envsubst ${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }} @@ -373,7 +598,7 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, - name: process.env.LABELf + name: process.env.LABEL }); console.log(`Removed '${process.env.LABEL}' label from PR #${prNumber}`); diff --git a/.github/workflows/ui_preview.yaml b/.github/workflows/ui_preview.yaml index 8b7b63f344..c6bc63c8ac 100644 --- a/.github/workflows/ui_preview.yaml +++ b/.github/workflows/ui_preview.yaml @@ -25,7 +25,8 @@ env: jobs: preview: - if: github.repository == 'sunnypilot/sunnypilot' + if: false # tmp disable due to GH API rate limiting flakiness + #if: github.repository == 'sunnypilot/sunnypilot' name: preview runs-on: ubuntu-latest timeout-minutes: 20 diff --git a/launch_chffrplus.sh b/launch_chffrplus.sh index 62ebffd27b..f30e03ca62 100755 --- a/launch_chffrplus.sh +++ b/launch_chffrplus.sh @@ -24,7 +24,9 @@ function agnos_init { if $AGNOS_PY --verify $MANIFEST; then sudo reboot fi - $DIR/openpilot/common/hardware/comma/updater $AGNOS_PY $MANIFEST + while true; do + $DIR/openpilot/common/hardware/comma/updater $AGNOS_PY $MANIFEST + done fi } diff --git a/opendbc_repo b/opendbc_repo index 16ec8fb7ab..cc4b08625a 160000 --- a/opendbc_repo +++ b/opendbc_repo @@ -1 +1 @@ -Subproject commit 16ec8fb7ab91394f5df53c89f12f479f9eaa0134 +Subproject commit cc4b08625a98e94b318cab15e45e05dad58042bd diff --git a/openpilot/common/params_keys.h b/openpilot/common/params_keys.h index be4a4dc4a2..111099fce9 100644 --- a/openpilot/common/params_keys.h +++ b/openpilot/common/params_keys.h @@ -59,7 +59,7 @@ inline static std::unordered_map keys = { {"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}}, {"IsEngaged", {PERSISTENT, BOOL}}, {"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}}, - {"IsLiveStreaming", {CLEAR_ON_MANAGER_START, BOOL}}, + {"IsLiveStreaming", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}}, {"IsMetric", {PERSISTENT | BACKUP, BOOL}}, {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}}, {"IsRhdDetected", {PERSISTENT, BOOL}}, @@ -196,6 +196,8 @@ inline static std::unordered_map keys = { // Model Manager params {"ModelManager_ActiveBundle", {PERSISTENT, JSON}}, {"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, STRING}}, + {"ModelManager_PrevBundle", {PERSISTENT, JSON}}, + {"ModelManager_PrevBundle_USBGPU", {PERSISTENT, JSON}}, {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}}, {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}}, {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}}, diff --git a/openpilot/common/version.py b/openpilot/common/version.py index 77ee79d5ef..a8ff3aba2b 100755 --- a/openpilot/common/version.py +++ b/openpilot/common/version.py @@ -16,6 +16,15 @@ MASTER_SP_BRANCHES = ['master'] RELEASE_BRANCHES = ['release-tizi-staging', 'release-mici-staging', 'release-tizi', 'release-mici', 'nightly'] TESTED_BRANCHES = RELEASE_BRANCHES + ['devel-staging', 'nightly-dev'] + RELEASE_SP_BRANCHES + TESTED_SP_BRANCHES +CHESTNUT_BRANCHES = { + "staging": "staging-chestnut", + "dev": "dev-chestnut", + "release-mici": "release-chestnut", + "release-tizi": "release-chestnut", + "release-mici-staging": "release-chestnut-staging", + "release-tizi-staging": "release-chestnut-staging", +} + SP_BRANCH_MIGRATIONS = { ("tici", "staging-c3-new"): "staging-tici", ("tici", "dev-c3-new"): "staging-tici", diff --git a/openpilot/selfdrive/controls/lib/longitudinal_planner.py b/openpilot/selfdrive/controls/lib/longitudinal_planner.py index 8b62808dc0..bf91c5e1f9 100755 --- a/openpilot/selfdrive/controls/lib/longitudinal_planner.py +++ b/openpilot/selfdrive/controls/lib/longitudinal_planner.py @@ -49,9 +49,8 @@ def get_cruise_accel(e2e, v_cruise, v_ego, a_cruise_prev, angle_steers, CP, dt, max_accel = min(max_accel, coast_limit) target_accel = np.clip(v_cruise - v_ego, A_CRUISE_MIN, max_accel) - if not e2e: - j_cruise = np.interp(v_ego, A_CRUISE_MAX_BP, J_CRUISE_VALS) - target_accel = float(np.clip(target_accel, a_cruise_prev - j_cruise * dt, a_cruise_prev + j_cruise * dt)) + j_cruise = np.interp(v_ego, A_CRUISE_MAX_BP, J_CRUISE_VALS) + target_accel = float(np.clip(target_accel, a_cruise_prev - j_cruise * dt, a_cruise_prev + j_cruise * dt)) return target_accel @@ -65,10 +64,9 @@ class LongitudinalPlanner(LongitudinalPlannerSP): self.dt = dt self.allow_throttle = True - self.a_desired = init_a self.v_desired_filter = FirstOrderFilter(init_v, 2.0, self.dt) - self.a_cruise = 0.0 - self.output_a_target = 0.0 + self.a_cruise = init_a + self.output_a_target = init_a self.output_should_stop = False self.v_desired_trajectory = np.zeros(CONTROL_N) @@ -105,7 +103,8 @@ class LongitudinalPlanner(LongitudinalPlannerSP): if reset_state: self.v_desired_filter.x = v_ego - self.a_desired = np.clip(sm['carState'].aEgo, ACCEL_MIN, ACCEL_MAX) + self.output_a_target = np.clip(sm['carState'].aEgo, ACCEL_MIN, ACCEL_MAX) + self.a_cruise = self.output_a_target # Prevent divergence, smooth in current v_ego self.v_desired_filter.x = max(0.0, self.v_desired_filter.update(v_ego)) @@ -113,11 +112,11 @@ class LongitudinalPlanner(LongitudinalPlannerSP): # No change cost when user is controlling the speed, or when standstill prev_accel_constraint = not (reset_state or sm['carState'].standstill) - # Get new v_cruise and a_desired from Smart Cruise Control and Speed Limit Assist - v_cruise, self.a_desired = LongitudinalPlannerSP.update_targets(self, sm, self.v_desired_filter.x, self.a_desired, v_cruise) + # Get new v_cruise and a_target from Smart Cruise Control and Speed Limit Assist + v_cruise, self.output_a_target = LongitudinalPlannerSP.update_targets(self, sm, self.v_desired_filter.x, self.output_a_target, v_cruise) self.mpc.set_weights(prev_accel_constraint, personality=sm['selfdriveState'].personality) - self.mpc.set_cur_state(self.v_desired_filter.x, self.a_desired) + self.mpc.set_cur_state(self.v_desired_filter.x, self.output_a_target) self.mpc.update(sm['radarState'], personality=sm['selfdriveState'].personality) self.v_desired_trajectory = np.interp(CONTROL_N_T_IDX, T_IDXS_MPC, self.mpc.v_solution) @@ -130,7 +129,7 @@ class LongitudinalPlanner(LongitudinalPlannerSP): cloudlog.info("FCW triggered") # Save starting point for next iteration - a_prev = self.a_desired + a_prev = self.output_a_target action_t = self.CP.longitudinalActuatorDelay + DT_MDL output_a_target_mpc = get_accel_from_plan(self.v_desired_trajectory, self.a_desired_trajectory, CONTROL_N_T_IDX, @@ -155,7 +154,6 @@ class LongitudinalPlanner(LongitudinalPlannerSP): self.output_should_stop = any(should_stop for _, _, should_stop in candidates) self.output_a_target = np.clip(output_a_target, ACCEL_MIN, ACCEL_MAX) - self.a_desired = float(self.output_a_target) self.v_desired_filter.x = self.v_desired_filter.x + self.dt * (self.output_a_target + a_prev) / 2.0 def publish(self, sm, pm): diff --git a/openpilot/selfdrive/modeld/SConscript b/openpilot/selfdrive/modeld/SConscript index 30a31aae27..19ce7d5e00 100644 --- a/openpilot/selfdrive/modeld/SConscript +++ b/openpilot/selfdrive/modeld/SConscript @@ -73,44 +73,45 @@ compile_modeld_script = [ model_w, model_h = MEDMODEL_INPUT_SIZE frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ -for usbgpu in [False, True] if USBGPU else [False]: - target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath - # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU - file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags) - driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath) - camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS) - # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it. - taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else '' - cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py ' - f'--model-size {model_w}x{model_h} ' - f'--camera-resolutions {camera_res_args} ' - f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} ' - f'--output {target_pkl_path} --frame-skip {frame_skip}') - onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps) - chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum)) - def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets): - from openpilot.system.hardware.chestnut.flash import link_up - # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars - for _ in range(10): - if link_up(): - break - time.sleep(1) - else: - print("Chestnut not ready, skipping big model build") - return - if ret := env.Execute(command): - return ret - chunk_file(pkl, chunks) - def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets): - chunk_file(pkl, chunks) - actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")] - node = lenv.Command( - chunk_targets, - tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file], - actions, - ) - if usbgpu: - lenv.SideEffect(usbgpu_lock, node) +if not os.getenv('SKIP_TINYGRAD_COMPILE'): + for usbgpu in [False, True] if USBGPU else [False]: + target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath + # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU + file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags) + driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath) + camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS) + # CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it. + taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else '' + cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py ' + f'--model-size {model_w}x{model_h} ' + f'--camera-resolutions {camera_res_args} ' + f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} ' + f'--output {target_pkl_path} --frame-skip {frame_skip}') + onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps) + chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum)) + def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets): + from openpilot.system.hardware.chestnut.flash import link_up + # chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars + for _ in range(10): + if link_up(): + break + time.sleep(1) + else: + print("Chestnut not ready, skipping big model build") + return + if ret := env.Execute(command): + return ret + chunk_file(pkl, chunks) + def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets): + chunk_file(pkl, chunks) + actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")] + node = lenv.Command( + chunk_targets, + tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file], + actions, + ) + if usbgpu: + lenv.SideEffect(usbgpu_lock, node) # get model metadata fn = File(f"models/dmonitoring_model").abspath @@ -142,4 +143,5 @@ def tg_compile(flags, model_name): Action(do_chunk, " [CHUNK] $TARGET")], ) -tg_compile(tg_flags, 'dmonitoring_model') +if not os.getenv('SKIP_TINYGRAD_COMPILE'): + tg_compile(tg_flags, 'dmonitoring_model') diff --git a/openpilot/selfdrive/selfdrived/alerts_offroad.json b/openpilot/selfdrive/selfdrived/alerts_offroad.json index 226be06683..91a7ec8ab3 100644 --- a/openpilot/selfdrive/selfdrived/alerts_offroad.json +++ b/openpilot/selfdrive/selfdrived/alerts_offroad.json @@ -18,7 +18,7 @@ "_comment": "Set extra field to the failed reason." }, "Offroad_ChestnutBranch": { - "text": "Chestnut detected! Switch to the release-chestnut branch to use chestnut-class models.", + "text": "Chestnut detected! Switch to the %1 branch to use chestnut-class models.", "severity": 0 }, "Offroad_UnregisteredHardware": { diff --git a/openpilot/selfdrive/ui/mici/widgets/button.py b/openpilot/selfdrive/ui/mici/widgets/button.py index cad40d7d01..e2912d00cf 100644 --- a/openpilot/selfdrive/ui/mici/widgets/button.py +++ b/openpilot/selfdrive/ui/mici/widgets/button.py @@ -149,11 +149,15 @@ class BigButton(Widget): def set_touch_valid_callback(self, touch_callback: Callable[[], bool]) -> None: super().set_touch_valid_callback(lambda: touch_callback() and self._grow_animation_until is None) - def _width_hint(self) -> int: - # A value moves the title to the top, where it shares space with the icon. + def _title_width_hint(self) -> int: + # A value moves the title to the top, where it shares space with the icon icon_size = self._txt_icon.width if self._txt_icon and self.value else 0 return int(self._rect.width - self.LABEL_HORIZONTAL_PADDING * 2 - icon_size) + def _subtitle_width_hint(self) -> int: + # Bottom aligned, so it sits below the icon + return int(self._rect.width - self.LABEL_HORIZONTAL_PADDING * 2) + def _get_label_font_size(self): if len(self.text) <= 18: return 48 @@ -228,14 +232,14 @@ class BigButton(Widget): label_color = LABEL_COLOR if self.enabled else rl.Color(255, 255, 255, int(255 * 0.35)) self._label.set_color(label_color) - label_rect = rl.Rectangle(label_x, btn_y + self.LABEL_VERTICAL_PADDING, self._width_hint(), + label_rect = rl.Rectangle(label_x, btn_y + self.LABEL_VERTICAL_PADDING, self._title_width_hint(), self._rect.height - self.LABEL_VERTICAL_PADDING * 2) self._label.render(label_rect) if self.value: - label_y = btn_y + self.LABEL_VERTICAL_PADDING + self._label.get_content_height(self._width_hint()) + label_y = label_rect.y + self._label.get_content_height(int(label_rect.width)) sub_label_height = btn_y + self._rect.height - self.LABEL_VERTICAL_PADDING - label_y - sub_label_rect = rl.Rectangle(label_x, label_y, self._width_hint(), sub_label_height) + sub_label_rect = rl.Rectangle(label_x, label_y, self._subtitle_width_hint(), sub_label_height) self._sub_label.render(sub_label_rect) # ICON ------------------------------------------------------------------- @@ -312,9 +316,6 @@ class BigMultiToggle(BigToggle): self.set_value(self._options[0]) - def _width_hint(self) -> int: - return int(self._rect.width - self.LABEL_HORIZONTAL_PADDING * 2 - self._txt_enabled_toggle.width) - def _handle_mouse_release(self, mouse_pos: MousePos): super()._handle_mouse_release(mouse_pos) cur_idx = self._options.index(self.value) @@ -363,9 +364,6 @@ class GreyBigButton(BigButton): def LABEL_VERTICAL_PADDING(self): return BigButton.LABEL_VERTICAL_PADDING if self._label.text else 18 - def _width_hint(self) -> int: - return int(self._rect.width - self.LABEL_HORIZONTAL_PADDING * 2) - def _get_label_font_size(self): return 36 diff --git a/openpilot/selfdrive/ui/onroad/model_renderer.py b/openpilot/selfdrive/ui/onroad/model_renderer.py index def1c644af..f5a39a2a5b 100644 --- a/openpilot/selfdrive/ui/onroad/model_renderer.py +++ b/openpilot/selfdrive/ui/onroad/model_renderer.py @@ -192,7 +192,7 @@ class ModelRenderer(Widget, ChevronMetrics, ModelRendererSP): max_idx = self._get_path_length_idx(path_x_array, max_distance) self._path.projected_points = self._map_line_to_polygon( - self._path.raw_points, 0.9, self._path_offset_z, max_idx, max_distance, allow_invert=False + self._path.raw_points, self._get_path_half_width(), self._path_offset_z, max_idx, max_distance, allow_invert=False ) self._update_experimental_gradient() @@ -292,7 +292,7 @@ class ModelRenderer(Widget, ChevronMetrics, ModelRendererSP): allow_throttle = sm['longitudinalPlan'].allowThrottle or not self._longitudinal_control self._blend_filter.update(int(allow_throttle)) - if ui_state.rainbow_path: + if ui_state.rainbow_path and self._lateral_active: self.rainbow_path.draw_rainbow_path(self._rect, self._path) return diff --git a/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py b/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py index ae4ff802f3..e4a6bea6e0 100644 --- a/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py +++ b/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py @@ -10,7 +10,7 @@ import time import pyray as rl from openpilot.cereal import custom -from openpilot.sunnypilot.models.default_model import DEFAULT_MODEL +from openpilot.sunnypilot.models.default_model import get_default_model from openpilot.common.constants import CV from openpilot.selfdrive.ui.ui_state import device, ui_state from openpilot.system.ui.lib.multilang import tr @@ -22,9 +22,9 @@ from openpilot.system.ui.widgets.toggle import ON_COLOR from openpilot.sunnypilot.models.runners.constants import CUSTOM_MODEL_PATH from openpilot.system.ui.sunnypilot.lib.styles import style -from openpilot.system.ui.sunnypilot.lib.utils import NoElideButtonAction +from openpilot.system.ui.sunnypilot.lib.utils import NoElideButtonAction, ScrollingButtonAction from openpilot.system.ui.sunnypilot.widgets.list_view import ListItemSP, toggle_item_sp, option_item_sp -from openpilot.system.ui.sunnypilot.widgets.progress_bar import progress_item +from openpilot.system.ui.sunnypilot.widgets.download_status import download_status_item from openpilot.system.ui.sunnypilot.widgets.tree_dialog import TreeOptionDialog, TreeNode, TreeFolder if gui_app.sunnypilot_ui(): @@ -35,9 +35,8 @@ class ModelsLayout(Widget): def __init__(self): super().__init__() self.model_manager = None - self.download_status = None - self.prev_download_status = None self.model_dialog = None + self._downloading = False self.last_cache_calc_time = 0 self._initialize_items() @@ -52,15 +51,11 @@ class ModelsLayout(Widget): self.current_model_item = ListItemSP( title=tr("Current Model"), description="", - action_item=NoElideButtonAction(tr("SELECT")), + action_item=ScrollingButtonAction(tr("SELECT")), callback=self._handle_current_model_clicked ) - self.supercombo_label = progress_item(tr("Driving Model")) - self.vision_label = progress_item(tr("Vision Model")) - self.policy_label = progress_item(tr("Policy Model")) - self.off_policy_label = progress_item(tr("Off-Policy Model")) - self.on_policy_label = progress_item(tr("On-Policy Model")) + self.download_item = download_status_item(lambda: tr("Download") if self._downloading else tr("Model Status")) self.refresh_item = button_item(tr("Refresh Model List"), tr("REFRESH"), "", lambda: (ui_state.params.put("ModelManager_LastSyncTime", 0), @@ -98,8 +93,7 @@ class ModelsLayout(Widget): 1, None, True, "", style.BUTTON_ACTION_WIDTH, None, True, lambda v: f"{v / 100:.2f} m") - self.items = [self.current_model_item, self.cancel_download_item, self.supercombo_label, self.vision_label, - self.policy_label, self.off_policy_label, self.on_policy_label, self.refresh_item, self.clear_cache_item, + self.items = [self.current_model_item, self.cancel_download_item, self.download_item, self.refresh_item, self.clear_cache_item, self.lane_turn_desire_toggle, self.lane_turn_value_control, self.lagd_toggle, self.delay_control, self.camera_offset] def _update_lagd_description(self, lagd_toggle: bool): @@ -121,8 +115,12 @@ class ModelsLayout(Widget): def calculate_cache_size(): cache_size = 0.0 if os.path.exists(CUSTOM_MODEL_PATH): - cache_size = sum(os.path.getsize(os.path.join(CUSTOM_MODEL_PATH, file)) for file in os.listdir(CUSTOM_MODEL_PATH)) / (1024**2) - return cache_size + for file in os.listdir(CUSTOM_MODEL_PATH): + try: + cache_size += os.path.getsize(os.path.join(CUSTOM_MODEL_PATH, file)) + except OSError: + continue + return cache_size / (1024**2) def _clear_cache(self): def _callback(response): @@ -135,14 +133,9 @@ class ModelsLayout(Widget): gui_app.push_widget(dialog) def _handle_bundle_download_progress(self): - labels = {custom.ModelManagerSP.Model.Type.supercombo: self.supercombo_label, - custom.ModelManagerSP.Model.Type.vision: self.vision_label, - custom.ModelManagerSP.Model.Type.policy: self.policy_label, - custom.ModelManagerSP.Model.Type.offPolicy: self.off_policy_label, - custom.ModelManagerSP.Model.Type.onPolicy: self.on_policy_label} - for label in labels.values(): - label.set_visible(False) + self.download_item.set_visible(False) self.cancel_download_item.set_visible(False) + self._downloading = False if not self.model_manager or (not self.model_manager.selectedBundle and not self.model_manager.activeBundle): return @@ -153,42 +146,41 @@ class ModelsLayout(Widget): if not bundle: return - self.download_status = bundle.status - status_changed = self.prev_download_status != self.download_status - self.prev_download_status = self.download_status - self.cancel_download_item.set_visible(bool(self.model_manager.selectedBundle) and ui_state.params.get("ModelManager_DownloadIndex") is not None) if (current_time := time.monotonic()) - self.last_cache_calc_time > 0.5: self.last_cache_calc_time = current_time self.clear_cache_item.action_item.set_value(f"{self.calculate_cache_size():.2f} MB") - if self.download_status == custom.ModelManagerSP.DownloadStatus.downloading: + if bundle.status == custom.ModelManagerSP.DownloadStatus.downloading: device._reset_interactive_timeout() - for model in bundle.models: - if label := labels.get(getattr(model.type, 'raw', model.type)): - label.set_visible(True) - p = model.artifact.downloadProgress - text, show, color = f"pending - {bundle.displayName}", False, rl.GRAY - if p.status == custom.ModelManagerSP.DownloadStatus.downloading: - text, show = f"{int(p.progress)}% - {bundle.displayName}", True - elif p.status in (custom.ModelManagerSP.DownloadStatus.downloaded, custom.ModelManagerSP.DownloadStatus.cached): - status_text = tr("from cache" if p.status == custom.ModelManagerSP.DownloadStatus.cached else "downloaded") - text, color = f"{bundle.displayName} - {status_text if status_changed else tr('ready')}", ON_COLOR - elif p.status == custom.ModelManagerSP.DownloadStatus.failed: - text, color = f"download failed - {bundle.displayName}", rl.RED - label.action_item.update(p.progress, text, show, color) + # every bundle is a single chunked artifact now + progresses = [model.artifact.downloadProgress for model in bundle.models if model.artifact.fileName] + if not progresses: + return + + self.download_item.set_visible(True) + self.download_item.action_item.update(**self._download_row_state(progresses, bundle.internalName)) + self._downloading = self.download_item.action_item.downloading @staticmethod - def _show_reset_params_dialog(): - def _callback(response): - if response == DialogResult.CONFIRM: - ui_state.params.remove("CalibrationParams") - ui_state.params.remove("LiveTorqueParameters") - msg = tr("Model download has started in the background. We suggest resetting calibration. Would you like to do that now?") - dialog = ConfirmDialog(msg, tr("Reset Calibration"), callback=_callback) - gui_app.push_widget(dialog) + def _download_row_state(progresses, name: str) -> dict: + """Maps a bundle's artifact progress to DownloadStatusAction.update kwargs.""" + # .raw: _DynamicEnum equals its int but does not hash like it + statuses = {getattr(p.status, 'raw', p.status) for p in progresses} + progress = sum(p.progress for p in progresses) / len(progresses) + ds = custom.ModelManagerSP.DownloadStatus + + if ds.failed in statuses: + # close.png is authored black and a tint cannot lift it, hence close2 + return {"name": name, "status_text": tr("download failed"), "text_color": rl.RED, "icon": "icons/close2.png"} + if ds.downloading in statuses: + return {"name": name, "downloading": True, "progress": progress} + if statuses <= {ds.downloaded, ds.cached}: + return {"name": name, "text_color": ON_COLOR, "icon": "icons/checkmark.png"} + # circled_slash is authored grey; tinting it again only darkens it + return {"name": name, "text_color": rl.GRAY, "icon": "icons/circled_slash.png", "icon_color": rl.WHITE} def _on_model_selected(self, result): if result != DialogResult.CONFIRM: @@ -196,11 +188,8 @@ class ModelsLayout(Widget): selected_ref = self.model_dialog.selection_ref if selected_ref == "Default": ui_state.params.remove("ModelManager_ActiveBundle") - self._show_reset_params_dialog() elif selected_bundle := next((bundle for bundle in self.model_manager.availableBundles if bundle.ref == selected_ref), None): ui_state.params.put("ModelManager_DownloadIndex", selected_bundle.index) - if self.model_manager.activeBundle and selected_bundle.generation != self.model_manager.activeBundle.generation: - self._show_reset_params_dialog() self.model_dialog = None @staticmethod @@ -213,7 +202,8 @@ class ModelsLayout(Widget): for bundle in bundles: folders.setdefault(next((ov_ride.value for ov_ride in bundle.overrides if ov_ride.key == "folder"), ""), []).append(bundle) - folders_list = [TreeFolder("", [TreeNode("Default", {'display_name': f"{DEFAULT_MODEL} (Default)", 'short_name': "Default"})])] + folders_list = [TreeFolder("", [TreeNode("Default", {'display_name': f"{get_default_model()} (Default)", + 'short_name': "Default"})])] for folder, folder_bundles in sorted(folders.items(), key=lambda x: max((bundle.index for bundle in x[1]), default=-1), reverse=True): folder_bundles.sort(key=lambda bundle: bundle.index, reverse=True) name = folder + (f" - (Updated: {m.group(1)})" if folder_bundles and (m := re.search(r'\(([^)]*)\)[^(]*$', folder_bundles[0].displayName)) else "") @@ -251,7 +241,8 @@ class ModelsLayout(Widget): self._update_lagd_description(live_delay) self.model_manager = ui_state.sm["modelManagerSP"] self._handle_bundle_download_progress() - active_name = self.model_manager.activeBundle.internalName if self.model_manager and self.model_manager.activeBundle.ref else f"{DEFAULT_MODEL} (Default)" + default_label = f"{get_default_model()} (Default)" + active_name = self.model_manager.activeBundle.displayName if self.model_manager and self.model_manager.activeBundle.ref else default_label self.current_model_item.action_item.set_value(active_name) if not ui_state.is_offroad(): diff --git a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py index af8f348f72..6eff456559 100644 --- a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py +++ b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py @@ -7,7 +7,7 @@ See the LICENSE.md file in the root directory for more details. import pyray as rl from openpilot.cereal import custom -from openpilot.sunnypilot.models.default_model import DEFAULT_MODEL +from openpilot.sunnypilot.models.default_model import get_default_model from openpilot.selfdrive.ui.mici.widgets.button import BigButton from openpilot.selfdrive.ui.sunnypilot.layouts.settings.models import ModelsLayout from openpilot.selfdrive.ui.ui_state import ui_state, device @@ -27,7 +27,7 @@ class CurrentModelInfo(Widget): subheader_color = rl.Color(255, 255, 255, int(255 * 0.9 * 0.65)) max_width = int(self._rect.width - 20) self.current_model_header = UnifiedLabel(tr("active model"), 48, max_width=max_width, text_color=header_color, font_weight=FontWeight.DISPLAY) - default_text = f"{DEFAULT_MODEL} (Default)".lower() + default_text = f"{get_default_model()} (Default)".lower() self.current_model_text = UnifiedLabel(default_text, 32, max_width=max_width, text_color=subheader_color, font_weight=FontWeight.ROMAN, scroll=True) self.info_header = UnifiedLabel("cache size", 48, max_width=max_width, text_color=header_color, font_weight=FontWeight.DISPLAY) @@ -95,7 +95,7 @@ class ModelsLayoutMici(NavScroller): folders = self._get_grouped_bundles(favorites) folder_buttons = [] - default_btn = BigButton(f"{DEFAULT_MODEL} (Default)".lower()) + default_btn = BigButton(f"{get_default_model()} (Default)".lower()) default_btn.set_click_callback(self._select_default) folder_buttons.append(default_btn) @@ -162,7 +162,8 @@ class ModelsLayoutMici(NavScroller): self._was_downloading = is_downloading self.current_model_info.current_model_header.set_text(tr("active model")) - model_text = manager.activeBundle.displayName.lower() if manager.activeBundle.ref else f"{DEFAULT_MODEL} (Default)".lower() + default_model_text = f"{get_default_model()} (Default)".lower() + model_text = manager.activeBundle.displayName.lower() if manager.activeBundle.ref else default_model_text self.current_model_info.current_model_text.set_text(model_text) self.current_model_info.info_header.set_text(tr("cache size")) self.current_model_info.info_text.set_text(f"{ModelsLayout.calculate_cache_size():.2f} MB") @@ -191,4 +192,3 @@ class ModelsLayoutMici(NavScroller): self.current_model_info.info_header.set_text(tr("progress") + self._download_progress) self.current_model_info.info_header._shimmer = True self.current_model_info.info_text.set_text(f"{progress/count:.2f}%") - diff --git a/openpilot/selfdrive/ui/sunnypilot/onroad/model_renderer.py b/openpilot/selfdrive/ui/sunnypilot/onroad/model_renderer.py index 5d78997662..4be82bef4a 100644 --- a/openpilot/selfdrive/ui/sunnypilot/onroad/model_renderer.py +++ b/openpilot/selfdrive/ui/sunnypilot/onroad/model_renderer.py @@ -4,11 +4,29 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. This file is part of sunnypilot and is licensed under the MIT License. See the LICENSE.md file in the root directory for more details. """ +from openpilot.common.filter_simple import FirstOrderFilter +from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus from openpilot.selfdrive.ui.sunnypilot.onroad.chevron_metrics import ChevronMetrics from openpilot.selfdrive.ui.sunnypilot.onroad.rainbow_path import RainbowPath +from openpilot.selfdrive.ui.sunnypilot.ui_state import MADSState +from openpilot.system.ui.lib.application import gui_app class ModelRendererSP: def __init__(self): self.rainbow_path = RainbowPath() self.chevron_metrics = ChevronMetrics() + self._width_filter = FirstOrderFilter(0.9, 0.1, 1 / gui_app.target_fps) + + @property + def _lateral_active(self) -> bool: + sm = ui_state.sm + if sm.valid["selfdriveStateSP"]: + mads = sm["selfdriveStateSP"].mads + if mads.available: + return mads.enabled and mads.state != MADSState.paused + return ui_state.status in (UIStatus.ENGAGED, UIStatus.LAT_ONLY) + + def _get_path_half_width(self) -> float: + target = 0.9 if self._lateral_active else 0.40 + return self._width_filter.update(target) diff --git a/openpilot/sunnypilot/livedelay/helpers.py b/openpilot/sunnypilot/livedelay/helpers.py index 0f7437ceea..7b36b57af1 100644 --- a/openpilot/sunnypilot/livedelay/helpers.py +++ b/openpilot/sunnypilot/livedelay/helpers.py @@ -8,7 +8,10 @@ from openpilot.common.params import Params def get_lat_delay(params: Params, stock_lat_delay: float) -> float: - if params.get_bool("LagdToggle"): - return float(params.get("LagdValueCache", return_default=True)) +# live learning on: use what lagd publishes. +# off: use the fixed steerActuatorDelay + software delay sum that LagdToggle caches. - return stock_lat_delay + if params.get_bool("LagdToggle"): + return stock_lat_delay + + return float(params.get("LagdValueCache", return_default=True)) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 4397209005..17687908c8 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -7,6 +7,7 @@ See the LICENSE.md file in the root directory for more details. """ import argparse +import math import os import tempfile import time @@ -66,14 +67,15 @@ def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tu if desire_key: shapes['desire'] = (input_shapes[desire_key][2],) - if is_supercombo and 'features_buffer' in input_shapes: - fb = input_shapes['features_buffer'] - shapes['prev_feat'] = (fb[0], fb[2]) - for key, shape in input_shapes.items(): if key not in (desire_key, 'features_buffer') and 'img' not in key: shapes[key] = tuple(shape) + if is_supercombo and 'features_buffer' in input_shapes: + fb = input_shapes['features_buffer'] + feat_dim = math.prod(fb[2:]) + shapes['prev_feat'] = (fb[0], feat_dim) + sizes = [int(np.prod(size)) for size in shapes.values()] return shapes, sizes @@ -117,7 +119,9 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D } if features_buffer: - queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), + feat_dim = math.prod(features_buffer[2:]) + feat_q_len = frame_skip * features_buffer[1] if is_supercombo else frame_skip * (features_buffer[1] - 1) + 1 + queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], feat_dim), dtype=np.float32), device=device).contiguous().realize() queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')}) @@ -182,14 +186,14 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, warped_dev = warped.to(Device.DEFAULT) Tensor.realize(packed_npy_inputs_dev, warped_dev) - img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn).realize() - big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn).realize() + img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn) + big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn) unpacked_tensors = [tensor.reshape(shape) for tensor, shape in zip(packed_npy_inputs_dev.split(npy_sizes), npy_shapes.values(), strict=True)] unpacked_dict = dict(zip(npy_shapes.keys(), unpacked_tensors, strict=True)) desire_dev = unpacked_dict['desire'] - desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn).realize() + desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn) inputs = {desire_key: desire_buf} for key, tensor_val in unpacked_dict.items(): @@ -198,7 +202,7 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, if 'prev_feat' in unpacked_dict: prev_feat_dev = unpacked_dict['prev_feat'] - inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).realize() + inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).reshape(input_shapes['features_buffer']) if vision_runner: vision_out_cast = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32').realize() @@ -210,7 +214,7 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, inputs.update({road_key: img, wide_key: big_img}) if 'features_buffer' not in inputs: - inputs['features_buffer'] = sample_skip_fn(feat_q) + inputs['features_buffer'] = sample_skip_fn(feat_q).reshape(input_shapes['features_buffer']) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32').realize() if 'features_buffer' not in inputs and features_slice is not None: @@ -271,18 +275,17 @@ def _parse_size(size_str: str) -> tuple[int, int]: return int(width), int(height) -def read_file_chunked_to_shm(path): +def read_file_chunked_to_disk(path): if not path: return None import atexit import shutil from openpilot.common.file_chunker import open_file_chunked - from openpilot.common.hardware.hw import Paths - shm_path = os.path.join(Paths.shm_path(), os.path.basename(path)) - atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path)) - with open(shm_path, 'wb') as dst, open_file_chunked(path) as src: - shutil.copyfileobj(src, dst) - return shm_path + tmp_path = f'{path}.unchunked' + with open(tmp_path, 'wb') as f, open_file_chunked(path) as src: + shutil.copyfileobj(src, f) + atexit.register(lambda: os.path.exists(tmp_path) and os.remove(tmp_path)) + return tmp_path def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]: @@ -326,11 +329,11 @@ if __name__ == "__main__": model_w, model_h = args.model_size output_data = {} - args.vision_onnx = read_file_chunked_to_shm(args.vision_onnx) - args.policy_onnx = read_file_chunked_to_shm(args.policy_onnx) - args.off_policy_onnx = read_file_chunked_to_shm(args.off_policy_onnx) - args.on_policy_onnx = read_file_chunked_to_shm(args.on_policy_onnx) - args.supercombo_onnx = read_file_chunked_to_shm(args.supercombo_onnx) + args.vision_onnx = read_file_chunked_to_disk(args.vision_onnx) + args.policy_onnx = read_file_chunked_to_disk(args.policy_onnx) + args.off_policy_onnx = read_file_chunked_to_disk(args.off_policy_onnx) + args.on_policy_onnx = read_file_chunked_to_disk(args.on_policy_onnx) + args.supercombo_onnx = read_file_chunked_to_disk(args.supercombo_onnx) vision_runner = OnnxRunner(args.vision_onnx) if args.vision_onnx else None diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py b/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py index 885696b853..86974b14f1 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_compile_modeld.py @@ -5,10 +5,15 @@ This file is part of sunnypilot and is licensed under the MIT License. See the LICENSE.md file in the root directory for more details. """ +import os +import tempfile +from pathlib import Path + import numpy as np from openpilot.common.parameterized import parameterized -from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key +from openpilot.common.file_chunker import chunk_file, get_chunk_targets +from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key, read_file_chunked_to_disk from openpilot.common.test import OpenpilotTestCase @@ -160,3 +165,115 @@ class TestOutputSlicePreservation(OpenpilotTestCase): policy_slices = {'plan': slice(0, 495), 'meta': slice(495, 550)} assert set(vision_slices.keys()) & set(policy_slices.keys()) == set(), \ "vision and policy slices should not overlap in keys" + + +class TestReadFileChunkedToDisk(OpenpilotTestCase): + def test_none_passthrough(self): + assert read_file_chunked_to_disk(None) is None + + def test_unchunked_source_staged_on_disk(self): + with tempfile.TemporaryDirectory() as d: + src = Path(d) / "driving_supercombo.onnx" + payload = os.urandom(1024) + src.write_bytes(payload) + + out = Path(read_file_chunked_to_disk(str(src))) + + assert out.parent == Path(d) + assert out.name == "driving_supercombo.onnx.unchunked" + assert out.read_bytes() == payload + + def test_chunked_source_reassembled_on_disk(self): + with tempfile.TemporaryDirectory() as d: + src = Path(d) / "driving_supercombo.onnx" + payload = os.urandom(4096) + src.write_bytes(payload) + chunk_file(str(src), get_chunk_targets(str(src), len(payload))) + assert not src.exists() + + out = Path(read_file_chunked_to_disk(str(src))) + + assert out.parent == Path(d) + assert out.read_bytes() == payload + + +class Test4DFeaturesBuffer(OpenpilotTestCase): + def test_get_policy_npy_shapes_4d(self): + from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes + input_shapes = { + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 32, 512), # compare 4d to 3d for regression + 'traffic_convention': (1, 2), + 'action_t': (1, 2) + } + shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True) + assert shapes['prev_feat'] == (1, 16384) + assert sizes == [8, 2, 2, 16384] + + def test_get_policy_npy_shapes_3d(self): + from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes + input_shapes = { + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 512), + 'traffic_convention': (1, 2), + 'action_t': (1, 2) + } + shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True) + assert shapes['prev_feat'] == (1, 512) + assert sizes == [8, 2, 2, 512] + + +class TestStockCompileModeldEquivalence(OpenpilotTestCase): + def test_get_policy_npy_shapes_matches_stock(self): + from openpilot.selfdrive.modeld.compile_modeld import get_policy_npy_shapes as stock_get_policy_npy_shapes + from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes as sunny_get_policy_npy_shapes + + stock_input_shapes = { + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 512), # see below comment + 'traffic_convention': (1, 2), + 'action_t': (1, 2), + } + + stock_shapes, stock_sizes = stock_get_policy_npy_shapes(stock_input_shapes) + sunny_shapes, sunny_sizes = sunny_get_policy_npy_shapes(stock_input_shapes, is_supercombo=True) + + assert sunny_shapes == stock_shapes + assert sunny_sizes == stock_sizes + assert sunny_shapes['prev_feat'] == (1, 512) + + def test_make_input_queues_full_stock_equivalence(self): + from openpilot.selfdrive.modeld.compile_modeld import make_input_queues as stock_make_input_queues + from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues as sunny_make_supercombo_input_queues + input_shapes = { + 'img': (1, 12, 128, 256), + 'desire_pulse': (1, 25, 8), + 'features_buffer': (1, 24, 512), # when https://github.com/commaai/openpilot/pull/38681 merges, update to 1,24,32,512 + 'traffic_convention': (1, 2), + 'action_t': (1, 2), + } + frame_skip = 4 + + stock_queues, stock_npy = stock_make_input_queues(input_shapes, frame_skip, device='NPY') + sunny_queues, sunny_npy = sunny_make_supercombo_input_queues(input_shapes, frame_skip, device='NPY') + assert set(sunny_queues.keys()) == set(stock_queues.keys()) + for key in stock_queues: + assert sunny_queues[key].shape == stock_queues[key].shape, \ + f"Queue shape mismatch for {key}: sunny {sunny_queues[key].shape} != stock {stock_queues[key].shape}" + assert set(sunny_npy.keys()) == set(stock_npy.keys()) + for key in stock_npy: + assert sunny_npy[key].shape == stock_npy[key].shape, \ + f"Numpy array shape mismatch for {key}: sunny {sunny_npy[key].shape} != stock {stock_npy[key].shape}" + + def test_make_warp_queues_stock_equivalence(self): + from openpilot.selfdrive.modeld.compile_modeld import make_warp_input_queues as stock_make_warp_queues + from openpilot.sunnypilot.modeld_v2.compile_modeld import make_warp_queues as sunny_make_warp_queues + stock_vision_shapes = {'img': (1, 12, 128, 256)} # for now? + stock_queues, stock_npy = stock_make_warp_queues(stock_vision_shapes, frame_skip=4, device='NPY') + sunny_queues, sunny_npy = sunny_make_warp_queues(device='NPY') + + assert set(sunny_npy.keys()) == set(stock_npy.keys()) == {'tfm', 'big_tfm'} + for key in sunny_npy: + assert sunny_npy[key].shape == stock_npy[key].shape == (3, 3) + + diff --git a/openpilot/sunnypilot/models/default_model.py b/openpilot/sunnypilot/models/default_model.py index 62b6831402..a8e55f8e69 100755 --- a/openpilot/sunnypilot/models/default_model.py +++ b/openpilot/sunnypilot/models/default_model.py @@ -3,8 +3,17 @@ import os import hashlib from openpilot.common.basedir import BASEDIR +from openpilot.selfdrive.ui.ui_state import ui_state from openpilot.sunnypilot import get_file_hash -from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL +from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL, DEFAULT_BIG_MODEL + + +def get_default_model() -> str: + show_big_model = (ui_state.usbgpu + and (ui_state.usbgpu_active or ui_state.usbgpu_loading or ui_state.is_offroad())) + + return DEFAULT_BIG_MODEL if show_big_model else DEFAULT_MODEL + DEFAULT_MODEL_NAME_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "model_name.py") MODEL_HASH_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "tests", "model_hash") @@ -13,7 +22,6 @@ SUPERCOMBO_ONNX_PATH = os.path.join(BASEDIR, "openpilot", "selfdrive", "modeld", def update_model_hash(): supercombo_hash = get_file_hash(SUPERCOMBO_ONNX_PATH) - combined_hash = hashlib.sha256(supercombo_hash.encode()).hexdigest() with open(MODEL_HASH_PATH, "w") as f: @@ -22,40 +30,28 @@ def update_model_hash(): print(f"Generated and updated new combined model hash to {MODEL_HASH_PATH}") -def get_current_default_model_name(): - print("[GET DEFAULT MODEL NAME]") - name = DEFAULT_MODEL - print(f'Current default model name: "{name}"') - - return name - - -def update_default_model_name(name: str): - print("[CHANGE DEFAULT MODEL NAME]") +def update_default_model_names(default_model_name: str, default_big_model_name: str): + print("[CHANGE DEFAULT MODEL NAMES]") with open(DEFAULT_MODEL_NAME_PATH, "w") as f: - f.write(f'DEFAULT_MODEL = "{name}"\n') - print(f'New default model name: "{name}"') + f.write(f'DEFAULT_MODEL = "{default_model_name}"\n') + f.write(f'DEFAULT_BIG_MODEL = "{default_big_model_name}"\n') + + print(f'New default small model name: "{default_model_name}"') + print(f'New default big model name: "{default_big_model_name}"') print("[DONE]") if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Update default model name and hash") - parser.add_argument("--new_name", type=str, help="New default model name") + parser = argparse.ArgumentParser(description="Update default model names and hash") + parser.add_argument("--new_small_model_name", type=str, help="New default small model name") + parser.add_argument("--new_big_model_name", type=str, help="New default big model name") args = parser.parse_args() - if not args.new_name: - print("Warning: No new default model name provided. Use --new_name to specify") - print("Default model name and hash will not be updated! (aborted)") - exit(0) + if args.new_small_model_name is None and args.new_big_model_name is None: + new_name = input(f'Enter new default small model name (current: "{DEFAULT_MODEL}", leave empty to keep): ').strip() + new_big_model_name = input(f'Enter new default big model name (current: "{DEFAULT_BIG_MODEL}", leave empty to keep): ').strip() + else: + new_name, new_big_model_name = args.new_small_model_name, args.new_big_model_name - current_name = get_current_default_model_name() - new_name = args.new_name - if current_name == new_name: - print(f'Proposed default model name: "{new_name}"') - confirm = input("Proposed default model name is the same as the current default model name. Confirm? (y/n): ").upper().strip() - if confirm != "Y": - print("Default model name and hash will not be updated! (aborted)") - exit(0) - - update_default_model_name(new_name) + update_default_model_names(new_name or DEFAULT_MODEL, new_big_model_name or DEFAULT_BIG_MODEL) update_model_hash() diff --git a/openpilot/sunnypilot/models/fetcher.py b/openpilot/sunnypilot/models/fetcher.py index e60af2925f..c9e86edd0c 100644 --- a/openpilot/sunnypilot/models/fetcher.py +++ b/openpilot/sunnypilot/models/fetcher.py @@ -13,8 +13,6 @@ from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog from openpilot.common.hardware.hw import Paths from openpilot.sunnypilot.models.helpers import is_bundle_version_compatible -from openpilot.selfdrive.modeld.helpers import usbgpu_present - from openpilot.cereal import custom @@ -140,8 +138,8 @@ class ModelCache: class ModelFetcher: """Handles fetching and caching of model data from remote source""" - MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v19.json" - MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v19.json" + MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v21.json" + MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v22.json" def __init__(self, params: Params): self.params = params @@ -149,11 +147,10 @@ class ModelFetcher: self._is_usbgpu: bool | None = None self.model_cache = ModelCache(params) self.model_url = self.MODEL_URL - self._update_model_source() - def _update_model_source(self) -> None: - """Updates what json to use based on usbgpu availability""" - is_usbgpu = usbgpu_present() + def _update_model_source(self, chestnut_present: bool) -> None: + """Updates what json to use based on chestnut hardware presence via deviceState""" + is_usbgpu = chestnut_present if is_usbgpu != self._is_usbgpu: self._is_usbgpu = is_usbgpu self.model_cache = ModelCache(self.params, suffix="_USBGPU" if is_usbgpu else "") @@ -191,9 +188,9 @@ class ModelFetcher: return None - def get_available_bundles(self) -> list[custom.ModelManagerSP.ModelBundle]: + def get_available_bundles(self, chestnut_present: bool = False) -> list[custom.ModelManagerSP.ModelBundle]: """Gets the list of available models, with smart cache handling""" - self._update_model_source() + self._update_model_source(chestnut_present) cached_data, is_expired = self.model_cache.get() if cached_data and not is_expired: @@ -210,10 +207,12 @@ class ModelFetcher: cloudlog.warning("Failed to fetch fresh data. Using expired cache as fallback") return self.model_parser.parse_models(cached_data) + if __name__ == "__main__": + from openpilot.selfdrive.modeld.helpers import usbgpu_present params = Params() model_fetcher = ModelFetcher(params) - bundles = model_fetcher.get_available_bundles() + bundles = model_fetcher.get_available_bundles(chestnut_present=usbgpu_present()) for bundle in bundles: for model in bundle.models: model_overrides = {override.key: override.value for override in bundle.overrides} diff --git a/openpilot/sunnypilot/models/helpers.py b/openpilot/sunnypilot/models/helpers.py index b5c97467d3..e33cc445d1 100644 --- a/openpilot/sunnypilot/models/helpers.py +++ b/openpilot/sunnypilot/models/helpers.py @@ -18,12 +18,11 @@ from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRai from openpilot.common.hardware.hw import Paths # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO -REQUIRED_JSON_VERSION = 17 +REQUIRED_JSON_VERSION = 18 CUSTOM_MODEL_PATH = Paths.model_root() METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl' ModelManager = custom.ModelManagerSP -_LAST_VALIDATED_RAW = None def _compute_hash(file_path: str) -> str | None: @@ -86,11 +85,11 @@ def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, availa if available_bundles is not None: matching_bundle = None for bundle in available_bundles: - if getattr(active_bundle, 'ref', None) and getattr(bundle, 'ref', None): + if active_bundle.ref and bundle.ref: if active_bundle.ref == bundle.ref: matching_bundle = bundle break - elif getattr(active_bundle, 'internalName', None) == getattr(bundle, 'internalName', None): + elif active_bundle.internalName == bundle.internalName: matching_bundle = bundle break @@ -98,36 +97,42 @@ def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, availa return True if active_bundle.minimumSelectorVersion != matching_bundle.minimumSelectorVersion: return True - - active_runner = getattr(active_bundle, 'runner', None) - matching_runner = getattr(matching_bundle, 'runner', None) - if active_runner is not None and matching_runner is not None: - if getattr(active_runner, 'raw', active_runner) != getattr(matching_runner, 'raw', matching_runner): - return True + if active_bundle.runner.raw != matching_bundle.runner.raw: + return True if set(_bundle_artifacts(active_bundle)) != set(_bundle_artifacts(matching_bundle)): return True - return not _bundle_is_valid_locally(active_bundle) + # missing files trigger re-download, not selection reset + return False -def validate_active_bundle(params: Params, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None) -> None: - global _LAST_VALIDATED_RAW +def _prev_bundle_key(is_usbgpu: bool) -> str: + return "ModelManager_PrevBundle_USBGPU" if is_usbgpu else "ModelManager_PrevBundle" + +def validate_active_bundle(params: Params, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None, + is_usbgpu: bool = False) -> None: raw_bundle = params.get("ModelManager_ActiveBundle") if not raw_bundle: - return - - if raw_bundle == _LAST_VALIDATED_RAW: + prev = params.get(_prev_bundle_key(is_usbgpu)) + if prev and (prev_bundle := get_active_bundle(params, raw_bundle_dict=prev)) is not None: + if not _bundle_needs_reset(prev_bundle, available_bundles): + params.put("ModelManager_ActiveBundle", prev, block=True) return active_bundle = get_active_bundle(params, raw_bundle_dict=raw_bundle) if active_bundle is None or _bundle_needs_reset(active_bundle, available_bundles): cloudlog.warning("Active model bundle invalid; resetting to default") + params.put(_prev_bundle_key(not is_usbgpu), raw_bundle, block=True) + + prev = params.get(_prev_bundle_key(is_usbgpu)) + if prev and (prev_bundle := get_active_bundle(params, raw_bundle_dict=prev)) is not None: + if not _bundle_needs_reset(prev_bundle, available_bundles): + params.put("ModelManager_ActiveBundle", prev, block=True) + return + params.remove("ModelManager_ActiveBundle") params.put("ModelRunnerTypeCache", int(custom.ModelManagerSP.Runner.stock), block=True) - _LAST_VALIDATED_RAW = None - else: - _LAST_VALIDATED_RAW = raw_bundle def get_active_bundle(params: Params | None = None, raw_bundle_dict: dict | bytes | None = None) -> "custom.ModelManagerSP.ModelBundle | None": diff --git a/openpilot/sunnypilot/models/manager.py b/openpilot/sunnypilot/models/manager.py index 7ba805ff95..e47cf7536c 100644 --- a/openpilot/sunnypilot/models/manager.py +++ b/openpilot/sunnypilot/models/manager.py @@ -30,6 +30,7 @@ class ModelManagerSP: self.params = Params() self.model_fetcher = ModelFetcher(self.params) self.pm = messaging.PubMaster(["modelManagerSP"]) + self.sm = messaging.SubMaster(["deviceState"]) self.available_models: list[custom.ModelManagerSP.ModelBundle] = [] self.selected_bundle: custom.ModelManagerSP.ModelBundle = None self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params) @@ -143,13 +144,17 @@ class ModelManagerSP: is_cached = False if len(artifact.chunks) > 0: from openpilot.common.file_chunker import get_chunk_name + num_chunks = len(artifact.chunks) chunks_valid = True for i, chunk in enumerate(artifact.chunks): - chunk_path = get_chunk_name(full_path, i, len(artifact.chunks)) + chunk_path = get_chunk_name(full_path, i, num_chunks) if not await verify_file(chunk_path, chunk.sha256): chunks_valid = False break - if chunks_valid and len(artifact.chunks) > 0: + artifact.downloadProgress.progress = ((i + 1) / num_chunks) * 100 + self._sync_artifact_progress(artifact) + self._report_status() + if chunks_valid and num_chunks > 0: is_cached = True else: if await verify_file(full_path, expected_hash): @@ -216,6 +221,9 @@ class ModelManagerSP: """Downloads all models in a bundle""" self.selected_bundle = model_bundle self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloading + for model in self.selected_bundle.models: + model.artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloading + self._report_status() os.makedirs(destination_path, exist_ok=True) try: @@ -249,18 +257,27 @@ class ModelManagerSP: """Main entry point for downloading a model bundle""" asyncio.run(self._download_bundle(model_bundle, destination_path)) + BOOT_SETTLE_TICKS = 10 # seconds at 1 Hz before validating active bundle + def main_thread(self) -> None: """Main thread for model management""" rk = Ratekeeper(1, print_delay_threshold=None) + boot_ticks = 0 while True: try: - self.available_models = self.model_fetcher.get_available_bundles() - validate_active_bundle(self.params, self.available_models) + self.sm.update(0) + chestnut_present = self.sm['deviceState'].chestnutPresent + self.available_models = self.model_fetcher.get_available_bundles(chestnut_present) + if boot_ticks >= self.BOOT_SETTLE_TICKS: + validate_active_bundle(self.params, self.available_models, is_usbgpu=chestnut_present) + boot_ticks = min(boot_ticks + 1, self.BOOT_SETTLE_TICKS) self.active_bundle = get_active_bundle(self.params) if (index_to_download := self.params.get("ModelManager_DownloadIndex")) is not None: - if model_to_download := next((model for model in self.available_models if model.index == index_to_download), None): + if self.active_bundle and self.active_bundle.index == index_to_download: + self.params.remove("ModelManager_DownloadIndex") + elif model_to_download := next((model for model in self.available_models if model.index == index_to_download), None): try: self.download(model_to_download, Paths.model_root()) except Exception as e: diff --git a/openpilot/sunnypilot/models/model_name.py b/openpilot/sunnypilot/models/model_name.py index 02a6c2bac2..374e8473df 100644 --- a/openpilot/sunnypilot/models/model_name.py +++ b/openpilot/sunnypilot/models/model_name.py @@ -1 +1,2 @@ DEFAULT_MODEL = "CD210" +DEFAULT_BIG_MODEL = "Lebowski" diff --git a/openpilot/sunnypilot/models/tests/test_default_model.py b/openpilot/sunnypilot/models/tests/test_default_model.py index 450237e0e9..b72c2b4c89 100644 --- a/openpilot/sunnypilot/models/tests/test_default_model.py +++ b/openpilot/sunnypilot/models/tests/test_default_model.py @@ -20,4 +20,4 @@ class TestDefaultModel(OpenpilotTestCase): with open(MODEL_HASH_PATH) as f: current_hash = f.read().strip() - assert combined_hash == current_hash, "Run sunnypilot/models/default_model.py to update the default model name and hash" + assert combined_hash == current_hash, "Run openpilot/sunnypilot/models/default_model.py to update the default model name and hash" diff --git a/openpilot/sunnypilot/selfdrive/locationd/tests/test_locationd.py b/openpilot/sunnypilot/selfdrive/locationd/tests/test_locationd.py index ac6ab6aaea..2e99122bc7 100644 --- a/openpilot/sunnypilot/selfdrive/locationd/tests/test_locationd.py +++ b/openpilot/sunnypilot/selfdrive/locationd/tests/test_locationd.py @@ -83,9 +83,14 @@ class TestLocationdProc(OpenpilotTestCase): self.pm.send(msg.which(), msg) if msg.which() == "cameraOdometry": self.pm.wait_for_readers_to_update(msg.which(), timeout=1, dt=0.005) - time.sleep(1) # wait for async params write + for _ in range(50): + val = self.params.get('LastGPSPositionLLK') + if val is not None: + break + time.sleep(0.1) - lastGPS = json.loads(self.params.get('LastGPSPositionLLK')) + self.assertIsNotNone(val, "LastGPSPositionLLK not written within 5s") + lastGPS = json.loads(val) self.assertAlmostEqual(lastGPS['latitude'], self.lat, delta=0.001) self.assertAlmostEqual(lastGPS['longitude'], self.lon, delta=0.001) self.assertAlmostEqual(lastGPS['altitude'], self.alt, delta=0.001) diff --git a/openpilot/sunnypilot/sunnylink/athena/sunnylinkd.py b/openpilot/sunnypilot/sunnylink/athena/sunnylinkd.py index ac168db7d8..1ab2f373ed 100755 --- a/openpilot/sunnypilot/sunnylink/athena/sunnylinkd.py +++ b/openpilot/sunnypilot/sunnylink/athena/sunnylinkd.py @@ -28,7 +28,7 @@ from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutExce create_connection, WebSocketConnectionClosedException) import openpilot.cereal.messaging as messaging -from openpilot.sunnypilot.models.default_model import DEFAULT_MODEL +from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL, DEFAULT_BIG_MODEL from openpilot.sunnypilot.selfdrive.car.sync_sunnylink_params import update_car_list_param from openpilot.sunnypilot.sunnylink.api import SunnylinkApi from openpilot.sunnypilot.sunnylink.utils import sunnylink_need_register, sunnylink_ready, get_param_as_byte, save_param_from_base64_encoded_string @@ -182,6 +182,8 @@ def getParamsMetadata() -> str: schema["capabilities"] = generate_capabilities() schema["capability_labels"] = CAPABILITY_LABELS schema["default_model"] = DEFAULT_MODEL + schema["default_big_model"] = DEFAULT_BIG_MODEL + schema["usbgpu_active"] = params.get_bool("UsbGpuActive") raw = json.dumps(schema, separators=(",", ":")).encode("utf-8") return base64.b64encode(gzip.compress(raw)).decode("utf-8") except Exception: diff --git a/openpilot/system/athena/athenad.py b/openpilot/system/athena/athenad.py index 89a98dd646..4d4735b461 100755 --- a/openpilot/system/athena/athenad.py +++ b/openpilot/system/athena/athenad.py @@ -828,20 +828,22 @@ def startStream(sdp: str, enabled: bool) -> dict: bridge_services_in = [] # stale car params case taken care of by webrtcd being shut off on ignition - cp_bytes = Params().get("CarParamsPersistent") + cp_bytes = params.get("CarParamsPersistent") if cp_bytes is not None: with car.CarParams.from_bytes(cp_bytes) as CP: if CP.notCar: bridge_services_in.append("testJoystick") - else: - raise Exception("failed to get CarParamsPersistent") if params.get_bool("IsOffroad"): # manager owns camerad/stream_encoderd/webrtcd; flip the param and let it bring them up. # webrtcd clears IsLiveStreaming when the session ends params.put_bool("IsLiveStreaming", True) # wait for webrtcd end points to wake up - wait_for_webrtcd() + try: + wait_for_webrtcd() + except TimeoutError: + cloudlog.event("athena.startStream.webrtcd_offroad_start_timeout", error=True) + raise return post_stream_request(StreamRequestBody(sdp, ["wideRoad"], enabled, bridge_services_in, ["carState", "deviceState"])) diff --git a/openpilot/system/hardware/hardwared.py b/openpilot/system/hardware/hardwared.py index 7250cd4d63..8aed8be5b9 100755 --- a/openpilot/system/hardware/hardwared.py +++ b/openpilot/system/hardware/hardwared.py @@ -16,6 +16,7 @@ from openpilot.common.utils import strip_deprecated_keys from openpilot.common.filter_simple import FirstOrderFilter from openpilot.common.params import Params from openpilot.common.realtime import DT_HW +from openpilot.selfdrive.modeld.helpers import MODELS_DIR, usbgpu_compiled from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert from openpilot.common.hardware import HARDWARE, COMMA_HARDWARE from openpilot.common.basedir import BASEDIR @@ -26,7 +27,7 @@ from openpilot.common.swaglog import cloudlog from openpilot.sunnypilot.system.statsd import statlog from openpilot.system.hardware.power_monitoring import PowerMonitoring from openpilot.system.hardware.fan_controller import FanController -from openpilot.common.version import terms_version, training_version, get_build_metadata, terms_version_sp +from openpilot.common.version import terms_version, training_version, get_build_metadata, terms_version_sp, CHESTNUT_BRANCHES ThermalStatus = log.DeviceState.ThermalStatus @@ -238,8 +239,7 @@ def hardware_thread(end_event, hw_queue) -> None: fan_controller = FanController(int(1./DT_HW)) chestnut = Chestnut() - big_model_available = os.path.isfile(os.path.join(BASEDIR, "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx")) or \ - os.path.isfile(os.path.join(BASEDIR, "openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl.chunkmanifest")) + big_model_available = (MODELS_DIR / 'big_driving_supercombo.onnx').is_file() or usbgpu_compiled() while not end_event.is_set(): sm.update(PANDA_STATES_TIMEOUT) @@ -301,7 +301,11 @@ def hardware_thread(end_event, hw_queue) -> None: set_usb_state(msg.deviceState, last_hw_state.usb_state) chestnut.update(started_ts is None, last_hw_state.usb_state) - set_offroad_alert_if_changed("Offroad_ChestnutBranch", msg.deviceState.chestnutPresent and not big_model_available) + current_channel = get_build_metadata().channel + chestnut_target = CHESTNUT_BRANCHES.get(current_channel) + chestnut_needs_switch = msg.deviceState.chestnutPresent and not big_model_available and chestnut_target is not None + set_offroad_alert_if_changed("Offroad_ChestnutBranch", chestnut_needs_switch, + extra_text=chestnut_target if chestnut_needs_switch else None) # this subset is only used for offroad temp_sources = [ diff --git a/openpilot/system/manager/github_runner.sh b/openpilot/system/manager/github_runner.sh deleted file mode 100755 index f2170cfc70..0000000000 --- a/openpilot/system/manager/github_runner.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# Define the service name -SERVICE_NAME="actions.runner.sunnypilot.$(uname -n)" - -# Function to control the service -control_service() { - local action=$1 # Store the function argument in a local variable - sudo systemctl $action ${SERVICE_NAME} -} - -service_exists_and_is_loaded() { - sudo systemctl status ${SERVICE_NAME} &>/dev/null - if [[ $? -ne 4 ]]; then - return 0 # Service is known to systemd (i.e., loaded) - else - return 1 # Service is unknown to systemd (i.e., not loaded) - fi -} - -# Check for required argument -if [[ -z $1 ]] || { [[ $1 != "start" ]] && [[ $1 != "stop" ]]; }; then - echo "Usage: $0 {start|stop}" - exit 1 -fi - -# Store the script argument in a descriptive variable -ACTION=$1 - -# Trap EXIT signal (Ctrl+C) and stop the service -trap 'control_service stop ; exit' SIGINT SIGKILL EXIT - -# Enter the main loop -while true; do - # Check if the service is actually present on the system - if service_exists_and_is_loaded; then - control_service $ACTION # Call the function with the specified action - fi - sleep 1 # Pause before the next iteration -done \ No newline at end of file diff --git a/openpilot/system/manager/process_config.py b/openpilot/system/manager/process_config.py index c9e939c432..f776a85923 100644 --- a/openpilot/system/manager/process_config.py +++ b/openpilot/system/manager/process_config.py @@ -68,10 +68,6 @@ def only_offroad(started: bool, params: Params, CP: car.CarParams) -> bool: def livestream(started: bool, params: Params, CP: car.CarParams) -> bool: return params.get_bool("IsLiveStreaming") -def use_github_runner(started, params, CP: car.CarParams) -> bool: - return not PC and params.get_bool("EnableGithubRunner") and ( - not params.get_bool("NetworkMetered") and not params.get_bool("GithubRunnerSufficientVoltage")) - def use_copyparty(started, params, CP: car.CarParams) -> bool: return bool(params.get_bool("EnableCopyparty")) @@ -110,15 +106,12 @@ def or_(*fns): def and_(*fns): return lambda *args: operator.and_(*(fn(*args) for fn in fns)) -def not_(*fns): - return lambda *args: operator.not_(*(fn(*args) for fn in fns)) - procs = [ DaemonProcess("manage_athenad", "openpilot.system.athena.manage_athenad", "AthenadPid"), NativeProcess("loggerd", "openpilot/system/loggerd", ["./loggerd"], logging), NativeProcess("encoderd", "openpilot/system/loggerd", ["./encoderd"], only_onroad), - NativeProcess("stream_encoderd", "openpilot/system/loggerd", ["./encoderd", "--stream"], or_(and_(livestream, not_(iscar)), notcar)), + NativeProcess("stream_encoderd", "openpilot/system/loggerd", ["./encoderd", "--stream"], or_(livestream, notcar)), PythonProcess("logmessaged", "openpilot.system.logmessaged", always_run), NativeProcess("camerad", "openpilot/system/camerad", ["./camerad"], or_(driverview, livestream), enabled=not WEBCAM), @@ -163,7 +156,7 @@ procs = [ # debug procs NativeProcess("bridge", "openpilot/cereal/messaging", ["./bridge"], notcar), - PythonProcess("webrtcd", "openpilot.system.webrtc.webrtcd", or_(and_(livestream, not_(iscar)), notcar)), + PythonProcess("webrtcd", "openpilot.system.webrtc.webrtcd", or_(livestream, notcar)), PythonProcess("joystick", "openpilot.tools.joystick.joystick_control", and_(joystick, iscar)), # sunnylink <3 @@ -189,10 +182,6 @@ procs += [ NativeProcess("locationd_llk", "openpilot/sunnypilot/selfdrive/locationd", ["./locationd"], only_onroad), ] -if os.path.exists("./github_runner.sh"): - procs += [NativeProcess("github_runner_start", "openpilot/system/manager", - ["./github_runner.sh", "start"], and_(only_offroad, use_github_runner), sigkill=False)] - if os.path.exists("../../sunnypilot/sunnylink/uploader.py"): procs += [PythonProcess("sunnylink_uploader", "openpilot.sunnypilot.sunnylink.uploader", use_sunnylink_uploader_shim)] diff --git a/openpilot/system/ui/sunnypilot/lib/utils.py b/openpilot/system/ui/sunnypilot/lib/utils.py index ecaba86f4b..b9ed152aff 100644 --- a/openpilot/system/ui/sunnypilot/lib/utils.py +++ b/openpilot/system/ui/sunnypilot/lib/utils.py @@ -4,7 +4,15 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. This file is part of sunnypilot and is licensed under the MIT License. See the LICENSE.md file in the root directory for more details. """ +from collections.abc import Callable + +import pyray as rl + +from openpilot.system.ui.lib.application import FontWeight +from openpilot.system.ui.sunnypilot.lib.styles import style from openpilot.system.ui.sunnypilot.widgets.list_view import ButtonActionSP +from openpilot.system.ui.widgets.label import UnifiedLabel +from openpilot.system.ui.widgets.list_view import BUTTON_WIDTH, BUTTON_HEIGHT, TEXT_PADDING, _resolve_value class NoElideButtonAction(ButtonActionSP): @@ -12,6 +20,38 @@ class NoElideButtonAction(ButtonActionSP): return super().get_width_hint() + 1 +class ScrollingButtonAction(ButtonActionSP): + """ButtonActionSP whose value scrolls instead of eliding when it doesn't fit.""" + + def __init__(self, text: str | Callable[[], str], width: int = style.BUTTON_ACTION_WIDTH, + enabled: bool | Callable[[], bool] = True): + super().__init__(text=text, width=width, enabled=enabled) + self._value_label = UnifiedLabel("", font_size=style.ITEM_TEXT_FONT_SIZE, font_weight=FontWeight.NORMAL, + text_color=self._value_color, scroll=True, + alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) + + def set_value(self, value: str | Callable[[], str], color: rl.Color = style.ITEM_TEXT_VALUE_COLOR): + if self.value != _resolve_value(value, ""): + self._value_label.reset_scroll() + super().set_value(value, color) + self._value_label.set_text(value) + self._value_label.set_text_color(color) + + def _render(self, rect: rl.Rectangle) -> bool: + """Duplicate of ButtonActionSP._render, with the value drawn by a scrolling label""" + self._button.set_text(self.text) + self._button.set_enabled(_resolve_value(self.enabled)) + button_rect = rl.Rectangle(rect.x + rect.width - BUTTON_WIDTH, rect.y + (rect.height - BUTTON_HEIGHT) / 2, BUTTON_WIDTH, BUTTON_HEIGHT) + self._button.render(button_rect) + + if self.value: + self._value_label.render(rl.Rectangle(rect.x, rect.y, rect.width - BUTTON_WIDTH - TEXT_PADDING, rect.height)) + + pressed = self._pressed + self._pressed = False + return pressed + + class AlertFadeAnimator: def __init__(self, target_fps: int, duration_on: float = 0.75, rc: float = 0.05): from openpilot.common.filter_simple import FirstOrderFilter diff --git a/openpilot/system/ui/sunnypilot/widgets/download_status.py b/openpilot/system/ui/sunnypilot/widgets/download_status.py new file mode 100644 index 0000000000..b299c464f1 --- /dev/null +++ b/openpilot/system/ui/sunnypilot/widgets/download_status.py @@ -0,0 +1,166 @@ +""" +Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + +This file is part of sunnypilot and is licensed under the MIT License. +See the LICENSE.md file in the root directory for more details. +""" +import math + +import numpy as np +import pyray as rl + +from openpilot.common.filter_simple import FirstOrderFilter +from openpilot.system.ui.lib.application import gui_app, FontWeight +from openpilot.system.ui.lib.shader_polygon import draw_polygon, Gradient +from openpilot.system.ui.lib.text_measure import measure_text_cached +from openpilot.system.ui.sunnypilot.lib.styles import style +from openpilot.system.ui.sunnypilot.widgets.list_view import ListItemSP +from openpilot.system.ui.widgets.label import UnifiedLabel +from openpilot.system.ui.widgets.list_view import ItemAction + +FONT_SIZE = style.ITEM_TEXT_FONT_SIZE +ICON_SIZE = 56 +ICON_PADDING = 12 + +BAR_WIDTH = 1100 +BAR_HEIGHT = 20 +BAR_GAP = 16 +BAR_RADIUS = BAR_HEIGHT / 2 +CAPSULE_POINTS = 24 + +RAIL_COLOR = rl.Color(60, 60, 60, 255) +FILL_COLOR = rl.Color(30, 121, 232, 255) +# rl.WHITE is a tuple; the shimmer path reads .a off the color +TEXT_COLOR = rl.Color(255, 255, 255, 255) + +SWEEP_SPEED = 550.0 # px/s +SWEEP_BAND = 240.0 # highlight half-width, px +SWEEP_DIM = 0.65 + + +class DownloadStatusAction(ItemAction): + """Model download row: a name + percent over a progress rail while downloading, a name + icon otherwise.""" + + def __init__(self): + super().__init__(width=BAR_WIDTH) + self.name = "" + self.status_text = "" + self.downloading = False + self.text_color = rl.GRAY + self.icon: str | None = None + self.icon_color: rl.Color | None = None + self._font = gui_app.font(FontWeight.NORMAL) + # raw progress arrives in steps, one per 128KB chunk the manager publishes + self._progress = FirstOrderFilter(0.0, 0.5, 1 / gui_app.target_fps) + # integrated per frame; (t * speed) % span jumps whenever the fill width changes + self._sweep = 0.0 + + self._name_label = UnifiedLabel("", font_size=FONT_SIZE, font_weight=FontWeight.NORMAL, text_color=TEXT_COLOR, + alignment=rl.GuiTextAlignment.TEXT_ALIGN_LEFT, + alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) + self._percent_label = UnifiedLabel("", font_size=FONT_SIZE, font_weight=FontWeight.NORMAL, text_color=TEXT_COLOR, + alignment=rl.GuiTextAlignment.TEXT_ALIGN_RIGHT, + alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) + + def update(self, name, downloading=False, progress=0.0, status_text="", text_color=rl.GRAY, icon=None, icon_color=None): + if downloading and not self.downloading: + self._name_label.reset_shimmer() + self._progress.x = progress + self._sweep = 0.0 + self.name = name + self.downloading = downloading + self.status_text = status_text + self.text_color = text_color + self.icon = icon + self.icon_color = icon_color + self._name_label._shimmer = downloading + if downloading: + self._progress.update(progress) + self._sweep += SWEEP_SPEED / gui_app.target_fps + + @property + def _idle_text(self) -> str: + return f"{self.name} - {self.status_text}" if self.status_text else self.name + + def get_width_hint(self) -> float: + if self.downloading: + return BAR_WIDTH + width = measure_text_cached(self._font, self._idle_text, FONT_SIZE).x + if self.icon: + width += ICON_SIZE + ICON_PADDING + return width + + def _render(self, rect: rl.Rectangle): + if self.downloading: + self._render_downloading(rect) + else: + self._render_idle(rect) + + def _sweep_gradient(self, width: float) -> Gradient: + # clearance at both ends keeps the wrap offscreen + center = (self._sweep % (width + 2 * SWEEP_BAND)) - SWEEP_BAND + + def band(x: float) -> float: + return max(0.0, 1.0 - abs(x - center) / SWEEP_BAND) + + # sampling the corners is exact for a piecewise linear band + xs = sorted({0.0, width} | {min(max(center + o, 0.0), width) for o in (-SWEEP_BAND, 0.0, SWEEP_BAND)}, reverse=True) + # the gradient axis runs right-to-left in screen space + stops = [1.0 - x / width for x in xs] + # alpha here is the lift over the SWEEP_DIM base, not the final opacity + colors = [rl.Color(FILL_COLOR.r, FILL_COLOR.g, FILL_COLOR.b, int(255 * band(x))) for x in xs] + return Gradient(start=(0.0, 0.0), end=(1.0, 0.0), colors=colors, stops=stops) + + @staticmethod + def _capsule(rect: rl.Rectangle) -> np.ndarray: + """Rounded-end ribbon so the gradient covers the caps.""" + r = rect.height / 2 + cy = rect.y + r + top, bottom = [], [] + for i in range(CAPSULE_POINTS): + x = rect.x + rect.width * i / (CAPSULE_POINTS - 1) + d = min(x - rect.x, rect.x + rect.width - x, r) + h = math.sqrt(max(r * r - (r - d) ** 2, 0.0)) + top.append((x, cy - h)) + bottom.append((x, cy + h)) + return np.array(top + bottom[::-1], dtype=np.float32) + + def _draw_fill(self, rail: rl.Rectangle, fill_width: float): + if fill_width <= 0: + return + fill = rl.Rectangle(rail.x, rail.y, fill_width, rail.height) + rl.draw_rectangle_rounded(fill, 1.0, 10, rl.Color(FILL_COLOR.r, FILL_COLOR.g, FILL_COLOR.b, int(255 * SWEEP_DIM))) + draw_polygon(fill, self._capsule(fill), gradient=self._sweep_gradient(fill_width)) + + def _render_downloading(self, rect: rl.Rectangle): + percent = f"{int(self._progress.x)}%" + text_height = measure_text_cached(self._font, percent, FONT_SIZE).y + top = rect.y + (rect.height - (text_height + BAR_GAP + BAR_HEIGHT)) / 2 + + text_rect = rl.Rectangle(rect.x, top, rect.width, text_height) + self._name_label.set_text(self.name) + self._name_label.render(text_rect) + self._percent_label.set_text(percent) + self._percent_label.render(text_rect) + + rail = rl.Rectangle(rect.x, top + text_height + BAR_GAP, rect.width, BAR_HEIGHT) + rl.draw_rectangle_rounded(rail, 1.0, 10, RAIL_COLOR) + self._draw_fill(rail, max(0.0, min(rect.width, rect.width * (self._progress.x / 100.0)))) + + def _render_idle(self, rect: rl.Rectangle): + text = self._idle_text + text_size = measure_text_cached(self._font, text, FONT_SIZE) + right = rect.x + rect.width + + if self.icon: + texture = gui_app.texture(self.icon, ICON_SIZE, ICON_SIZE, keep_aspect_ratio=True) + rl.draw_texture_v(texture, rl.Vector2(right - texture.width, rect.y + (rect.height - texture.height) / 2), + self.icon_color or self.text_color) + right -= texture.width + ICON_PADDING + + rl.draw_text_ex(self._font, text, rl.Vector2(right - text_size.x, rect.y + (rect.height - text_size.y) / 2), + FONT_SIZE, 0, self.text_color) + + +def download_status_item(title): + return ListItemSP(title=title, action_item=DownloadStatusAction(), title_color=style.ITEM_TEXT_COLOR) diff --git a/openpilot/system/webrtc/helpers.py b/openpilot/system/webrtc/helpers.py index de45e1c6c9..87fe20eb20 100644 --- a/openpilot/system/webrtc/helpers.py +++ b/openpilot/system/webrtc/helpers.py @@ -23,9 +23,9 @@ def post_stream_request(body: StreamRequestBody) -> dict: ret["time"] = (t_end - t_start) * 1000 return ret except requests.ConnectTimeout as e: - raise Exception("webrtc took too long to respond.") from e + raise Exception("device took too long to respond.") from e except requests.ConnectionError as e: - raise Exception("webrtc server on device is not running.") from e + raise Exception("turn car ignition off to use livestreaming.") from e def wait_for_webrtcd(max_retries: float = 10) -> None: @@ -37,4 +37,4 @@ def wait_for_webrtcd(max_retries: float = 10) -> None: except requests.ConnectionError: attempts += 1 time.sleep(0.5) - raise TimeoutError("webrtcd did not initialize in time.") + raise TimeoutError("livestreaming service did not initialize in time.") diff --git a/openpilot/system/webrtc/tests/test_stream_session.py b/openpilot/system/webrtc/tests/test_stream_session.py index d64cf6aed6..03d540c3c2 100644 --- a/openpilot/system/webrtc/tests/test_stream_session.py +++ b/openpilot/system/webrtc/tests/test_stream_session.py @@ -7,7 +7,7 @@ from openpilot.common.test import OpenpilotTestCase from openpilot.cereal import messaging, log from teleoprtc.tracks import VIDEO_CLOCK_RATE -from openpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy +from openpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy, ServerState, handle_get_stream from openpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack @@ -80,3 +80,8 @@ class TestStreamSession(OpenpilotTestCase): start_pts = packet.pts assert abs(i + packet.pts - (start_pts + (((time.monotonic_ns() - start_ns) * VIDEO_CLOCK_RATE) // 1_000_000_000))) < 450 #5ms assert bytes(packet) == b"" + + def test_stream_rejects_non_json_content_type(self): + response = self.loop.run_until_complete(handle_get_stream(ServerState(), b"{}", "text/plain")) + + assert response == (415, b'{"error": "unsupported media type"}', "application/json; charset=utf-8") diff --git a/openpilot/system/webrtc/webrtcd.py b/openpilot/system/webrtc/webrtcd.py index 8e022eda1c..9481e077ab 100755 --- a/openpilot/system/webrtc/webrtcd.py +++ b/openpilot/system/webrtc/webrtcd.py @@ -21,10 +21,16 @@ from typing import Any from openpilot.system.webrtc.helpers import StreamRequestBody from openpilot.system.webrtc.schema import generate_field from openpilot.common.params import Params +from openpilot.common.swaglog import cloudlog from openpilot.cereal import messaging, log SESSION_TIMEOUT_SECONDS = 300 + +# ice candidate parser for logging +def _ice_candidates(sdp: str) -> list[str]: + return [line.removeprefix("a=") for line in sdp.splitlines() if line.startswith("a=candidate:")] + # socket trick: route lookup for 8.8.8.8 (nothing is sent or actually connected to) # return the source interfaces IP which is the default interface of the device def _default_route_ip() -> str | None: @@ -253,7 +259,7 @@ class StreamSession: self._cleanup_lock = asyncio.Lock() self._cleanup_done = False self.logger = logging.getLogger("webrtcd") - self.logger.info( + cloudlog.warning( "New stream session (%s), video cameras %s, video enabled %s, incoming services %s, outgoing services %s", self.identifier, [t.id for t in self.video_tracks], body.enabled, body.bridge_services_in, body.bridge_services_out, ) @@ -329,9 +335,12 @@ class StreamSession: async def run(self): try: self.params.put("LivestreamRequestKeyframe", True) + + # avoid datachannel race by adding messange_handler immediately + self.stream.set_message_handler(self.message_handler) + await asyncio.wait_for(self.stream.wait_for_connection(), timeout=15) if self.stream.has_messaging_channel(): - self.stream.set_message_handler(self.message_handler) if self.incoming_bridge is not None: await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services) if self.outgoing_bridge is not None: @@ -341,14 +350,18 @@ class StreamSession: if self.bitrate_controller is not None: self.bitrate_controller.start() - self.logger.info("Stream session (%s) connected", self.identifier) + with cloudlog.ctx(session_id=self.identifier): + cloudlog.warning("webrtcd.session.connected") if self.is_body: await self.run_body_session() else: await self.run_normal_session() - self.logger.info("Stream session (%s) ended", self.identifier) + with cloudlog.ctx(session_id=self.identifier): + cloudlog.warning("webrtcd.session.ended") except Exception: self.logger.exception("Stream session failure") + with cloudlog.ctx(session_id=self.identifier): + cloudlog.exception("webrtcd.session.exception") finally: await self.post_run_cleanup() @@ -395,7 +408,10 @@ def _text_response(text: str, status: int = 200) -> tuple[int, bytes, str]: return (status, text.encode(), "text/plain; charset=utf-8") -async def handle_get_stream(state: ServerState, raw_body: bytes) -> tuple[int, bytes, str]: +async def handle_get_stream(state: ServerState, raw_body: bytes, content_type: str) -> tuple[int, bytes, str]: + if content_type != "application/json": + return _json_response({"error": "unsupported media type"}, status=415) + stream_dict = state.streams body = StreamRequestBody(**json.loads(raw_body)) @@ -419,15 +435,25 @@ async def handle_get_stream(state: ServerState, raw_body: bytes) -> tuple[int, b stream_dict[session.identifier] = session try: answer = await asyncio.wait_for(session.get_answer(), timeout=30) + cloudlog.event( + "webrtcd.session.ice_candidates", + session_id=session.identifier, + offer_candidates=_ice_candidates(body.sdp), + answer_candidates=_ice_candidates(answer.sdp), + ) except TimeoutError: await session.stop() stream_dict.pop(session.identifier, None) logging.getLogger("webrtcd").exception("Timed out creating stream answer") + with cloudlog.ctx(session_id=session.identifier): + cloudlog.warning("webrtcd.session.answer_timeout") raise except Exception: await session.stop() stream_dict.pop(session.identifier, None) logging.getLogger("webrtcd").exception("Failed to create stream answer") + with cloudlog.ctx(session_id=session.identifier): + cloudlog.exception("webrtcd.session.answer_exception") raise session.start() @@ -508,7 +534,7 @@ class WebrtcdHandler(BaseHTTPRequestHandler): services = parse_qs(parsed.query).get("services", [""])[0] result = self._run(handle_get_schema(self.server.state, services)) elif parsed.path == "/stream": - result = self._run(handle_get_stream(self.server.state, self._read_body())) + result = self._run(handle_get_stream(self.server.state, self._read_body(), self.headers.get_content_type())) else: # /notify try: payload = json.loads(self._read_body()) @@ -611,7 +637,7 @@ def webrtcd_thread(host: str, port: int): def main(): parser = argparse.ArgumentParser(description="WebRTC daemon") - parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to listen on") + parser.add_argument("--host", type=str, default="127.0.0.1", help="Host to listen on") parser.add_argument("--port", type=int, default=5001, help="Port to listen on") args = parser.parse_args() diff --git a/pyproject.toml b/pyproject.toml index dcc1024479..6a13796d70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "tqdm", # cars (fw_versions.py) on start + many one-off uses # core - "scons", + "scons==4.10.1", # 4.11 removed the qt3 tool still used to build Cabana "pycapnp==2.1.0", # 2.2 introduces a memory leak due to cyclic references "numpy >=2.0", diff --git a/release/ci/install_github_runner.sh b/release/ci/install_github_runner.sh deleted file mode 100755 index 9f11e4841c..0000000000 --- a/release/ci/install_github_runner.sh +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Default values -DEFAULT_REPO_URL="https://github.com/sunnypilot" -START_AT_BOOT=false -RESTORE_MODE=false -RUNNER_VERSION="2.325.0" - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case $1 in - --start-at-boot) - START_AT_BOOT=true - shift - ;; - --token) - GITHUB_TOKEN="$2" - shift 2 - ;; - --repo) - REPO_URL="$2" - shift 2 - ;; - --restore) - RESTORE_MODE=true - shift - ;; - *) - if [ -z "$GITHUB_TOKEN" ]; then - GITHUB_TOKEN="$1" - elif [ -z "$REPO_URL" ]; then - REPO_URL="$1" - fi - shift - ;; - esac -done - -# Determine BASE_DIR based on mount point -if mountpoint -q /data/media; then - BASE_DIR="/data/media/0/github" -else - BASE_DIR="/data/github" -fi - -# Constants -RUNNER_USER="github-runner" -USER_GROUPS="comma,gpu,gpio,sudo" -RUNNER_DIR="${BASE_DIR}/runner" -BUILDS_DIR="${BASE_DIR}/builds" -LOGS_DIR="${BASE_DIR}/logs" -CACHE_DIR="${BASE_DIR}/cache" -OPENPILOT_DIR="${BASE_DIR}/openpilot" - -# Basic utility functions (no dependencies) -remount_rw() { - sudo mount -o remount,rw / -} - -remount_ro() { - sync || true # Try to sync but continue even if it fails - sudo mount -o remount,ro / # Always try to remount as read-only -} - -# Always ensure we try to remount as read-only on exit -trap remount_ro EXIT - -setup_runner_user() { - sudo useradd --comment 'GitHub Runner' --create-home --home-dir ${BASE_DIR} ${RUNNER_USER} --shell /bin/bash -G ${USER_GROUPS} || sudo usermod -aG ${USER_GROUPS} ${RUNNER_USER} -} - -create_sudoers_entry() { - sudo grep -qxF "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" /etc/sudoers || echo "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers -} - -set_directory_permissions() { - sudo chown -R ${RUNNER_USER}:comma "$BASE_DIR" - sudo chmod -R g+rwx "$BASE_DIR" - sudo find "$BASE_DIR" -type d -exec chmod g+s {} + -} - -setup_directories() { - echo "Creating necessary directories..." - sudo mkdir -p "$RUNNER_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR" - mkdir -p "/data/openpilot" - sudo chown -R comma:comma "/data/openpilot" - sync -} - -wipe_bash_logout() { - export BASE_DIR - sudo -u ${RUNNER_USER} bash -c "touch ${BASE_DIR}/.bash_logout" - sudo -u ${RUNNER_USER} bash -c "truncate -s 0 '${BASE_DIR}/.bash_logout'" -} - -# System configuration functions (depends on basic utility functions) -setup_system_configs() { - echo "Setting up system configurations..." - remount_rw - setup_runner_user - create_sudoers_entry - remount_ro - set_directory_permissions - wipe_bash_logout -} - -# Runner setup functions -install_runner() { - echo "Downloading and setting up runner..." - cd "$RUNNER_DIR" - curl -o actions-runner-linux-arm64-${RUNNER_VERSION}.tar.gz -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-arm64-${RUNNER_VERSION}.tar.gz - sudo -u ${RUNNER_USER} tar -xzf ./actions-runner-linux-arm64-${RUNNER_VERSION}.tar.gz - sudo rm ./actions-runner-linux-arm64-${RUNNER_VERSION}.tar.gz - sudo chmod +x ./config.sh -} - -configure_runner() { - remount_rw - echo "Configuring runner..." - cd "$RUNNER_DIR" - sudo -u ${RUNNER_USER} ./config.sh --url "$REPO_URL" --token "$GITHUB_TOKEN" --name $(hostname) --runnergroup "tici-tizi" --labels "tici" --work "$BUILDS_DIR" --unattended - remount_ro -} - -create_service_template() { - echo "Creating service template..." - cat < "$RUNNER_DIR/bin/actions.runner.service.template" -[Unit] -Description={{Description}} -After=network-online.target nss-lookup.target time-sync.target -Wants=network-online.target nss-lookup.target time-sync.target -StartLimitInterval=5 -StartLimitBurst=10 - -[Service] -Type=simple -User=root -ExecStart=/usr/bin/unshare -m -- /bin/bash -c 'mount --bind ${OPENPILOT_DIR} /data/openpilot && setpriv --reuid={{User}} --regid={{User}} --init-groups env HOME=${BASE_DIR} USER={{User}} LOGNAME={{User}} MAIL=/var/mail/{{User}} {{RunnerRoot}}/runsvc.sh' -WorkingDirectory={{RunnerRoot}} -KillMode=process -KillSignal=SIGTERM -TimeoutStopSec=5min -Restart=always -RestartSec=120 - -[Install] -WantedBy=multi-user.target -EOL -} - -install_service() { - local service_name - if [ -f "${RUNNER_DIR}/.service" ]; then - service_name=$(cat "${RUNNER_DIR}/.service") - else - service_name="actions.runner.sunnypilot.$(uname -n)" - fi - - create_service_template - remount_rw - local service_path="/etc/systemd/system/${service_name}" - echo "Installing systemd service..." - if [ -f "${service_path}" ]; then - echo "Service ${service_path} found in systemd, we will delete it" - sudo rm -f "${service_path}" - fi - - cd "$RUNNER_DIR" - sudo ./svc.sh install $RUNNER_USER - - if [ "$START_AT_BOOT" = false ]; then - sudo systemctl disable "${service_name}" - fi - remount_ro -} - -check_restore_prerequisites() { - local can_restore=false - local service_name="" - - # Check if base runner directory exists - if [ ! -d "${RUNNER_DIR}" ]; then - echo "ERROR: Runner directory ${RUNNER_DIR} does not exist" - echo "This directory is required for restore operations" - exit 1 - fi - - # First check if we have the required files for restoration - if [ -f "${RUNNER_DIR}/.credentials" ] && [ -f "${RUNNER_DIR}/.service" ]; then - can_restore=true - service_name=$(cat "${RUNNER_DIR}/.service") - echo "Found required runner configuration files" - else - echo "Missing required runner configuration files" - echo "Required: .credentials and .service files in ${RUNNER_DIR}" - exit 1 - fi - - if ! id "${RUNNER_USER}" &>/dev/null; then - echo "User ${RUNNER_USER} does not exist" - fi - - # Only proceed if we can restore AND need to restore - if [ "$can_restore" = true ]; then - echo "Restoration is possible" - return 0 - else - echo "No restoration possible" - exit 0 - fi -} - -perform_restore() { - echo "Starting runner restoration..." - setup_directories - setup_system_configs - install_service - echo "Runner restoration completed successfully" -} - -perform_install() { - echo "Starting fresh installation..." - setup_directories - setup_system_configs - install_runner - set_directory_permissions - configure_runner - install_service - echo "Installation completed successfully" -} - -main() { - if [ "$RESTORE_MODE" = true ]; then - echo "Running in restore mode - will only restore system configurations..." - check_restore_prerequisites - perform_restore - else - # Check required arguments for normal installation - if [ -z "$GITHUB_TOKEN" ]; then - echo "Usage: $0 [--start-at-boot] [--token ] [--repo ] [--restore]" - echo "Required argument (except for --restore): github_token" - echo "Optional arguments:" - echo " --start-at-boot Enable auto-start at boot (default: false)" - echo " --repo Repository URL (default: ${DEFAULT_REPO_URL})" - echo " --restore Restore existing runner configuration" - exit 1 - fi - - # Set repository URL if not provided - REPO_URL="${REPO_URL:-$DEFAULT_REPO_URL}" - perform_install - fi - - echo "Starting runner service..." - cd "$RUNNER_DIR" - sudo ./svc.sh start -} - -main diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py index 607260f145..2d35d319c2 100755 --- a/release/ci/model_generator.py +++ b/release/ci/model_generator.py @@ -53,24 +53,28 @@ def create_pkl_name(full_name: str) -> str: return pkl -def _read_pkl_bytes(pkl_path: Path) -> bytes: +def _hash_pkl(pkl_path: Path) -> str: manifest = Path(f"{pkl_path}.chunkmanifest") if manifest.exists(): num_chunks = int(manifest.read_text().strip()) - parts = [] - for i in range(num_chunks): - chunk = Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}") - parts.append(chunk.read_bytes()) - return b''.join(parts) - return pkl_path.read_bytes() + paths = [Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}") for i in range(num_chunks)] + else: + paths = [pkl_path] + + digest = hashlib.sha256() + for path in paths: + with path.open('rb') as f: + while block := f.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() def _find_driving_pkl(output_path: Path) -> Path | None: - for pattern in ('driving_tinygrad.pkl', 'driving_*_tinygrad.pkl'): + for pattern in ('*driving_tinygrad.pkl', '*driving_*_tinygrad.pkl'): matches = sorted(output_path.glob(pattern)) if matches: return matches[0] - for pattern in ('driving_tinygrad.pkl.chunkmanifest', 'driving_*_tinygrad.pkl.chunkmanifest'): + for pattern in ('*driving_tinygrad.pkl.chunkmanifest', '*driving_*_tinygrad.pkl.chunkmanifest'): matches = sorted(output_path.glob(pattern)) if matches: return Path(str(matches[0]).removesuffix('.chunkmanifest')) @@ -86,8 +90,20 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path: return old_pkl.rename(new_pkl) +def _hash_onnx_files(model_dir: Path) -> str | None: + onnx_files = sorted(model_dir.glob("*.onnx")) + if not onnx_files: + return None + digest = hashlib.sha256() + for f in onnx_files: + with f.open('rb') as fh: + while block := fh.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + def generate_chunked_model(driving_pkl: Path) -> dict: - tinygrad_hash = hashlib.sha256(_read_pkl_bytes(driving_pkl)).hexdigest() + tinygrad_hash = _hash_pkl(driving_pkl) chunks_config = [] manifest_file = Path(f"{driving_pkl}.chunkmanifest") @@ -119,7 +135,8 @@ def generate_chunked_model(driving_pkl: Path) -> dict: } -def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown") -> None: +def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown", + onnx_sha256=None, is_big=False) -> None: bundle_json = { "short_name": short_name, "display_name": custom_name or upstream_branch, @@ -132,9 +149,13 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short "generation": "-1", "build_time": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "overrides": {}, + "is_big": is_big, "models": models, } + if onnx_sha256: + bundle_json["onnx_sha256"] = onnx_sha256 + # Write metadata to output_dir metadata_json = { "bundles": [bundle_json] @@ -166,6 +187,8 @@ if __name__ == "__main__": print(f"No driving_tinygrad.pkl found in {_output_dir}", file=sys.stderr) sys.exit(1) + is_big = _driving_pkl.name.startswith('big_') + if _pkl: new_pkl = _output_dir / f"driving_{_pkl}_tinygrad.pkl" if not new_pkl.exists(): @@ -174,4 +197,6 @@ if __name__ == "__main__": _driving_pkl = new_pkl _model_metadata = generate_chunked_model(_driving_pkl) - create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch) + _onnx_sha256 = _hash_onnx_files(Path(args.model_dir)) + create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch, + onnx_sha256=_onnx_sha256, is_big=is_big) diff --git a/release/ci/publish.sh b/release/ci/publish.sh index 27904caf5d..4b328a035c 100755 --- a/release/ci/publish.sh +++ b/release/ci/publish.sh @@ -47,11 +47,17 @@ git rm -rf $OUTPUT_DIR/.git || true # Doing cleanup, but it might fail if the .g git remote remove origin || true # ensure cleanup git remote add origin $GIT_ORIGIN #git push origin -d $DEV_BRANCH || true # Ensuring we delete the remote branch if it exists as we are wiping it out -git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH) +git fetch --depth 1 origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH) echo "[-] committing version $VERSION T=$SECONDS" git add -f . +# gitlinks break the release tree on device +if git ls-files -s | awk '$1 == "160000" { found = 1; print } END { exit !found }'; then + echo "Error: submodules found in release tree." + exit 1 +fi + # include source commit hash and build date in commit GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD) DATETIME=$(date '+%Y-%m-%dT%H:%M:%S') diff --git a/release/ci/uninstall_github_runner.sh b/release/ci/uninstall_github_runner.sh deleted file mode 100755 index 5f3acfbafd..0000000000 --- a/release/ci/uninstall_github_runner.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# Determine BASE_DIR based on mount point -if mountpoint -q /data/media; then - GITHUB_BASE_DIR="/data/media/0/github" -else - GITHUB_BASE_DIR="/data/github" -fi - -# Define directories and user -BIN_DIR="$GITHUB_BASE_DIR/bin" -BUILDS_DIR="$GITHUB_BASE_DIR/builds" -OPENPILOT_DIR="$GITHUB_BASE_DIR/openpilot" -LOGS_DIR="$GITHUB_BASE_DIR/logs" -CACHE_DIR="$GITHUB_BASE_DIR/cache" -RUNNER_USERNAME="github-runner" -# Define the systemd service name -SERVICE_NAME="github-runner" -USER_GROUPS="comma,gpu,gpio,sudo" - -# Function to stop and disable the systemd service -stop_and_uninstall_service() { - cd $GITHUB_BASE_DIR/runner - sudo ./svc.sh stop - sudo ./svc.sh uninstall -} - -# Function to remove the systemd service file -remove_runner() { - cd $GITHUB_BASE_DIR/runner - sudo rm .runner - sudo su -c './config.sh remove' github-runner -} - -# Function to delete the Github Runner directories -delete_directories() { - sudo rm -rf "$BIN_DIR/github-runner" - sudo rm -rf "$GITHUB_BASE_DIR" "$BIN_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR" -} - -# Function to remove the Github Runner user -delete_user() { - for group in ${USER_GROUPS//,/ } - do - sudo gpasswd -d ${RUNNER_USERNAME} ${group} - done - sudo userdel -r ${RUNNER_USERNAME} -} - -# Function to remove sudoers entry -remove_sudoers_entry() { - sudo sed -i.bak "/${RUNNER_USERNAME} ALL=(ALL) NOPASSWD: ALL/d" /etc/sudoers -} - -# Make filesystem writable -sudo mount -o remount rw / - -# Ensure filesystem is remounted as read-only on script exit -trap "sudo mount -o remount ro /" EXIT - -# Call functions -stop_and_uninstall_service -remove_runner -delete_directories -delete_user -remove_sudoers_entry -# End of uninstall script diff --git a/release/ci/upload_default_model.py b/release/ci/upload_default_model.py new file mode 100644 index 0000000000..eac48e3be4 --- /dev/null +++ b/release/ci/upload_default_model.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + +This file is part of sunnypilot and is licensed under the MIT License. +See the LICENSE.md file in the root directory for more details. +""" + +import argparse +import hashlib +import json +import tempfile + +from huggingface_hub import HfApi, hf_hub_download + + +def hash_file(path: str) -> str: + digest = hashlib.sha256() + with open(path, 'rb') as f: + while block := f.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--hf-repo", required=True) + parser.add_argument("--hf-defaults-path", required=True) + parser.add_argument("--artifact-name", required=True) + parser.add_argument("--model-dir", required=True) + parser.add_argument("--onnx-path", required=True) + parser.add_argument("--onnx-ref", required=True) + parser.add_argument("--model-name", required=True) + parser.add_argument("--tinygrad-ref", required=True) + parser.add_argument("--run-number", required=True) + args = parser.parse_args() + + api = HfApi() + onnx_sha256 = hash_file(args.onnx_path) + short_ref = args.onnx_ref[:8] + folder_name = f"model-{args.model_name}-{short_ref}-{args.run_number}" + + print(f"ONNX hash: {onnx_sha256}") + print(f"ONNX ref: {args.onnx_ref} (short: {short_ref})") + print(f"Folder: {folder_name}") + + metadata_path = f"{args.model_dir}/metadata.json" + with open(metadata_path) as f: + metadata = json.load(f) + + bundle = metadata['bundles'][0] + bundle['display_name'] = args.model_name + bundle['onnx_sha256'] = onnx_sha256 + bundle['onnx_ref'] = args.onnx_ref + + artifact = bundle['models'][0]['artifact'] + hf_base = f"https://huggingface.co/datasets/{args.hf_repo}/resolve/main/{args.hf_defaults_path}/{folder_name}" + artifact['download_uri']['url'] = f"{hf_base}/{artifact['file_name']}" + for chunk in artifact.get('chunks', []): + chunk['url'] = f"{hf_base}/{chunk['file_name']}" + + print(f"Uploading model to {args.hf_defaults_path}/{folder_name}/") + api.upload_folder( + folder_path=args.model_dir, + path_in_repo=f"{args.hf_defaults_path}/{folder_name}", + repo_id=args.hf_repo, + repo_type="dataset", + ) + + json_filename = f"{args.hf_defaults_path}/default_models.json" + try: + local_path = hf_hub_download(repo_id=args.hf_repo, repo_type='dataset', filename=json_filename) + with open(local_path) as f: + defaults_json = json.load(f) + except Exception: + defaults_json = {"tinygrad_ref": args.tinygrad_ref, "bundles": []} + + defaults_json['tinygrad_ref'] = args.tinygrad_ref + + existing_idx = next((i for i, b in enumerate(defaults_json['bundles']) + if b.get('onnx_sha256') == onnx_sha256), None) + if existing_idx is not None: + defaults_json['bundles'][existing_idx] = bundle + else: + defaults_json['bundles'].append(bundle) + + print(json.dumps(defaults_json, indent=2)) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(defaults_json, f, indent=2) + tmp_path = f.name + + api.upload_file( + path_or_fileobj=tmp_path, + path_in_repo=json_filename, + repo_id=args.hf_repo, + repo_type="dataset", + ) + + print(f"Updated {json_filename}") + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index 8f3d4ca96c..b12a3283a1 100644 --- a/uv.lock +++ b/uv.lock @@ -381,20 +381,20 @@ wheels = [ [[package]] name = "deepmerge" -version = "2.1.0" +version = "3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2a/78/6e9e20106224083cfb817d2d3c26e80e72258d617b616721a169b87081e0/deepmerge-2.1.0.tar.gz", hash = "sha256:07ca7a7b8935df596c512fa8161877c0487ac61f691c07766e7d71d2b23bdd2f", size = 21449, upload-time = "2026-06-22T05:46:07.669Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/6c/9f4577a36d5f463a3a3f8322bd65d33e1a1a6b6ba1d692a5ebc3cba19015/deepmerge-3.0.tar.gz", hash = "sha256:14ed69f063de64b7743985c732ccff5d6c34ff4560946e7fbfd99086b853b9ce", size = 22279, upload-time = "2026-08-17T05:50:53.161Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/51/25/2a75b47cb057b1e164c604fb81ab690a6cdb5e2260ce651194eae90f64a3/deepmerge-2.1.0-py3-none-any.whl", hash = "sha256:8f148339a91d680a75ecb74ade235d9e759a93df373a0b04e9d31c8666cfeb75", size = 14345, upload-time = "2026-06-22T05:46:06.742Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d7/7f19bedd30b90b72865aeec3a29127bed6dee6c9ef0324bb5b4d424bb0e3/deepmerge-3.0-py3-none-any.whl", hash = "sha256:c8541c3e186dc88d19a5513ad3a0b2d0b22beaa780969fc0c13b995a64265365", size = 14855, upload-time = "2026-08-17T05:50:52.218Z" }, ] [[package]] name = "filelock" -version = "3.32.2" +version = "3.32.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/64/a02e6765de08964ed371eca577870593245afc9dfac16d037de7c10d18e6/filelock-3.32.3.tar.gz", hash = "sha256:0ffa185a3540854c95caa7fa76b76cb219d907415e2c5dc9af25fd970563487f", size = 218135, upload-time = "2026-08-13T16:00:05.577Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z" }, + { url = "https://files.pythonhosted.org/packages/a7/8e/50f46a9c0ce8d2861a394c1347caae037ea0431d2f67d7feb151cbc4649a/filelock-3.32.3-py3-none-any.whl", hash = "sha256:7f0ca4bcc0e181c60dbbd8aa9ab5b120ebb99e4e064e83636340056f833a1f09", size = 98901, upload-time = "2026-08-13T16:00:03.974Z" }, ] [[package]] @@ -478,7 +478,7 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "1.27.0" +version = "1.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -491,18 +491,18 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3e/9b/ddf3d02a8681f1b9ce52fda03d755dad6b74c4f8172304c4c8d2975450f9/huggingface_hub-1.27.0.tar.gz", hash = "sha256:c1fed40ea82a6b41b477f5243546549b792ae0a93abcea608cff66089bf8f8df", size = 942668, upload-time = "2026-08-07T12:48:05.161Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/ae/222a91937ebee7f62c0ca8f5ee0afd97577caf24c0abb927d1f5c7e9f6d2/huggingface_hub-1.28.0.tar.gz", hash = "sha256:46a2e950c09234de54093d587d1675382f0d08dbd600d9fb599b5932f5b2c6cb", size = 959609, upload-time = "2026-08-18T12:27:15.101Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/d8/95b735e183957c1f26d94c52977f09d466d55119cbbc1558ea4975e4c216/huggingface_hub-1.27.0-py3-none-any.whl", hash = "sha256:7df6827c2f956c60fbaa64646e979e566db76f619dd0a9729dfb8c5a3eb4f68d", size = 784926, upload-time = "2026-08-07T12:48:02.905Z" }, + { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, ] [[package]] name = "idna" -version = "3.18" +version = "3.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, + { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, ] [[package]] @@ -847,7 +847,7 @@ requires-dist = [ { name = "rednose", marker = "extra == 'submodules'", editable = "rednose_repo" }, { name = "requests" }, { name = "ruff", marker = "extra == 'testing'" }, - { name = "scons" }, + { name = "scons", specifier = "==4.10.1" }, { name = "sentry-sdk" }, { name = "setproctitle" }, { name = "sounddevice" }, @@ -971,11 +971,11 @@ wheels = [ [[package]] name = "pygments" -version = "2.20.0" +version = "2.21.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] [[package]] @@ -1136,11 +1136,11 @@ wheels = [ [[package]] name = "scons" -version = "4.11.0" +version = "4.10.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dd/82/3c4e089ac8df2eaee8a7f14e489b2a76f94f4c1d8defa4e46c8ad15cae86/scons-4.11.0.tar.gz", hash = "sha256:5ba48f9e2eb6b9178cabdc9893792418e6970c84f43f4b027e4468e20616a89c", size = 3269126, upload-time = "2026-08-11T04:29:45.62Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/c9/2f430bb39e4eccba32ce8008df4a3206df651276422204e177a09e12b30b/scons-4.10.1.tar.gz", hash = "sha256:99c0e94a42a2c1182fa6859b0be697953db07ba936ecc9817ae0d218ced20b15", size = 3258403, upload-time = "2025-11-16T22:43:39.258Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/ac/a4445bbbd58a5fa6a5c8b3b0458ffbee04e4acaff87677058eab9c6af682/scons-4.11.0-py3-none-any.whl", hash = "sha256:2edc077aaeafc43377ba46ce1fa3e7b40edea59c62db9ef7e39e07dc88b754fa", size = 4123742, upload-time = "2026-08-11T04:29:42.881Z" }, + { url = "https://files.pythonhosted.org/packages/ce/bf/931fb9fbb87234c32b8b1b1c15fba23472a10777c12043336675633809a7/scons-4.10.1-py3-none-any.whl", hash = "sha256:bd9d1c52f908d874eba92a8c0c0a8dcf2ed9f3b88ab956d0fce1da479c4e7126", size = 4136069, upload-time = "2025-11-16T22:43:35.933Z" }, ] [[package]] @@ -1194,18 +1194,18 @@ wheels = [ [[package]] name = "sounddevice" -version = "0.5.5" +version = "0.5.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2a/f9/2592608737553638fca98e21e54bfec40bf577bb98a61b2770c912aab25e/sounddevice-0.5.5.tar.gz", hash = "sha256:22487b65198cb5bf2208755105b524f78ad173e5ab6b445bdab1c989f6698df3", size = 143191, upload-time = "2026-01-23T18:36:43.529Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/db/0c890e2d9aab9ba284021efc02e1d3aebfecab1b611762d7434602209bcf/sounddevice-0.5.6.tar.gz", hash = "sha256:8ec9fbfde2e32f020b167e348f3ab3bac6625a5f15af524d790108ac7147a410", size = 1120094, upload-time = "2026-08-17T07:55:05.048Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/0a/478e441fd049002cf308520c0d62dd8333e7c6cc8d997f0dda07b9fbcc46/sounddevice-0.5.5-py3-none-any.whl", hash = "sha256:30ff99f6c107f49d25ad16a45cacd8d91c25a1bcdd3e81a206b921a3a6405b1f", size = 32807, upload-time = "2026-01-23T18:36:35.649Z" }, - { url = "https://files.pythonhosted.org/packages/56/f9/c037c35f6d0b6bc3bc7bfb314f1d6f1f9a341328ef47cd63fc4f850a7b27/sounddevice-0.5.5-py3-none-macosx_10_6_x86_64.macosx_10_6_universal2.whl", hash = "sha256:05eb9fd6c54c38d67741441c19164c0dae8ce80453af2d8c4ad2e7823d15b722", size = 108557, upload-time = "2026-01-23T18:36:37.41Z" }, - { url = "https://files.pythonhosted.org/packages/88/a1/d19dd9889cd4bce2e233c4fac007cd8daaf5b9fe6e6a5d432cf17be0b807/sounddevice-0.5.5-py3-none-win32.whl", hash = "sha256:1234cc9b4c9df97b6cbe748146ae0ec64dd7d6e44739e8e42eaa5b595313a103", size = 317765, upload-time = "2026-01-23T18:36:39.047Z" }, - { url = "https://files.pythonhosted.org/packages/c3/0e/002ed7c4c1c2ab69031f78989d3b789fee3a7fba9e586eb2b81688bf4961/sounddevice-0.5.5-py3-none-win_amd64.whl", hash = "sha256:cfc6b2c49fb7f555591c78cb8ecf48d6a637fd5b6e1db5fec6ed9365d64b3519", size = 365324, upload-time = "2026-01-23T18:36:40.496Z" }, - { url = "https://files.pythonhosted.org/packages/4e/39/a61d4b83a7746b70d23d9173be688c0c6bfc7173772344b7442c2c155497/sounddevice-0.5.5-py3-none-win_arm64.whl", hash = "sha256:3861901ddd8230d2e0e8ae62ac320cdd4c688d81df89da036dcb812f757bb3e6", size = 317115, upload-time = "2026-01-23T18:36:42.235Z" }, + { url = "https://files.pythonhosted.org/packages/72/1f/62eef605172bddc1017508469a12f75bc7c4194ece35c734f822795f53b1/sounddevice-0.5.6-py3-none-any.whl", hash = "sha256:de099612311ad81e55d31ccbd83f43ea6bf4d87b48f9b6ea55a1fbcde0eee4e0", size = 32793, upload-time = "2026-08-17T07:54:57.507Z" }, + { url = "https://files.pythonhosted.org/packages/b6/84/85e719d49cf98b2f406d9ac9c338892286c4448eb42ef0b2625ccf159616/sounddevice-0.5.6-py3-none-macosx_10_6_x86_64.macosx_10_6_universal2.whl", hash = "sha256:e3aef00ad8b1d1740eb66d9a7671eab88a4d2b8fa4ab33498d742e63b65c309c", size = 1009647, upload-time = "2026-08-17T07:54:58.814Z" }, + { url = "https://files.pythonhosted.org/packages/c5/6f/6292145099f72a153a710245f46ae43e5fb6c77bec1b6086cb76c12dc280/sounddevice-0.5.6-py3-none-win32.whl", hash = "sha256:b36b807eb02abd257198bf84b2af05e4fea199a9d2f0019014169c7136d45e9c", size = 1009627, upload-time = "2026-08-17T07:55:00.401Z" }, + { url = "https://files.pythonhosted.org/packages/8d/3e/cbc593c31a5f0d817b3fe97e64aa8461bd0f55cb07b67ce1b776296ae336/sounddevice-0.5.6-py3-none-win_amd64.whl", hash = "sha256:7f4162f514f007b0bf25a3ccfed3f1705bc2ec311888a90232729eec4f57a4f4", size = 1009630, upload-time = "2026-08-17T07:55:02.088Z" }, + { url = "https://files.pythonhosted.org/packages/60/a4/b0c21c9f215a6fd9606b8f8748c21212dc098e5d5a2d93068c50edcf19b4/sounddevice-0.5.6-py3-none-win_arm64.whl", hash = "sha256:c8ae19173e5f27f8c12d4b5eee2dbfe542cee125d591e663e0fb4dfb75246d45", size = 1009630, upload-time = "2026-08-17T07:55:03.689Z" }, ] [[package]] @@ -1285,7 +1285,6 @@ requires-dist = [ { name = "pylint", marker = "extra == 'linting'" }, { name = "pytest", marker = "extra == 'testing-minimal'" }, { name = "pytest-split", marker = "extra == 'testing-minimal'" }, - { name = "pytest-timeout", marker = "extra == 'testing-minimal'" }, { name = "pytest-xdist", marker = "extra == 'testing-minimal'" }, { name = "ruff", marker = "extra == 'linting'", specifier = "==0.14.10" }, { name = "safetensors", marker = "extra == 'testing-unit'" }, @@ -1294,6 +1293,7 @@ requires-dist = [ { name = "tiktoken", marker = "extra == 'testing'" }, { name = "tinygrad", extras = ["testing-minimal"], marker = "extra == 'testing-unit'" }, { name = "tinygrad", extras = ["testing-unit"], marker = "extra == 'testing'" }, + { name = "tinymesa", marker = "extra == 'mesa'", specifier = "==25.2.7.2" }, { name = "torch", marker = "extra == 'testing-minimal'", specifier = "==2.9.1" }, { name = "tqdm", marker = "extra == 'testing-unit'" }, { name = "transformers", marker = "extra == 'testing'" }, @@ -1301,7 +1301,7 @@ requires-dist = [ { name = "typing-extensions", marker = "extra == 'linting'" }, { name = "z3-solver", marker = "extra == 'testing-minimal'", specifier = "<4.15.4" }, ] -provides-extras = ["linting", "testing-minimal", "testing-unit", "testing", "docs"] +provides-extras = ["linting", "testing-minimal", "testing-unit", "testing", "docs", "mesa"] [[package]] name = "tomli" @@ -1335,27 +1335,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.72" +version = "0.0.73" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d5/df/656e684bafb13c1d146e7d5b5f3e7978ca177232acc84998ff36427e9462/ty-0.0.72.tar.gz", hash = "sha256:ec2b8066b618df18cab4cb8e992f8da45d360332acb23fa34df7fa29cd1b9d3a", size = 6654939, upload-time = "2026-08-14T21:35:42.612Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/90/c4e1bb4cead3b644c3e258a27f9b05c7dc5eb0ec96a4f5282194edae9e0d/ty-0.0.73.tar.gz", hash = "sha256:823d4ce0d237bfc7eb6bcee70842f2c0706113813a16951077840743712f4b74", size = 6712739, upload-time = "2026-08-19T03:12:43.381Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/3b/f51461239a4e66565d4b362f97a3b55fe7fdba2e944068341f87c62f6743/ty-0.0.72-py3-none-linux_armv6l.whl", hash = "sha256:fda86db153ffd85ee52000cf175d6a3f1c0223772cf7c5b6f726200bf92c7b44", size = 12621989, upload-time = "2026-08-14T21:35:01.676Z" }, - { url = "https://files.pythonhosted.org/packages/ca/fb/79ddf683affc679ca856f3510b5640ec3a88a842ba5f654f5d4bc78f1786/ty-0.0.72-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ceb944c612529b9023acfdc9cf4c0dcbb722549f9d17d46baecd1141baf01d7f", size = 12233910, upload-time = "2026-08-14T21:35:04.334Z" }, - { url = "https://files.pythonhosted.org/packages/5d/45/10562a0d84802158db8fa4ec46de54aa9fdcecdeeaabbfe3639ae7042b66/ty-0.0.72-py3-none-macosx_11_0_arm64.whl", hash = "sha256:108d76218333d6c092e5f1cebf8e9b06f25738613a0236a28e2dd47c936ee52c", size = 12084108, upload-time = "2026-08-14T21:35:06.686Z" }, - { url = "https://files.pythonhosted.org/packages/a1/dc/1fe1aef8d697e3509face271a5331700c7aa1d1e44a4b622707bdfa41d4b/ty-0.0.72-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f3943f186f741a2499a31053872169250c9264a9a49684920e48d8fcf4ef4f5", size = 12132640, upload-time = "2026-08-14T21:35:09.305Z" }, - { url = "https://files.pythonhosted.org/packages/14/46/41ceb265e96969487311a2014bd0e53abb4fbc1395efb2ebe411fcb4db62/ty-0.0.72-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf283c07dc3cc52ca48a3ad8ab100fb5aec3aebbd03ef6a12d5f910b8e596fc5", size = 12402489, upload-time = "2026-08-14T21:35:11.555Z" }, - { url = "https://files.pythonhosted.org/packages/2b/45/30bf43cb4fd505c5c2dd30fda27dde5f05208686cd21217adec77c954204/ty-0.0.72-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95f3b6462c38f9f115d10cee21f47fedf715fcf2040daf36eef210359300bc7c", size = 13130835, upload-time = "2026-08-14T21:35:13.746Z" }, - { url = "https://files.pythonhosted.org/packages/31/2f/03bba754d2613f640df168335c41f83f41db150bb515839c60d80e3a7880/ty-0.0.72-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30caf658feb8ffb250d9e9e47107657a78f5f3425c227df1664d8df2ebe38880", size = 13590392, upload-time = "2026-08-14T21:35:16.839Z" }, - { url = "https://files.pythonhosted.org/packages/04/c7/03c67f00e63005ec41585653dc3096064570b1e6273742baae2798cd242f/ty-0.0.72-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27bdc012ddfbeec8948e4a6036c0dc39ac7cf2c8ec7c7d48dc7d2fd56d57b399", size = 13309629, upload-time = "2026-08-14T21:35:19.169Z" }, - { url = "https://files.pythonhosted.org/packages/c1/df/102d3b264eb7f2a58dd11952f229bb5150bb5668d176a6154976a6675981/ty-0.0.72-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:802c5970a77d7739e6f499921fbb6984fb7ad8a31d95e1ff42fd46f3642e4f3b", size = 12734028, upload-time = "2026-08-14T21:35:22.099Z" }, - { url = "https://files.pythonhosted.org/packages/61/85/d0737c8c54d0ba67366ddfb9f31d88edf0b02299e65923e6945ae60ebcb5/ty-0.0.72-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:47dce65114fdc615c68ca0edb393b433df0956447e4267df0e264137a789598d", size = 13174832, upload-time = "2026-08-14T21:35:24.71Z" }, - { url = "https://files.pythonhosted.org/packages/1e/31/497f5a96c36d9b586ab6afe0574986835c6fd5b835a89773d2bec4711b49/ty-0.0.72-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:325144fa07e2675d0faa337fcc864213c272a499eb0cfe5bde2fdc62282d27bc", size = 12215005, upload-time = "2026-08-14T21:35:26.892Z" }, - { url = "https://files.pythonhosted.org/packages/df/7d/46e65b17b4966c7cd0140f134380d33d8e84fe6efccd761533ce793dc502/ty-0.0.72-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a5c9f15d0f58e43707d8848274be1821a0ef408eccb8aa7dda28a4a9eddf7640", size = 12421298, upload-time = "2026-08-14T21:35:29.301Z" }, - { url = "https://files.pythonhosted.org/packages/08/2a/12ada4ec17700b3cb1d4fd3bc3e5b1852df9e6885288429318cade87b3c1/ty-0.0.72-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee508d64b381871529cc22c412b41071bf5e908b7aa5d66a38f3f6b2573a806", size = 12669242, upload-time = "2026-08-14T21:35:31.444Z" }, - { url = "https://files.pythonhosted.org/packages/1c/1a/4692536880790fb550ed6d44a6096778dc71bb112f2c6d615cebb01a57e5/ty-0.0.72-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3699e2ec7921d44da79d6b089f7bf239b2cc53c4e45a5a38430adc34ee9e9a55", size = 12988199, upload-time = "2026-08-14T21:35:33.749Z" }, - { url = "https://files.pythonhosted.org/packages/9a/0d/f5e5a50322e9c45865e7b7a428ba6cd6527387cf0f2472492ac3cf746243/ty-0.0.72-py3-none-win32.whl", hash = "sha256:f25f72a67bd36cd247707c4784e52fad0b6b4f42a1b7dd14804110fa95c486ed", size = 11939708, upload-time = "2026-08-14T21:35:36.006Z" }, - { url = "https://files.pythonhosted.org/packages/3f/4e/8af3534b2e4214e6184a5a59c34101e94a68d578f081f97b995866bab1bf/ty-0.0.72-py3-none-win_amd64.whl", hash = "sha256:cdeee869341717e1736cea2e2d7856738c6957c320f584ed2f68c8f90100d2f5", size = 12643876, upload-time = "2026-08-14T21:35:38.141Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ea/a2606e654c7276bd08586391a2525b0af3f3bf60228a8c57b2d248f273f9/ty-0.0.72-py3-none-win_arm64.whl", hash = "sha256:1bd3ac3ed4424a6d6990a85dc388556aea012bd752de21349a84b685951de0d8", size = 12394857, upload-time = "2026-08-14T21:35:40.277Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0f/f5e1801e55cc631f2db193276675b30561b963a2403da832bffb5d100267/ty-0.0.73-py3-none-linux_armv6l.whl", hash = "sha256:90a946082bf9bc446b5e72973d9f4ff1222a240b2ca4c9e6eed61eb913e30810", size = 12715452, upload-time = "2026-08-19T03:12:06.673Z" }, + { url = "https://files.pythonhosted.org/packages/54/32/515dd05074c213b433524ab97eb003b0132ae7e358e0d75633ba7a314ed8/ty-0.0.73-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b7d6b5c6a6db7ea95fbbc16af514ef44a27a29a2fe1dc798900790364d170209", size = 12301870, upload-time = "2026-08-19T03:12:08.924Z" }, + { url = "https://files.pythonhosted.org/packages/50/4d/085b4889f0d4bbe4af8b96242d4a1cb209fff95967cfa239ea141983719b/ty-0.0.73-py3-none-macosx_11_0_arm64.whl", hash = "sha256:dd6f657f463e01372d8688f235be164750c8db722c97da27fa4903aa8d40b203", size = 12111741, upload-time = "2026-08-19T03:12:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/95/f6/d6ec277cadfecf03ad4c18551b67c4c6eb7807a0560d801db14be99d7a89/ty-0.0.73-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2de468e33fd44c9ff1c43473a7316f4289480f5cba8995a67b6d22aee39ca9", size = 12196124, upload-time = "2026-08-19T03:12:13.14Z" }, + { url = "https://files.pythonhosted.org/packages/75/b7/ce78d8707563af9cae9bbd25328bfbc4931035085bd20089adf0c418f70e/ty-0.0.73-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2942fa0ef795a66034cdc8d75a72f453442f3b58ff2f69b4da05b7b954765b55", size = 12488557, upload-time = "2026-08-19T03:12:15.252Z" }, + { url = "https://files.pythonhosted.org/packages/d8/e8/329b9851b23502758c5c98e8cc875ea2a1b4c9674b4ca3a86da56a5063d3/ty-0.0.73-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e0f1ef14f642e18ac4e7a616a2796dcf7a5d82e28cd17f9796494acc7c4aabb", size = 13215606, upload-time = "2026-08-19T03:12:17.225Z" }, + { url = "https://files.pythonhosted.org/packages/36/38/67fedfd2cb77516ef0066b1642f487dba0eb3006493cf3475b15f5b8b228/ty-0.0.73-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16981e15fdceedb37d0aff76c5ac25914595dfee2675af95335550064251ad22", size = 13665497, upload-time = "2026-08-19T03:12:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/8e/b3/154f4dd48ec5eebc186ab4b822c6e62f982fc5ddfd262d6e3903c2acba44/ty-0.0.73-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644b2bec8a2e2e4957a942ae81d6cff5571c489bb5a8675e4d3886de537a694d", size = 13351231, upload-time = "2026-08-19T03:12:21.353Z" }, + { url = "https://files.pythonhosted.org/packages/35/5f/d462496903fbe453fb76363f8478be929c8e6ff21e6928c57dcd7e5fa21f/ty-0.0.73-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338d565be3186f50ff8e9d10483685549c2d23f0754485d5ede3b54f4319188a", size = 12782586, upload-time = "2026-08-19T03:12:23.667Z" }, + { url = "https://files.pythonhosted.org/packages/87/52/ec6d24b74abe3ec324204c1c71e6d0c6c76a17ffc15fd51d603b0a302abe/ty-0.0.73-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:11c7b6d839309d2c102cb3a4c03d817176bbfab5b2fccc95a75ec5c9597421c9", size = 13247134, upload-time = "2026-08-19T03:12:25.956Z" }, + { url = "https://files.pythonhosted.org/packages/26/20/cc74650fec56a54786c6d7c89e09576fcad3092be34cf21715d39a406a9b/ty-0.0.73-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:488572db7ff97fb50ea36a76250f2d617c9727d143da6c7bf0623276eb0fc507", size = 12309344, upload-time = "2026-08-19T03:12:28.122Z" }, + { url = "https://files.pythonhosted.org/packages/89/bd/4b0a9087f4315d7fbadf77a3ce44c816cc9ffabed1ced06cc5be81fbc414/ty-0.0.73-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1b958ebceefbbf594e59eb8d3d55bbd033ce634026fcba3e4bc3179e78e45bb7", size = 12502319, upload-time = "2026-08-19T03:12:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/11/80/0a925074911fe111912ea29d9eed309bcc183f43d2fb3eef07db056a0beb/ty-0.0.73-py3-none-musllinux_1_2_i686.whl", hash = "sha256:91a32993b3c34e42c3f323ad6c0399cb596bd1c27e9b7f20db7cd64c1067b68e", size = 12753688, upload-time = "2026-08-19T03:12:32.433Z" }, + { url = "https://files.pythonhosted.org/packages/24/6b/aeccaf89efbc2e112bd415340a22e2669ec998aa397242503e747b712ca4/ty-0.0.73-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bab8a19fbf51f479bddb2a12c5fabfe52f918a5590362321ed5d89b44eb62c15", size = 13069050, upload-time = "2026-08-19T03:12:35.398Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3e/eae485fd86c1585943fd4e1746b0757b2da01e2c43136ebe8c686fe1c7f1/ty-0.0.73-py3-none-win32.whl", hash = "sha256:03347a612f0fa020b19bfd8dbd521db6ecc75d377a3e4d4f6e6c2e62871da4cc", size = 12053187, upload-time = "2026-08-19T03:12:37.565Z" }, + { url = "https://files.pythonhosted.org/packages/a7/01/9b8b983786e3ce34924e372e8b76b92b508273ab65c589fc7e88cc03ee17/ty-0.0.73-py3-none-win_amd64.whl", hash = "sha256:cedd05122ded0b5dcc55431a370e974b747f99c41c290a3d2ab8c1867f197519", size = 12693838, upload-time = "2026-08-19T03:12:39.483Z" }, + { url = "https://files.pythonhosted.org/packages/ea/88/25333bbfea6a5dc064371d2002d3d4807db90b84d5448f9106b2712b0fbc/ty-0.0.73-py3-none-win_arm64.whl", hash = "sha256:e47068f8369dea5d641a26a2ad0a947a320b02ff87099b07e95de0323245a4dc", size = 12443573, upload-time = "2026-08-19T03:12:41.449Z" }, ] [[package]] @@ -1387,7 +1387,7 @@ wheels = [ [[package]] name = "zensical" -version = "0.0.54" +version = "0.0.56" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -1399,20 +1399,20 @@ dependencies = [ { name = "pyyaml" }, { name = "tomli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/7e/343a78c0c9da1954d2f0a4d47ca778baac48bb18c6b9c0c6260e7974976e/zensical-0.0.54.tar.gz", hash = "sha256:4de205dbb323d0a443e2ebf3fef77e93e3c1493c34a58d205e7f3631dd7745af", size = 3992024, upload-time = "2026-08-13T16:04:49.297Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/7d/18bb725a659352e9af0940a3d879c5edcff5d86fec3ac15ce500d484d9d3/zensical-0.0.56.tar.gz", hash = "sha256:c359163800d1c3a8c39af48f4e2869fcfc2b4fc00d28652bd2a5b0330c36530c", size = 3997416, upload-time = "2026-08-18T15:46:47.283Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/cf/13e887c303fd5c786c09f83362382a42d10292fca633a95067de6a6591a3/zensical-0.0.54-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f7177a3b6647e4ee47864ab02e5141f9e923dd57b9fa96e5dc5da4228daff505", size = 12893082, upload-time = "2026-08-13T16:04:17.337Z" }, - { url = "https://files.pythonhosted.org/packages/82/ee/7fe1418fa31bc120cf9eb0fbce9e021c40752206ce993a7bc652c011f890/zensical-0.0.54-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7b23c3b0c720885891b220c3e562074d93eb940314823d91875c6246976bd9b2", size = 12778626, upload-time = "2026-08-13T16:04:19.906Z" }, - { url = "https://files.pythonhosted.org/packages/61/b8/0420115270c1a22a2d4a1598f89dadc4e933eb7aa85539b72f3532acc5b6/zensical-0.0.54-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c012eb0ec20fda5794e90b4906b7401c77b712e2acc7f9f65026935368539da", size = 13225462, upload-time = "2026-08-13T16:04:22.663Z" }, - { url = "https://files.pythonhosted.org/packages/8d/48/0dfdd3e00fb3b807de702383ef5f4e9cf0d2791fee0e8a56f39bd590f16b/zensical-0.0.54-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f0e851d26ba4f7397db3b3532e534c0572a9826bbd451ee0a00e649c030dcb38", size = 13158184, upload-time = "2026-08-13T16:04:24.958Z" }, - { url = "https://files.pythonhosted.org/packages/b1/2c/f1fb1f5387108b206f985cdb394bbdc730557cc339e63067ded9b3565263/zensical-0.0.54-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48583da6f485e2373845d4332882c9e9d504302dbd2731c589543584df5b087", size = 13536772, upload-time = "2026-08-13T16:04:27.373Z" }, - { url = "https://files.pythonhosted.org/packages/a3/97/3224b3dd5d76cebddf9725ae871a5a6a8f2de177be59d802232d00073d8b/zensical-0.0.54-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da7781a906623fb7bc278cc874ec6f80691c8753b90fbdf90a451abb046ae204", size = 13191901, upload-time = "2026-08-13T16:04:30.295Z" }, - { url = "https://files.pythonhosted.org/packages/b2/00/b1f55530c8df4331e1f209ec605816a578142f124b5f8078a9d984514d63/zensical-0.0.54-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:75aff1c01f6104dd0e79d08c87bd595da13db3e1f6da55fc9933ebff3e94fdd5", size = 13402956, upload-time = "2026-08-13T16:04:32.899Z" }, - { url = "https://files.pythonhosted.org/packages/c7/6e/58df496c2742600df3a2f273ba52b8ced1ce357340b825af9f1bb0f2042e/zensical-0.0.54-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1469c5d551a0ea2d0fcf922046a263d2afcff5f03ce3bb443e286970d04ff9f2", size = 13431462, upload-time = "2026-08-13T16:04:35.518Z" }, - { url = "https://files.pythonhosted.org/packages/5b/85/70ae775db7865be2434bc39e9b4ff1c7988978d796a7ff9d49294e7410c7/zensical-0.0.54-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:31c354f98b3374b9bcab65278a278dbec66125c997f7f1a5ee9501f413b87b9c", size = 13587289, upload-time = "2026-08-13T16:04:37.954Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ca/05e6b3e04323810bbf4da9240b8b710740fac82ec01659b14c86e44aa88e/zensical-0.0.54-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85ef75654e7845aa65f1392af771771566f33faac452a2aed004ba367454f812", size = 13534594, upload-time = "2026-08-13T16:04:41.006Z" }, - { url = "https://files.pythonhosted.org/packages/14/70/be34910c13632f85911f505bd9a4d1bb53d46c8498d20ebb18570bbe4b7b/zensical-0.0.54-cp310-abi3-win32.whl", hash = "sha256:b56c80a8cd234666afb917fb1ed8467104f6857b1acabd66f60ef1b8a0daa66f", size = 12448180, upload-time = "2026-08-13T16:04:43.486Z" }, - { url = "https://files.pythonhosted.org/packages/a1/c6/a965265946555023dc0e159a41038502190882669d177dfa59b1dd3d580b/zensical-0.0.54-cp310-abi3-win_amd64.whl", hash = "sha256:f5a602986c4123a349cfd075c8d494096a2a2db74422169d65f8092872e48dfa", size = 12712264, upload-time = "2026-08-13T16:04:46.155Z" }, + { url = "https://files.pythonhosted.org/packages/8b/4b/6810aec6e670451f39639039b570a43d90bc1d4a3b93cf13316ccf6bad11/zensical-0.0.56-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:5135ea3aa5d1358503fc1903e866c191873b138028bc1ab170abac3a4b537ffe", size = 12874712, upload-time = "2026-08-18T15:46:18.378Z" }, + { url = "https://files.pythonhosted.org/packages/97/98/23445d8ed708088dd6d9d51674f8836b77c53aab280360d7aa7a206bea8e/zensical-0.0.56-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:a22ae2329ba755c6e58e1fe5967ca429d580d346a73cf467ad5185377e2cf809", size = 12764552, upload-time = "2026-08-18T15:46:20.746Z" }, + { url = "https://files.pythonhosted.org/packages/14/bd/ab89450728b0a55e6a52a86060b38a362a52a1b9f02eda7fef68b729eb2e/zensical-0.0.56-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03c70ef328e31cce0e31739acdd6679e9272bc7a3016ca5eafaa16dcfda460c9", size = 13212920, upload-time = "2026-08-18T15:46:22.977Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5a/969fd9a461204392a9266a544c185fbb223714b306534661dcbb7d290be7/zensical-0.0.56-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ffc50153a50292078357a5052e7a6b3e20a818523792ffcbeace70418b761eb", size = 13146652, upload-time = "2026-08-18T15:46:25.773Z" }, + { url = "https://files.pythonhosted.org/packages/22/b3/28a7c8dea8fe2edbba75a664efbde797b5922651ea92ffc480947ab22197/zensical-0.0.56-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88bba0e339d36647ce638a42b4ba96f2521b59ea54be74c314340be7e401f94b", size = 13530946, upload-time = "2026-08-18T15:46:28.19Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a6/d55a18c6e041b788af1d19e4d8c61ee9b8a7de5b9cc79ffa7bc9565e3a28/zensical-0.0.56-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b7a37f6ac38ac218e3e0dd311b58c468603963e38ee23f2e25672dcd6e9c17b", size = 13178741, upload-time = "2026-08-18T15:46:30.378Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4f/a07da2f761cfd6a27faf9e8d5a9475c396a9a0ca37424a0b67cab7ae4d2c/zensical-0.0.56-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5f6d850bce3184422b37b9d98ef4d123047a47446844793251cabc04bd55f8f0", size = 13389836, upload-time = "2026-08-18T15:46:32.847Z" }, + { url = "https://files.pythonhosted.org/packages/34/aa/697ef9846b0e2071de4d03b0472e2658380ea9271bd01235f4ffaeef1978/zensical-0.0.56-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d18944a4a111050b8f571e73d4e2475d7ce62ce78b64f09bcf33bb11cc346e1c", size = 13419551, upload-time = "2026-08-18T15:46:35.112Z" }, + { url = "https://files.pythonhosted.org/packages/97/6b/20e1b2443951d5182b3fc2ee54c200ea00c09bd8901a479be4147c94361b/zensical-0.0.56-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3937029ec5091d577c2a05ebccd38fde97fab28662d165ead66d566689b2a6df", size = 13579878, upload-time = "2026-08-18T15:46:37.381Z" }, + { url = "https://files.pythonhosted.org/packages/53/c7/9c3b400b8a7d78cc169b7a78d4f74a90f9114209034a604f04051f48037c/zensical-0.0.56-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:346a1cdbad157633d185f79039a97c8b4f8c2b40be7d7efd56d72622f40247b0", size = 13526142, upload-time = "2026-08-18T15:46:39.949Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f6/7a6f2a513054071e44a504d7157684f00ac5e5e5f741eadc78ab6c80642c/zensical-0.0.56-cp310-abi3-win32.whl", hash = "sha256:a06681046f74b5bdc506d22af8fcef505b2450e45538c7f01a5e028f4e50c6f7", size = 12433218, upload-time = "2026-08-18T15:46:42.329Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ed/5b497c75fb1fd5845f3ec2f0b11b5e01f18f14c297041cd2f20d30d8b6a4/zensical-0.0.56-cp310-abi3-win_amd64.whl", hash = "sha256:c557985f12d042c15dcb7d577543d81ceee9281f96d591d265310b673437a325", size = 12702823, upload-time = "2026-08-18T15:46:44.836Z" }, ] [[package]]