mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-08-27 00:43:47 +08:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1c3558e017 | |||
| bf567847de | |||
| bd1d54e239 | |||
| 8d49466b62 |
@@ -0,0 +1,83 @@
|
||||
name: Build default big model
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
HF_REPO: sunnypilot/sunnypilot_models_v1
|
||||
HF_DEFAULTS_PATH: models/defaults/big
|
||||
|
||||
jobs:
|
||||
resolve_name:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
model_name: ${{ steps.name.outputs.model_name }}
|
||||
onnx_ref: ${{ steps.name.outputs.onnx_ref }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- id: name
|
||||
run: |
|
||||
NAME=$(PYTHONPATH=${{ github.workspace }} python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)")
|
||||
ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx)
|
||||
echo "model_name=${NAME}" >> $GITHUB_OUTPUT
|
||||
echo "onnx_ref=$ONNX_REF" >> $GITHUB_OUTPUT
|
||||
|
||||
build_model:
|
||||
needs: resolve_name
|
||||
uses: ./.github/workflows/sunnypilot-build-model.yaml
|
||||
with:
|
||||
upstream_branch: ${{ needs.resolve_name.outputs.onnx_ref }}
|
||||
custom_name: ${{ needs.resolve_name.outputs.model_name }}
|
||||
target_hardware: usbgpu
|
||||
secrets: inherit
|
||||
|
||||
upload_defaults:
|
||||
needs: [ resolve_name, build_model ]
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
|
||||
|
||||
- name: Install huggingface_hub
|
||||
run: pip install --upgrade "huggingface_hub>=0.22.0"
|
||||
|
||||
- name: Download artifact name
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: artifact-name-${{ needs.resolve_name.outputs.model_name }}
|
||||
path: artifact_name
|
||||
|
||||
- name: Read artifact name
|
||||
id: artifact
|
||||
run: |
|
||||
ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
|
||||
echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Download model artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.artifact.outputs.artifact_name }}
|
||||
path: output
|
||||
|
||||
- name: Upload to HF and update default_models.json
|
||||
env:
|
||||
HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
|
||||
ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
|
||||
run: |
|
||||
rm -f output/artifact_name.txt
|
||||
export PYTHONPATH=$(pwd)
|
||||
python3 release/ci/upload_default_model.py \
|
||||
--hf-repo "${{ env.HF_REPO }}" \
|
||||
--hf-defaults-path "${{ env.HF_DEFAULTS_PATH }}" \
|
||||
--artifact-name "$ARTIFACT_NAME" \
|
||||
--model-dir output \
|
||||
--onnx-path "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" \
|
||||
--onnx-ref "${{ needs.resolve_name.outputs.onnx_ref }}" \
|
||||
--model-name "${{ needs.resolve_name.outputs.model_name }}" \
|
||||
--tinygrad-ref "$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)" \
|
||||
--run-number "${{ github.run_number }}"
|
||||
@@ -1,501 +0,0 @@
|
||||
name: Build default models
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target:
|
||||
description: 'Model target to build'
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- small
|
||||
- big
|
||||
- dm
|
||||
workflow_call:
|
||||
inputs:
|
||||
target:
|
||||
description: 'Model target to build (small, big, or dm)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: build-default-models-${{ inputs.target }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
HF_REPO: sunnypilot/sunnypilot_models_v1
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
model_name: ${{ steps.resolve.outputs.model_name }}
|
||||
onnx_ref: ${{ steps.resolve.outputs.onnx_ref }}
|
||||
onnx_path: ${{ steps.resolve.outputs.onnx_path }}
|
||||
hf_defaults_path: ${{ steps.resolve.outputs.hf_defaults_path }}
|
||||
tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- id: resolve
|
||||
run: |
|
||||
export PYTHONPATH=${{ github.workspace }}
|
||||
|
||||
if [ "${{ inputs.target }}" = "big" ]; then
|
||||
NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)")
|
||||
ONNX_PATH="openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
|
||||
HF_DEFAULTS_PATH="models/defaults/big"
|
||||
elif [ "${{ inputs.target }}" = "dm" ]; then
|
||||
ONNX_PATH="openpilot/selfdrive/modeld/models/dmonitoring_model.onnx"
|
||||
HF_DEFAULTS_PATH="models/defaults/dm"
|
||||
NAME="dmonitoring_model ($(git log -1 --format=%cd --date=format:'%B %d, %Y' -- "$ONNX_PATH"))"
|
||||
else
|
||||
NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL; print(DEFAULT_MODEL)")
|
||||
ONNX_PATH="openpilot/selfdrive/modeld/models/driving_supercombo.onnx"
|
||||
HF_DEFAULTS_PATH="models/defaults/small"
|
||||
fi
|
||||
|
||||
ONNX_REF=$(git log -1 --format='%H' -- "$ONNX_PATH")
|
||||
TINYGRAD_REF=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)
|
||||
if [ -z "$TINYGRAD_REF" ]; then
|
||||
echo "::error::Failed to resolve tinygrad ref"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "model_name=${NAME}" >> $GITHUB_OUTPUT
|
||||
echo "onnx_ref=${ONNX_REF}" >> $GITHUB_OUTPUT
|
||||
echo "onnx_path=${ONNX_PATH}" >> $GITHUB_OUTPUT
|
||||
echo "hf_defaults_path=${HF_DEFAULTS_PATH}" >> $GITHUB_OUTPUT
|
||||
echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT
|
||||
|
||||
build_small_model:
|
||||
needs: resolve
|
||||
if: ${{ inputs.target == 'small' }}
|
||||
runs-on: [self-hosted, tici]
|
||||
env:
|
||||
SMALL_ONNX: openpilot/selfdrive/modeld/models/driving_supercombo.onnx
|
||||
SMALL_PKL: openpilot/selfdrive/modeld/models/driving_tinygrad.pkl
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Pull ONNX via LFS
|
||||
run: git lfs pull -I "${{ env.SMALL_ONNX }}"
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
source /etc/profile
|
||||
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
||||
export UV_PYTHON_PREFERENCE=managed
|
||||
export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
|
||||
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
||||
uv sync --frozen
|
||||
printenv >> $GITHUB_ENV
|
||||
|
||||
- name: Disable powersave
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
|
||||
|
||||
- name: Compile small model with stock compiler
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"
|
||||
|
||||
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
|
||||
CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
|
||||
FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)")
|
||||
|
||||
TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
|
||||
|
||||
env ${TG_FLAGS} python3 \
|
||||
${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \
|
||||
--onnx ${{ github.workspace }}/${{ env.SMALL_ONNX }} \
|
||||
--model-size $MODEL_SIZE \
|
||||
--camera-resolutions $CAMERA_RES \
|
||||
--frame-skip $FRAME_SKIP \
|
||||
--output ${{ github.workspace }}/${{ env.SMALL_PKL }}
|
||||
|
||||
- name: Chunk small pkl
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH=${{ github.workspace }}
|
||||
python3 -c "
|
||||
from openpilot.common.file_chunker import chunk_file, get_chunk_targets
|
||||
import os
|
||||
pkl = '${{ github.workspace }}/${{ env.SMALL_PKL }}'
|
||||
size = os.path.getsize(pkl)
|
||||
targets = get_chunk_targets(pkl, size)
|
||||
chunk_file(pkl, targets)
|
||||
print(f'Chunked into {len(targets)} files')
|
||||
"
|
||||
|
||||
- name: Prepare output
|
||||
env:
|
||||
MODEL_NAME: ${{ needs.resolve.outputs.model_name }}
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH=${{ github.workspace }}
|
||||
MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models"
|
||||
OUTPUT_DIR="${{ github.workspace }}/small_output"
|
||||
PKL_BASE="driving_tinygrad.pkl"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/"
|
||||
cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/"
|
||||
|
||||
python3 "${{ github.workspace }}/release/ci/model_generator.py" \
|
||||
--model-dir "$MODELS_DIR" \
|
||||
--output-dir "$OUTPUT_DIR" \
|
||||
--custom-name "$MODEL_NAME" \
|
||||
--upstream-branch "${{ needs.resolve.outputs.onnx_ref }}"
|
||||
|
||||
echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt"
|
||||
|
||||
- name: Upload small model artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }}
|
||||
path: ${{ github.workspace }}/small_output/
|
||||
|
||||
- name: Upload artifact name file
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact-name-${{ needs.resolve.outputs.model_name }}
|
||||
path: ${{ github.workspace }}/small_output/artifact_name.txt
|
||||
|
||||
- name: Re-enable powersave
|
||||
if: always()
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
|
||||
|
||||
build_big_model:
|
||||
needs: resolve
|
||||
if: ${{ inputs.target == 'big' }}
|
||||
runs-on: [self-hosted, usbgpu]
|
||||
env:
|
||||
BIG_ONNX: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
|
||||
BIG_PKL: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Pull big ONNX via LFS
|
||||
run: git lfs pull -I "${{ env.BIG_ONNX }}"
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
source /etc/profile
|
||||
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
||||
export UV_PYTHON_PREFERENCE=managed
|
||||
export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
|
||||
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
||||
uv sync --frozen
|
||||
printenv >> $GITHUB_ENV
|
||||
|
||||
- name: Disable powersave
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
|
||||
|
||||
- name: Wait for chestnut PCIe link
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"
|
||||
python3 -c "
|
||||
import time
|
||||
from openpilot.system.hardware.chestnut.flash import link_up
|
||||
for i in range(10):
|
||||
if link_up():
|
||||
print(f'PCIe link up after {i+1} attempt(s)')
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError('Chestnut PCIe link not ready after 10 attempts')
|
||||
"
|
||||
|
||||
- name: Compile big model with stock compiler
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"
|
||||
|
||||
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
|
||||
CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
|
||||
FRAME_SKIP=$(python3 -c "from openpilot.selfdrive.modeld.constants import ModelConstants as MC; print(MC.MODEL_RUN_FREQ // MC.MODEL_CONTEXT_FREQ)")
|
||||
|
||||
TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2"
|
||||
|
||||
env ${TG_FLAGS} python3 \
|
||||
${{ github.workspace }}/openpilot/selfdrive/modeld/compile_modeld.py \
|
||||
--onnx ${{ github.workspace }}/${{ env.BIG_ONNX }} \
|
||||
--model-size $MODEL_SIZE \
|
||||
--camera-resolutions $CAMERA_RES \
|
||||
--frame-skip $FRAME_SKIP \
|
||||
--output ${{ github.workspace }}/${{ env.BIG_PKL }}
|
||||
|
||||
- name: Chunk big pkl
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH=${{ github.workspace }}
|
||||
python3 -c "
|
||||
from openpilot.common.file_chunker import chunk_file, get_chunk_targets
|
||||
import os
|
||||
pkl = '${{ github.workspace }}/${{ env.BIG_PKL }}'
|
||||
size = os.path.getsize(pkl)
|
||||
targets = get_chunk_targets(pkl, size)
|
||||
chunk_file(pkl, targets)
|
||||
print(f'Chunked into {len(targets)} files')
|
||||
"
|
||||
|
||||
- name: Prepare output
|
||||
env:
|
||||
MODEL_NAME: ${{ needs.resolve.outputs.model_name }}
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH=${{ github.workspace }}
|
||||
MODELS_DIR="${{ github.workspace }}/openpilot/selfdrive/modeld/models"
|
||||
OUTPUT_DIR="${{ github.workspace }}/big_output"
|
||||
PKL_BASE="big_driving_tinygrad.pkl"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
cp "$MODELS_DIR/${PKL_BASE}".chunk* "$OUTPUT_DIR/"
|
||||
cp "$MODELS_DIR/${PKL_BASE}.chunkmanifest" "$OUTPUT_DIR/"
|
||||
|
||||
python3 "${{ github.workspace }}/release/ci/model_generator.py" \
|
||||
--model-dir "$MODELS_DIR" \
|
||||
--output-dir "$OUTPUT_DIR" \
|
||||
--custom-name "$MODEL_NAME" \
|
||||
--upstream-branch "${{ needs.resolve.outputs.onnx_ref }}"
|
||||
|
||||
echo "model-${MODEL_NAME}-${{ github.run_number }}" > "$OUTPUT_DIR/artifact_name.txt"
|
||||
|
||||
- name: Upload big model artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }}
|
||||
path: ${{ github.workspace }}/big_output/
|
||||
|
||||
- name: Upload artifact name file
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact-name-${{ needs.resolve.outputs.model_name }}
|
||||
path: ${{ github.workspace }}/big_output/artifact_name.txt
|
||||
|
||||
- name: Re-enable powersave
|
||||
if: always()
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
|
||||
|
||||
upload_defaults:
|
||||
needs: [ resolve, build_small_model, build_big_model, build_dm_model ]
|
||||
if: |
|
||||
${{
|
||||
!cancelled() &&
|
||||
(inputs.target == 'big' && needs.build_big_model.result == 'success' ||
|
||||
inputs.target == 'small' && needs.build_small_model.result == 'success' ||
|
||||
inputs.target == 'dm' && needs.build_dm_model.result == 'success')
|
||||
}}
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Pull ONNX via LFS
|
||||
run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}"
|
||||
|
||||
- name: Install huggingface_hub
|
||||
run: pip install --upgrade "huggingface_hub>=0.22.0"
|
||||
|
||||
- name: Download artifact name
|
||||
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: artifact-name-${{ needs.resolve.outputs.model_name }}
|
||||
path: artifact_name
|
||||
|
||||
- name: Read artifact name
|
||||
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
|
||||
id: artifact
|
||||
run: |
|
||||
ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
|
||||
echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Download model artifact
|
||||
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.artifact.outputs.artifact_name }}
|
||||
path: output
|
||||
|
||||
- name: Upload model to HF
|
||||
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
|
||||
env:
|
||||
HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
|
||||
ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
|
||||
run: |
|
||||
rm -f output/artifact_name.txt
|
||||
export PYTHONPATH=$(pwd)
|
||||
python3 release/ci/upload_default_model.py \
|
||||
--hf-repo "${{ env.HF_REPO }}" \
|
||||
--hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \
|
||||
--artifact-name "$ARTIFACT_NAME" \
|
||||
--model-dir output \
|
||||
--onnx-path "${{ needs.resolve.outputs.onnx_path }}" \
|
||||
--onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \
|
||||
--model-name "${{ needs.resolve.outputs.model_name }}" \
|
||||
--tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \
|
||||
--run-number "${{ github.run_number }}"
|
||||
|
||||
- name: Download DM artifact
|
||||
if: ${{ inputs.target == 'dm' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dm-model-${{ github.run_number }}
|
||||
path: dm_output
|
||||
|
||||
- name: Generate DM metadata and upload to HF
|
||||
if: ${{ inputs.target == 'dm' }}
|
||||
env:
|
||||
HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
|
||||
run: |
|
||||
export PYTHONPATH=$(pwd)
|
||||
python3 -c "
|
||||
import json, hashlib
|
||||
from pathlib import Path
|
||||
from datetime import datetime, UTC
|
||||
|
||||
dm_dir = Path('dm_output')
|
||||
manifest = list(dm_dir.glob('*.chunkmanifest'))
|
||||
assert manifest, 'No chunkmanifest found'
|
||||
pkl_name = manifest[0].name.removesuffix('.chunkmanifest')
|
||||
num_chunks = int(manifest[0].read_text().strip())
|
||||
|
||||
chunks = []
|
||||
for i in range(num_chunks):
|
||||
chunk = dm_dir / f'{pkl_name}.chunk{i+1:02d}of{num_chunks:02d}'
|
||||
chunks.append({
|
||||
'file_name': chunk.name,
|
||||
'sha256': hashlib.sha256(chunk.read_bytes()).hexdigest()
|
||||
})
|
||||
|
||||
digest = hashlib.sha256()
|
||||
for c in chunks:
|
||||
with open(dm_dir / c['file_name'], 'rb') as f:
|
||||
while block := f.read(1024*1024):
|
||||
digest.update(block)
|
||||
|
||||
metadata = {
|
||||
'bundles': [{
|
||||
'short_name': 'DMMODEL',
|
||||
'display_name': '${{ needs.resolve.outputs.model_name }}',
|
||||
'ref': '${{ needs.resolve.outputs.onnx_ref }}',
|
||||
'runner': 'tinygrad',
|
||||
'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'models': [{
|
||||
'type': 'chunked',
|
||||
'artifact': {
|
||||
'file_name': pkl_name,
|
||||
'download_uri': {'url': '', 'sha256': digest.hexdigest()},
|
||||
'chunks': chunks
|
||||
}
|
||||
}]
|
||||
}]
|
||||
}
|
||||
with open(dm_dir / 'metadata.json', 'w') as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
print('Generated DM metadata.json')
|
||||
"
|
||||
|
||||
python3 release/ci/upload_default_model.py \
|
||||
--hf-repo "${{ env.HF_REPO }}" \
|
||||
--hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \
|
||||
--artifact-name "dm-model-${{ github.run_number }}" \
|
||||
--model-dir dm_output \
|
||||
--onnx-path "${{ needs.resolve.outputs.onnx_path }}" \
|
||||
--onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \
|
||||
--model-name "${{ needs.resolve.outputs.model_name }}" \
|
||||
--tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \
|
||||
--run-number "${{ github.run_number }}"
|
||||
|
||||
build_dm_model:
|
||||
needs: resolve
|
||||
if: ${{ inputs.target == 'dm' }}
|
||||
runs-on: [self-hosted, tici]
|
||||
env:
|
||||
DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx
|
||||
DM_PKL: openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Pull DM ONNX via LFS
|
||||
run: git lfs pull -I "${{ env.DM_ONNX }}"
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
source /etc/profile
|
||||
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
||||
export UV_PYTHON_PREFERENCE=managed
|
||||
export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
|
||||
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
||||
uv sync --frozen
|
||||
printenv >> $GITHUB_ENV
|
||||
|
||||
- name: Disable powersave
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
|
||||
|
||||
- name: Compile DM model
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"
|
||||
|
||||
TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
|
||||
|
||||
taskset -c 7 env ${TG_FLAGS} python3 \
|
||||
${{ github.workspace }}/tinygrad_repo/examples/openpilot/compile3.py \
|
||||
${{ github.workspace }}/${{ env.DM_ONNX }} \
|
||||
${{ github.workspace }}/${{ env.DM_PKL }}
|
||||
|
||||
- name: Chunk DM pkl
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH=${{ github.workspace }}
|
||||
python3 -c "
|
||||
from openpilot.common.file_chunker import chunk_file, get_chunk_targets
|
||||
import os
|
||||
pkl = '${{ github.workspace }}/${{ env.DM_PKL }}'
|
||||
size = os.path.getsize(pkl)
|
||||
targets = get_chunk_targets(pkl, size)
|
||||
chunk_file(pkl, targets)
|
||||
print(f'Chunked {pkl} into {len(targets)} chunks')
|
||||
"
|
||||
|
||||
- name: Prepare DM output
|
||||
run: |
|
||||
mkdir -p dm_output
|
||||
cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunk* dm_output/
|
||||
cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunkmanifest dm_output/
|
||||
|
||||
- name: Upload DM artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dm-model-${{ github.run_number }}
|
||||
path: dm_output/
|
||||
|
||||
- name: Re-enable powersave
|
||||
if: always()
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
name: Download HF model chunks
|
||||
description: Resolve and download model chunks from HuggingFace in parallel
|
||||
|
||||
inputs:
|
||||
hf_repo:
|
||||
description: HuggingFace dataset repo
|
||||
required: true
|
||||
models:
|
||||
description: 'JSON array of {hf_path, onnx_hash, canonical} objects'
|
||||
required: true
|
||||
dest_dir:
|
||||
description: Destination directory for downloaded chunks
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Download model chunks
|
||||
shell: bash
|
||||
env:
|
||||
HF_REPO: ${{ inputs.hf_repo }}
|
||||
MODELS_JSON: ${{ inputs.models }}
|
||||
DEST_DIR: ${{ inputs.dest_dir }}
|
||||
run: |
|
||||
set -eo pipefail
|
||||
DOWNLOAD_LIST=$(mktemp)
|
||||
|
||||
resolve_chunks() {
|
||||
local HF_PATH="$1" ONNX_HASH="$2" CANONICAL="$3" DEST_DIR="$4"
|
||||
local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_PATH}/default_models.json"
|
||||
local DEFAULTS BUNDLE ARTIFACT BASE_URL NUM_CHUNKS
|
||||
DEFAULTS=$(curl -fsSL "$JSON_URL")
|
||||
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
|
||||
ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
|
||||
BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
|
||||
NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
|
||||
|
||||
mkdir -p "$DEST_DIR"
|
||||
while IFS= read -r CHUNK_NAME; do
|
||||
CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true)
|
||||
if [ -z "$CHUNK_IDX" ]; then
|
||||
echo "::error::Failed to parse chunk index from: $CHUNK_NAME"
|
||||
return 1
|
||||
fi
|
||||
ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
|
||||
printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}.chunk${CHUNK_IDX}" >> "$DOWNLOAD_LIST"
|
||||
done < <(echo "$ARTIFACT" | jq -r '.chunks[].file_name')
|
||||
echo "$NUM_CHUNKS" > "${DEST_DIR}/${CANONICAL}.chunkmanifest"
|
||||
}
|
||||
|
||||
echo "$MODELS_JSON" | jq -c '.[]' | while IFS= read -r model; do
|
||||
HF_PATH=$(echo "$model" | jq -r '.hf_path')
|
||||
ONNX_HASH=$(echo "$model" | jq -r '.onnx_hash')
|
||||
CANONICAL=$(echo "$model" | jq -r '.canonical')
|
||||
resolve_chunks "$HF_PATH" "$ONNX_HASH" "$CANONICAL" "$DEST_DIR"
|
||||
done
|
||||
|
||||
TOTAL=$(wc -l < "$DOWNLOAD_LIST")
|
||||
echo "Downloading $TOTAL chunks with 8 parallel connections..."
|
||||
xargs -P8 -d'\n' -I{} bash -c '
|
||||
URL="${1%% *}"
|
||||
DEST="${1#* }"
|
||||
echo "Downloading $(basename "$DEST")"
|
||||
curl -fsSL --retry 3 --retry-delay 5 -o "$DEST" "$URL"
|
||||
' _ {} < "$DOWNLOAD_LIST"
|
||||
rm -f "$DOWNLOAD_LIST"
|
||||
@@ -188,7 +188,7 @@ jobs:
|
||||
if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
|
||||
echo "USBGPU build"
|
||||
export USBGPU=1
|
||||
TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2"
|
||||
TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2"
|
||||
OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
|
||||
else
|
||||
echo "QCOM build"
|
||||
|
||||
@@ -39,8 +39,6 @@ jobs:
|
||||
include_big_model: ${{ steps.strategy.outputs.include_big_model }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- name: Extract deploy strategy
|
||||
id: strategy
|
||||
run: |
|
||||
@@ -98,8 +96,6 @@ jobs:
|
||||
}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- name: Wait for Tests
|
||||
uses: ./.github/workflows/wait-for-action # Path to where you place the action
|
||||
with:
|
||||
@@ -123,7 +119,6 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
submodules: recursive
|
||||
ref: ${{ env.SOURCE_BRANCH }}
|
||||
repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }}
|
||||
@@ -170,7 +165,7 @@ jobs:
|
||||
scons -j1 cache_dir="$SCONS_CACHE" --minimal \
|
||||
openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd
|
||||
echo "Building rest of sunnypilot"
|
||||
SKIP_TINYGRAD_COMPILE=1 /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
|
||||
/usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
|
||||
touch ${BUILD_DIR}/prebuilt
|
||||
if [[ "${{ runner.debug }}" == "1" ]]; then
|
||||
ls -la ${BUILD_DIR}
|
||||
@@ -219,174 +214,59 @@ jobs:
|
||||
outputs:
|
||||
onnx_sha256: ${{ steps.resolve.outputs.onnx_sha256 }}
|
||||
env:
|
||||
GH_REPO: ${{ github.repository }}
|
||||
HF_REPO: sunnypilot/sunnypilot_models_v1
|
||||
HF_DEFAULTS_PATH: models/defaults/big
|
||||
steps:
|
||||
- name: Resolve ONNX hash and tinygrad ref via API
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.head_ref || github.ref_name }}
|
||||
|
||||
- run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
|
||||
|
||||
- name: Check HF defaults and build if needed
|
||||
id: resolve
|
||||
run: |
|
||||
REF="${{ github.head_ref || github.ref_name }}"
|
||||
|
||||
ONNX_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2)
|
||||
echo "ONNX hash: $ONNX_HASH"
|
||||
echo "onnx_sha256=$ONNX_HASH" >> $GITHUB_OUTPUT
|
||||
|
||||
TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha')
|
||||
echo "tinygrad ref: $TINYGRAD_REF"
|
||||
ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
|
||||
echo "Repo ONNX hash: $ACTUAL_ONNX_HASH"
|
||||
echo "onnx_sha256=$ACTUAL_ONNX_HASH" >> $GITHUB_OUTPUT
|
||||
|
||||
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
|
||||
|
||||
check_defaults() {
|
||||
check_hash() {
|
||||
DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
|
||||
TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null)
|
||||
[ "$TINYGRAD_MATCH" = "true" ] || return 1
|
||||
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
|
||||
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
|
||||
[ -n "$BUNDLE" ] && [ "$BUNDLE" != "null" ]
|
||||
}
|
||||
|
||||
if check_defaults; then
|
||||
echo "HF defaults match repo ONNX hash and tinygrad ref"
|
||||
exit 0
|
||||
fi
|
||||
if check_hash; then
|
||||
echo "HF defaults match repo ONNX"
|
||||
else
|
||||
echo "No matching model on HF — triggering build"
|
||||
gh workflow run build-default-big-model.yaml --ref "${{ github.head_ref || github.ref_name }}"
|
||||
|
||||
echo "No matching model on HF — dispatching build"
|
||||
gh workflow run build-default-models.yaml --ref "$REF" -f target=big
|
||||
echo "Waiting for build to start..."
|
||||
sleep 120
|
||||
|
||||
echo "Polling HF for big model availability..."
|
||||
for i in $(seq 1 90); do
|
||||
sleep 30
|
||||
if check_defaults; then
|
||||
echo "Big model available on HF after $((i * 30))s"
|
||||
exit 0
|
||||
RUN_ID=$(gh run list --workflow=build-default-big-model.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId')
|
||||
if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
|
||||
echo "::error::Failed to find build-default-big-model run"
|
||||
exit 1
|
||||
fi
|
||||
echo "Poll $i/90: not yet available"
|
||||
done
|
||||
|
||||
echo "::error::Big model not available on HF after 45 minutes"
|
||||
exit 1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
echo "Waiting for run $RUN_ID..."
|
||||
gh run watch "$RUN_ID"
|
||||
|
||||
- name: Cancel run on failure
|
||||
if: failure()
|
||||
run: gh run cancel ${{ github.run_id }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
prepare_small_model:
|
||||
needs: [ prepare_strategy ]
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
driving_onnx_sha256: ${{ steps.resolve.outputs.driving_onnx_sha256 }}
|
||||
env:
|
||||
GH_REPO: ${{ github.repository }}
|
||||
HF_REPO: sunnypilot/sunnypilot_models_v1
|
||||
HF_DEFAULTS_PATH: models/defaults/small
|
||||
steps:
|
||||
- name: Resolve ONNX hash and tinygrad ref via API
|
||||
id: resolve
|
||||
run: |
|
||||
REF="${{ github.head_ref || github.ref_name }}"
|
||||
|
||||
DRIVING_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/driving_supercombo.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2)
|
||||
echo "Driving ONNX hash: $DRIVING_HASH"
|
||||
echo "driving_onnx_sha256=$DRIVING_HASH" >> $GITHUB_OUTPUT
|
||||
|
||||
TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha')
|
||||
echo "tinygrad ref: $TINYGRAD_REF"
|
||||
|
||||
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
|
||||
|
||||
check_defaults() {
|
||||
DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
|
||||
TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null)
|
||||
[ "$TINYGRAD_MATCH" = "true" ] || return 1
|
||||
DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
|
||||
[ -n "$DRIVING" ] && [ "$DRIVING" != "null" ] || return 1
|
||||
}
|
||||
|
||||
if check_defaults; then
|
||||
echo "HF defaults match repo ONNX hash and tinygrad ref"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "No matching model on HF — dispatching build"
|
||||
gh workflow run build-default-models.yaml --ref "$REF" -f target=small
|
||||
|
||||
echo "Polling HF for model availability..."
|
||||
for i in $(seq 1 60); do
|
||||
sleep 30
|
||||
if check_defaults; then
|
||||
echo "Model available on HF after $((i * 30))s"
|
||||
exit 0
|
||||
CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
|
||||
if [ "$CONCLUSION" != "success" ]; then
|
||||
echo "::error::build-default-big-model failed: $CONCLUSION"
|
||||
exit 1
|
||||
fi
|
||||
echo "Poll $i/60: not yet available"
|
||||
done
|
||||
|
||||
echo "::error::Small driving model not available on HF after 30 minutes"
|
||||
exit 1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Cancel run on failure
|
||||
if: failure()
|
||||
run: gh run cancel ${{ github.run_id }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
prepare_dm_model:
|
||||
needs: [ prepare_strategy ]
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }}
|
||||
env:
|
||||
GH_REPO: ${{ github.repository }}
|
||||
HF_REPO: sunnypilot/sunnypilot_models_v1
|
||||
HF_DEFAULTS_PATH: models/defaults/dm
|
||||
steps:
|
||||
- name: Resolve ONNX hash and tinygrad ref via API
|
||||
id: resolve
|
||||
run: |
|
||||
REF="${{ github.head_ref || github.ref_name }}"
|
||||
|
||||
DM_HASH=$(gh api "repos/${GH_REPO}/contents/openpilot/selfdrive/modeld/models/dmonitoring_model.onnx?ref=${REF}" --jq '.content' | base64 -d | grep '^oid sha256:' | cut -d: -f2)
|
||||
echo "DM ONNX hash: $DM_HASH"
|
||||
echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT
|
||||
|
||||
TINYGRAD_REF=$(gh api "repos/${GH_REPO}/contents/tinygrad_repo?ref=${REF}" --jq '.sha')
|
||||
echo "tinygrad ref: $TINYGRAD_REF"
|
||||
|
||||
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
|
||||
|
||||
check_defaults() {
|
||||
DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
|
||||
TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null)
|
||||
[ "$TINYGRAD_MATCH" = "true" ] || return 1
|
||||
DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
|
||||
[ -n "$DM" ] && [ "$DM" != "null" ] || return 1
|
||||
}
|
||||
|
||||
if check_defaults; then
|
||||
echo "HF defaults match DM ONNX hash and tinygrad ref"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "No matching DM model on HF — dispatching build"
|
||||
gh workflow run build-default-models.yaml --ref "$REF" -f target=dm
|
||||
|
||||
echo "Polling HF for DM model availability..."
|
||||
for i in $(seq 1 60); do
|
||||
sleep 30
|
||||
if check_defaults; then
|
||||
echo "DM model available on HF after $((i * 30))s"
|
||||
exit 0
|
||||
if ! check_hash; then
|
||||
echo "::error::HF defaults still don't match after build"
|
||||
exit 1
|
||||
fi
|
||||
echo "Poll $i/60: not yet available"
|
||||
done
|
||||
|
||||
echo "::error::DM model not available on HF after 30 minutes"
|
||||
exit 1
|
||||
fi
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -398,24 +278,23 @@ jobs:
|
||||
|
||||
publish:
|
||||
concurrency:
|
||||
# We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
|
||||
# This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
|
||||
# Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
|
||||
group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
|
||||
cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
|
||||
if: ${{
|
||||
always() && !cancelled() &&
|
||||
needs.build.result == 'success' &&
|
||||
needs.prepare_strategy.result == 'success' &&
|
||||
needs.prepare_small_model.result == 'success' &&
|
||||
needs.prepare_dm_model.result == 'success' &&
|
||||
(!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
|
||||
(needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
|
||||
}}
|
||||
needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ]
|
||||
needs: [ build, prepare_strategy, prepare_chestnut ]
|
||||
runs-on: ubuntu-24.04
|
||||
environment: ${{ needs.prepare_strategy.outputs.environment }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Download prebuilt artifact
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -427,16 +306,43 @@ jobs:
|
||||
mkdir -p ${{ env.OUTPUT_DIR }}
|
||||
tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
|
||||
|
||||
- name: Download model chunks from HF
|
||||
uses: ./.github/workflows/download-hf-model-chunks
|
||||
with:
|
||||
hf_repo: sunnypilot/sunnypilot_models_v1
|
||||
dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models
|
||||
models: |
|
||||
[
|
||||
{"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"},
|
||||
{"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"}
|
||||
]
|
||||
- name: Prepare chestnut output
|
||||
if: ${{ needs.prepare_chestnut.result == 'success' }}
|
||||
run: |
|
||||
mkdir -p "${{ github.workspace }}/chestnut_output"
|
||||
tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output"
|
||||
|
||||
- name: Download big model chunks from HF
|
||||
if: ${{ needs.prepare_chestnut.result == 'success' }}
|
||||
env:
|
||||
HF_REPO: sunnypilot/sunnypilot_models_v1
|
||||
HF_DEFAULTS_PATH: models/defaults/big
|
||||
run: |
|
||||
ONNX_HASH="${{ needs.prepare_chestnut.outputs.onnx_sha256 }}"
|
||||
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
|
||||
DEFAULTS=$(curl -fsSL "$JSON_URL")
|
||||
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
|
||||
|
||||
mkdir -p big_model_chunks
|
||||
ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
|
||||
BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
|
||||
NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
|
||||
|
||||
CANONICAL="big_driving_tinygrad.pkl"
|
||||
echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
|
||||
CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+')
|
||||
CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
|
||||
ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
|
||||
echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
|
||||
curl -fsSL -o "big_model_chunks/${CANONICAL_CHUNK}" "$ENCODED_URL"
|
||||
done
|
||||
|
||||
echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest"
|
||||
|
||||
- name: Inject big model into chestnut
|
||||
if: ${{ needs.prepare_chestnut.result == 'success' }}
|
||||
run: |
|
||||
cp big_model_chunks/* "${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models/"
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
@@ -458,6 +364,22 @@ jobs:
|
||||
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
|
||||
"${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
|
||||
|
||||
- name: Publish chestnut branch
|
||||
if: ${{ needs.prepare_chestnut.result == 'success' }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
|
||||
CHESTNUT_DIR="${{ github.workspace }}/chestnut_output"
|
||||
|
||||
${{ env.CI_DIR }}/publish.sh \
|
||||
"${{ github.workspace }}" \
|
||||
"$CHESTNUT_DIR" \
|
||||
"$CHESTNUT_BRANCH" \
|
||||
"${{ needs.prepare_strategy.outputs.version }}" \
|
||||
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
|
||||
"${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
|
||||
|
||||
- name: Tag ${{ needs.prepare_strategy.outputs.environment }}
|
||||
if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
|
||||
run: |
|
||||
@@ -465,77 +387,12 @@ jobs:
|
||||
git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}."
|
||||
git push -f origin ${TAG}
|
||||
|
||||
publish_chestnut:
|
||||
concurrency:
|
||||
group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}-chestnut
|
||||
cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
|
||||
if: ${{
|
||||
always() && !cancelled() &&
|
||||
needs.build.result == 'success' &&
|
||||
needs.prepare_strategy.result == 'success' &&
|
||||
needs.prepare_small_model.result == 'success' &&
|
||||
needs.prepare_dm_model.result == 'success' &&
|
||||
needs.prepare_chestnut.result == 'success' &&
|
||||
(!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
|
||||
}}
|
||||
needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ]
|
||||
runs-on: ubuntu-24.04
|
||||
environment: ${{ needs.prepare_strategy.outputs.environment }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Download prebuilt artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: prebuilt
|
||||
|
||||
- name: Untar prebuilt
|
||||
run: |
|
||||
mkdir -p ${{ env.OUTPUT_DIR }}
|
||||
tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
|
||||
|
||||
- name: Download model chunks from HF
|
||||
uses: ./.github/workflows/download-hf-model-chunks
|
||||
with:
|
||||
hf_repo: sunnypilot/sunnypilot_models_v1
|
||||
dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models
|
||||
models: |
|
||||
[
|
||||
{"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"},
|
||||
{"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"},
|
||||
{"hf_path": "models/defaults/big", "onnx_hash": "${{ needs.prepare_chestnut.outputs.onnx_sha256 }}", "canonical": "big_driving_tinygrad.pkl"}
|
||||
]
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
- name: Publish chestnut branch
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
|
||||
|
||||
${{ env.CI_DIR }}/publish.sh \
|
||||
"${{ github.workspace }}" \
|
||||
"${{ env.OUTPUT_DIR }}" \
|
||||
"$CHESTNUT_BRANCH" \
|
||||
"${{ needs.prepare_strategy.outputs.version }}" \
|
||||
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
|
||||
"${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
|
||||
|
||||
notify:
|
||||
needs:
|
||||
- prepare_strategy
|
||||
- build
|
||||
- publish
|
||||
- publish_chestnut
|
||||
- prepare_chestnut
|
||||
- prepare_small_model
|
||||
- prepare_dm_model
|
||||
runs-on: ubuntu-24.04
|
||||
if: ${{ (always() && !cancelled() && !failure())
|
||||
&& needs.publish.result == 'success'
|
||||
@@ -543,8 +400,6 @@ jobs:
|
||||
&& (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Prepare notification message
|
||||
id: message
|
||||
|
||||
@@ -195,10 +195,9 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
|
||||
// Model Manager params
|
||||
{"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
|
||||
{"ModelManager_ActiveBundleUSBGPU", {PERSISTENT, JSON}},
|
||||
{"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"ModelManager_DownloadRef", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
|
||||
{"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}},
|
||||
{"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
|
||||
{"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
|
||||
{"ModelManager_LastSyncTime_USBGPU", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
|
||||
|
||||
@@ -73,45 +73,44 @@ compile_modeld_script = [
|
||||
model_w, model_h = MEDMODEL_INPUT_SIZE
|
||||
frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ
|
||||
|
||||
if not os.getenv('SKIP_TINYGRAD_COMPILE'):
|
||||
for usbgpu in [False, True] if USBGPU else [False]:
|
||||
target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath
|
||||
# BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU
|
||||
file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags)
|
||||
driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath)
|
||||
camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS)
|
||||
# CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it.
|
||||
taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else ''
|
||||
cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py '
|
||||
f'--model-size {model_w}x{model_h} '
|
||||
f'--camera-resolutions {camera_res_args} '
|
||||
f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} '
|
||||
f'--output {target_pkl_path} --frame-skip {frame_skip}')
|
||||
onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps)
|
||||
chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum))
|
||||
def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets):
|
||||
from openpilot.system.hardware.chestnut.flash import link_up
|
||||
# chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars
|
||||
for _ in range(10):
|
||||
if link_up():
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
print("Chestnut not ready, skipping big model build")
|
||||
return
|
||||
if ret := env.Execute(command):
|
||||
return ret
|
||||
chunk_file(pkl, chunks)
|
||||
def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets):
|
||||
chunk_file(pkl, chunks)
|
||||
actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")]
|
||||
node = lenv.Command(
|
||||
chunk_targets,
|
||||
tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file],
|
||||
actions,
|
||||
)
|
||||
if usbgpu:
|
||||
lenv.SideEffect(usbgpu_lock, node)
|
||||
for usbgpu in [False, True] if USBGPU else [False]:
|
||||
target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath
|
||||
# BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU
|
||||
file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags)
|
||||
driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath)
|
||||
camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS)
|
||||
# CPU 7 is isolated with isolcpus on AGNOS, so explicitly pin the compiler to it.
|
||||
taskset = 'taskset -c 7 ' if arch == 'comma_arm64' else ''
|
||||
cmd = (f'{cmd_flags} {mac_brew_string} {taskset}python3 {modeld_dir}/compile_modeld.py '
|
||||
f'--model-size {model_w}x{model_h} '
|
||||
f'--camera-resolutions {camera_res_args} '
|
||||
f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} '
|
||||
f'--output {target_pkl_path} --frame-skip {frame_skip}')
|
||||
onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps)
|
||||
chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum))
|
||||
def do_compile(target, source, env, command=cmd, pkl=target_pkl_path, chunks=chunk_targets):
|
||||
from openpilot.system.hardware.chestnut.flash import link_up
|
||||
# chestnut can enumerate before its PCIe link is up due to varying 12V power behavior across cars
|
||||
for _ in range(10):
|
||||
if link_up():
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
print("Chestnut not ready, skipping big model build")
|
||||
return
|
||||
if ret := env.Execute(command):
|
||||
return ret
|
||||
chunk_file(pkl, chunks)
|
||||
def do_chunk(target, source, env, pkl=target_pkl_path, chunks=chunk_targets):
|
||||
chunk_file(pkl, chunks)
|
||||
actions = Action(do_compile, " [USBGPU] $TARGET") if usbgpu else [cmd, Action(do_chunk, " [CHUNK] $TARGET")]
|
||||
node = lenv.Command(
|
||||
chunk_targets,
|
||||
tinygrad_files + compile_modeld_script + driving_onnx_deps + [Value(chunk_targets), chunker_file],
|
||||
actions,
|
||||
)
|
||||
if usbgpu:
|
||||
lenv.SideEffect(usbgpu_lock, node)
|
||||
|
||||
# get model metadata
|
||||
fn = File(f"models/dmonitoring_model").abspath
|
||||
@@ -143,5 +142,4 @@ def tg_compile(flags, model_name):
|
||||
Action(do_chunk, " [CHUNK] $TARGET")],
|
||||
)
|
||||
|
||||
if not os.getenv('SKIP_TINYGRAD_COMPILE'):
|
||||
tg_compile(tg_flags, 'dmonitoring_model')
|
||||
tg_compile(tg_flags, 'dmonitoring_model')
|
||||
|
||||
@@ -575,7 +575,7 @@ class SelfdriveD(CruiseHelper):
|
||||
clear_event_types = set()
|
||||
if ET.WARNING not in self.state_machine.current_alert_types:
|
||||
clear_event_types.add(ET.WARNING)
|
||||
if self.enabled:
|
||||
if self.enabled or (ET.NO_ENTRY not in (c := self.state_machine.current_alert_types) and (ET.ENABLE in c or ET.USER_DISABLE in c)):
|
||||
clear_event_types.add(ET.NO_ENTRY)
|
||||
|
||||
pers = LONGITUDINAL_PERSONALITY_MAP[self.personality]
|
||||
|
||||
@@ -168,16 +168,9 @@ class Sidebar(Widget, SidebarSP):
|
||||
# Home/Flag button
|
||||
flag_pressed = mouse_down and rl.check_collision_point_rec(mouse_pos, HOME_BTN)
|
||||
button_img = self._flag_img if ui_state.started else self._home_img
|
||||
button_pos = rl.Vector2(HOME_BTN.x, HOME_BTN.y)
|
||||
icon_opacity = 1.0
|
||||
|
||||
if gui_app.sunnypilot_ui():
|
||||
button_img, button_pos, icon_opacity = SidebarSP._get_home_icon(self, button_img)
|
||||
|
||||
tint = Colors.BUTTON_PRESSED if (ui_state.started and flag_pressed) else Colors.BUTTON_NORMAL
|
||||
if icon_opacity < 1.0:
|
||||
tint = rl.Color(tint[0], tint[1], tint[2], int(255 * icon_opacity))
|
||||
rl.draw_texture_ex(button_img, button_pos, 0.0, 1.0, tint)
|
||||
rl.draw_texture_ex(button_img, rl.Vector2(HOME_BTN.x, HOME_BTN.y), 0.0, 1.0, tint)
|
||||
|
||||
# Microphone button
|
||||
if self._recording_audio:
|
||||
|
||||
@@ -248,11 +248,8 @@ class MiciHomeLayout(Widget):
|
||||
|
||||
# ***** Center-aligned bottom section icons *****
|
||||
self._experimental_icon.set_visible(ui_state.experimental_mode)
|
||||
if gui_app.sunnypilot_ui():
|
||||
self._set_egpu_visibility()
|
||||
else:
|
||||
self._egpu_icon.set_visible(ui_state.sm["deviceState"].chestnutPresent and ui_state.usbgpu_compiled)
|
||||
self._egpu_icon_gray.set_visible(ui_state.sm["deviceState"].chestnutPresent and not ui_state.usbgpu_compiled)
|
||||
self._egpu_icon.set_visible(ui_state.sm["deviceState"].chestnutPresent and ui_state.usbgpu_compiled)
|
||||
self._egpu_icon_gray.set_visible(ui_state.sm["deviceState"].chestnutPresent and not ui_state.usbgpu_compiled)
|
||||
self._mic_icon.set_visible(ui_state.recording_audio)
|
||||
self._body_icon.set_visible(bool(ui_state.is_body))
|
||||
|
||||
|
||||
@@ -11,8 +11,6 @@ import pyray as rl
|
||||
|
||||
from openpilot.cereal import custom
|
||||
from openpilot.sunnypilot.models.default_model import get_default_model
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher
|
||||
from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS
|
||||
from openpilot.common.constants import CV
|
||||
from openpilot.selfdrive.ui.ui_state import device, ui_state
|
||||
from openpilot.system.ui.lib.multilang import tr
|
||||
@@ -70,7 +68,7 @@ class ModelsLayout(Widget):
|
||||
callback=self._clear_cache
|
||||
)
|
||||
|
||||
self.cancel_download_item = button_item(tr("Cancel Download"), tr("Cancel"), "", lambda: ui_state.params.remove("ModelManager_DownloadRef"))
|
||||
self.cancel_download_item = button_item(tr("Cancel Download"), tr("Cancel"), "", lambda: ui_state.params.remove("ModelManager_DownloadIndex"))
|
||||
|
||||
self.lane_turn_value_control = option_item_sp(tr("Adjust Lane Turn Speed"), "LaneTurnValue", 500, 2000,
|
||||
tr("Set the maximum speed for lane turn desires. Default is 19 mph."),
|
||||
@@ -117,12 +115,8 @@ class ModelsLayout(Widget):
|
||||
def calculate_cache_size():
|
||||
cache_size = 0.0
|
||||
if os.path.exists(CUSTOM_MODEL_PATH):
|
||||
for file in os.listdir(CUSTOM_MODEL_PATH):
|
||||
try:
|
||||
cache_size += os.path.getsize(os.path.join(CUSTOM_MODEL_PATH, file))
|
||||
except OSError:
|
||||
continue
|
||||
return cache_size / (1024**2)
|
||||
cache_size = sum(os.path.getsize(os.path.join(CUSTOM_MODEL_PATH, file)) for file in os.listdir(CUSTOM_MODEL_PATH)) / (1024**2)
|
||||
return cache_size
|
||||
|
||||
def _clear_cache(self):
|
||||
def _callback(response):
|
||||
@@ -148,7 +142,7 @@ class ModelsLayout(Widget):
|
||||
if not bundle:
|
||||
return
|
||||
|
||||
self.cancel_download_item.set_visible(bool(self.model_manager.selectedBundle) and ui_state.params.get("ModelManager_DownloadRef") is not None)
|
||||
self.cancel_download_item.set_visible(bool(self.model_manager.selectedBundle) and ui_state.params.get("ModelManager_DownloadIndex") is not None)
|
||||
|
||||
if (current_time := time.monotonic()) - self.last_cache_calc_time > 0.5:
|
||||
self.last_cache_calc_time = current_time
|
||||
@@ -189,10 +183,9 @@ class ModelsLayout(Widget):
|
||||
return
|
||||
selected_ref = self.model_dialog.selection_ref
|
||||
if selected_ref == "Default":
|
||||
source = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent)
|
||||
ui_state.params.remove(ACTIVE_BUNDLE_KEYS[source])
|
||||
ui_state.params.remove("ModelManager_ActiveBundle")
|
||||
elif selected_bundle := next((bundle for bundle in self.model_manager.availableBundles if bundle.ref == selected_ref), None):
|
||||
ui_state.params.put("ModelManager_DownloadRef", selected_bundle.ref)
|
||||
ui_state.params.put("ModelManager_DownloadIndex", selected_bundle.index)
|
||||
self.model_dialog = None
|
||||
|
||||
@staticmethod
|
||||
@@ -230,7 +223,7 @@ class ModelsLayout(Widget):
|
||||
advanced_controls: bool = ui_state.params.get_bool("ShowAdvancedControls")
|
||||
turn_desire: bool = ui_state.params.get_bool("LaneTurnDesire")
|
||||
live_delay: bool = ui_state.params.get_bool("LagdToggle")
|
||||
camera_offset: bool = ui_state.active_bundle is not None
|
||||
camera_offset: bool = ui_state.params.get("ModelManager_ActiveBundle") is not None
|
||||
|
||||
self.lane_turn_desire_toggle.action_item.set_state(turn_desire)
|
||||
self.lane_turn_value_control.set_visible(turn_desire and advanced_controls)
|
||||
@@ -244,9 +237,8 @@ class ModelsLayout(Widget):
|
||||
self._update_lagd_description(live_delay)
|
||||
self.model_manager = ui_state.sm["modelManagerSP"]
|
||||
self._handle_bundle_download_progress()
|
||||
# read the slot through ui_state, not modelManagerSP: the manager republishes a
|
||||
# tick after a chestnut change, and the stale bundle flashes the wrong model
|
||||
active_name = (ui_state.active_bundle or {}).get("displayName") or f"{get_default_model()} (Default)"
|
||||
default_label = f"{get_default_model()} (Default)"
|
||||
active_name = self.model_manager.activeBundle.displayName if self.model_manager and self.model_manager.activeBundle.ref else default_label
|
||||
self.current_model_item.action_item.set_value(active_name)
|
||||
|
||||
if not ui_state.is_offroad():
|
||||
|
||||
@@ -4,14 +4,11 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
import math
|
||||
|
||||
import pyray as rl
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state
|
||||
from openpilot.sunnypilot.sunnylink.api import UNREGISTERED_SUNNYLINK_DONGLE_ID
|
||||
from openpilot.system.ui.lib.application import gui_app
|
||||
from openpilot.system.ui.lib.multilang import tr_noop
|
||||
|
||||
|
||||
@@ -21,9 +18,6 @@ METRIC_MARGIN = 30
|
||||
METRIC_START_Y = 300
|
||||
HOME_BTN = rl.Rectangle(60, 860, 180, 180)
|
||||
|
||||
EGPU_ICON_WIDTH = 180
|
||||
EGPU_ICON_HEIGHT = 133
|
||||
|
||||
|
||||
# Color scheme
|
||||
class Colors:
|
||||
@@ -59,10 +53,6 @@ class MetricData:
|
||||
class SidebarSP:
|
||||
def __init__(self):
|
||||
self._sunnylink_status = MetricData(tr_noop("SUNNYLINK"), tr_noop("OFFLINE"), Colors.WARNING)
|
||||
self._egpu_green_img = gui_app.texture("icons_mici/egpu_green.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT)
|
||||
self._egpu_default_img = gui_app.texture("icons_mici/egpu.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT)
|
||||
self._egpu_orange_img = gui_app.texture("icons_mici/egpu_orange.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT)
|
||||
self._egpu_gray_img = gui_app.texture("icons_mici/egpu_gray.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT)
|
||||
|
||||
def _update_sunnylink_status(self):
|
||||
if not ui_state.params.get_bool("SunnylinkEnabled"):
|
||||
@@ -88,29 +78,6 @@ class SidebarSP:
|
||||
|
||||
self._sunnylink_status.update(tr_noop("SUNNYLINK"), status, color)
|
||||
|
||||
def _get_home_icon(self, default_img: rl.Texture) -> tuple[rl.Texture, rl.Vector2, float]:
|
||||
default_pos = rl.Vector2(HOME_BTN.x, HOME_BTN.y)
|
||||
if not ui_state.sm["deviceState"].chestnutPresent:
|
||||
return default_img, default_pos, 1.0
|
||||
|
||||
big_model_selected = ui_state.usbgpu_compiled or ui_state.model_runner_tinygrad
|
||||
big_model_failed = ui_state.started and ui_state.big_model_failed
|
||||
loading = ui_state.usbgpu_loading or (big_model_selected and ui_state.started and ui_state.usbgpu_active is None)
|
||||
|
||||
if loading:
|
||||
icon = self._egpu_default_img
|
||||
opacity = 0.35 + 0.65 * (0.5 - 0.5 * math.cos(rl.get_time() * 6.0))
|
||||
elif big_model_selected and big_model_failed:
|
||||
icon, opacity = self._egpu_orange_img, 1.0
|
||||
elif big_model_selected:
|
||||
icon, opacity = self._egpu_green_img, 1.0
|
||||
else:
|
||||
icon, opacity = self._egpu_gray_img, 1.0
|
||||
|
||||
x = HOME_BTN.x + (HOME_BTN.width - icon.width) / 2
|
||||
y = HOME_BTN.y + (HOME_BTN.height - icon.height) / 2
|
||||
return icon, rl.Vector2(x, y), opacity
|
||||
|
||||
def _draw_metrics_w_sunnylink(self, rect: rl.Rectangle, _temp, _panda, _connect):
|
||||
metrics = [_temp, _panda, _connect, self._sunnylink_status]
|
||||
start_y = int(rect.y) + METRIC_START_Y
|
||||
|
||||
@@ -4,14 +4,8 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
import math
|
||||
|
||||
import pyray as rl
|
||||
|
||||
from openpilot.selfdrive.ui.mici.layouts.home import MiciHomeLayout
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state
|
||||
from openpilot.system.ui.lib.application import FontWeight
|
||||
from openpilot.system.ui.widgets.icon_widget import IconWidget
|
||||
from openpilot.system.ui.widgets.label import UnifiedLabel
|
||||
|
||||
|
||||
@@ -19,35 +13,3 @@ class MiciHomeLayoutSP(MiciHomeLayout):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._openpilot_label = UnifiedLabel("sunnypilot", font_size=88, font_weight=FontWeight.AUDIOWIDE, max_width=480, wrap_text=False)
|
||||
self._egpu_icon_default = IconWidget("icons_mici/egpu.png", (50, 37))
|
||||
self._egpu_icon_default.set_visible(False)
|
||||
self._egpu_icon_orange = IconWidget("icons_mici/egpu_orange.png", (50, 37))
|
||||
self._egpu_icon_orange.set_visible(False)
|
||||
gray_idx = self._status_bar_layout.widgets.index(self._egpu_icon_gray)
|
||||
self._status_bar_layout.widgets.insert(gray_idx + 1, self._egpu_icon_default)
|
||||
self._status_bar_layout.widgets.insert(gray_idx + 2, self._egpu_icon_orange)
|
||||
|
||||
def _set_egpu_visibility(self):
|
||||
chestnut = ui_state.sm["deviceState"].chestnutPresent
|
||||
if not chestnut:
|
||||
self._egpu_icon.set_visible(False)
|
||||
self._egpu_icon_default.set_visible(False)
|
||||
self._egpu_icon_orange.set_visible(False)
|
||||
self._egpu_icon_gray.set_visible(False)
|
||||
return
|
||||
|
||||
big_model_selected = ui_state.usbgpu_compiled or ui_state.model_runner_tinygrad
|
||||
big_model_failed = ui_state.started and ui_state.big_model_failed
|
||||
loading = ui_state.usbgpu_loading or (big_model_selected and ui_state.started and ui_state.usbgpu_active is None)
|
||||
|
||||
if loading:
|
||||
self._egpu_icon_default._opacity = 0.35 + 0.65 * (0.5 - 0.5 * math.cos(rl.get_time() * 6.0))
|
||||
self._egpu_icon_default.set_visible(True)
|
||||
self._egpu_icon.set_visible(False)
|
||||
self._egpu_icon_orange.set_visible(False)
|
||||
self._egpu_icon_gray.set_visible(False)
|
||||
else:
|
||||
self._egpu_icon_default.set_visible(False)
|
||||
self._egpu_icon.set_visible(big_model_selected and not big_model_failed)
|
||||
self._egpu_icon_orange.set_visible(big_model_selected and big_model_failed)
|
||||
self._egpu_icon_gray.set_visible(not big_model_selected)
|
||||
|
||||
@@ -8,8 +8,6 @@ import pyray as rl
|
||||
|
||||
from openpilot.cereal import custom
|
||||
from openpilot.sunnypilot.models.default_model import get_default_model
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher
|
||||
from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS
|
||||
from openpilot.selfdrive.ui.mici.widgets.button import BigButton
|
||||
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.models import ModelsLayout
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state, device
|
||||
@@ -62,7 +60,7 @@ class ModelsLayoutMici(NavScroller):
|
||||
self.select_model_btn.set_click_callback(self._show_folders)
|
||||
|
||||
self.cancel_download_btn = BigButton(tr("cancel download"))
|
||||
self.cancel_download_btn.set_click_callback(lambda: ui_state.params.remove("ModelManager_DownloadRef"))
|
||||
self.cancel_download_btn.set_click_callback(lambda: ui_state.params.remove("ModelManager_DownloadIndex"))
|
||||
|
||||
self.main_items = [self.current_model_info, self.select_model_btn, self.cancel_download_btn]
|
||||
self._scroller.add_widgets(self.main_items)
|
||||
@@ -115,12 +113,11 @@ class ModelsLayoutMici(NavScroller):
|
||||
gui_app.pop_widgets_to(self)
|
||||
|
||||
def _select_model(self, bundle):
|
||||
ui_state.params.put("ModelManager_DownloadRef", bundle.ref)
|
||||
ui_state.params.put("ModelManager_DownloadIndex", bundle.index)
|
||||
self._pop_to_main()
|
||||
|
||||
def _select_default(self):
|
||||
source = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent)
|
||||
ui_state.params.remove(ACTIVE_BUNDLE_KEYS[source])
|
||||
ui_state.params.remove("ModelManager_ActiveBundle")
|
||||
self._pop_to_main()
|
||||
|
||||
def _select_folder(self, folder_name):
|
||||
@@ -165,9 +162,8 @@ class ModelsLayoutMici(NavScroller):
|
||||
self._was_downloading = is_downloading
|
||||
|
||||
self.current_model_info.current_model_header.set_text(tr("active model"))
|
||||
# read the slot through ui_state, not modelManagerSP: the manager republishes a
|
||||
# tick after a chestnut change, and the stale bundle flashes the wrong model
|
||||
model_text = ((ui_state.active_bundle or {}).get("displayName") or f"{get_default_model()} (Default)").lower()
|
||||
default_model_text = f"{get_default_model()} (Default)".lower()
|
||||
model_text = manager.activeBundle.displayName.lower() if manager.activeBundle.ref else default_model_text
|
||||
self.current_model_info.current_model_text.set_text(model_text)
|
||||
self.current_model_info.info_header.set_text(tr("cache size"))
|
||||
self.current_model_info.info_text.set_text(f"{ModelsLayout.calculate_cache_size():.2f} MB")
|
||||
|
||||
@@ -7,7 +7,6 @@ See the LICENSE.md file in the root directory for more details.
|
||||
import pyray as rl
|
||||
|
||||
from openpilot.selfdrive.ui.mici.onroad.hud_renderer import HudRenderer
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state
|
||||
from openpilot.selfdrive.ui.sunnypilot.onroad.blind_spot_indicators import BlindSpotIndicators
|
||||
|
||||
|
||||
@@ -22,8 +21,6 @@ class HudRendererSP(HudRenderer):
|
||||
|
||||
def _render(self, rect: rl.Rectangle) -> None:
|
||||
super()._render(rect)
|
||||
if ui_state.usbgpu and not ui_state.usbgpu_compiled and ui_state.model_runner_tinygrad:
|
||||
self._draw_model_source(rect)
|
||||
self.blind_spot_indicators.render(rect)
|
||||
|
||||
def _has_blind_spot_detected(self) -> bool:
|
||||
|
||||
@@ -10,7 +10,6 @@ from openpilot.cereal import messaging, log, custom
|
||||
from opendbc.car.structs import car
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.display import OnroadBrightness
|
||||
from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_active_source
|
||||
from openpilot.sunnypilot.sunnylink.sunnylink_state import SunnylinkState
|
||||
from openpilot.system.ui.lib.application import gui_app
|
||||
from openpilot.system.ui.sunnypilot.widgets.screen_saver import ScreenSaverSP
|
||||
@@ -44,7 +43,6 @@ class UIStateSP:
|
||||
self.screensaver_enabled: bool = False
|
||||
|
||||
self.active_bundle = None
|
||||
self.model_runner_tinygrad: bool = False
|
||||
self.blindspot: bool = False
|
||||
self.chevron_metrics = None
|
||||
self.custom_interactive_timeout: int = 0
|
||||
@@ -152,10 +150,7 @@ class UIStateSP:
|
||||
self.has_icbm = self.CP_SP.intelligentCruiseButtonManagementAvailable and self.params.get_bool("IntelligentCruiseButtonManagement")
|
||||
|
||||
self._enforce_constraints()
|
||||
source = get_active_source(usbgpu=self.usbgpu, usbgpu_active=self.usbgpu_active,
|
||||
usbgpu_loading=self.usbgpu_loading, offroad=self.is_offroad())
|
||||
self.active_bundle = self.params.get(ACTIVE_BUNDLE_KEYS[source])
|
||||
self.model_runner_tinygrad = self.active_bundle is not None and self.active_bundle.get("runner") == "tinygrad"
|
||||
self.active_bundle = self.params.get("ModelManager_ActiveBundle")
|
||||
self.blindspot = self.params.get_bool("BlindSpot")
|
||||
self.chevron_metrics = self.params.get("ChevronInfo")
|
||||
self.custom_interactive_timeout = self.params.get("InteractivityTimeout", return_default=True)
|
||||
|
||||
@@ -112,15 +112,6 @@ class UIState(UIStateSP):
|
||||
def add_on_body_changed_callbacks(self, callback: Callable[[], None]):
|
||||
self._on_body_changed_callbacks.append(callback)
|
||||
|
||||
@property
|
||||
def big_model_failed(self) -> bool:
|
||||
# Mirrors the onroad HUD's four-condition check so sidebar and home icons reflect the same failure states
|
||||
return (self.usbgpu_active is False or
|
||||
not self.sm['deviceState'].chestnutPresent or
|
||||
(self.usbgpu_active is True and self.sm.recv_frame['modelV2'] > self.started_frame and
|
||||
not self.sm.alive['modelV2']) or
|
||||
(self.usbgpu_active is None and self.sm.recv_frame['modelV2'] > self.started_frame))
|
||||
|
||||
@property
|
||||
def engaged(self) -> bool:
|
||||
return self.started and (self.sm["selfdriveState"].enabled or self.sm["selfdriveStateSP"].mads.enabled)
|
||||
|
||||
@@ -7,7 +7,6 @@ See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
@@ -67,15 +66,14 @@ def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tu
|
||||
if desire_key:
|
||||
shapes['desire'] = (input_shapes[desire_key][2],)
|
||||
|
||||
if is_supercombo and 'features_buffer' in input_shapes:
|
||||
fb = input_shapes['features_buffer']
|
||||
shapes['prev_feat'] = (fb[0], fb[2])
|
||||
|
||||
for key, shape in input_shapes.items():
|
||||
if key not in (desire_key, 'features_buffer') and 'img' not in key:
|
||||
shapes[key] = tuple(shape)
|
||||
|
||||
if is_supercombo and 'features_buffer' in input_shapes:
|
||||
fb = input_shapes['features_buffer']
|
||||
feat_dim = math.prod(fb[2:])
|
||||
shapes['prev_feat'] = (fb[0], feat_dim)
|
||||
|
||||
sizes = [int(np.prod(size)) for size in shapes.values()]
|
||||
return shapes, sizes
|
||||
|
||||
@@ -119,9 +117,8 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D
|
||||
}
|
||||
|
||||
if features_buffer:
|
||||
feat_dim = math.prod(features_buffer[2:])
|
||||
feat_q_len = frame_skip * features_buffer[1] if is_supercombo else frame_skip * (features_buffer[1] - 1) + 1
|
||||
queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], feat_dim),
|
||||
queues['feat_q'] = Tensor(np.zeros((feat_q_len, features_buffer[0], features_buffer[2]),
|
||||
dtype=np.float32), device=device).contiguous().realize()
|
||||
|
||||
queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')})
|
||||
@@ -186,14 +183,14 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
|
||||
warped_dev = warped.to(Device.DEFAULT)
|
||||
Tensor.realize(packed_npy_inputs_dev, warped_dev)
|
||||
|
||||
img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn)
|
||||
big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn)
|
||||
img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn).realize()
|
||||
big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn).realize()
|
||||
|
||||
unpacked_tensors = [tensor.reshape(shape) for tensor, shape in zip(packed_npy_inputs_dev.split(npy_sizes), npy_shapes.values(), strict=True)]
|
||||
unpacked_dict = dict(zip(npy_shapes.keys(), unpacked_tensors, strict=True))
|
||||
|
||||
desire_dev = unpacked_dict['desire']
|
||||
desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn)
|
||||
desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn).realize()
|
||||
|
||||
inputs = {desire_key: desire_buf}
|
||||
for key, tensor_val in unpacked_dict.items():
|
||||
@@ -202,7 +199,7 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
|
||||
|
||||
if 'prev_feat' in unpacked_dict:
|
||||
prev_feat_dev = unpacked_dict['prev_feat']
|
||||
inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).reshape(input_shapes['features_buffer'])
|
||||
inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).realize()
|
||||
|
||||
if vision_runner:
|
||||
vision_out_cast = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32').realize()
|
||||
@@ -214,7 +211,7 @@ def make_run_policy(vision_runner, policy_runners: list, features_slice: slice,
|
||||
|
||||
inputs.update({road_key: img, wide_key: big_img})
|
||||
if 'features_buffer' not in inputs:
|
||||
inputs['features_buffer'] = sample_skip_fn(feat_q).reshape(input_shapes['features_buffer'])
|
||||
inputs['features_buffer'] = sample_skip_fn(feat_q)
|
||||
|
||||
policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32').realize()
|
||||
if 'features_buffer' not in inputs and features_slice is not None:
|
||||
|
||||
@@ -91,7 +91,7 @@ class ModelState(ModelStateBase):
|
||||
if env_pkl and os.path.exists(env_pkl):
|
||||
model_bundle = None
|
||||
else:
|
||||
model_bundle = get_active_bundle(usbgpu=usbgpu)
|
||||
model_bundle = get_active_bundle()
|
||||
self.generation = model_bundle.generation if model_bundle is not None else None
|
||||
overrides = {override.key: override.value for override in model_bundle.overrides} if model_bundle else {}
|
||||
|
||||
|
||||
@@ -190,8 +190,8 @@ def tmp_path():
|
||||
|
||||
def patch_modeld(monkeypatch):
|
||||
def _patch(bundle):
|
||||
monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle)
|
||||
monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle)
|
||||
monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None: bundle)
|
||||
monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None: bundle)
|
||||
|
||||
return _patch
|
||||
|
||||
|
||||
@@ -59,8 +59,8 @@ class TestFindDrivingPkl(OpenpilotTestCase):
|
||||
class TestModelStateCombinedInit(OpenpilotTestCase):
|
||||
def test_asserts_when_no_pkl(self, monkeypatch):
|
||||
bundle = DummyBundle(models=[], is_20hz=True)
|
||||
monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle)
|
||||
monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle)
|
||||
monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None: bundle)
|
||||
monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None: bundle)
|
||||
with self.assertRaisesRegex(AssertionError, "No driving pkl found"):
|
||||
ModelState(cam_w=CAM_W, cam_h=CAM_H)
|
||||
|
||||
|
||||
@@ -195,85 +195,3 @@ class TestReadFileChunkedToDisk(OpenpilotTestCase):
|
||||
|
||||
assert out.parent == Path(d)
|
||||
assert out.read_bytes() == payload
|
||||
|
||||
|
||||
class Test4DFeaturesBuffer(OpenpilotTestCase):
|
||||
def test_get_policy_npy_shapes_4d(self):
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes
|
||||
input_shapes = {
|
||||
'desire_pulse': (1, 25, 8),
|
||||
'features_buffer': (1, 24, 32, 512), # compare 4d to 3d for regression
|
||||
'traffic_convention': (1, 2),
|
||||
'action_t': (1, 2)
|
||||
}
|
||||
shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True)
|
||||
assert shapes['prev_feat'] == (1, 16384)
|
||||
assert sizes == [8, 2, 2, 16384]
|
||||
|
||||
def test_get_policy_npy_shapes_3d(self):
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes
|
||||
input_shapes = {
|
||||
'desire_pulse': (1, 25, 8),
|
||||
'features_buffer': (1, 24, 512),
|
||||
'traffic_convention': (1, 2),
|
||||
'action_t': (1, 2)
|
||||
}
|
||||
shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=True)
|
||||
assert shapes['prev_feat'] == (1, 512)
|
||||
assert sizes == [8, 2, 2, 512]
|
||||
|
||||
|
||||
class TestStockCompileModeldEquivalence(OpenpilotTestCase):
|
||||
def test_get_policy_npy_shapes_matches_stock(self):
|
||||
from openpilot.selfdrive.modeld.compile_modeld import get_policy_npy_shapes as stock_get_policy_npy_shapes
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import get_policy_npy_shapes as sunny_get_policy_npy_shapes
|
||||
|
||||
stock_input_shapes = {
|
||||
'desire_pulse': (1, 25, 8),
|
||||
'features_buffer': (1, 24, 512), # see below comment
|
||||
'traffic_convention': (1, 2),
|
||||
'action_t': (1, 2),
|
||||
}
|
||||
|
||||
stock_shapes, stock_sizes = stock_get_policy_npy_shapes(stock_input_shapes)
|
||||
sunny_shapes, sunny_sizes = sunny_get_policy_npy_shapes(stock_input_shapes, is_supercombo=True)
|
||||
|
||||
assert sunny_shapes == stock_shapes
|
||||
assert sunny_sizes == stock_sizes
|
||||
assert sunny_shapes['prev_feat'] == (1, 512)
|
||||
|
||||
def test_make_input_queues_full_stock_equivalence(self):
|
||||
from openpilot.selfdrive.modeld.compile_modeld import make_input_queues as stock_make_input_queues
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues as sunny_make_supercombo_input_queues
|
||||
input_shapes = {
|
||||
'img': (1, 12, 128, 256),
|
||||
'desire_pulse': (1, 25, 8),
|
||||
'features_buffer': (1, 24, 512), # when https://github.com/commaai/openpilot/pull/38681 merges, update to 1,24,32,512
|
||||
'traffic_convention': (1, 2),
|
||||
'action_t': (1, 2),
|
||||
}
|
||||
frame_skip = 4
|
||||
|
||||
stock_queues, stock_npy = stock_make_input_queues(input_shapes, frame_skip, device='NPY')
|
||||
sunny_queues, sunny_npy = sunny_make_supercombo_input_queues(input_shapes, frame_skip, device='NPY')
|
||||
assert set(sunny_queues.keys()) == set(stock_queues.keys())
|
||||
for key in stock_queues:
|
||||
assert sunny_queues[key].shape == stock_queues[key].shape, \
|
||||
f"Queue shape mismatch for {key}: sunny {sunny_queues[key].shape} != stock {stock_queues[key].shape}"
|
||||
assert set(sunny_npy.keys()) == set(stock_npy.keys())
|
||||
for key in stock_npy:
|
||||
assert sunny_npy[key].shape == stock_npy[key].shape, \
|
||||
f"Numpy array shape mismatch for {key}: sunny {sunny_npy[key].shape} != stock {stock_npy[key].shape}"
|
||||
|
||||
def test_make_warp_queues_stock_equivalence(self):
|
||||
from openpilot.selfdrive.modeld.compile_modeld import make_warp_input_queues as stock_make_warp_queues
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import make_warp_queues as sunny_make_warp_queues
|
||||
stock_vision_shapes = {'img': (1, 12, 128, 256)} # for now?
|
||||
stock_queues, stock_npy = stock_make_warp_queues(stock_vision_shapes, frame_skip=4, device='NPY')
|
||||
sunny_queues, sunny_npy = sunny_make_warp_queues(device='NPY')
|
||||
|
||||
assert set(sunny_npy.keys()) == set(stock_npy.keys()) == {'tfm', 'big_tfm'}
|
||||
for key in sunny_npy:
|
||||
assert sunny_npy[key].shape == stock_npy[key].shape == (3, 3)
|
||||
|
||||
|
||||
|
||||
@@ -13,6 +13,8 @@ from openpilot.common.params import Params
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.common.hardware.hw import Paths
|
||||
from openpilot.sunnypilot.models.helpers import is_bundle_version_compatible
|
||||
from openpilot.selfdrive.modeld.helpers import usbgpu_present
|
||||
|
||||
from openpilot.cereal import custom
|
||||
|
||||
|
||||
@@ -138,53 +140,45 @@ class ModelCache:
|
||||
|
||||
class ModelFetcher:
|
||||
"""Handles fetching and caching of model data from remote source"""
|
||||
MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v21.json"
|
||||
MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v22.json"
|
||||
|
||||
MODEL_SOURCES = {
|
||||
"qcom": (MODEL_URL, ""),
|
||||
"usbgpu": (MODEL_URL_USBGPU, "_USBGPU"),
|
||||
}
|
||||
MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v20.json"
|
||||
MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v21.json"
|
||||
|
||||
def __init__(self, params: Params):
|
||||
self.params = params
|
||||
self.model_parser = ModelParser()
|
||||
self.model_caches = {
|
||||
source: ModelCache(params, suffix=suffix)
|
||||
for source, (_, suffix) in self.MODEL_SOURCES.items()
|
||||
}
|
||||
self._refetched: set[str] = set()
|
||||
self.params.put("ModelManager_ActiveJson", {
|
||||
"qcom": self.MODEL_URL,
|
||||
"usbgpu": self.MODEL_URL_USBGPU,
|
||||
}, block=True)
|
||||
self._is_usbgpu: bool | None = None
|
||||
self.model_cache = ModelCache(params)
|
||||
self.model_url = self.MODEL_URL
|
||||
self._update_model_source()
|
||||
|
||||
@staticmethod
|
||||
def active_source(chestnut_present: bool) -> str:
|
||||
return "usbgpu" if chestnut_present else "qcom"
|
||||
def _update_model_source(self) -> None:
|
||||
"""Updates what json to use based on usbgpu availability"""
|
||||
is_usbgpu = usbgpu_present()
|
||||
if is_usbgpu != self._is_usbgpu:
|
||||
self._is_usbgpu = is_usbgpu
|
||||
self.model_cache = ModelCache(self.params, suffix="_USBGPU" if is_usbgpu else "")
|
||||
self.model_url = self.MODEL_URL_USBGPU if is_usbgpu else self.MODEL_URL
|
||||
self.params.put("ModelManager_ActiveJson", self.model_url, block=True)
|
||||
|
||||
def _fetch_and_cache_models(self, source: str) -> list[custom.ModelManagerSP.ModelBundle] | None:
|
||||
def _fetch_and_cache_models(self) -> list[custom.ModelManagerSP.ModelBundle] | None:
|
||||
"""Fetches fresh model data from remote and updates cache.
|
||||
Returns None on transport errors. Raises on 404 and other fatal HTTP errors.
|
||||
"""
|
||||
model_url, _ = self.MODEL_SOURCES[source]
|
||||
try:
|
||||
response = requests.get(model_url, timeout=10)
|
||||
response = requests.get(self.model_url, timeout=10)
|
||||
|
||||
# Explicitly handle 404 differently
|
||||
if response.status_code == 404:
|
||||
cloudlog.error(f"Models URL returned 404 Not Found: {model_url}")
|
||||
raise HTTPError(f"404 Not Found: {model_url}", response=response)
|
||||
cloudlog.error(f"Models URL returned 404 Not Found: {self.model_url}")
|
||||
raise HTTPError(f"404 Not Found: {self.model_url}", response=response)
|
||||
|
||||
# Raise for any other 4xx/5xx
|
||||
response.raise_for_status()
|
||||
|
||||
json_data = response.json()
|
||||
parsed = self.model_parser.parse_models(json_data)
|
||||
if parsed:
|
||||
self.model_caches[source].set(json_data)
|
||||
cloudlog.debug(f"Successfully updated models cache for {source}")
|
||||
return parsed
|
||||
self.model_cache.set(json_data)
|
||||
cloudlog.debug("Successfully updated models cache")
|
||||
return self.model_parser.parse_models(json_data)
|
||||
|
||||
except ConnectionError as e:
|
||||
cloudlog.warning(f"DNS/connection error while fetching models: {e}")
|
||||
@@ -197,40 +191,16 @@ class ModelFetcher:
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _cache_matches_source(source: str, cached_data: dict) -> bool:
|
||||
bundles = cached_data.get("bundles", [])
|
||||
if source == "usbgpu":
|
||||
return any(bundle.get("is_big") is True for bundle in bundles)
|
||||
return not any(bundle.get("is_big") is True for bundle in bundles)
|
||||
|
||||
def get_bundles_for_source(self, source: str) -> list[custom.ModelManagerSP.ModelBundle]:
|
||||
if source not in self.MODEL_SOURCES:
|
||||
cloudlog.warning(f"Unknown model source: {source}")
|
||||
return []
|
||||
|
||||
cached_data, is_expired = self.model_caches[source].get()
|
||||
def get_available_bundles(self) -> list[custom.ModelManagerSP.ModelBundle]:
|
||||
"""Gets the list of available models, with smart cache handling"""
|
||||
self._update_model_source()
|
||||
cached_data, is_expired = self.model_cache.get()
|
||||
|
||||
if cached_data and not is_expired:
|
||||
# a source is refetched over a mismatch at most once per process: if the fresh
|
||||
# manifest still mismatches, the URL is authoritative and the cache is trusted
|
||||
if self._cache_matches_source(source, cached_data) or source in self._refetched:
|
||||
try:
|
||||
parsed = self.model_parser.parse_models(cached_data)
|
||||
except Exception:
|
||||
cloudlog.warning(f"Failed to parse cached models for {source}; refetching", exc_info=True)
|
||||
else:
|
||||
if parsed:
|
||||
cloudlog.debug(f"Using valid cached models data for source {source}")
|
||||
return parsed
|
||||
# a source-matching cache that yields no valid bundles is stale (e.g. an old
|
||||
# manifest version) - do not trust it, refetch so the source is repopulated
|
||||
cloudlog.warning(f"Cached models for {source} have no valid bundles; refetching")
|
||||
else:
|
||||
self._refetched.add(source)
|
||||
cloudlog.warning(f"Cached models for {source} not valid; refetching once")
|
||||
cloudlog.debug("Using valid cached models data")
|
||||
return self.model_parser.parse_models(cached_data)
|
||||
|
||||
fetched_bundles = self._fetch_and_cache_models(source)
|
||||
fetched_bundles = self._fetch_and_cache_models()
|
||||
if fetched_bundles is not None:
|
||||
return fetched_bundles
|
||||
|
||||
@@ -238,33 +208,12 @@ class ModelFetcher:
|
||||
cloudlog.warning("Failed to fetch fresh data and no cache available")
|
||||
|
||||
cloudlog.warning("Failed to fetch fresh data. Using expired cache as fallback")
|
||||
try:
|
||||
return self.model_parser.parse_models(cached_data)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def get_cached_bundles(params: Params, source: str) -> list[custom.ModelManagerSP.ModelBundle]:
|
||||
|
||||
if source not in ModelFetcher.MODEL_SOURCES:
|
||||
cloudlog.warning(f"Unknown model source: {source}")
|
||||
return []
|
||||
_, suffix = ModelFetcher.MODEL_SOURCES[source]
|
||||
cached_data = params.get(f"ModelManager_ModelsCache{suffix}")
|
||||
if not cached_data:
|
||||
return []
|
||||
try:
|
||||
return ModelParser.parse_models(cached_data)
|
||||
except Exception as e:
|
||||
cloudlog.warning(f"Failed to parse cached models for source {source}: {e}")
|
||||
return []
|
||||
|
||||
return self.model_parser.parse_models(cached_data)
|
||||
|
||||
if __name__ == "__main__":
|
||||
from openpilot.selfdrive.modeld.helpers import usbgpu_present
|
||||
params = Params()
|
||||
model_fetcher = ModelFetcher(params)
|
||||
bundles = model_fetcher.get_bundles_for_source(ModelFetcher.active_source(usbgpu_present()))
|
||||
bundles = model_fetcher.get_available_bundles()
|
||||
for bundle in bundles:
|
||||
for model in bundle.models:
|
||||
model_overrides = {override.key: override.value for override in bundle.overrides}
|
||||
|
||||
@@ -16,20 +16,14 @@ from openpilot.common.params import Params
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRaider
|
||||
from openpilot.common.hardware.hw import Paths
|
||||
from openpilot.selfdrive.modeld.helpers import usbgpu_present
|
||||
|
||||
# SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO
|
||||
REQUIRED_JSON_VERSION = 18
|
||||
REQUIRED_JSON_VERSION = 17
|
||||
|
||||
CUSTOM_MODEL_PATH = Paths.model_root()
|
||||
METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl'
|
||||
ModelManager = custom.ModelManagerSP
|
||||
|
||||
ACTIVE_BUNDLE_KEYS = {
|
||||
"qcom": "ModelManager_ActiveBundle",
|
||||
"usbgpu": "ModelManager_ActiveBundleUSBGPU",
|
||||
}
|
||||
_LAST_VALIDATED_RAW: dict[str, dict | None] = {}
|
||||
_LAST_VALIDATED_RAW = None
|
||||
|
||||
|
||||
def _compute_hash(file_path: str) -> str | None:
|
||||
@@ -92,11 +86,11 @@ def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, availa
|
||||
if available_bundles is not None:
|
||||
matching_bundle = None
|
||||
for bundle in available_bundles:
|
||||
if active_bundle.ref and bundle.ref:
|
||||
if getattr(active_bundle, 'ref', None) and getattr(bundle, 'ref', None):
|
||||
if active_bundle.ref == bundle.ref:
|
||||
matching_bundle = bundle
|
||||
break
|
||||
elif active_bundle.internalName == bundle.internalName:
|
||||
elif getattr(active_bundle, 'internalName', None) == getattr(bundle, 'internalName', None):
|
||||
matching_bundle = bundle
|
||||
break
|
||||
|
||||
@@ -104,79 +98,47 @@ def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, availa
|
||||
return True
|
||||
if active_bundle.minimumSelectorVersion != matching_bundle.minimumSelectorVersion:
|
||||
return True
|
||||
if active_bundle.runner != matching_bundle.runner:
|
||||
return True
|
||||
|
||||
active_runner = getattr(active_bundle, 'runner', None)
|
||||
matching_runner = getattr(matching_bundle, 'runner', None)
|
||||
if active_runner is not None and matching_runner is not None:
|
||||
if getattr(active_runner, 'raw', active_runner) != getattr(matching_runner, 'raw', matching_runner):
|
||||
return True
|
||||
if set(_bundle_artifacts(active_bundle)) != set(_bundle_artifacts(matching_bundle)):
|
||||
return True
|
||||
|
||||
return not _bundle_is_valid_locally(active_bundle)
|
||||
|
||||
|
||||
def _parse_active_bundle(raw_bundle) -> "custom.ModelManagerSP.ModelBundle | None":
|
||||
try:
|
||||
if isinstance(raw_bundle, dict) and raw_bundle and is_bundle_version_compatible(raw_bundle):
|
||||
return custom.ModelManagerSP.ModelBundle(**raw_bundle)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_selected_bundle(params: Params | None = None, source: str = "qcom") -> "custom.ModelManagerSP.ModelBundle | None":
|
||||
params = params or Params()
|
||||
return _parse_active_bundle(params.get(ACTIVE_BUNDLE_KEYS[source]))
|
||||
|
||||
|
||||
def get_active_source(usbgpu: bool | None = None, usbgpu_active: bool | None = None,
|
||||
usbgpu_loading: bool | None = None, offroad: bool | None = None) -> str:
|
||||
if usbgpu is None:
|
||||
usbgpu = usbgpu_present()
|
||||
state_valid = usbgpu_active is not None or usbgpu_loading is not None or offroad is not None
|
||||
big_active = usbgpu and (not state_valid or usbgpu_active or usbgpu_loading or offroad)
|
||||
return "usbgpu" if big_active else "qcom"
|
||||
|
||||
|
||||
def get_active_bundle(params: Params | None = None, *, usbgpu: bool | None = None) -> "custom.ModelManagerSP.ModelBundle | None":
|
||||
# no cross-slot fallback: an empty active slot means the hardware default, which
|
||||
# only stock modeld can run - modeld_v2 requires a real bundle
|
||||
params = params or Params()
|
||||
return get_selected_bundle(params, get_active_source(usbgpu=usbgpu))
|
||||
|
||||
|
||||
def resolve_bundle_by_ref(
|
||||
ref: str, source_bundles: dict[str, list[custom.ModelManagerSP.ModelBundle]],
|
||||
) -> "tuple[custom.ModelManagerSP.ModelBundle, str] | None":
|
||||
for source, bundles in source_bundles.items():
|
||||
for bundle in bundles:
|
||||
if bundle.ref == ref:
|
||||
return bundle, source
|
||||
return None
|
||||
|
||||
|
||||
def _validate_active_bundle(params: Params, source: str, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None) -> None:
|
||||
def validate_active_bundle(params: Params, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None) -> None:
|
||||
global _LAST_VALIDATED_RAW
|
||||
|
||||
key = ACTIVE_BUNDLE_KEYS[source]
|
||||
raw_bundle = params.get(key)
|
||||
raw_bundle = params.get("ModelManager_ActiveBundle")
|
||||
if not raw_bundle:
|
||||
return
|
||||
|
||||
if _LAST_VALIDATED_RAW.get(key) == raw_bundle:
|
||||
if raw_bundle == _LAST_VALIDATED_RAW:
|
||||
return
|
||||
|
||||
active_bundle = _parse_active_bundle(raw_bundle)
|
||||
active_bundle = get_active_bundle(params, raw_bundle_dict=raw_bundle)
|
||||
if active_bundle is None or _bundle_needs_reset(active_bundle, available_bundles):
|
||||
cloudlog.warning(f"Active model bundle invalid for {source}; resetting to default")
|
||||
params.remove(key)
|
||||
_LAST_VALIDATED_RAW[key] = None
|
||||
cloudlog.warning("Active model bundle invalid; resetting to default")
|
||||
params.remove("ModelManager_ActiveBundle")
|
||||
params.put("ModelRunnerTypeCache", int(custom.ModelManagerSP.Runner.stock), block=True)
|
||||
_LAST_VALIDATED_RAW = None
|
||||
else:
|
||||
_LAST_VALIDATED_RAW[key] = raw_bundle
|
||||
_LAST_VALIDATED_RAW = raw_bundle
|
||||
|
||||
|
||||
def validate_active_bundles(params: Params, source_bundles: dict[str, list[custom.ModelManagerSP.ModelBundle]]) -> None:
|
||||
# an empty list means the fetch failed, not that the catalog dropped the bundle
|
||||
for source, bundles in source_bundles.items():
|
||||
_validate_active_bundle(params, source, bundles or None)
|
||||
get_active_model_runner(params, force_check=True)
|
||||
def get_active_bundle(params: Params | None = None, raw_bundle_dict: dict | bytes | None = None) -> "custom.ModelManagerSP.ModelBundle | None":
|
||||
params = params or Params()
|
||||
try:
|
||||
active_bundle_dict = raw_bundle_dict if raw_bundle_dict is not None else (params.get("ModelManager_ActiveBundle") or {})
|
||||
if isinstance(active_bundle_dict, dict) and active_bundle_dict and is_bundle_version_compatible(active_bundle_dict):
|
||||
return custom.ModelManagerSP.ModelBundle(**active_bundle_dict)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_active_model_runner(params: Params | None = None, force_check: bool = False) -> int:
|
||||
|
||||
@@ -17,8 +17,7 @@ from openpilot.common.hardware.hw import Paths
|
||||
|
||||
from openpilot.cereal import messaging, custom
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher
|
||||
from openpilot.sunnypilot.models.helpers import (ACTIVE_BUNDLE_KEYS, get_active_bundle, get_selected_bundle,
|
||||
resolve_bundle_by_ref, validate_active_bundles, verify_file)
|
||||
from openpilot.sunnypilot.models.helpers import get_active_bundle, validate_active_bundle, verify_file
|
||||
|
||||
# (connect, read) seconds. read is per-request inactivity, not a total cap
|
||||
DOWNLOAD_TIMEOUT = (30, 30)
|
||||
@@ -31,12 +30,9 @@ class ModelManagerSP:
|
||||
self.params = Params()
|
||||
self.model_fetcher = ModelFetcher(self.params)
|
||||
self.pm = messaging.PubMaster(["modelManagerSP"])
|
||||
self.sm = messaging.SubMaster(["deviceState"])
|
||||
self.chestnut_present = False
|
||||
self.available_models: list[custom.ModelManagerSP.ModelBundle] = []
|
||||
self.source_models: dict[str, list[custom.ModelManagerSP.ModelBundle]] = {}
|
||||
self.selected_bundle: custom.ModelManagerSP.ModelBundle = None
|
||||
self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params, usbgpu=self.chestnut_present)
|
||||
self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params)
|
||||
self._chunk_size = 128 * 1000 # 128 KB chunks
|
||||
self._download_start_times: dict[str, float] = {} # Track start time per model
|
||||
|
||||
@@ -80,7 +76,7 @@ class ModelManagerSP:
|
||||
f.write(chunk)
|
||||
bytes_downloaded += len(chunk)
|
||||
|
||||
if self.params.get("ModelManager_DownloadRef") is None:
|
||||
if self.params.get("ModelManager_DownloadIndex") is None:
|
||||
raise Exception("Download cancelled")
|
||||
|
||||
if total_size > 0:
|
||||
@@ -118,7 +114,7 @@ class ModelManagerSP:
|
||||
for data in response.iter_content(chunk_size=self._chunk_size):
|
||||
f.write(data)
|
||||
chunk_downloaded += len(data)
|
||||
if self.params.get("ModelManager_DownloadRef") is None:
|
||||
if self.params.get("ModelManager_DownloadIndex") is None:
|
||||
raise Exception("Download cancelled")
|
||||
intra = chunk_downloaded / max(chunk_size, 1)
|
||||
progress = min(99.0, ((i + intra) / num_chunks) * 100)
|
||||
@@ -220,7 +216,8 @@ class ModelManagerSP:
|
||||
model_manager_state.availableBundles = self.available_models
|
||||
self.pm.send('modelManagerSP', msg)
|
||||
|
||||
async def _download_bundle(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str, source: str) -> None:
|
||||
async def _download_bundle(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None:
|
||||
"""Downloads all models in a bundle"""
|
||||
self.selected_bundle = model_bundle
|
||||
self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloading
|
||||
for model in self.selected_bundle.models:
|
||||
@@ -242,9 +239,10 @@ class ModelManagerSP:
|
||||
seen_artifacts.add(artifact.fileName)
|
||||
await self._process_artifact(artifact, destination_path)
|
||||
|
||||
self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
self.params.put(ACTIVE_BUNDLE_KEYS[source], model_bundle.to_dict(), block=True)
|
||||
self.active_bundle = get_active_bundle(self.params, usbgpu=self.chestnut_present)
|
||||
self.active_bundle = self.selected_bundle
|
||||
self.active_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
self.params.put("ModelManager_ActiveBundle", self.active_bundle.to_dict(), block=True)
|
||||
self.selected_bundle = None
|
||||
|
||||
except Exception:
|
||||
if self.selected_bundle is not None:
|
||||
@@ -254,9 +252,9 @@ class ModelManagerSP:
|
||||
finally:
|
||||
self._report_status()
|
||||
|
||||
def download(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str, source: str) -> None:
|
||||
def download(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None:
|
||||
"""Main entry point for downloading a model bundle"""
|
||||
asyncio.run(self._download_bundle(model_bundle, destination_path, source))
|
||||
asyncio.run(self._download_bundle(model_bundle, destination_path))
|
||||
|
||||
def main_thread(self) -> None:
|
||||
"""Main thread for model management"""
|
||||
@@ -264,22 +262,20 @@ class ModelManagerSP:
|
||||
|
||||
while True:
|
||||
try:
|
||||
self.sm.update(0)
|
||||
self.chestnut_present = self.sm['deviceState'].chestnutPresent
|
||||
self.source_models = {source: self.model_fetcher.get_bundles_for_source(source) for source in ModelFetcher.MODEL_SOURCES}
|
||||
self.available_models = self.source_models[ModelFetcher.active_source(self.chestnut_present)]
|
||||
validate_active_bundles(self.params, self.source_models)
|
||||
self.active_bundle = get_active_bundle(self.params, usbgpu=self.chestnut_present)
|
||||
self.available_models = self.model_fetcher.get_available_bundles()
|
||||
validate_active_bundle(self.params, self.available_models)
|
||||
self.active_bundle = get_active_bundle(self.params)
|
||||
|
||||
if (ref_to_download := self.params.get("ModelManager_DownloadRef")) is not None:
|
||||
if resolved := resolve_bundle_by_ref(ref_to_download, self.source_models):
|
||||
model_to_download, source = resolved
|
||||
if (index_to_download := self.params.get("ModelManager_DownloadIndex")) is not None:
|
||||
if self.active_bundle and self.active_bundle.index == index_to_download:
|
||||
self.params.remove("ModelManager_DownloadIndex")
|
||||
elif model_to_download := next((model for model in self.available_models if model.index == index_to_download), None):
|
||||
try:
|
||||
self.download(model_to_download, Paths.model_root(), source)
|
||||
self.download(model_to_download, Paths.model_root())
|
||||
except Exception as e:
|
||||
cloudlog.exception(e)
|
||||
finally:
|
||||
self.params.remove("ModelManager_DownloadRef")
|
||||
self.params.remove("ModelManager_DownloadIndex")
|
||||
self.selected_bundle = None
|
||||
|
||||
if self.params.get("ModelManager_ClearCache"):
|
||||
@@ -298,14 +294,12 @@ class ModelManagerSP:
|
||||
Clears the model cache directory of all files except those in the active model bundle.
|
||||
"""
|
||||
|
||||
# Get list of files used by both slots' selected bundles (either may become
|
||||
# the truly active bundle depending on hardware availability)
|
||||
# Get list of files used by active model bundle
|
||||
active_files = []
|
||||
for source in ACTIVE_BUNDLE_KEYS:
|
||||
if selected_bundle := get_selected_bundle(self.params, source):
|
||||
for model in selected_bundle.models:
|
||||
if model.artifact.fileName:
|
||||
active_files.append(model.artifact.fileName)
|
||||
if self.active_bundle is not None: # When the default model is active
|
||||
for model in self.active_bundle.models:
|
||||
if hasattr(model, 'artifact') and model.artifact.fileName:
|
||||
active_files.append(model.artifact.fileName)
|
||||
|
||||
# Remove all files except active ones (including their chunk files)
|
||||
model_dir = Paths.model_root()
|
||||
|
||||
@@ -11,7 +11,6 @@ import http.server
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
@@ -24,10 +23,6 @@ from openpilot.common.test import OpenpilotTestCase
|
||||
from openpilot.common.file_chunker import get_chunk_name, get_manifest_path
|
||||
from openpilot.selfdrive.test.helpers import http_server_context
|
||||
from openpilot.sunnypilot.models import manager as manager_module
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher, get_cached_bundles
|
||||
from openpilot.sunnypilot.models import helpers
|
||||
from openpilot.sunnypilot.models.helpers import (get_active_bundle, get_active_source, get_selected_bundle,
|
||||
resolve_bundle_by_ref, validate_active_bundles)
|
||||
from openpilot.sunnypilot.models.manager import ModelManagerSP
|
||||
|
||||
CHUNK_BODIES = [b'A' * 5000, b'B' * 5000, b'C' * 3000]
|
||||
@@ -108,7 +103,6 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
|
||||
self.manager.selected_bundle = None
|
||||
self.manager.active_bundle = None
|
||||
self.manager.available_models = []
|
||||
self.manager.chestnut_present = False
|
||||
self.manager._chunk_size = 1024
|
||||
self.manager._download_start_times = {}
|
||||
|
||||
@@ -255,85 +249,6 @@ class TestManagerDownload(ManagerDownloadTestBase):
|
||||
assert self.manager._download_start_times == {}
|
||||
self.run_with_server(body)
|
||||
|
||||
def test_download_ref_present_keeps_download_alive(self):
|
||||
"""A pending download request (DownloadRef set) must not be cancelled mid-transfer."""
|
||||
def body():
|
||||
artifact = self.make_artifact(chunked=True)
|
||||
base_path = os.path.join(self.dest, artifact.fileName)
|
||||
self.manager.params.get.side_effect = lambda key: b"ref" if key == "ModelManager_DownloadRef" else None
|
||||
asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact))
|
||||
assert os.path.isfile(get_manifest_path(base_path))
|
||||
self.run_with_server(body)
|
||||
|
||||
def test_cancellation_via_download_ref(self):
|
||||
"""Removing DownloadRef mid-transfer cancels the download."""
|
||||
def body():
|
||||
artifact = self.make_artifact(chunked=True)
|
||||
base_path = os.path.join(self.dest, artifact.fileName)
|
||||
checks = {"n": 0}
|
||||
|
||||
def get(key):
|
||||
if key == "ModelManager_DownloadRef":
|
||||
checks["n"] += 1
|
||||
return b"ref" if checks["n"] <= 2 else None
|
||||
return b"0"
|
||||
|
||||
self.manager.params.get.side_effect = get
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact))
|
||||
assert 'cancelled' in str(ctx.exception).lower()
|
||||
assert not os.path.isfile(get_manifest_path(base_path))
|
||||
self.run_with_server(body)
|
||||
|
||||
def _make_params_with_store(self):
|
||||
params = mock.MagicMock()
|
||||
store = {}
|
||||
|
||||
def get(key, *args, **kwargs):
|
||||
return store.get(key, b"0") # b"0" -> download not cancelled
|
||||
|
||||
def put(key, value, *args, **kwargs):
|
||||
store[key] = value
|
||||
|
||||
params.get.side_effect = get
|
||||
params.put.side_effect = put
|
||||
return params, store
|
||||
|
||||
def test_download_writes_qcom_slot(self):
|
||||
"""A download resolved to the qcom source writes the qcom active bundle slot only."""
|
||||
def body():
|
||||
artifact = self.make_artifact(chunked=True)
|
||||
self._bundle.ref = "test-ref"
|
||||
self._bundle.minimumSelectorVersion = 18
|
||||
params, store = self._make_params_with_store()
|
||||
self.manager.params = params
|
||||
asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "qcom"))
|
||||
|
||||
assert "ModelManager_ActiveBundle" in store, "qcom download must write the qcom slot"
|
||||
assert "ModelManager_ActiveBundleUSBGPU" not in store, "qcom download must not touch the usbgpu slot"
|
||||
assert self.manager.selected_bundle.status == custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
assert self.manager.active_bundle is not None and self.manager.active_bundle.ref == "test-ref"
|
||||
assert self.manager.active_bundle.status == custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
chunk_names = [get_chunk_name(artifact.fileName, i, len(artifact.chunks)) for i in range(len(artifact.chunks))]
|
||||
missing = [c for c in chunk_names if not os.path.isfile(os.path.join(self.dest, c))]
|
||||
assert missing == [], f"chunks missing from the cache: {missing}"
|
||||
self.run_with_server(body)
|
||||
|
||||
def test_download_writes_usbgpu_slot(self):
|
||||
"""A download resolved to the usbgpu source writes the usbgpu active bundle slot only."""
|
||||
def body():
|
||||
self.make_artifact(chunked=True)
|
||||
self._bundle.ref = "big-ref"
|
||||
self._bundle.minimumSelectorVersion = 18
|
||||
params, store = self._make_params_with_store()
|
||||
self.manager.params = params
|
||||
asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "usbgpu"))
|
||||
|
||||
assert "ModelManager_ActiveBundleUSBGPU" in store, "usbgpu download must write the usbgpu slot"
|
||||
assert "ModelManager_ActiveBundle" not in store, "usbgpu download must not touch the qcom slot"
|
||||
assert self.manager.selected_bundle.status == custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
self.run_with_server(body)
|
||||
|
||||
|
||||
class TestManagerImports(OpenpilotTestCase):
|
||||
"""Catches undeclared dependencies. aiohttp lived only in the AGNOS venv; 19.6 dropped
|
||||
@@ -352,352 +267,6 @@ class TestManagerImports(OpenpilotTestCase):
|
||||
assert connect > 0 and read > 0, "requests defaults to no timeout; downloads would hang forever"
|
||||
|
||||
|
||||
class TestResolveBundleByRef(OpenpilotTestCase):
|
||||
"""A ref resolves to (bundle, source) across both hardware manifests. Refs are
|
||||
unique per manifest and never overlap across sources, so a ref maps to exactly
|
||||
one slot. Shared by the manager's download flow and the settings UI."""
|
||||
|
||||
@staticmethod
|
||||
def _bundle(ref: str):
|
||||
bundle = custom.ModelManagerSP.ModelBundle.new_message()
|
||||
bundle.ref = ref
|
||||
return bundle
|
||||
|
||||
def test_qcom_ref_resolves_to_qcom_slot(self):
|
||||
small = self._bundle("small")
|
||||
assert resolve_bundle_by_ref("small", {"qcom": [small], "usbgpu": []}) == (small, "qcom")
|
||||
|
||||
def test_usbgpu_ref_resolves_to_usbgpu_slot(self):
|
||||
big = self._bundle("big")
|
||||
assert resolve_bundle_by_ref("big", {"qcom": [], "usbgpu": [big]}) == (big, "usbgpu")
|
||||
|
||||
def test_unknown_ref_returns_none(self):
|
||||
source_bundles = {"qcom": [self._bundle("small")], "usbgpu": []}
|
||||
assert resolve_bundle_by_ref("nope", source_bundles) is None
|
||||
|
||||
|
||||
def manifest_bundle(short_name: str, ref: str, index: int = 0, is_big: bool = False) -> dict:
|
||||
"""Minimal manifest bundle dict, version-compatible (no chunks to avoid disk side effects).
|
||||
Big (usbgpu) bundles carry `is_big: true` in the manifest JSON."""
|
||||
return {
|
||||
"index": index,
|
||||
"short_name": short_name,
|
||||
"display_name": short_name.upper(),
|
||||
"generation": 1,
|
||||
"environment": "release",
|
||||
"runner": "tinygrad",
|
||||
"is_big": is_big,
|
||||
"minimum_selector_version": "18",
|
||||
"ref": ref,
|
||||
"models": [{
|
||||
"type": "supercombo",
|
||||
"artifact": {
|
||||
"file_name": f"{short_name}.pkl",
|
||||
"download_uri": {"url": f"https://example.com/{short_name}.pkl", "sha256": "s"},
|
||||
},
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
def fresh_sync_time() -> int:
|
||||
return int(time.monotonic() * 1e9)
|
||||
|
||||
|
||||
class TestModelFetcherSources(OpenpilotTestCase):
|
||||
"""Both manifests are always maintained: get_bundles_for_source exposes either
|
||||
source by name, and active_source picks which one matches the attached hardware."""
|
||||
|
||||
def _make_params(self, qcom_manifest, usbgpu_manifest):
|
||||
params = mock.MagicMock()
|
||||
|
||||
def get(key):
|
||||
if key == "ModelManager_ModelsCache":
|
||||
return qcom_manifest
|
||||
if key == "ModelManager_ModelsCache_USBGPU":
|
||||
return usbgpu_manifest
|
||||
if key in ("ModelManager_LastSyncTime", "ModelManager_LastSyncTime_USBGPU"):
|
||||
return fresh_sync_time()
|
||||
return None
|
||||
|
||||
params.get.side_effect = get
|
||||
return params
|
||||
|
||||
def test_active_source_follows_chestnut_presence(self):
|
||||
assert ModelFetcher.active_source(False) == "qcom"
|
||||
assert ModelFetcher.active_source(True) == "usbgpu"
|
||||
|
||||
def test_get_bundles_for_source_returns_each_source(self):
|
||||
params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]},
|
||||
{"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
assert [bundle.ref for bundle in fetcher.get_bundles_for_source("qcom")] == ["aaa"]
|
||||
assert [bundle.ref for bundle in fetcher.get_bundles_for_source("usbgpu")] == ["bbb"]
|
||||
|
||||
def test_get_bundles_for_source_unknown(self):
|
||||
assert ModelFetcher(mock.MagicMock()).get_bundles_for_source("bogus") == []
|
||||
|
||||
def test_get_cached_bundles_parses_source(self):
|
||||
params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]},
|
||||
{"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
qcom_bundles = get_cached_bundles(params, "qcom")
|
||||
usbgpu_bundles = get_cached_bundles(params, "usbgpu")
|
||||
assert [b.ref for b in qcom_bundles] == ["aaa"]
|
||||
assert [b.ref for b in usbgpu_bundles] == ["bbb"]
|
||||
assert qcom_bundles[0].displayName == "SMALL"
|
||||
|
||||
def test_get_cached_bundles_empty_when_missing(self):
|
||||
params = mock.MagicMock()
|
||||
params.get.return_value = None
|
||||
assert get_cached_bundles(params, "qcom") == []
|
||||
assert get_cached_bundles(params, "usbgpu") == []
|
||||
|
||||
def test_get_cached_bundles_unknown_source(self):
|
||||
assert get_cached_bundles(mock.MagicMock(), "bogus") == []
|
||||
|
||||
def test_active_json_has_both_urls(self):
|
||||
params = mock.MagicMock()
|
||||
ModelFetcher(params)
|
||||
active_json_calls = [call for call in params.put.call_args_list if call.args[0] == "ModelManager_ActiveJson"]
|
||||
assert active_json_calls, "expected ModelManager_ActiveJson to be written"
|
||||
assert active_json_calls[-1].args[1] == {
|
||||
"qcom": ModelFetcher.MODEL_URL,
|
||||
"usbgpu": ModelFetcher.MODEL_URL_USBGPU,
|
||||
}
|
||||
|
||||
|
||||
|
||||
class TestSourceCacheIntegrity(OpenpilotTestCase):
|
||||
"""Each source's cached manifest must contain only that source's models; the
|
||||
`is_big` flag in the JSON marks the big (usbgpu) models. A mismatched cache is
|
||||
legacy data from before the per-source split (the active manifest was cached
|
||||
under the unsuffixed key regardless of hardware) and is refetched. This
|
||||
replaces the old one-time bundle migration."""
|
||||
|
||||
def _make_params(self, qcom_manifest, usbgpu_manifest):
|
||||
params = mock.MagicMock()
|
||||
|
||||
def get(key):
|
||||
if key == "ModelManager_ModelsCache":
|
||||
return qcom_manifest
|
||||
if key == "ModelManager_ModelsCache_USBGPU":
|
||||
return usbgpu_manifest
|
||||
if key in ("ModelManager_LastSyncTime", "ModelManager_LastSyncTime_USBGPU"):
|
||||
return fresh_sync_time()
|
||||
return None
|
||||
|
||||
params.get.side_effect = get
|
||||
return params
|
||||
|
||||
def _fetched(self, *bundles):
|
||||
return ModelFetcher(mock.MagicMock()).model_parser.parse_models({"bundles": list(bundles)})
|
||||
|
||||
def test_qcom_cache_with_big_models_is_refetched(self):
|
||||
"""Legacy: the unsuffixed cache holds the big manifest. is_big confirms it is
|
||||
the wrong set for qcom, so a fresh fetch replaces it."""
|
||||
params = self._make_params({"bundles": [manifest_bundle("big", "bbb", is_big=True)]},
|
||||
{"bundles": [manifest_bundle("big2", "ccc", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
fetched = self._fetched(manifest_bundle("small", "aaa"))
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched):
|
||||
bundles = fetcher.get_bundles_for_source("qcom")
|
||||
assert [bundle.ref for bundle in bundles] == ["aaa"]
|
||||
|
||||
def test_usbgpu_cache_without_big_models_is_refetched(self):
|
||||
params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]},
|
||||
{"bundles": [manifest_bundle("big2", "ccc")]})
|
||||
fetcher = ModelFetcher(params)
|
||||
fetched = self._fetched(manifest_bundle("big", "bbb", is_big=True))
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched):
|
||||
bundles = fetcher.get_bundles_for_source("usbgpu")
|
||||
assert [bundle.ref for bundle in bundles] == ["bbb"]
|
||||
|
||||
def test_matching_caches_are_used_without_fetch(self):
|
||||
params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]},
|
||||
{"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", side_effect=AssertionError("cache should be used")):
|
||||
assert [bundle.ref for bundle in fetcher.get_bundles_for_source("qcom")] == ["aaa"]
|
||||
assert [bundle.ref for bundle in fetcher.get_bundles_for_source("usbgpu")] == ["bbb"]
|
||||
|
||||
def test_stale_version_cache_is_refetched(self):
|
||||
"""A source-matching cache whose bundles are all filtered by the selector version
|
||||
check parses to zero valid bundles; it is stale (e.g. an old manifest) and must be
|
||||
refetched instead of silently returning an empty list forever."""
|
||||
stale = manifest_bundle("small", "aaa")
|
||||
stale["minimum_selector_version"] = "16"
|
||||
params = self._make_params({"bundles": [stale]},
|
||||
{"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
fetched = self._fetched(manifest_bundle("small2", "ddd"))
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched) as fetch:
|
||||
bundles = fetcher.get_bundles_for_source("qcom")
|
||||
fetch.assert_called_once_with("qcom")
|
||||
assert [bundle.ref for bundle in bundles] == ["ddd"]
|
||||
|
||||
def test_mismatched_refetch_happens_once(self):
|
||||
"""If the fresh manifest still fails the source check, the URL is authoritative:
|
||||
trust it instead of refetching at 1 Hz forever."""
|
||||
params = self._make_params({"bundles": [manifest_bundle("big", "bbb", is_big=True)]},
|
||||
{"bundles": [manifest_bundle("big2", "ccc", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
fetched = self._fetched(manifest_bundle("big", "bbb", is_big=True))
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched) as fetch:
|
||||
first = fetcher.get_bundles_for_source("qcom")
|
||||
second = fetcher.get_bundles_for_source("qcom")
|
||||
fetch.assert_called_once_with("qcom")
|
||||
assert [bundle.ref for bundle in first] == ["bbb"]
|
||||
assert [bundle.ref for bundle in second] == ["bbb"]
|
||||
|
||||
def test_corrupt_cache_is_refetched(self):
|
||||
"""A cache that fails to parse (e.g. truncated/foreign JSON) must trigger a
|
||||
refetch instead of raising every loop and never recovering."""
|
||||
corrupt = {"bundles": [{"short_name": "broken"}]} # missing required fields
|
||||
params = self._make_params(corrupt, {"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
fetched = self._fetched(manifest_bundle("small", "aaa"))
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched) as fetch:
|
||||
bundles = fetcher.get_bundles_for_source("qcom")
|
||||
fetch.assert_called_once_with("qcom")
|
||||
assert [bundle.ref for bundle in bundles] == ["aaa"]
|
||||
|
||||
|
||||
class TestActiveBundleValidation(OpenpilotTestCase):
|
||||
"""Validation is per-slot: a failed fetch (empty bundle list) must not reset a slot,
|
||||
and resetting one slot must not stomp the runner cache derived from the other."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
helpers._LAST_VALIDATED_RAW.clear()
|
||||
|
||||
@staticmethod
|
||||
def _raw_bundle(ref: str, runner: int | None = None) -> dict:
|
||||
bundle = custom.ModelManagerSP.ModelBundle.new_message()
|
||||
bundle.ref = ref
|
||||
bundle.minimumSelectorVersion = 18
|
||||
if runner is not None:
|
||||
bundle.runner = runner
|
||||
return bundle.to_dict()
|
||||
|
||||
def _params(self, qcom=None, usbgpu=None):
|
||||
params = mock.MagicMock()
|
||||
|
||||
def get(key, *args, **kwargs):
|
||||
return {"ModelManager_ActiveBundle": qcom, "ModelManager_ActiveBundleUSBGPU": usbgpu}.get(key)
|
||||
|
||||
params.get.side_effect = get
|
||||
return params
|
||||
|
||||
def test_empty_catalog_does_not_reset_slot(self):
|
||||
params = self._params(qcom=self._raw_bundle("small"))
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False):
|
||||
validate_active_bundles(params, {"qcom": [], "usbgpu": []})
|
||||
params.remove.assert_not_called()
|
||||
|
||||
def test_reset_recomputes_runner_from_surviving_slot(self):
|
||||
tinygrad = int(custom.ModelManagerSP.Runner.tinygrad)
|
||||
big_raw = self._raw_bundle("big", runner=tinygrad)
|
||||
params = self._params(qcom=self._raw_bundle("gone"), usbgpu=big_raw)
|
||||
catalog = {"qcom": [custom.ModelManagerSP.ModelBundle(**self._raw_bundle("other"))],
|
||||
"usbgpu": [custom.ModelManagerSP.ModelBundle(**big_raw)]}
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True):
|
||||
validate_active_bundles(params, catalog)
|
||||
params.remove.assert_called_once_with("ModelManager_ActiveBundle")
|
||||
runner_puts = [call for call in params.put.call_args_list if call.args[0] == "ModelRunnerTypeCache"]
|
||||
assert [call.args[1] for call in runner_puts] == [tinygrad]
|
||||
|
||||
|
||||
class TestActiveBundleSelection(OpenpilotTestCase):
|
||||
"""The effective active bundle is the active source's slot: usbgpu when a GPU is
|
||||
present, qcom otherwise. An empty active slot means the hardware default (stock
|
||||
runner), never the other slot's pick - modeld_v2 requires a real bundle."""
|
||||
|
||||
@staticmethod
|
||||
def _raw_bundle(ref: str) -> dict:
|
||||
bundle = custom.ModelManagerSP.ModelBundle.new_message()
|
||||
bundle.ref = ref
|
||||
bundle.minimumSelectorVersion = 18
|
||||
return bundle.to_dict()
|
||||
|
||||
def _params(self, qcom=None, usbgpu=None):
|
||||
params = mock.MagicMock()
|
||||
|
||||
def get(key, *args, **kwargs):
|
||||
if key == "ModelManager_ActiveBundle":
|
||||
return qcom
|
||||
if key == "ModelManager_ActiveBundleUSBGPU":
|
||||
return usbgpu
|
||||
return None
|
||||
|
||||
params.get.side_effect = get
|
||||
return params
|
||||
|
||||
def test_selected_bundle_is_per_slot(self):
|
||||
params = self._params(qcom=self._raw_bundle("small"), usbgpu=self._raw_bundle("big"))
|
||||
assert get_selected_bundle(params, "qcom").ref == "small"
|
||||
assert get_selected_bundle(params, "usbgpu").ref == "big"
|
||||
|
||||
def test_no_gpu_uses_qcom_slot(self):
|
||||
params = self._params(qcom=self._raw_bundle("small"), usbgpu=self._raw_bundle("big"))
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False):
|
||||
assert get_active_bundle(params).ref == "small"
|
||||
|
||||
def test_gpu_uses_usbgpu_slot(self):
|
||||
params = self._params(qcom=self._raw_bundle("small"), usbgpu=self._raw_bundle("big"))
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True):
|
||||
assert get_active_bundle(params).ref == "big"
|
||||
|
||||
def test_gpu_without_big_selection_is_hardware_default(self):
|
||||
params = self._params(qcom=self._raw_bundle("small"), usbgpu=None)
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True):
|
||||
assert get_active_bundle(params) is None
|
||||
|
||||
|
||||
class TestEffectiveSource(OpenpilotTestCase):
|
||||
"""One gate decides the active source. With no flags it is runtime truth (GPU
|
||||
attached); display callers (mici) pass the ui_state flags, which additionally
|
||||
require the big model to be loading, active, or the device offroad. The active
|
||||
bundle is simply the selected bundle of that source."""
|
||||
|
||||
@staticmethod
|
||||
def _raw_bundle(ref: str) -> dict:
|
||||
bundle = custom.ModelManagerSP.ModelBundle.new_message()
|
||||
bundle.ref = ref
|
||||
bundle.minimumSelectorVersion = 18
|
||||
return bundle.to_dict()
|
||||
|
||||
def test_runtime_no_gpu(self):
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False):
|
||||
assert get_active_source() == "qcom"
|
||||
|
||||
def test_runtime_gpu_present(self):
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True):
|
||||
assert get_active_source() == "usbgpu"
|
||||
|
||||
def test_display_offroad_gpu_present_shows_big(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=False, usbgpu_loading=False, offroad=True) == "usbgpu"
|
||||
|
||||
def test_display_onroad_gpu_loading_shows_big(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=False, usbgpu_loading=True, offroad=False) == "usbgpu"
|
||||
|
||||
def test_display_onroad_gpu_active_shows_big(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=True, usbgpu_loading=False, offroad=False) == "usbgpu"
|
||||
|
||||
def test_display_onroad_gpu_idle_shows_small(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=False, usbgpu_loading=False, offroad=False) == "qcom"
|
||||
|
||||
def test_display_active_none_is_idle(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=None, usbgpu_loading=False, offroad=False) == "qcom"
|
||||
|
||||
def test_active_bundle_follows_source(self):
|
||||
params = mock.MagicMock()
|
||||
params.get.side_effect = lambda key: {"ModelManager_ActiveBundle": self._raw_bundle("small"),
|
||||
"ModelManager_ActiveBundleUSBGPU": self._raw_bundle("big")}.get(key)
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False):
|
||||
assert get_active_bundle(params).ref == "small"
|
||||
assert get_selected_bundle(params, get_active_source(usbgpu=True, usbgpu_active=False,
|
||||
usbgpu_loading=False, offroad=True)).ref == "big"
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get('RUN_INTEGRATION_TESTS'), 'requires external network')
|
||||
class TestLiveModelManifest(OpenpilotTestCase):
|
||||
"""Every artifact and chunk URL in the published manifest must resolve."""
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import requests
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.sunnypilot.models.tinygrad_ref import get_tinygrad_ref
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher
|
||||
from openpilot.common.test import OpenpilotTestCase
|
||||
|
||||
def fetch_tinygrad_ref():
|
||||
response = requests.get(ModelFetcher.MODEL_URL, timeout=10)
|
||||
fetcher = ModelFetcher(Params())
|
||||
response = requests.get(fetcher.model_url, timeout=10)
|
||||
response.raise_for_status()
|
||||
json_data = response.json()
|
||||
return json_data.get("tinygrad_ref")
|
||||
|
||||
@@ -28,6 +28,7 @@ from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutExce
|
||||
create_connection, WebSocketConnectionClosedException)
|
||||
|
||||
import openpilot.cereal.messaging as messaging
|
||||
from openpilot.selfdrive.modeld.helpers import usbgpu_present
|
||||
from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL, DEFAULT_BIG_MODEL
|
||||
from openpilot.sunnypilot.selfdrive.car.sync_sunnylink_params import update_car_list_param
|
||||
from openpilot.sunnypilot.sunnylink.api import SunnylinkApi
|
||||
@@ -181,8 +182,10 @@ def getParamsMetadata() -> str:
|
||||
schema = generate_schema()
|
||||
schema["capabilities"] = generate_capabilities()
|
||||
schema["capability_labels"] = CAPABILITY_LABELS
|
||||
schema["default_model"] = DEFAULT_MODEL
|
||||
schema["default_big_model"] = DEFAULT_BIG_MODEL
|
||||
# mirrors get_default_model() — ui_state unavailable in sunnylinkd process
|
||||
show_big = (usbgpu_present()
|
||||
and (params.get_bool("UsbGpuActive") or params.get_bool("UsbGpuLoading") or params.get_bool("IsOffroad")))
|
||||
schema["default_model"] = DEFAULT_BIG_MODEL if show_big else DEFAULT_MODEL
|
||||
schema["usbgpu_active"] = params.get_bool("UsbGpuActive")
|
||||
raw = json.dumps(schema, separators=(",", ":")).encode("utf-8")
|
||||
return base64.b64encode(gzip.compress(raw)).decode("utf-8")
|
||||
|
||||
@@ -65,7 +65,6 @@ def sp_stats(end_event):
|
||||
'MadsSteeringMode',
|
||||
'MadsUnifiedEngagementMode',
|
||||
'ModelManager_ActiveBundle',
|
||||
'ModelManager_ActiveBundleUSBGPU',
|
||||
'ModelManager_Favs',
|
||||
'EnableSunnylinkUploader',
|
||||
'SunnylinkEnabled',
|
||||
|
||||
@@ -84,21 +84,6 @@ def _migrate_tesla_mads_screen_button(_params):
|
||||
cloudlog.exception(f"Error migrating TeslaMadsScreenButton: {e}")
|
||||
|
||||
|
||||
def _migrate_model_bundle_slots(_params):
|
||||
# Pre-split, a chestnut user's big-model selection lived in the single
|
||||
# ActiveBundle. Seed both slots; validation drops whichever does not match
|
||||
# its own manifest.
|
||||
try:
|
||||
if _params.get("ModelManager_ActiveBundleUSBGPU") is not None:
|
||||
return
|
||||
if (bundle := _params.get("ModelManager_ActiveBundle")) is None:
|
||||
return
|
||||
_params.put("ModelManager_ActiveBundleUSBGPU", bundle, block=True)
|
||||
cloudlog.info("params_migration: seeded ModelManager_ActiveBundleUSBGPU from ModelManager_ActiveBundle")
|
||||
except Exception as e:
|
||||
cloudlog.exception(f"Error migrating model bundle slots: {e}")
|
||||
|
||||
|
||||
def run_migration(_params):
|
||||
# migrate OnroadScreenOffBrightness
|
||||
if _params.get("OnroadScreenOffBrightnessMigrated") != ONROAD_BRIGHTNESS_MIGRATION_VERSION:
|
||||
@@ -135,6 +120,3 @@ def run_migration(_params):
|
||||
|
||||
# seed TeslaMadsScreenButton for existing Tesla installs
|
||||
_migrate_tesla_mads_screen_button(_params)
|
||||
|
||||
# seed the usbgpu model slot from the pre-split single slot
|
||||
_migrate_model_bundle_slots(_params)
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
"""
|
||||
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.test import OpenpilotTestCase
|
||||
from openpilot.sunnypilot.system.params_migration import _migrate_model_bundle_slots
|
||||
|
||||
|
||||
class TestModelBundleSlotMigration(OpenpilotTestCase):
|
||||
"""Pre-split, a chestnut user's big-model selection lived in the single ActiveBundle.
|
||||
The migration seeds both slots; per-source validation later drops whichever does not
|
||||
match its own manifest."""
|
||||
|
||||
def test_seeds_usbgpu_slot_from_active_bundle(self):
|
||||
params = Params()
|
||||
bundle = {"ref": "big", "minimumSelectorVersion": 18}
|
||||
params.put("ModelManager_ActiveBundle", bundle, block=True)
|
||||
_migrate_model_bundle_slots(params)
|
||||
assert params.get("ModelManager_ActiveBundleUSBGPU") == bundle
|
||||
assert params.get("ModelManager_ActiveBundle") == bundle
|
||||
|
||||
def test_noop_when_usbgpu_slot_already_set(self):
|
||||
params = Params()
|
||||
params.put("ModelManager_ActiveBundle", {"ref": "small"}, block=True)
|
||||
params.put("ModelManager_ActiveBundleUSBGPU", {"ref": "big"}, block=True)
|
||||
_migrate_model_bundle_slots(params)
|
||||
assert params.get("ModelManager_ActiveBundleUSBGPU") == {"ref": "big"}
|
||||
|
||||
def test_noop_when_no_selection(self):
|
||||
params = Params()
|
||||
_migrate_model_bundle_slots(params)
|
||||
assert params.get("ModelManager_ActiveBundleUSBGPU") is None
|
||||
@@ -8,26 +8,12 @@ from collections.abc import Callable
|
||||
|
||||
import pyray as rl
|
||||
|
||||
from openpilot.system.ui.lib.application import gui_app, FontWeight
|
||||
from openpilot.system.ui.lib.application import FontWeight
|
||||
from openpilot.system.ui.sunnypilot.lib.styles import style
|
||||
from openpilot.system.ui.sunnypilot.widgets.list_view import ButtonActionSP
|
||||
from openpilot.system.ui.widgets.label import ScrollState, UnifiedLabel
|
||||
from openpilot.system.ui.widgets.label import UnifiedLabel
|
||||
from openpilot.system.ui.widgets.list_view import BUTTON_WIDTH, BUTTON_HEIGHT, TEXT_PADDING, _resolve_value
|
||||
|
||||
SCROLL_SPEED = 1.2 # stock is 0.8, boosted 50% to compensate for larger font (50 vs 32)
|
||||
SCROLL_REFERENCE_FPS = 60.
|
||||
|
||||
|
||||
class UnifiedLabelSP(UnifiedLabel):
|
||||
# stock scroll formula (0.8 / 60 * fps) is inverted — pre-correct so speed is constant px/sec
|
||||
def _render(self, _):
|
||||
if self._needs_scroll and self._scroll_state == ScrollState.SCROLLING:
|
||||
fps = gui_app.target_fps
|
||||
wrong_step = 0.8 / SCROLL_REFERENCE_FPS * fps
|
||||
correct_step = SCROLL_SPEED * SCROLL_REFERENCE_FPS / fps
|
||||
self._scroll_offset -= (correct_step - wrong_step)
|
||||
super()._render(_)
|
||||
|
||||
|
||||
class NoElideButtonAction(ButtonActionSP):
|
||||
def get_width_hint(self):
|
||||
@@ -35,12 +21,14 @@ class NoElideButtonAction(ButtonActionSP):
|
||||
|
||||
|
||||
class ScrollingButtonAction(ButtonActionSP):
|
||||
"""ButtonActionSP whose value scrolls instead of eliding when it doesn't fit."""
|
||||
|
||||
def __init__(self, text: str | Callable[[], str], width: int = style.BUTTON_ACTION_WIDTH,
|
||||
enabled: bool | Callable[[], bool] = True):
|
||||
super().__init__(text=text, width=width, enabled=enabled)
|
||||
self._value_label = UnifiedLabelSP("", font_size=style.ITEM_TEXT_FONT_SIZE, font_weight=FontWeight.NORMAL,
|
||||
text_color=self._value_color, scroll=True,
|
||||
alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE)
|
||||
self._value_label = UnifiedLabel("", font_size=style.ITEM_TEXT_FONT_SIZE, font_weight=FontWeight.NORMAL,
|
||||
text_color=self._value_color, scroll=True,
|
||||
alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE)
|
||||
|
||||
def set_value(self, value: str | Callable[[], str], color: rl.Color = style.ITEM_TEXT_VALUE_COLOR):
|
||||
if self.value != _resolve_value(value, ""):
|
||||
|
||||
@@ -136,7 +136,7 @@ def generate_chunked_model(driving_pkl: Path) -> dict:
|
||||
|
||||
|
||||
def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown",
|
||||
onnx_sha256=None, is_big=False) -> None:
|
||||
onnx_sha256=None) -> None:
|
||||
bundle_json = {
|
||||
"short_name": short_name,
|
||||
"display_name": custom_name or upstream_branch,
|
||||
@@ -149,7 +149,6 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short
|
||||
"generation": "-1",
|
||||
"build_time": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"overrides": {},
|
||||
"is_big": is_big,
|
||||
"models": models,
|
||||
}
|
||||
|
||||
@@ -187,8 +186,6 @@ if __name__ == "__main__":
|
||||
print(f"No driving_tinygrad.pkl found in {_output_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
is_big = _driving_pkl.name.startswith('big_')
|
||||
|
||||
if _pkl:
|
||||
new_pkl = _output_dir / f"driving_{_pkl}_tinygrad.pkl"
|
||||
if not new_pkl.exists():
|
||||
@@ -199,4 +196,4 @@ if __name__ == "__main__":
|
||||
_model_metadata = generate_chunked_model(_driving_pkl)
|
||||
_onnx_sha256 = _hash_onnx_files(Path(args.model_dir))
|
||||
create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch,
|
||||
onnx_sha256=_onnx_sha256, is_big=is_big)
|
||||
onnx_sha256=_onnx_sha256)
|
||||
|
||||
@@ -47,7 +47,7 @@ git rm -rf $OUTPUT_DIR/.git || true # Doing cleanup, but it might fail if the .g
|
||||
git remote remove origin || true # ensure cleanup
|
||||
git remote add origin $GIT_ORIGIN
|
||||
#git push origin -d $DEV_BRANCH || true # Ensuring we delete the remote branch if it exists as we are wiping it out
|
||||
git fetch --depth 1 origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH)
|
||||
git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH)
|
||||
|
||||
echo "[-] committing version $VERSION T=$SECONDS"
|
||||
git add -f .
|
||||
|
||||
Reference in New Issue
Block a user