mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-08-26 00:53:42 +08:00
Compare commits
37 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bbb3c01b06 | |||
| 9479242359 | |||
| bfe5c3a4f7 | |||
| 1db7675a98 | |||
| 66cf334067 | |||
| 94ed0608e6 | |||
| 0fbca979df | |||
| dcddb2a0bd | |||
| 699eaf7957 | |||
| c246e6318a | |||
| 718db8c62e | |||
| c2214d4c32 | |||
| 0de7fbf33d | |||
| 211f990f6b | |||
| 97468e4fa4 | |||
| 6c6fba9a14 | |||
| 34621cf816 | |||
| 086530b7c6 | |||
| 4f46433e2b | |||
| 5a8567e3e7 | |||
| 07558166c8 | |||
| ca9338812e | |||
| 4667241fe7 | |||
| 084747c75d | |||
| a49c260927 | |||
| 5ad2bfdb75 | |||
| b742557d62 | |||
| 5ecd05aedf | |||
| 5ae100aa1d | |||
| be76a88b80 | |||
| 049d225d5a | |||
| 555f48c5d2 | |||
| dcf9d25bf3 | |||
| a8d1a280c6 | |||
| 5b36799eec | |||
| 20fdc3d824 | |||
| 7bd6cad821 |
@@ -0,0 +1,279 @@
|
||||
name: Build default models
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target:
|
||||
description: 'Model target to build'
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- small
|
||||
- big
|
||||
workflow_call:
|
||||
inputs:
|
||||
target:
|
||||
description: 'Model target to build (small or big)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
env:
|
||||
HF_REPO: sunnypilot/sunnypilot_models_v1
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
model_name: ${{ steps.resolve.outputs.model_name }}
|
||||
onnx_ref: ${{ steps.resolve.outputs.onnx_ref }}
|
||||
onnx_path: ${{ steps.resolve.outputs.onnx_path }}
|
||||
hf_defaults_path: ${{ steps.resolve.outputs.hf_defaults_path }}
|
||||
target_hardware: ${{ steps.resolve.outputs.target_hardware }}
|
||||
tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }}
|
||||
dm_onnx_ref: ${{ steps.resolve.outputs.dm_onnx_ref }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- id: resolve
|
||||
run: |
|
||||
export PYTHONPATH=${{ github.workspace }}
|
||||
|
||||
if [ "${{ inputs.target }}" = "big" ]; then
|
||||
NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)")
|
||||
ONNX_PATH="openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
|
||||
HF_DEFAULTS_PATH="models/defaults/big"
|
||||
TARGET_HW="usbgpu"
|
||||
else
|
||||
NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL; print(DEFAULT_MODEL)")
|
||||
ONNX_PATH="openpilot/selfdrive/modeld/models/driving_supercombo.onnx"
|
||||
HF_DEFAULTS_PATH="models/defaults/small"
|
||||
TARGET_HW="qcom"
|
||||
fi
|
||||
|
||||
ONNX_REF=$(git log -1 --format='%H' -- "$ONNX_PATH")
|
||||
TINYGRAD_REF=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)
|
||||
if [ -z "$TINYGRAD_REF" ]; then
|
||||
echo "::error::Failed to resolve tinygrad ref"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DM_ONNX_REF=""
|
||||
if [ "${{ inputs.target }}" = "small" ]; then
|
||||
DM_ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx)
|
||||
fi
|
||||
|
||||
echo "model_name=${NAME}" >> $GITHUB_OUTPUT
|
||||
echo "onnx_ref=${ONNX_REF}" >> $GITHUB_OUTPUT
|
||||
echo "onnx_path=${ONNX_PATH}" >> $GITHUB_OUTPUT
|
||||
echo "hf_defaults_path=${HF_DEFAULTS_PATH}" >> $GITHUB_OUTPUT
|
||||
echo "target_hardware=${TARGET_HW}" >> $GITHUB_OUTPUT
|
||||
echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT
|
||||
echo "dm_onnx_ref=${DM_ONNX_REF}" >> $GITHUB_OUTPUT
|
||||
|
||||
build_driving_model:
|
||||
needs: resolve
|
||||
uses: ./.github/workflows/sunnypilot-build-model.yaml
|
||||
with:
|
||||
upstream_branch: ${{ needs.resolve.outputs.onnx_ref }}
|
||||
custom_name: ${{ needs.resolve.outputs.model_name }}
|
||||
target_hardware: ${{ needs.resolve.outputs.target_hardware }}
|
||||
secrets: inherit
|
||||
|
||||
upload_defaults:
|
||||
needs: [ resolve, build_driving_model, build_dm_model ]
|
||||
if: ${{ !cancelled() && needs.build_driving_model.result == 'success' && (inputs.target != 'small' || needs.build_dm_model.result == 'success') }}
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: write
|
||||
env:
|
||||
DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Pull ONNX via LFS
|
||||
run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}${{ inputs.target == 'small' && ',openpilot/selfdrive/modeld/models/dmonitoring_model.onnx' || '' }}"
|
||||
|
||||
- name: Install huggingface_hub
|
||||
run: pip install --upgrade "huggingface_hub>=0.22.0"
|
||||
|
||||
- name: Download driving artifact name
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: artifact-name-${{ needs.resolve.outputs.model_name }}
|
||||
path: artifact_name
|
||||
|
||||
- name: Read driving artifact name
|
||||
id: artifact
|
||||
run: |
|
||||
ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
|
||||
echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Download driving model artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.artifact.outputs.artifact_name }}
|
||||
path: output
|
||||
|
||||
- name: Upload driving model to HF
|
||||
env:
|
||||
HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
|
||||
ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
|
||||
run: |
|
||||
rm -f output/artifact_name.txt
|
||||
export PYTHONPATH=$(pwd)
|
||||
python3 release/ci/upload_default_model.py \
|
||||
--hf-repo "${{ env.HF_REPO }}" \
|
||||
--hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \
|
||||
--artifact-name "$ARTIFACT_NAME" \
|
||||
--model-dir output \
|
||||
--onnx-path "${{ needs.resolve.outputs.onnx_path }}" \
|
||||
--onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \
|
||||
--model-name "${{ needs.resolve.outputs.model_name }}" \
|
||||
--tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \
|
||||
--run-number "${{ github.run_number }}"
|
||||
|
||||
- name: Download DM artifact
|
||||
if: ${{ inputs.target == 'small' }}
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dm-model-${{ github.run_number }}
|
||||
path: dm_output
|
||||
|
||||
- name: Generate DM metadata and upload to HF
|
||||
if: ${{ inputs.target == 'small' }}
|
||||
env:
|
||||
HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
|
||||
run: |
|
||||
export PYTHONPATH=$(pwd)
|
||||
python3 -c "
|
||||
import json, hashlib
|
||||
from pathlib import Path
|
||||
from datetime import datetime, UTC
|
||||
|
||||
dm_dir = Path('dm_output')
|
||||
manifest = list(dm_dir.glob('*.chunkmanifest'))
|
||||
assert manifest, 'No chunkmanifest found'
|
||||
pkl_name = manifest[0].name.removesuffix('.chunkmanifest')
|
||||
num_chunks = int(manifest[0].read_text().strip())
|
||||
|
||||
chunks = []
|
||||
for i in range(num_chunks):
|
||||
chunk = dm_dir / f'{pkl_name}.chunk{i+1:02d}of{num_chunks:02d}'
|
||||
chunks.append({
|
||||
'file_name': chunk.name,
|
||||
'sha256': hashlib.sha256(chunk.read_bytes()).hexdigest()
|
||||
})
|
||||
|
||||
digest = hashlib.sha256()
|
||||
for c in chunks:
|
||||
with open(dm_dir / c['file_name'], 'rb') as f:
|
||||
while block := f.read(1024*1024):
|
||||
digest.update(block)
|
||||
|
||||
metadata = {
|
||||
'bundles': [{
|
||||
'short_name': 'DMMODEL',
|
||||
'display_name': 'dmonitoring_model',
|
||||
'ref': '${{ needs.resolve.outputs.dm_onnx_ref }}',
|
||||
'runner': 'tinygrad',
|
||||
'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'models': [{
|
||||
'type': 'chunked',
|
||||
'artifact': {
|
||||
'file_name': pkl_name,
|
||||
'download_uri': {'url': '', 'sha256': digest.hexdigest()},
|
||||
'chunks': chunks
|
||||
}
|
||||
}]
|
||||
}]
|
||||
}
|
||||
with open(dm_dir / 'metadata.json', 'w') as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
print('Generated DM metadata.json')
|
||||
"
|
||||
|
||||
python3 release/ci/upload_default_model.py \
|
||||
--hf-repo "${{ env.HF_REPO }}" \
|
||||
--hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \
|
||||
--artifact-name "dm-model-${{ github.run_number }}" \
|
||||
--model-dir dm_output \
|
||||
--onnx-path "${{ env.DM_ONNX }}" \
|
||||
--onnx-ref "${{ needs.resolve.outputs.dm_onnx_ref }}" \
|
||||
--model-name "dmonitoring_model" \
|
||||
--tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \
|
||||
--run-number "${{ github.run_number }}"
|
||||
|
||||
build_dm_model:
|
||||
needs: resolve
|
||||
if: ${{ inputs.target == 'small' }}
|
||||
runs-on: [self-hosted, tici]
|
||||
env:
|
||||
DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx
|
||||
DM_PKL: openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
source /etc/profile
|
||||
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
||||
export UV_PYTHON_PREFERENCE=managed
|
||||
export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
|
||||
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
||||
uv sync --frozen
|
||||
printenv >> $GITHUB_ENV
|
||||
|
||||
- name: Disable powersave
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
|
||||
|
||||
- name: Compile DM model
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}"
|
||||
|
||||
TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
|
||||
|
||||
taskset -c 7 env ${TG_FLAGS} python3 \
|
||||
${{ github.workspace }}/tinygrad_repo/examples/openpilot/compile3.py \
|
||||
${{ github.workspace }}/${{ env.DM_ONNX }} \
|
||||
${{ github.workspace }}/${{ env.DM_PKL }}
|
||||
|
||||
- name: Chunk DM pkl
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH=${{ github.workspace }}
|
||||
python3 -c "
|
||||
from openpilot.common.file_chunker import chunk_file, get_chunk_targets
|
||||
import os
|
||||
pkl = '${{ github.workspace }}/${{ env.DM_PKL }}'
|
||||
size = os.path.getsize(pkl)
|
||||
targets = get_chunk_targets(pkl, size)
|
||||
chunk_file(pkl, targets)
|
||||
print(f'Chunked {pkl} into {len(targets)} chunks')
|
||||
"
|
||||
|
||||
- name: Prepare DM output
|
||||
run: |
|
||||
mkdir -p dm_output
|
||||
cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunk* dm_output/
|
||||
cp ${{ github.workspace }}/${{ env.DM_PKL }}.chunkmanifest dm_output/
|
||||
|
||||
- name: Upload DM artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dm-model-${{ github.run_number }}
|
||||
path: dm_output/
|
||||
|
||||
- name: Re-enable powersave
|
||||
if: always()
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
|
||||
|
||||
@@ -103,20 +103,25 @@ jobs:
|
||||
- run: |
|
||||
cd ${{ github.workspace }}/openpilot/openpilot
|
||||
if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then
|
||||
git lfs pull -X "selfdrive/modeld/models/big_*.onnx" -X "selfdrive/modeld/models/dmonitoring_*.onnx"
|
||||
git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx,**/selfdrive/modeld/models/dmonitoring_*.onnx"
|
||||
rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx
|
||||
else
|
||||
git lfs pull -I "selfdrive/modeld/models/big_*.onnx"
|
||||
git lfs pull -I "**/selfdrive/modeld/models/big_*.onnx" -X ""
|
||||
find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete
|
||||
fi
|
||||
if grep -lIF "version https://git-lfs.github.com/spec/v1" selfdrive/modeld/models/*.onnx; then
|
||||
echo "::error::the ONNX files above are still LFS pointers, not real models"
|
||||
exit 1
|
||||
fi
|
||||
- name: 'Upload Artifact'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
|
||||
path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
|
||||
if-no-files-found: error
|
||||
|
||||
build_model:
|
||||
runs-on: [self-hosted, tici]
|
||||
runs-on: [self-hosted, usbgpu]
|
||||
needs: get_model
|
||||
env:
|
||||
MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
|
||||
@@ -127,7 +132,6 @@ jobs:
|
||||
fetch-depth: 1
|
||||
submodules: recursive
|
||||
|
||||
- run: git lfs pull
|
||||
|
||||
- name: Set environment variables
|
||||
id: set-env
|
||||
@@ -160,7 +164,7 @@ jobs:
|
||||
fi
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
|
||||
rm -rf ${{ env.MODELS_DIR }}/*.onnx
|
||||
rm -rf ${{ env.MODELS_DIR }}/*.onnx*
|
||||
|
||||
- name: Download model artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -180,6 +184,7 @@ jobs:
|
||||
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
|
||||
CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
|
||||
|
||||
TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
|
||||
if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
|
||||
echo "USBGPU build"
|
||||
export USBGPU=1
|
||||
@@ -187,27 +192,40 @@ jobs:
|
||||
OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
|
||||
else
|
||||
echo "QCOM build"
|
||||
TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
|
||||
TG_FLAGS="$TG_FLAGS_QCOM"
|
||||
OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
|
||||
fi
|
||||
|
||||
# Generate metadata for all ONNX files
|
||||
find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
|
||||
echo "Generating metadata: $onnx_file"
|
||||
env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
|
||||
env ${TG_FLAGS_QCOM} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
|
||||
done
|
||||
|
||||
# Detect model type and build compile args
|
||||
VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
|
||||
POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
|
||||
OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
|
||||
ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
|
||||
VISION_ONNX=""
|
||||
for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do
|
||||
[ -f "$f" ] && VISION_ONNX="$f" && break
|
||||
done
|
||||
|
||||
POLICY_ONNX=""
|
||||
for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do
|
||||
[ -f "$f" ] && POLICY_ONNX="$f" && break
|
||||
done
|
||||
|
||||
OFF_POLICY_ONNX=""
|
||||
for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do
|
||||
[ -f "$f" ] && OFF_POLICY_ONNX="$f" && break
|
||||
done
|
||||
|
||||
ON_POLICY_ONNX=""
|
||||
for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do
|
||||
[ -f "$f" ] && ON_POLICY_ONNX="$f" && break
|
||||
done
|
||||
|
||||
SUPERCOMBO_ONNX=""
|
||||
for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do
|
||||
if [ -f "$f" ]; then
|
||||
SUPERCOMBO_ONNX="$f"
|
||||
break
|
||||
fi
|
||||
for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do
|
||||
[ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break
|
||||
done
|
||||
|
||||
MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
|
||||
|
||||
@@ -36,6 +36,7 @@ jobs:
|
||||
publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
|
||||
is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
|
||||
build: ${{ steps.strategy.outputs.build }}
|
||||
include_big_model: ${{ steps.strategy.outputs.include_big_model }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Extract deploy strategy
|
||||
@@ -78,6 +79,9 @@ jobs:
|
||||
stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
|
||||
echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
|
||||
echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
|
||||
|
||||
include_big_model="$(echo "$CONFIG" | jq -r '.include_big_model // false')";
|
||||
echo "include_big_model=$include_big_model" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
echo "build=$BUILD" >> $GITHUB_OUTPUT
|
||||
cat $GITHUB_OUTPUT
|
||||
@@ -203,6 +207,74 @@ jobs:
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
|
||||
|
||||
prepare_chestnut:
|
||||
needs: [ prepare_strategy ]
|
||||
runs-on: ubuntu-24.04
|
||||
if: ${{ needs.prepare_strategy.outputs.include_big_model == 'true' }}
|
||||
outputs:
|
||||
onnx_sha256: ${{ steps.resolve.outputs.onnx_sha256 }}
|
||||
env:
|
||||
HF_REPO: sunnypilot/sunnypilot_models_v1
|
||||
HF_DEFAULTS_PATH: models/defaults/big
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.head_ref || github.ref_name }}
|
||||
|
||||
- run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
|
||||
|
||||
- name: Check HF defaults and build if needed
|
||||
id: resolve
|
||||
run: |
|
||||
ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
|
||||
echo "Repo ONNX hash: $ACTUAL_ONNX_HASH"
|
||||
echo "onnx_sha256=$ACTUAL_ONNX_HASH" >> $GITHUB_OUTPUT
|
||||
|
||||
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
|
||||
|
||||
check_hash() {
|
||||
DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
|
||||
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
|
||||
[ -n "$BUNDLE" ] && [ "$BUNDLE" != "null" ]
|
||||
}
|
||||
|
||||
if check_hash; then
|
||||
echo "HF defaults match repo ONNX"
|
||||
else
|
||||
echo "No matching model on HF — triggering build"
|
||||
gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=big
|
||||
|
||||
echo "Waiting for build to start..."
|
||||
sleep 120
|
||||
|
||||
RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId')
|
||||
if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
|
||||
echo "::error::Failed to find build-default-models run"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Waiting for run $RUN_ID..."
|
||||
gh run watch "$RUN_ID"
|
||||
|
||||
CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
|
||||
if [ "$CONCLUSION" != "success" ]; then
|
||||
echo "::error::build-default-models failed: $CONCLUSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! check_hash; then
|
||||
echo "::error::HF defaults still don't match after build"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Cancel run on failure
|
||||
if: failure()
|
||||
run: gh run cancel ${{ github.run_id }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
publish:
|
||||
concurrency:
|
||||
@@ -211,14 +283,20 @@ jobs:
|
||||
# Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
|
||||
group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
|
||||
cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
|
||||
if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
|
||||
needs: [ build, prepare_strategy ]
|
||||
if: ${{
|
||||
always() && !cancelled() &&
|
||||
needs.build.result == 'success' &&
|
||||
needs.prepare_strategy.result == 'success' &&
|
||||
(!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
|
||||
(needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
|
||||
}}
|
||||
needs: [ build, prepare_strategy, prepare_chestnut ]
|
||||
runs-on: ubuntu-24.04
|
||||
environment: ${{ needs.prepare_strategy.outputs.environment }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Download build artifacts
|
||||
- name: Download prebuilt artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: prebuilt
|
||||
@@ -228,6 +306,44 @@ jobs:
|
||||
mkdir -p ${{ env.OUTPUT_DIR }}
|
||||
tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
|
||||
|
||||
- name: Prepare chestnut output
|
||||
if: ${{ needs.prepare_chestnut.result == 'success' }}
|
||||
run: |
|
||||
mkdir -p "${{ github.workspace }}/chestnut_output"
|
||||
tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output"
|
||||
|
||||
- name: Download big model chunks from HF
|
||||
if: ${{ needs.prepare_chestnut.result == 'success' }}
|
||||
env:
|
||||
HF_REPO: sunnypilot/sunnypilot_models_v1
|
||||
HF_DEFAULTS_PATH: models/defaults/big
|
||||
run: |
|
||||
ONNX_HASH="${{ needs.prepare_chestnut.outputs.onnx_sha256 }}"
|
||||
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
|
||||
DEFAULTS=$(curl -fsSL "$JSON_URL")
|
||||
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
|
||||
|
||||
mkdir -p big_model_chunks
|
||||
ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
|
||||
BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
|
||||
NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
|
||||
|
||||
CANONICAL="big_driving_tinygrad.pkl"
|
||||
echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
|
||||
CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+')
|
||||
CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
|
||||
ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
|
||||
echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
|
||||
curl -fsSL -o "big_model_chunks/${CANONICAL_CHUNK}" "$ENCODED_URL"
|
||||
done
|
||||
|
||||
echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest"
|
||||
|
||||
- name: Inject big model into chestnut
|
||||
if: ${{ needs.prepare_chestnut.result == 'success' }}
|
||||
run: |
|
||||
cp big_model_chunks/* "${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models/"
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
||||
@@ -248,6 +364,22 @@ jobs:
|
||||
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
|
||||
"${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
|
||||
|
||||
- name: Publish chestnut branch
|
||||
if: ${{ needs.prepare_chestnut.result == 'success' }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
|
||||
CHESTNUT_DIR="${{ github.workspace }}/chestnut_output"
|
||||
|
||||
${{ env.CI_DIR }}/publish.sh \
|
||||
"${{ github.workspace }}" \
|
||||
"$CHESTNUT_DIR" \
|
||||
"$CHESTNUT_BRANCH" \
|
||||
"${{ needs.prepare_strategy.outputs.version }}" \
|
||||
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
|
||||
"${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
|
||||
|
||||
- name: Tag ${{ needs.prepare_strategy.outputs.environment }}
|
||||
if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
|
||||
run: |
|
||||
@@ -260,6 +392,7 @@ jobs:
|
||||
- prepare_strategy
|
||||
- build
|
||||
- publish
|
||||
- prepare_chestnut
|
||||
runs-on: ubuntu-24.04
|
||||
if: ${{ (always() && !cancelled() && !failure())
|
||||
&& needs.publish.result == 'success'
|
||||
@@ -279,6 +412,7 @@ jobs:
|
||||
export commit_short_sha="${commit_short_sha:0:7}"
|
||||
export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
|
||||
export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
|
||||
export chestnut_branch="${{ needs.prepare_chestnut.result == 'success' && format('{0}-chestnut', needs.prepare_strategy.outputs.new_branch) || '' }}"
|
||||
|
||||
MESSAGE=$(cat << 'EOF' | envsubst
|
||||
${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}
|
||||
|
||||
+3
-1
@@ -24,7 +24,9 @@ function agnos_init {
|
||||
if $AGNOS_PY --verify $MANIFEST; then
|
||||
sudo reboot
|
||||
fi
|
||||
$DIR/openpilot/common/hardware/comma/updater $AGNOS_PY $MANIFEST
|
||||
while true; do
|
||||
$DIR/openpilot/common/hardware/comma/updater $AGNOS_PY $MANIFEST
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
{"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"IsEngaged", {PERSISTENT, BOOL}},
|
||||
{"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"IsLiveStreaming", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"IsLiveStreaming", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
|
||||
{"IsMetric", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"IsRhdDetected", {PERSISTENT, BOOL}},
|
||||
@@ -195,9 +195,10 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
|
||||
// Model Manager params
|
||||
{"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
|
||||
{"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"ModelManager_ActiveBundleUSBGPU", {PERSISTENT, JSON}},
|
||||
{"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}},
|
||||
{"ModelManager_DownloadRef", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
|
||||
{"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
|
||||
{"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
|
||||
{"ModelManager_LastSyncTime_USBGPU", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
|
||||
|
||||
@@ -16,6 +16,15 @@ MASTER_SP_BRANCHES = ['master']
|
||||
RELEASE_BRANCHES = ['release-tizi-staging', 'release-mici-staging', 'release-tizi', 'release-mici', 'nightly']
|
||||
TESTED_BRANCHES = RELEASE_BRANCHES + ['devel-staging', 'nightly-dev'] + RELEASE_SP_BRANCHES + TESTED_SP_BRANCHES
|
||||
|
||||
CHESTNUT_BRANCHES = {
|
||||
"staging": "staging-chestnut",
|
||||
"dev": "dev-chestnut",
|
||||
"release-mici": "release-chestnut",
|
||||
"release-tizi": "release-chestnut",
|
||||
"release-mici-staging": "release-chestnut-staging",
|
||||
"release-tizi-staging": "release-chestnut-staging",
|
||||
}
|
||||
|
||||
SP_BRANCH_MIGRATIONS = {
|
||||
("tici", "staging-c3-new"): "staging-tici",
|
||||
("tici", "dev-c3-new"): "staging-tici",
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"_comment": "Set extra field to the failed reason."
|
||||
},
|
||||
"Offroad_ChestnutBranch": {
|
||||
"text": "Chestnut detected! Switch to the release-chestnut branch to use chestnut-class models.",
|
||||
"text": "Chestnut detected! Switch to the %1 branch to use chestnut-class models.",
|
||||
"severity": 0
|
||||
},
|
||||
"Offroad_UnregisteredHardware": {
|
||||
|
||||
@@ -149,11 +149,15 @@ class BigButton(Widget):
|
||||
def set_touch_valid_callback(self, touch_callback: Callable[[], bool]) -> None:
|
||||
super().set_touch_valid_callback(lambda: touch_callback() and self._grow_animation_until is None)
|
||||
|
||||
def _width_hint(self) -> int:
|
||||
# A value moves the title to the top, where it shares space with the icon.
|
||||
def _title_width_hint(self) -> int:
|
||||
# A value moves the title to the top, where it shares space with the icon
|
||||
icon_size = self._txt_icon.width if self._txt_icon and self.value else 0
|
||||
return int(self._rect.width - self.LABEL_HORIZONTAL_PADDING * 2 - icon_size)
|
||||
|
||||
def _subtitle_width_hint(self) -> int:
|
||||
# Bottom aligned, so it sits below the icon
|
||||
return int(self._rect.width - self.LABEL_HORIZONTAL_PADDING * 2)
|
||||
|
||||
def _get_label_font_size(self):
|
||||
if len(self.text) <= 18:
|
||||
return 48
|
||||
@@ -228,14 +232,14 @@ class BigButton(Widget):
|
||||
|
||||
label_color = LABEL_COLOR if self.enabled else rl.Color(255, 255, 255, int(255 * 0.35))
|
||||
self._label.set_color(label_color)
|
||||
label_rect = rl.Rectangle(label_x, btn_y + self.LABEL_VERTICAL_PADDING, self._width_hint(),
|
||||
label_rect = rl.Rectangle(label_x, btn_y + self.LABEL_VERTICAL_PADDING, self._title_width_hint(),
|
||||
self._rect.height - self.LABEL_VERTICAL_PADDING * 2)
|
||||
self._label.render(label_rect)
|
||||
|
||||
if self.value:
|
||||
label_y = btn_y + self.LABEL_VERTICAL_PADDING + self._label.get_content_height(self._width_hint())
|
||||
label_y = label_rect.y + self._label.get_content_height(int(label_rect.width))
|
||||
sub_label_height = btn_y + self._rect.height - self.LABEL_VERTICAL_PADDING - label_y
|
||||
sub_label_rect = rl.Rectangle(label_x, label_y, self._width_hint(), sub_label_height)
|
||||
sub_label_rect = rl.Rectangle(label_x, label_y, self._subtitle_width_hint(), sub_label_height)
|
||||
self._sub_label.render(sub_label_rect)
|
||||
|
||||
# ICON -------------------------------------------------------------------
|
||||
@@ -312,9 +316,6 @@ class BigMultiToggle(BigToggle):
|
||||
|
||||
self.set_value(self._options[0])
|
||||
|
||||
def _width_hint(self) -> int:
|
||||
return int(self._rect.width - self.LABEL_HORIZONTAL_PADDING * 2 - self._txt_enabled_toggle.width)
|
||||
|
||||
def _handle_mouse_release(self, mouse_pos: MousePos):
|
||||
super()._handle_mouse_release(mouse_pos)
|
||||
cur_idx = self._options.index(self.value)
|
||||
@@ -363,9 +364,6 @@ class GreyBigButton(BigButton):
|
||||
def LABEL_VERTICAL_PADDING(self):
|
||||
return BigButton.LABEL_VERTICAL_PADDING if self._label.text else 18
|
||||
|
||||
def _width_hint(self) -> int:
|
||||
return int(self._rect.width - self.LABEL_HORIZONTAL_PADDING * 2)
|
||||
|
||||
def _get_label_font_size(self):
|
||||
return 36
|
||||
|
||||
|
||||
@@ -192,7 +192,7 @@ class ModelRenderer(Widget, ChevronMetrics, ModelRendererSP):
|
||||
|
||||
max_idx = self._get_path_length_idx(path_x_array, max_distance)
|
||||
self._path.projected_points = self._map_line_to_polygon(
|
||||
self._path.raw_points, 0.9, self._path_offset_z, max_idx, max_distance, allow_invert=False
|
||||
self._path.raw_points, self._get_path_half_width(), self._path_offset_z, max_idx, max_distance, allow_invert=False
|
||||
)
|
||||
|
||||
self._update_experimental_gradient()
|
||||
@@ -292,7 +292,7 @@ class ModelRenderer(Widget, ChevronMetrics, ModelRendererSP):
|
||||
allow_throttle = sm['longitudinalPlan'].allowThrottle or not self._longitudinal_control
|
||||
self._blend_filter.update(int(allow_throttle))
|
||||
|
||||
if ui_state.rainbow_path:
|
||||
if ui_state.rainbow_path and self._lateral_active:
|
||||
self.rainbow_path.draw_rainbow_path(self._rect, self._path)
|
||||
return
|
||||
|
||||
|
||||
@@ -10,9 +10,11 @@ import time
|
||||
import pyray as rl
|
||||
|
||||
from openpilot.cereal import custom
|
||||
from openpilot.sunnypilot.models.default_model import DEFAULT_MODEL
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher, get_cached_bundles
|
||||
from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_selected_bundle, resolve_bundle_by_ref
|
||||
from openpilot.common.constants import CV
|
||||
from openpilot.selfdrive.ui.ui_state import device, ui_state
|
||||
from openpilot.selfdrive.ui.sunnypilot.model_info import model_info
|
||||
from openpilot.system.ui.lib.multilang import tr
|
||||
from openpilot.system.ui.lib.application import gui_app
|
||||
from openpilot.system.ui.widgets import DialogResult, Widget
|
||||
@@ -36,6 +38,7 @@ class ModelsLayout(Widget):
|
||||
super().__init__()
|
||||
self.model_manager = None
|
||||
self.model_dialog = None
|
||||
self._selection_source = None
|
||||
self._downloading = False
|
||||
self.last_cache_calc_time = 0
|
||||
|
||||
@@ -49,16 +52,23 @@ class ModelsLayout(Widget):
|
||||
|
||||
def _initialize_items(self):
|
||||
self.current_model_item = ListItemSP(
|
||||
title=tr("Current Model"),
|
||||
title=tr("Active Model"),
|
||||
description="",
|
||||
action_item=ScrollingButtonAction(tr("SELECT")),
|
||||
callback=self._handle_current_model_clicked
|
||||
)
|
||||
|
||||
self.other_model_item = ListItemSP(
|
||||
title=tr("Big Model"),
|
||||
action_item=ScrollingButtonAction(tr("SELECT")),
|
||||
callback=self._handle_other_model_clicked
|
||||
)
|
||||
|
||||
self.download_item = download_status_item(lambda: tr("Download") if self._downloading else tr("Model Status"))
|
||||
|
||||
self.refresh_item = button_item(tr("Refresh Model List"), tr("REFRESH"), "",
|
||||
lambda: (ui_state.params.put("ModelManager_LastSyncTime", 0),
|
||||
ui_state.params.put("ModelManager_LastSyncTime_USBGPU", 0),
|
||||
gui_app.push_widget(alert_dialog(tr("Fetching Latest Models")))))
|
||||
|
||||
self.clear_cache_item = ListItemSP(
|
||||
@@ -68,7 +78,8 @@ class ModelsLayout(Widget):
|
||||
callback=self._clear_cache
|
||||
)
|
||||
|
||||
self.cancel_download_item = button_item(tr("Cancel Download"), tr("Cancel"), "", lambda: ui_state.params.remove("ModelManager_DownloadIndex"))
|
||||
self.cancel_download_item = button_item(tr("Cancel Download"), tr("Cancel"), "",
|
||||
lambda: ui_state.params.remove("ModelManager_DownloadRef"))
|
||||
|
||||
self.lane_turn_value_control = option_item_sp(tr("Adjust Lane Turn Speed"), "LaneTurnValue", 500, 2000,
|
||||
tr("Set the maximum speed for lane turn desires. Default is 19 mph."),
|
||||
@@ -93,7 +104,7 @@ class ModelsLayout(Widget):
|
||||
1, None, True, "", style.BUTTON_ACTION_WIDTH, None, True,
|
||||
lambda v: f"{v / 100:.2f} m")
|
||||
|
||||
self.items = [self.current_model_item, self.cancel_download_item, self.download_item, self.refresh_item, self.clear_cache_item,
|
||||
self.items = [self.current_model_item, self.other_model_item, self.cancel_download_item, self.download_item, self.refresh_item, self.clear_cache_item,
|
||||
self.lane_turn_desire_toggle, self.lane_turn_value_control, self.lagd_toggle, self.delay_control, self.camera_offset]
|
||||
|
||||
def _update_lagd_description(self, lagd_toggle: bool):
|
||||
@@ -142,7 +153,7 @@ class ModelsLayout(Widget):
|
||||
if not bundle:
|
||||
return
|
||||
|
||||
self.cancel_download_item.set_visible(bool(self.model_manager.selectedBundle) and ui_state.params.get("ModelManager_DownloadIndex") is not None)
|
||||
self.cancel_download_item.set_visible(bool(self.model_manager.selectedBundle) and ui_state.params.get("ModelManager_DownloadRef") is not None)
|
||||
|
||||
if (current_time := time.monotonic()) - self.last_cache_calc_time > 0.5:
|
||||
self.last_cache_calc_time = current_time
|
||||
@@ -178,40 +189,39 @@ class ModelsLayout(Widget):
|
||||
# circled_slash is authored grey; tinting it again only darkens it
|
||||
return {"name": name, "text_color": rl.GRAY, "icon": "icons/circled_slash.png", "icon_color": rl.WHITE}
|
||||
|
||||
@staticmethod
|
||||
def _show_reset_params_dialog():
|
||||
def _callback(response):
|
||||
if response == DialogResult.CONFIRM:
|
||||
ui_state.params.remove("CalibrationParams")
|
||||
ui_state.params.remove("LiveTorqueParameters")
|
||||
msg = tr("Model download has started in the background. We suggest resetting calibration. Would you like to do that now?")
|
||||
dialog = ConfirmDialog(msg, tr("Reset Calibration"), callback=_callback)
|
||||
gui_app.push_widget(dialog)
|
||||
|
||||
def _on_model_selected(self, result):
|
||||
if result != DialogResult.CONFIRM:
|
||||
self.model_dialog = None
|
||||
return
|
||||
selected_ref = self.model_dialog.selection_ref
|
||||
if selected_ref == "Default":
|
||||
ui_state.params.remove("ModelManager_ActiveBundle")
|
||||
self._show_reset_params_dialog()
|
||||
elif selected_bundle := next((bundle for bundle in self.model_manager.availableBundles if bundle.ref == selected_ref), None):
|
||||
ui_state.params.put("ModelManager_DownloadIndex", selected_bundle.index)
|
||||
if self.model_manager.activeBundle and selected_bundle.generation != self.model_manager.activeBundle.generation:
|
||||
self._show_reset_params_dialog()
|
||||
self.model_dialog = None
|
||||
if selected_ref == "Default":
|
||||
if self._selection_source in ACTIVE_BUNDLE_KEYS:
|
||||
ui_state.params.remove(ACTIVE_BUNDLE_KEYS[self._selection_source])
|
||||
return
|
||||
if selected_bundle := self._resolve_selected_bundle(selected_ref):
|
||||
ui_state.params.put("ModelManager_DownloadRef", selected_bundle.ref)
|
||||
|
||||
def _resolve_selected_bundle(self, ref):
|
||||
"""Finds the bundle for a ref across both hardware manifests."""
|
||||
active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent)
|
||||
source_bundles = {
|
||||
source: self.model_manager.availableBundles if source == active else get_cached_bundles(ui_state.params, source)
|
||||
for source in ("qcom", "usbgpu")
|
||||
}
|
||||
resolved = resolve_bundle_by_ref(ref, source_bundles)
|
||||
return resolved[0] if resolved else None
|
||||
|
||||
@staticmethod
|
||||
def _bundle_to_node(bundle):
|
||||
return TreeNode(bundle.ref, {'display_name': bundle.displayName, 'short_name': bundle.internalName})
|
||||
|
||||
def _get_folders(self, favorites):
|
||||
bundles = self.model_manager.availableBundles
|
||||
def _get_folders(self, favorites, bundles):
|
||||
folders = {}
|
||||
for bundle in bundles:
|
||||
folders.setdefault(next((ov_ride.value for ov_ride in bundle.overrides if ov_ride.key == "folder"), ""), []).append(bundle)
|
||||
|
||||
folders_list = [TreeFolder("", [TreeNode("Default", {'display_name': f"{DEFAULT_MODEL} (Default)", 'short_name': "Default"})])]
|
||||
folders_list = []
|
||||
for folder, folder_bundles in sorted(folders.items(), key=lambda x: max((bundle.index for bundle in x[1]), default=-1), reverse=True):
|
||||
folder_bundles.sort(key=lambda bundle: bundle.index, reverse=True)
|
||||
name = folder + (f" - (Updated: {m.group(1)})" if folder_bundles and (m := re.search(r'\(([^)]*)\)[^(]*$', folder_bundles[0].displayName)) else "")
|
||||
@@ -222,20 +232,45 @@ class ModelsLayout(Widget):
|
||||
return folders_list
|
||||
|
||||
def _handle_current_model_clicked(self):
|
||||
self._open_source_dialog(ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent))
|
||||
|
||||
def _handle_other_model_clicked(self):
|
||||
active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent)
|
||||
self._open_source_dialog("qcom" if active == "usbgpu" else "usbgpu")
|
||||
|
||||
def _open_source_dialog(self, source):
|
||||
"""Opens the picker for one hardware: its model folders plus the Default reset entry."""
|
||||
self._selection_source = source
|
||||
favs = ui_state.params.get("ModelManager_Favs")
|
||||
favorites = set(favs.split(';')) if favs else set()
|
||||
folders_list = self._get_folders(favorites)
|
||||
|
||||
active_ref = self.model_manager.activeBundle.ref if self.model_manager.activeBundle else "Default"
|
||||
self.model_dialog = TreeOptionDialog(tr("Select a Model"), folders_list, active_ref, "ModelManager_Favs",
|
||||
get_folders_fn=self._get_folders, on_exit=self._on_model_selected)
|
||||
folders_list = self._source_folders(favorites, source)
|
||||
if not folders_list:
|
||||
gui_app.push_widget(alert_dialog(tr("No models are available for this hardware yet. Connect to the internet and refresh the model list.")))
|
||||
return
|
||||
self.model_dialog = TreeOptionDialog(tr("Select a Model"), folders_list, self._slot_active_ref(source), "ModelManager_Favs",
|
||||
get_folders_fn=lambda favs: self._source_folders(favs, source), on_exit=self._on_model_selected)
|
||||
gui_app.push_widget(self.model_dialog)
|
||||
|
||||
def _source_folders(self, favorites, source):
|
||||
"""Default reset entry on top, then the hardware's model folders."""
|
||||
active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent)
|
||||
bundles = self.model_manager.availableBundles if source == active else get_cached_bundles(ui_state.params, source)
|
||||
if not bundles:
|
||||
return []
|
||||
folders_list = [TreeFolder("", [TreeNode("Default", {'display_name': "Default"})])]
|
||||
folders_list.extend(self._get_folders(favorites, bundles))
|
||||
return folders_list
|
||||
|
||||
@staticmethod
|
||||
def _slot_active_ref(source: str) -> str:
|
||||
bundle = get_selected_bundle(ui_state.params, source)
|
||||
return bundle.ref if bundle else "Default"
|
||||
|
||||
def _update_state(self):
|
||||
advanced_controls: bool = ui_state.params.get_bool("ShowAdvancedControls")
|
||||
turn_desire: bool = ui_state.params.get_bool("LaneTurnDesire")
|
||||
live_delay: bool = ui_state.params.get_bool("LagdToggle")
|
||||
camera_offset: bool = ui_state.params.get("ModelManager_ActiveBundle") is not None
|
||||
camera_offset: bool = ui_state.active_bundle is not None
|
||||
|
||||
self.lane_turn_desire_toggle.action_item.set_state(turn_desire)
|
||||
self.lane_turn_value_control.set_visible(turn_desire and advanced_controls)
|
||||
@@ -249,8 +284,10 @@ class ModelsLayout(Widget):
|
||||
self._update_lagd_description(live_delay)
|
||||
self.model_manager = ui_state.sm["modelManagerSP"]
|
||||
self._handle_bundle_download_progress()
|
||||
active_name = self.model_manager.activeBundle.displayName if self.model_manager and self.model_manager.activeBundle.ref else f"{DEFAULT_MODEL} (Default)"
|
||||
source, active_name, other_name = model_info()
|
||||
self.current_model_item.action_item.set_value(active_name)
|
||||
self.other_model_item.set_title(tr("Big Model") if source == "qcom" else tr("Small Model"))
|
||||
self.other_model_item.action_item.set_value(other_name)
|
||||
|
||||
if not ui_state.is_offroad():
|
||||
self.current_model_item.action_item.set_enabled(False)
|
||||
|
||||
@@ -7,16 +7,25 @@ See the LICENSE.md file in the root directory for more details.
|
||||
import pyray as rl
|
||||
|
||||
from openpilot.cereal import custom
|
||||
from openpilot.sunnypilot.models.default_model import DEFAULT_MODEL
|
||||
from openpilot.selfdrive.ui.mici.widgets.dialog import BigDialog
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher, get_cached_bundles
|
||||
from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS
|
||||
from openpilot.selfdrive.ui.mici.widgets.button import BigButton
|
||||
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.models import ModelsLayout
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state, device
|
||||
from openpilot.selfdrive.ui.sunnypilot.model_info import model_info
|
||||
from openpilot.system.ui.lib.application import FontWeight, gui_app
|
||||
from openpilot.system.ui.lib.multilang import tr
|
||||
from openpilot.system.ui.widgets import Widget
|
||||
from openpilot.system.ui.widgets.label import UnifiedLabel
|
||||
from openpilot.system.ui.widgets.scroller import NavScroller
|
||||
|
||||
def _model_info() -> tuple[str, str, str]:
|
||||
"""(active model, other-model header, other-model text) for the panel."""
|
||||
source, active_name, other_name = model_info()
|
||||
header = tr("small model") if source == "usbgpu" else tr("big model")
|
||||
return active_name.lower(), header, other_name.lower()
|
||||
|
||||
|
||||
class CurrentModelInfo(Widget):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -26,12 +35,12 @@ class CurrentModelInfo(Widget):
|
||||
header_color = rl.Color(255, 255, 255, int(255 * 0.9))
|
||||
subheader_color = rl.Color(255, 255, 255, int(255 * 0.9 * 0.65))
|
||||
max_width = int(self._rect.width - 20)
|
||||
active_text, info_header, info_text = _model_info()
|
||||
self.current_model_header = UnifiedLabel(tr("active model"), 48, max_width=max_width, text_color=header_color, font_weight=FontWeight.DISPLAY)
|
||||
default_text = f"{DEFAULT_MODEL} (Default)".lower()
|
||||
self.current_model_text = UnifiedLabel(default_text, 32, max_width=max_width, text_color=subheader_color, font_weight=FontWeight.ROMAN, scroll=True)
|
||||
self.current_model_text = UnifiedLabel(active_text, 32, max_width=max_width, text_color=subheader_color, font_weight=FontWeight.ROMAN, scroll=True)
|
||||
|
||||
self.info_header = UnifiedLabel("cache size", 48, max_width=max_width, text_color=header_color, font_weight=FontWeight.DISPLAY)
|
||||
self.info_text = UnifiedLabel("0 mb", 32, max_width=max_width, text_color=subheader_color, font_weight=FontWeight.ROMAN)
|
||||
self.info_header = UnifiedLabel(info_header, 48, max_width=max_width, text_color=header_color, font_weight=FontWeight.DISPLAY)
|
||||
self.info_text = UnifiedLabel(info_text, 32, max_width=max_width, text_color=subheader_color, font_weight=FontWeight.ROMAN, scroll=True)
|
||||
|
||||
def _render(self, _):
|
||||
self.current_model_header.set_position(self._rect.x + 20, self._rect.y - 10)
|
||||
@@ -55,12 +64,13 @@ class ModelsLayoutMici(NavScroller):
|
||||
self._download_progress = "."
|
||||
self._download_frame = 0
|
||||
self._was_downloading = False
|
||||
self._selection_source: str | None = None
|
||||
|
||||
self.select_model_btn = BigButton(tr("select model"))
|
||||
self.select_model_btn.set_click_callback(self._show_folders)
|
||||
|
||||
self.cancel_download_btn = BigButton(tr("cancel download"))
|
||||
self.cancel_download_btn.set_click_callback(lambda: ui_state.params.remove("ModelManager_DownloadIndex"))
|
||||
self.cancel_download_btn.set_click_callback(lambda: ui_state.params.remove("ModelManager_DownloadRef"))
|
||||
|
||||
self.main_items = [self.current_model_info, self.select_model_btn, self.cancel_download_btn]
|
||||
self._scroller.add_widgets(self.main_items)
|
||||
@@ -69,8 +79,7 @@ class ModelsLayoutMici(NavScroller):
|
||||
def model_manager(self):
|
||||
return ui_state.sm["modelManagerSP"]
|
||||
|
||||
def _get_grouped_bundles(self, favorites = None):
|
||||
bundles = self.model_manager.availableBundles
|
||||
def _get_grouped_bundles(self, bundles, favorites = None):
|
||||
folders = {}
|
||||
for bundle in bundles:
|
||||
folder = next((override.value for override in bundle.overrides if override.key == "folder"), "")
|
||||
@@ -90,47 +99,74 @@ class ModelsLayoutMici(NavScroller):
|
||||
def _show_folders(self):
|
||||
self.focused_widget = self.select_model_btn
|
||||
|
||||
hardware_btns = []
|
||||
for source, label in (("qcom", tr("small models")), ("usbgpu", tr("big models"))):
|
||||
btn = BigButton(label.lower())
|
||||
btn.set_click_callback(lambda s=source: self._select_hardware(s))
|
||||
hardware_btns.append(btn)
|
||||
self._push_selection_view(hardware_btns)
|
||||
|
||||
def _select_hardware(self, source):
|
||||
self._selection_source = source
|
||||
|
||||
favs = ui_state.params.get("ModelManager_Favs")
|
||||
favorites = set(favs.split(';')) if favs else set()
|
||||
active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent)
|
||||
|
||||
if source != active:
|
||||
bundles = get_cached_bundles(ui_state.params, source)
|
||||
if not bundles:
|
||||
gui_app.push_widget(BigDialog(title=tr("No models available"),
|
||||
description=tr("No models are available for this hardware yet. Connect to the internet and refresh the model list.")))
|
||||
return
|
||||
else:
|
||||
bundles = self.model_manager.availableBundles
|
||||
folders = self._get_grouped_bundles(bundles, favorites)
|
||||
|
||||
folders = self._get_grouped_bundles(favorites)
|
||||
folder_buttons = []
|
||||
default_btn = BigButton(f"{DEFAULT_MODEL} (Default)".lower())
|
||||
default_btn.set_click_callback(self._select_default)
|
||||
default_btn = BigButton(tr("default"))
|
||||
default_btn.set_click_callback(lambda s=source: self._select_default(s))
|
||||
folder_buttons.append(default_btn)
|
||||
|
||||
for folder in sorted(folders.keys(), key=lambda f: max((bundle.index for bundle in folders[f]), default=-1), reverse=True):
|
||||
if folder.lower() in ["release models", "master models", "favorites"]:
|
||||
btn = BigButton(folder.lower())
|
||||
btn.set_click_callback(lambda f=folder: self._select_folder(f))
|
||||
if folder.lower() == "favorites":
|
||||
folder_buttons.insert(0, btn)
|
||||
else:
|
||||
folder_buttons.append(btn)
|
||||
btn = BigButton(folder.lower())
|
||||
btn.set_click_callback(lambda f=folder: self._select_folder(f))
|
||||
if folder.lower() == "favorites":
|
||||
folder_buttons.insert(0, btn)
|
||||
else:
|
||||
folder_buttons.append(btn)
|
||||
self._push_selection_view(folder_buttons)
|
||||
|
||||
def _pop_to_main(self):
|
||||
gui_app.pop_widgets_to(self)
|
||||
self._scroller.scroll_panel.set_offset(0.0)
|
||||
|
||||
def _select_model(self, bundle):
|
||||
ui_state.params.put("ModelManager_DownloadIndex", bundle.index)
|
||||
ui_state.params.put("ModelManager_DownloadRef", bundle.ref)
|
||||
self._pop_to_main()
|
||||
|
||||
def _select_default(self):
|
||||
ui_state.params.remove("ModelManager_ActiveBundle")
|
||||
def _select_default(self, source):
|
||||
ui_state.params.remove(ACTIVE_BUNDLE_KEYS[source])
|
||||
self._pop_to_main()
|
||||
|
||||
def _select_folder(self, folder_name):
|
||||
source = self._selection_source
|
||||
if source is None: # folders are only reachable after picking a hardware
|
||||
return
|
||||
favs = ui_state.params.get("ModelManager_Favs")
|
||||
favorites = set(favs.split(';')) if favs else set()
|
||||
active = ModelFetcher.active_source(ui_state.sm["deviceState"].chestnutPresent)
|
||||
|
||||
folders = self._get_grouped_bundles(favorites)
|
||||
if source != active:
|
||||
bundles = get_cached_bundles(ui_state.params, source)
|
||||
else:
|
||||
bundles = self.model_manager.availableBundles
|
||||
folders = self._get_grouped_bundles(bundles, favorites)
|
||||
bundles = sorted(folders.get(folder_name, []), key=lambda b: b.index, reverse=True)
|
||||
|
||||
btns = []
|
||||
for bundle in bundles:
|
||||
txt = bundle.displayName.lower()
|
||||
btn = BigButton(txt)
|
||||
btn = BigButton(bundle.displayName.lower())
|
||||
btn.set_click_callback(lambda b=bundle: self._select_model(b))
|
||||
btns.append(btn)
|
||||
self._push_selection_view(btns)
|
||||
@@ -162,10 +198,10 @@ class ModelsLayoutMici(NavScroller):
|
||||
self._was_downloading = is_downloading
|
||||
|
||||
self.current_model_info.current_model_header.set_text(tr("active model"))
|
||||
model_text = manager.activeBundle.displayName.lower() if manager.activeBundle.ref else f"{DEFAULT_MODEL} (Default)".lower()
|
||||
self.current_model_info.current_model_text.set_text(model_text)
|
||||
self.current_model_info.info_header.set_text(tr("cache size"))
|
||||
self.current_model_info.info_text.set_text(f"{ModelsLayout.calculate_cache_size():.2f} MB")
|
||||
active_text, info_header, info_text = _model_info()
|
||||
self.current_model_info.current_model_text.set_text(active_text)
|
||||
self.current_model_info.info_header.set_text(info_header)
|
||||
self.current_model_info.info_text.set_text(info_text)
|
||||
|
||||
if manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.failed:
|
||||
self.current_model_info.info_header.set_text(tr("error") + self._download_progress)
|
||||
@@ -192,3 +228,6 @@ class ModelsLayoutMici(NavScroller):
|
||||
self.current_model_info.info_header._shimmer = True
|
||||
self.current_model_info.info_text.set_text(f"{progress/count:.2f}%")
|
||||
|
||||
elif manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.downloaded:
|
||||
self.current_model_info.info_header.set_text(tr("downloaded"))
|
||||
self.current_model_info.info_text.set_text(tr("downloaded"))
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""
|
||||
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state
|
||||
from openpilot.sunnypilot.models.helpers import get_active_source, get_selected_bundle
|
||||
from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL, DEFAULT_MODEL
|
||||
|
||||
|
||||
def model_info() -> tuple[str, str, str]:
|
||||
"""returns (active source, active model name, other model name)"""
|
||||
source = get_active_source(usbgpu=ui_state.usbgpu,
|
||||
usbgpu_active=ui_state.usbgpu_active, usbgpu_loading=ui_state.usbgpu_loading,
|
||||
offroad=ui_state.is_offroad())
|
||||
other = "qcom" if source == "usbgpu" else "usbgpu"
|
||||
active_bundle = get_selected_bundle(ui_state.params, source)
|
||||
other_bundle = get_selected_bundle(ui_state.params, other)
|
||||
|
||||
active_name = active_bundle.displayName if active_bundle \
|
||||
else f"{DEFAULT_BIG_MODEL if source == 'usbgpu' else DEFAULT_MODEL} (Default)"
|
||||
other_name = other_bundle.displayName if other_bundle \
|
||||
else f"{DEFAULT_MODEL if source == 'usbgpu' else DEFAULT_BIG_MODEL} (Default)"
|
||||
return source, active_name, other_name
|
||||
@@ -4,11 +4,29 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
from openpilot.common.filter_simple import FirstOrderFilter
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus
|
||||
from openpilot.selfdrive.ui.sunnypilot.onroad.chevron_metrics import ChevronMetrics
|
||||
from openpilot.selfdrive.ui.sunnypilot.onroad.rainbow_path import RainbowPath
|
||||
from openpilot.selfdrive.ui.sunnypilot.ui_state import MADSState
|
||||
from openpilot.system.ui.lib.application import gui_app
|
||||
|
||||
|
||||
class ModelRendererSP:
|
||||
def __init__(self):
|
||||
self.rainbow_path = RainbowPath()
|
||||
self.chevron_metrics = ChevronMetrics()
|
||||
self._width_filter = FirstOrderFilter(0.9, 0.1, 1 / gui_app.target_fps)
|
||||
|
||||
@property
|
||||
def _lateral_active(self) -> bool:
|
||||
sm = ui_state.sm
|
||||
if sm.valid["selfdriveStateSP"]:
|
||||
mads = sm["selfdriveStateSP"].mads
|
||||
if mads.available:
|
||||
return mads.enabled and mads.state != MADSState.paused
|
||||
return ui_state.status in (UIStatus.ENGAGED, UIStatus.LAT_ONLY)
|
||||
|
||||
def _get_path_half_width(self) -> float:
|
||||
target = 0.9 if self._lateral_active else 0.40
|
||||
return self._width_filter.update(target)
|
||||
|
||||
@@ -10,6 +10,7 @@ from openpilot.cereal import messaging, log, custom
|
||||
from opendbc.car.structs import car
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.display import OnroadBrightness
|
||||
from openpilot.sunnypilot.models.helpers import get_active_bundle
|
||||
from openpilot.sunnypilot.sunnylink.sunnylink_state import SunnylinkState
|
||||
from openpilot.system.ui.lib.application import gui_app
|
||||
from openpilot.system.ui.sunnypilot.widgets.screen_saver import ScreenSaverSP
|
||||
@@ -150,7 +151,7 @@ class UIStateSP:
|
||||
self.has_icbm = self.CP_SP.intelligentCruiseButtonManagementAvailable and self.params.get_bool("IntelligentCruiseButtonManagement")
|
||||
|
||||
self._enforce_constraints()
|
||||
self.active_bundle = self.params.get("ModelManager_ActiveBundle")
|
||||
self.active_bundle = get_active_bundle(self.params)
|
||||
self.blindspot = self.params.get_bool("BlindSpot")
|
||||
self.chevron_metrics = self.params.get("ChevronInfo")
|
||||
self.custom_interactive_timeout = self.params.get("InteractivityTimeout", return_default=True)
|
||||
|
||||
@@ -272,18 +272,17 @@ def _parse_size(size_str: str) -> tuple[int, int]:
|
||||
return int(width), int(height)
|
||||
|
||||
|
||||
def read_file_chunked_to_shm(path):
|
||||
def read_file_chunked_to_disk(path):
|
||||
if not path:
|
||||
return None
|
||||
import atexit
|
||||
import shutil
|
||||
from openpilot.common.file_chunker import open_file_chunked
|
||||
from openpilot.common.hardware.hw import Paths
|
||||
shm_path = os.path.join(Paths.shm_path(), os.path.basename(path))
|
||||
atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path))
|
||||
with open(shm_path, 'wb') as dst, open_file_chunked(path) as src:
|
||||
shutil.copyfileobj(src, dst)
|
||||
return shm_path
|
||||
tmp_path = f'{path}.unchunked'
|
||||
with open(tmp_path, 'wb') as f, open_file_chunked(path) as src:
|
||||
shutil.copyfileobj(src, f)
|
||||
atexit.register(lambda: os.path.exists(tmp_path) and os.remove(tmp_path))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]:
|
||||
@@ -327,11 +326,11 @@ if __name__ == "__main__":
|
||||
model_w, model_h = args.model_size
|
||||
output_data = {}
|
||||
|
||||
args.vision_onnx = read_file_chunked_to_shm(args.vision_onnx)
|
||||
args.policy_onnx = read_file_chunked_to_shm(args.policy_onnx)
|
||||
args.off_policy_onnx = read_file_chunked_to_shm(args.off_policy_onnx)
|
||||
args.on_policy_onnx = read_file_chunked_to_shm(args.on_policy_onnx)
|
||||
args.supercombo_onnx = read_file_chunked_to_shm(args.supercombo_onnx)
|
||||
args.vision_onnx = read_file_chunked_to_disk(args.vision_onnx)
|
||||
args.policy_onnx = read_file_chunked_to_disk(args.policy_onnx)
|
||||
args.off_policy_onnx = read_file_chunked_to_disk(args.off_policy_onnx)
|
||||
args.on_policy_onnx = read_file_chunked_to_disk(args.on_policy_onnx)
|
||||
args.supercombo_onnx = read_file_chunked_to_disk(args.supercombo_onnx)
|
||||
|
||||
vision_runner = OnnxRunner(args.vision_onnx) if args.vision_onnx else None
|
||||
|
||||
|
||||
@@ -5,10 +5,15 @@ This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from openpilot.common.parameterized import parameterized
|
||||
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key
|
||||
from openpilot.common.file_chunker import chunk_file, get_chunk_targets
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, _detect_desire_key, read_file_chunked_to_disk
|
||||
from openpilot.common.test import OpenpilotTestCase
|
||||
|
||||
|
||||
@@ -160,3 +165,33 @@ class TestOutputSlicePreservation(OpenpilotTestCase):
|
||||
policy_slices = {'plan': slice(0, 495), 'meta': slice(495, 550)}
|
||||
assert set(vision_slices.keys()) & set(policy_slices.keys()) == set(), \
|
||||
"vision and policy slices should not overlap in keys"
|
||||
|
||||
|
||||
class TestReadFileChunkedToDisk(OpenpilotTestCase):
|
||||
def test_none_passthrough(self):
|
||||
assert read_file_chunked_to_disk(None) is None
|
||||
|
||||
def test_unchunked_source_staged_on_disk(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
src = Path(d) / "driving_supercombo.onnx"
|
||||
payload = os.urandom(1024)
|
||||
src.write_bytes(payload)
|
||||
|
||||
out = Path(read_file_chunked_to_disk(str(src)))
|
||||
|
||||
assert out.parent == Path(d)
|
||||
assert out.name == "driving_supercombo.onnx.unchunked"
|
||||
assert out.read_bytes() == payload
|
||||
|
||||
def test_chunked_source_reassembled_on_disk(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
src = Path(d) / "driving_supercombo.onnx"
|
||||
payload = os.urandom(4096)
|
||||
src.write_bytes(payload)
|
||||
chunk_file(str(src), get_chunk_targets(str(src), len(payload)))
|
||||
assert not src.exists()
|
||||
|
||||
out = Path(read_file_chunked_to_disk(str(src)))
|
||||
|
||||
assert out.parent == Path(d)
|
||||
assert out.read_bytes() == payload
|
||||
|
||||
@@ -3,8 +3,17 @@ import os
|
||||
import hashlib
|
||||
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state
|
||||
from openpilot.sunnypilot import get_file_hash
|
||||
from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL
|
||||
from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL, DEFAULT_BIG_MODEL
|
||||
|
||||
|
||||
def get_default_model() -> str:
|
||||
show_big_model = (ui_state.usbgpu
|
||||
and (ui_state.usbgpu_active or ui_state.usbgpu_loading or ui_state.is_offroad()))
|
||||
|
||||
return DEFAULT_BIG_MODEL if show_big_model else DEFAULT_MODEL
|
||||
|
||||
|
||||
DEFAULT_MODEL_NAME_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "model_name.py")
|
||||
MODEL_HASH_PATH = os.path.join(BASEDIR, "openpilot", "sunnypilot", "models", "tests", "model_hash")
|
||||
@@ -13,7 +22,6 @@ SUPERCOMBO_ONNX_PATH = os.path.join(BASEDIR, "openpilot", "selfdrive", "modeld",
|
||||
|
||||
def update_model_hash():
|
||||
supercombo_hash = get_file_hash(SUPERCOMBO_ONNX_PATH)
|
||||
|
||||
combined_hash = hashlib.sha256(supercombo_hash.encode()).hexdigest()
|
||||
|
||||
with open(MODEL_HASH_PATH, "w") as f:
|
||||
@@ -22,40 +30,28 @@ def update_model_hash():
|
||||
print(f"Generated and updated new combined model hash to {MODEL_HASH_PATH}")
|
||||
|
||||
|
||||
def get_current_default_model_name():
|
||||
print("[GET DEFAULT MODEL NAME]")
|
||||
name = DEFAULT_MODEL
|
||||
print(f'Current default model name: "{name}"')
|
||||
|
||||
return name
|
||||
|
||||
|
||||
def update_default_model_name(name: str):
|
||||
print("[CHANGE DEFAULT MODEL NAME]")
|
||||
def update_default_model_names(default_model_name: str, default_big_model_name: str):
|
||||
print("[CHANGE DEFAULT MODEL NAMES]")
|
||||
with open(DEFAULT_MODEL_NAME_PATH, "w") as f:
|
||||
f.write(f'DEFAULT_MODEL = "{name}"\n')
|
||||
print(f'New default model name: "{name}"')
|
||||
f.write(f'DEFAULT_MODEL = "{default_model_name}"\n')
|
||||
f.write(f'DEFAULT_BIG_MODEL = "{default_big_model_name}"\n')
|
||||
|
||||
print(f'New default small model name: "{default_model_name}"')
|
||||
print(f'New default big model name: "{default_big_model_name}"')
|
||||
print("[DONE]")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Update default model name and hash")
|
||||
parser.add_argument("--new_name", type=str, help="New default model name")
|
||||
parser = argparse.ArgumentParser(description="Update default model names and hash")
|
||||
parser.add_argument("--new_small_model_name", type=str, help="New default small model name")
|
||||
parser.add_argument("--new_big_model_name", type=str, help="New default big model name")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.new_name:
|
||||
print("Warning: No new default model name provided. Use --new_name to specify")
|
||||
print("Default model name and hash will not be updated! (aborted)")
|
||||
exit(0)
|
||||
if args.new_small_model_name is None and args.new_big_model_name is None:
|
||||
new_name = input(f'Enter new default small model name (current: "{DEFAULT_MODEL}", leave empty to keep): ').strip()
|
||||
new_big_model_name = input(f'Enter new default big model name (current: "{DEFAULT_BIG_MODEL}", leave empty to keep): ').strip()
|
||||
else:
|
||||
new_name, new_big_model_name = args.new_small_model_name, args.new_big_model_name
|
||||
|
||||
current_name = get_current_default_model_name()
|
||||
new_name = args.new_name
|
||||
if current_name == new_name:
|
||||
print(f'Proposed default model name: "{new_name}"')
|
||||
confirm = input("Proposed default model name is the same as the current default model name. Confirm? (y/n): ").upper().strip()
|
||||
if confirm != "Y":
|
||||
print("Default model name and hash will not be updated! (aborted)")
|
||||
exit(0)
|
||||
|
||||
update_default_model_name(new_name)
|
||||
update_default_model_names(new_name or DEFAULT_MODEL, new_big_model_name or DEFAULT_BIG_MODEL)
|
||||
update_model_hash()
|
||||
|
||||
@@ -13,8 +13,6 @@ from openpilot.common.params import Params
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.common.hardware.hw import Paths
|
||||
from openpilot.sunnypilot.models.helpers import is_bundle_version_compatible
|
||||
from openpilot.selfdrive.modeld.helpers import usbgpu_present
|
||||
|
||||
from openpilot.cereal import custom
|
||||
|
||||
|
||||
@@ -141,44 +139,59 @@ class ModelCache:
|
||||
class ModelFetcher:
|
||||
"""Handles fetching and caching of model data from remote source"""
|
||||
MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v20.json"
|
||||
MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v20.json"
|
||||
MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v21.json"
|
||||
|
||||
MODEL_SOURCES = {
|
||||
"qcom": (MODEL_URL, ""),
|
||||
"usbgpu": (MODEL_URL_USBGPU, "_USBGPU"),
|
||||
}
|
||||
|
||||
def __init__(self, params: Params):
|
||||
self.params = params
|
||||
self.model_parser = ModelParser()
|
||||
self._is_usbgpu: bool | None = None
|
||||
self.model_cache = ModelCache(params)
|
||||
self._active_json_published = False
|
||||
self.model_caches = {
|
||||
source: ModelCache(params, suffix=suffix)
|
||||
for source, (_, suffix) in self.MODEL_SOURCES.items()
|
||||
}
|
||||
self.model_url = self.MODEL_URL
|
||||
self._update_model_source()
|
||||
|
||||
def _update_model_source(self) -> None:
|
||||
"""Updates what json to use based on usbgpu availability"""
|
||||
is_usbgpu = usbgpu_present()
|
||||
if is_usbgpu != self._is_usbgpu:
|
||||
self._is_usbgpu = is_usbgpu
|
||||
self.model_cache = ModelCache(self.params, suffix="_USBGPU" if is_usbgpu else "")
|
||||
self.model_url = self.MODEL_URL_USBGPU if is_usbgpu else self.MODEL_URL
|
||||
self.params.put("ModelManager_ActiveJson", self.model_url, block=True)
|
||||
@staticmethod
|
||||
def active_source(chestnut_present: bool) -> str:
|
||||
return "usbgpu" if chestnut_present else "qcom"
|
||||
|
||||
def _fetch_and_cache_models(self) -> list[custom.ModelManagerSP.ModelBundle] | None:
|
||||
def _update_model_source(self) -> None:
|
||||
"""Publishes the manifest URLs for both sources"""
|
||||
if not self._active_json_published:
|
||||
self._active_json_published = True
|
||||
self.params.put("ModelManager_ActiveJson", {
|
||||
"qcom": self.MODEL_URL,
|
||||
"usbgpu": self.MODEL_URL_USBGPU,
|
||||
}, block=True)
|
||||
|
||||
def _fetch_and_cache_models(self, source: str) -> list[custom.ModelManagerSP.ModelBundle] | None:
|
||||
"""Fetches fresh model data from remote and updates cache.
|
||||
Returns None on transport errors. Raises on 404 and other fatal HTTP errors.
|
||||
"""
|
||||
model_url, _ = self.MODEL_SOURCES[source]
|
||||
try:
|
||||
response = requests.get(self.model_url, timeout=10)
|
||||
response = requests.get(model_url, timeout=10)
|
||||
|
||||
# Explicitly handle 404 differently
|
||||
if response.status_code == 404:
|
||||
cloudlog.error(f"Models URL returned 404 Not Found: {self.model_url}")
|
||||
raise HTTPError(f"404 Not Found: {self.model_url}", response=response)
|
||||
cloudlog.error(f"Models URL returned 404 Not Found: {model_url}")
|
||||
raise HTTPError(f"404 Not Found: {model_url}", response=response)
|
||||
|
||||
# Raise for any other 4xx/5xx
|
||||
response.raise_for_status()
|
||||
|
||||
json_data = response.json()
|
||||
self.model_cache.set(json_data)
|
||||
cloudlog.debug("Successfully updated models cache")
|
||||
return self.model_parser.parse_models(json_data)
|
||||
parsed = self.model_parser.parse_models(json_data)
|
||||
if parsed:
|
||||
self.model_caches[source].set(json_data)
|
||||
cloudlog.debug(f"Successfully updated models cache for {source}")
|
||||
return parsed
|
||||
|
||||
except ConnectionError as e:
|
||||
cloudlog.warning(f"DNS/connection error while fetching models: {e}")
|
||||
@@ -191,16 +204,34 @@ class ModelFetcher:
|
||||
|
||||
return None
|
||||
|
||||
def get_available_bundles(self) -> list[custom.ModelManagerSP.ModelBundle]:
|
||||
"""Gets the list of available models, with smart cache handling"""
|
||||
self._update_model_source()
|
||||
cached_data, is_expired = self.model_cache.get()
|
||||
@staticmethod
|
||||
def _cache_matches_source(source: str, cached_data: dict) -> bool:
|
||||
"""Confirms a cached manifest contains requested source's models."""
|
||||
bundles = cached_data.get("bundles", [])
|
||||
if source == "usbgpu":
|
||||
return any(bundle.get("is_big") is True for bundle in bundles)
|
||||
return not any(bundle.get("is_big") is True for bundle in bundles)
|
||||
|
||||
def _get_source_bundles(self, source: str) -> list[custom.ModelManagerSP.ModelBundle]:
|
||||
cached_data, is_expired = self.model_caches[source].get()
|
||||
|
||||
if cached_data and not is_expired:
|
||||
cloudlog.debug("Using valid cached models data")
|
||||
return self.model_parser.parse_models(cached_data)
|
||||
if self._cache_matches_source(source, cached_data):
|
||||
try:
|
||||
parsed = self.model_parser.parse_models(cached_data)
|
||||
except Exception:
|
||||
cloudlog.warning(f"Failed to parse cached models for {source}; refetching", exc_info=True)
|
||||
else:
|
||||
if parsed:
|
||||
cloudlog.debug(f"Using valid cached models data for source {source}")
|
||||
return parsed
|
||||
# a source-matching cache that yields no valid bundles is stale (e.g. an old
|
||||
# manifest version) - do not trust it, refetch so the source is repopulated
|
||||
cloudlog.warning(f"Cached models for {source} have no valid bundles; refetching")
|
||||
else:
|
||||
cloudlog.warning(f"Cached models for {source} not valid; refetching")
|
||||
|
||||
fetched_bundles = self._fetch_and_cache_models()
|
||||
fetched_bundles = self._fetch_and_cache_models(source)
|
||||
if fetched_bundles is not None:
|
||||
return fetched_bundles
|
||||
|
||||
@@ -208,12 +239,41 @@ class ModelFetcher:
|
||||
cloudlog.warning("Failed to fetch fresh data and no cache available")
|
||||
|
||||
cloudlog.warning("Failed to fetch fresh data. Using expired cache as fallback")
|
||||
return self.model_parser.parse_models(cached_data)
|
||||
try:
|
||||
return self.model_parser.parse_models(cached_data)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def get_bundles_for_source(self, source: str) -> list[custom.ModelManagerSP.ModelBundle]:
|
||||
"""Gets the list of available models for a specific source, with smart cache handling."""
|
||||
if source not in self.MODEL_SOURCES:
|
||||
cloudlog.warning(f"Unknown model source: {source}")
|
||||
return []
|
||||
return self._get_source_bundles(source)
|
||||
|
||||
|
||||
def get_cached_bundles(params: Params, source: str) -> list[custom.ModelManagerSP.ModelBundle]:
|
||||
"""Reads a source's cached manifest from params and parses it into bundles."""
|
||||
|
||||
if source not in ModelFetcher.MODEL_SOURCES:
|
||||
cloudlog.warning(f"Unknown model source: {source}")
|
||||
return []
|
||||
_, suffix = ModelFetcher.MODEL_SOURCES[source]
|
||||
cached_data = params.get(f"ModelManager_ModelsCache{suffix}")
|
||||
if not cached_data:
|
||||
return []
|
||||
try:
|
||||
return ModelParser.parse_models(cached_data)
|
||||
except Exception as e:
|
||||
cloudlog.warning(f"Failed to parse cached models for source {source}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from openpilot.selfdrive.modeld.helpers import usbgpu_present
|
||||
params = Params()
|
||||
model_fetcher = ModelFetcher(params)
|
||||
bundles = model_fetcher.get_available_bundles()
|
||||
bundles = model_fetcher.get_bundles_for_source(ModelFetcher.active_source(usbgpu_present()))
|
||||
for bundle in bundles:
|
||||
for model in bundle.models:
|
||||
model_overrides = {override.key: override.value for override in bundle.overrides}
|
||||
|
||||
@@ -16,6 +16,7 @@ from openpilot.common.params import Params
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRaider
|
||||
from openpilot.common.hardware.hw import Paths
|
||||
from openpilot.selfdrive.modeld.helpers import usbgpu_present
|
||||
|
||||
# SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO
|
||||
REQUIRED_JSON_VERSION = 17
|
||||
@@ -23,7 +24,12 @@ REQUIRED_JSON_VERSION = 17
|
||||
CUSTOM_MODEL_PATH = Paths.model_root()
|
||||
METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl'
|
||||
ModelManager = custom.ModelManagerSP
|
||||
_LAST_VALIDATED_RAW = None
|
||||
|
||||
ACTIVE_BUNDLE_KEYS = {
|
||||
"qcom": "ModelManager_ActiveBundle",
|
||||
"usbgpu": "ModelManager_ActiveBundleUSBGPU",
|
||||
}
|
||||
_LAST_VALIDATED_RAW: dict[str, bytes | None] = {}
|
||||
|
||||
|
||||
def _compute_hash(file_path: str) -> str | None:
|
||||
@@ -110,37 +116,74 @@ def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, availa
|
||||
return not _bundle_is_valid_locally(active_bundle)
|
||||
|
||||
|
||||
def validate_active_bundle(params: Params, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None) -> None:
|
||||
global _LAST_VALIDATED_RAW
|
||||
|
||||
raw_bundle = params.get("ModelManager_ActiveBundle")
|
||||
if not raw_bundle:
|
||||
return
|
||||
|
||||
if raw_bundle == _LAST_VALIDATED_RAW:
|
||||
return
|
||||
|
||||
active_bundle = get_active_bundle(params, raw_bundle_dict=raw_bundle)
|
||||
if active_bundle is None or _bundle_needs_reset(active_bundle, available_bundles):
|
||||
cloudlog.warning("Active model bundle invalid; resetting to default")
|
||||
params.remove("ModelManager_ActiveBundle")
|
||||
params.put("ModelRunnerTypeCache", int(custom.ModelManagerSP.Runner.stock), block=True)
|
||||
_LAST_VALIDATED_RAW = None
|
||||
else:
|
||||
_LAST_VALIDATED_RAW = raw_bundle
|
||||
|
||||
|
||||
def get_active_bundle(params: Params | None = None, raw_bundle_dict: dict | bytes | None = None) -> "custom.ModelManagerSP.ModelBundle | None":
|
||||
params = params or Params()
|
||||
def _parse_active_bundle(raw_bundle) -> "custom.ModelManagerSP.ModelBundle | None":
|
||||
try:
|
||||
active_bundle_dict = raw_bundle_dict if raw_bundle_dict is not None else (params.get("ModelManager_ActiveBundle") or {})
|
||||
if isinstance(active_bundle_dict, dict) and active_bundle_dict and is_bundle_version_compatible(active_bundle_dict):
|
||||
return custom.ModelManagerSP.ModelBundle(**active_bundle_dict)
|
||||
if isinstance(raw_bundle, dict) and raw_bundle and is_bundle_version_compatible(raw_bundle):
|
||||
return custom.ModelManagerSP.ModelBundle(**raw_bundle)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_selected_bundle(params: Params | None = None, source: str = "qcom") -> "custom.ModelManagerSP.ModelBundle | None":
|
||||
params = params or Params()
|
||||
return _parse_active_bundle(params.get(ACTIVE_BUNDLE_KEYS.get(source, "ModelManager_ActiveBundle")))
|
||||
|
||||
|
||||
def get_active_source(usbgpu: bool | None = None, usbgpu_active: bool | None = None,
|
||||
usbgpu_loading: bool | None = None, offroad: bool | None = None) -> str:
|
||||
if usbgpu is None:
|
||||
usbgpu = usbgpu_present()
|
||||
state_valid = usbgpu_active is not None or usbgpu_loading is not None or offroad is not None
|
||||
big_active = usbgpu and (not state_valid or usbgpu_active or usbgpu_loading or offroad)
|
||||
return "usbgpu" if big_active else "qcom"
|
||||
|
||||
|
||||
def get_active_bundle(params: Params | None = None, *, usbgpu: bool | None = None) -> "custom.ModelManagerSP.ModelBundle | None":
|
||||
params = params or Params()
|
||||
if get_active_source(usbgpu=usbgpu) == "usbgpu":
|
||||
if bundle := get_selected_bundle(params, "usbgpu"):
|
||||
return bundle
|
||||
return get_selected_bundle(params, "qcom")
|
||||
|
||||
|
||||
def resolve_bundle_by_ref(
|
||||
ref: str, source_bundles: dict[str, list[custom.ModelManagerSP.ModelBundle]],
|
||||
) -> "tuple[custom.ModelManagerSP.ModelBundle, str] | None":
|
||||
"""Finds the bundle matching a ref across all sources."""
|
||||
for source, bundles in source_bundles.items():
|
||||
for bundle in bundles:
|
||||
if bundle.ref == ref:
|
||||
return bundle, source
|
||||
return None
|
||||
|
||||
|
||||
def _validate_active_bundle(params: Params, source: str, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None) -> None:
|
||||
global _LAST_VALIDATED_RAW
|
||||
|
||||
key = ACTIVE_BUNDLE_KEYS[source]
|
||||
raw_bundle = params.get(key)
|
||||
if not raw_bundle:
|
||||
return
|
||||
|
||||
if _LAST_VALIDATED_RAW.get(key) == raw_bundle:
|
||||
return
|
||||
|
||||
active_bundle = _parse_active_bundle(raw_bundle)
|
||||
if active_bundle is None or _bundle_needs_reset(active_bundle, available_bundles):
|
||||
cloudlog.warning(f"Active model bundle invalid for {source}; resetting to default")
|
||||
params.remove(key)
|
||||
params.put("ModelRunnerTypeCache", int(custom.ModelManagerSP.Runner.stock), block=True)
|
||||
_LAST_VALIDATED_RAW[key] = None
|
||||
else:
|
||||
_LAST_VALIDATED_RAW[key] = raw_bundle
|
||||
|
||||
|
||||
def validate_active_bundles(params: Params, source_bundles: dict[str, list[custom.ModelManagerSP.ModelBundle]]) -> None:
|
||||
for source, bundles in source_bundles.items():
|
||||
_validate_active_bundle(params, source, bundles)
|
||||
|
||||
|
||||
def get_active_model_runner(params: Params | None = None, force_check: bool = False) -> int:
|
||||
params = params or Params()
|
||||
cached_runner_type = params.get("ModelRunnerTypeCache")
|
||||
|
||||
@@ -17,7 +17,8 @@ from openpilot.common.hardware.hw import Paths
|
||||
|
||||
from openpilot.cereal import messaging, custom
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher
|
||||
from openpilot.sunnypilot.models.helpers import get_active_bundle, validate_active_bundle, verify_file
|
||||
from openpilot.sunnypilot.models.helpers import (ACTIVE_BUNDLE_KEYS, get_active_bundle, get_selected_bundle,
|
||||
resolve_bundle_by_ref, validate_active_bundles, verify_file)
|
||||
|
||||
# (connect, read) seconds. read is per-request inactivity, not a total cap
|
||||
DOWNLOAD_TIMEOUT = (30, 30)
|
||||
@@ -30,9 +31,12 @@ class ModelManagerSP:
|
||||
self.params = Params()
|
||||
self.model_fetcher = ModelFetcher(self.params)
|
||||
self.pm = messaging.PubMaster(["modelManagerSP"])
|
||||
self.sm = messaging.SubMaster(["deviceState"])
|
||||
self.chestnut_present = False
|
||||
self.available_models: list[custom.ModelManagerSP.ModelBundle] = []
|
||||
self.source_models: dict[str, list[custom.ModelManagerSP.ModelBundle]] = {}
|
||||
self.selected_bundle: custom.ModelManagerSP.ModelBundle = None
|
||||
self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params)
|
||||
self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params, usbgpu=self.chestnut_present)
|
||||
self._chunk_size = 128 * 1000 # 128 KB chunks
|
||||
self._download_start_times: dict[str, float] = {} # Track start time per model
|
||||
|
||||
@@ -76,7 +80,7 @@ class ModelManagerSP:
|
||||
f.write(chunk)
|
||||
bytes_downloaded += len(chunk)
|
||||
|
||||
if self.params.get("ModelManager_DownloadIndex") is None:
|
||||
if self.params.get("ModelManager_DownloadRef") is None:
|
||||
raise Exception("Download cancelled")
|
||||
|
||||
if total_size > 0:
|
||||
@@ -114,7 +118,7 @@ class ModelManagerSP:
|
||||
for data in response.iter_content(chunk_size=self._chunk_size):
|
||||
f.write(data)
|
||||
chunk_downloaded += len(data)
|
||||
if self.params.get("ModelManager_DownloadIndex") is None:
|
||||
if self.params.get("ModelManager_DownloadRef") is None:
|
||||
raise Exception("Download cancelled")
|
||||
intra = chunk_downloaded / max(chunk_size, 1)
|
||||
progress = min(99.0, ((i + intra) / num_chunks) * 100)
|
||||
@@ -143,13 +147,17 @@ class ModelManagerSP:
|
||||
is_cached = False
|
||||
if len(artifact.chunks) > 0:
|
||||
from openpilot.common.file_chunker import get_chunk_name
|
||||
num_chunks = len(artifact.chunks)
|
||||
chunks_valid = True
|
||||
for i, chunk in enumerate(artifact.chunks):
|
||||
chunk_path = get_chunk_name(full_path, i, len(artifact.chunks))
|
||||
chunk_path = get_chunk_name(full_path, i, num_chunks)
|
||||
if not await verify_file(chunk_path, chunk.sha256):
|
||||
chunks_valid = False
|
||||
break
|
||||
if chunks_valid and len(artifact.chunks) > 0:
|
||||
artifact.downloadProgress.progress = ((i + 1) / num_chunks) * 100
|
||||
self._sync_artifact_progress(artifact)
|
||||
self._report_status()
|
||||
if chunks_valid and num_chunks > 0:
|
||||
is_cached = True
|
||||
else:
|
||||
if await verify_file(full_path, expected_hash):
|
||||
@@ -212,10 +220,13 @@ class ModelManagerSP:
|
||||
model_manager_state.availableBundles = self.available_models
|
||||
self.pm.send('modelManagerSP', msg)
|
||||
|
||||
async def _download_bundle(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None:
|
||||
"""Downloads all models in a bundle"""
|
||||
async def _download_bundle(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str, source: str) -> None:
|
||||
"""Downloads a bundle and sets it as the active bundle for its source"""
|
||||
self.selected_bundle = model_bundle
|
||||
self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloading
|
||||
for model in self.selected_bundle.models:
|
||||
model.artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloading
|
||||
self._report_status()
|
||||
os.makedirs(destination_path, exist_ok=True)
|
||||
|
||||
try:
|
||||
@@ -232,10 +243,9 @@ class ModelManagerSP:
|
||||
seen_artifacts.add(artifact.fileName)
|
||||
await self._process_artifact(artifact, destination_path)
|
||||
|
||||
self.active_bundle = self.selected_bundle
|
||||
self.active_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
self.params.put("ModelManager_ActiveBundle", self.active_bundle.to_dict(), block=True)
|
||||
self.selected_bundle = None
|
||||
self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
self.params.put(ACTIVE_BUNDLE_KEYS[source], model_bundle.to_dict(), block=True)
|
||||
self.active_bundle = get_active_bundle(self.params, usbgpu=self.chestnut_present)
|
||||
|
||||
except Exception:
|
||||
if self.selected_bundle is not None:
|
||||
@@ -245,9 +255,9 @@ class ModelManagerSP:
|
||||
finally:
|
||||
self._report_status()
|
||||
|
||||
def download(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None:
|
||||
def download(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str, source: str) -> None:
|
||||
"""Main entry point for downloading a model bundle"""
|
||||
asyncio.run(self._download_bundle(model_bundle, destination_path))
|
||||
asyncio.run(self._download_bundle(model_bundle, destination_path, source))
|
||||
|
||||
def main_thread(self) -> None:
|
||||
"""Main thread for model management"""
|
||||
@@ -255,18 +265,22 @@ class ModelManagerSP:
|
||||
|
||||
while True:
|
||||
try:
|
||||
self.available_models = self.model_fetcher.get_available_bundles()
|
||||
validate_active_bundle(self.params, self.available_models)
|
||||
self.active_bundle = get_active_bundle(self.params)
|
||||
self.sm.update(0)
|
||||
self.chestnut_present = self.sm['deviceState'].chestnutPresent
|
||||
self.source_models = {source: self.model_fetcher.get_bundles_for_source(source) for source in ModelFetcher.MODEL_SOURCES}
|
||||
self.available_models = self.source_models[ModelFetcher.active_source(self.chestnut_present)]
|
||||
validate_active_bundles(self.params, self.source_models)
|
||||
self.active_bundle = get_active_bundle(self.params, usbgpu=self.chestnut_present)
|
||||
|
||||
if (index_to_download := self.params.get("ModelManager_DownloadIndex")) is not None:
|
||||
if model_to_download := next((model for model in self.available_models if model.index == index_to_download), None):
|
||||
if (ref_to_download := self.params.get("ModelManager_DownloadRef")) is not None:
|
||||
if resolved := resolve_bundle_by_ref(ref_to_download, self.source_models):
|
||||
model_to_download, source = resolved
|
||||
try:
|
||||
self.download(model_to_download, Paths.model_root())
|
||||
self.download(model_to_download, Paths.model_root(), source)
|
||||
except Exception as e:
|
||||
cloudlog.exception(e)
|
||||
finally:
|
||||
self.params.remove("ModelManager_DownloadIndex")
|
||||
self.params.remove("ModelManager_DownloadRef")
|
||||
self.selected_bundle = None
|
||||
|
||||
if self.params.get("ModelManager_ClearCache"):
|
||||
@@ -285,12 +299,14 @@ class ModelManagerSP:
|
||||
Clears the model cache directory of all files except those in the active model bundle.
|
||||
"""
|
||||
|
||||
# Get list of files used by active model bundle
|
||||
# Get list of files used by both slots' selected bundles (either may become
|
||||
# the truly active bundle depending on hardware availability)
|
||||
active_files = []
|
||||
if self.active_bundle is not None: # When the default model is active
|
||||
for model in self.active_bundle.models:
|
||||
if hasattr(model, 'artifact') and model.artifact.fileName:
|
||||
active_files.append(model.artifact.fileName)
|
||||
for source in ACTIVE_BUNDLE_KEYS:
|
||||
if selected_bundle := get_selected_bundle(self.params, source):
|
||||
for model in selected_bundle.models:
|
||||
if hasattr(model, 'artifact') and model.artifact.fileName:
|
||||
active_files.append(model.artifact.fileName)
|
||||
|
||||
# Remove all files except active ones (including their chunk files)
|
||||
model_dir = Paths.model_root()
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
DEFAULT_MODEL = "CD210"
|
||||
DEFAULT_BIG_MODEL = "Lebowski"
|
||||
|
||||
@@ -20,4 +20,4 @@ class TestDefaultModel(OpenpilotTestCase):
|
||||
with open(MODEL_HASH_PATH) as f:
|
||||
current_hash = f.read().strip()
|
||||
|
||||
assert combined_hash == current_hash, "Run sunnypilot/models/default_model.py to update the default model name and hash"
|
||||
assert combined_hash == current_hash, "Run openpilot/sunnypilot/models/default_model.py to update the default model name and hash"
|
||||
|
||||
@@ -11,6 +11,7 @@ import http.server
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
@@ -23,6 +24,8 @@ from openpilot.common.test import OpenpilotTestCase
|
||||
from openpilot.common.file_chunker import get_chunk_name, get_manifest_path
|
||||
from openpilot.selfdrive.test.helpers import http_server_context
|
||||
from openpilot.sunnypilot.models import manager as manager_module
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher, get_cached_bundles
|
||||
from openpilot.sunnypilot.models.helpers import get_active_bundle, get_active_source, get_selected_bundle, resolve_bundle_by_ref
|
||||
from openpilot.sunnypilot.models.manager import ModelManagerSP
|
||||
|
||||
CHUNK_BODIES = [b'A' * 5000, b'B' * 5000, b'C' * 3000]
|
||||
@@ -103,6 +106,7 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
|
||||
self.manager.selected_bundle = None
|
||||
self.manager.active_bundle = None
|
||||
self.manager.available_models = []
|
||||
self.manager.chestnut_present = False
|
||||
self.manager._chunk_size = 1024
|
||||
self.manager._download_start_times = {}
|
||||
|
||||
@@ -249,6 +253,85 @@ class TestManagerDownload(ManagerDownloadTestBase):
|
||||
assert self.manager._download_start_times == {}
|
||||
self.run_with_server(body)
|
||||
|
||||
def test_download_ref_present_keeps_download_alive(self):
|
||||
"""A pending download request (DownloadRef set) must not be cancelled mid-transfer."""
|
||||
def body():
|
||||
artifact = self.make_artifact(chunked=True)
|
||||
base_path = os.path.join(self.dest, artifact.fileName)
|
||||
self.manager.params.get.side_effect = lambda key: b"ref" if key == "ModelManager_DownloadRef" else None
|
||||
asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact))
|
||||
assert os.path.isfile(get_manifest_path(base_path))
|
||||
self.run_with_server(body)
|
||||
|
||||
def test_cancellation_via_download_ref(self):
|
||||
"""Removing DownloadRef mid-transfer cancels the download."""
|
||||
def body():
|
||||
artifact = self.make_artifact(chunked=True)
|
||||
base_path = os.path.join(self.dest, artifact.fileName)
|
||||
checks = {"n": 0}
|
||||
|
||||
def get(key):
|
||||
if key == "ModelManager_DownloadRef":
|
||||
checks["n"] += 1
|
||||
return b"ref" if checks["n"] <= 2 else None
|
||||
return b"0"
|
||||
|
||||
self.manager.params.get.side_effect = get
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact))
|
||||
assert 'cancelled' in str(ctx.exception).lower()
|
||||
assert not os.path.isfile(get_manifest_path(base_path))
|
||||
self.run_with_server(body)
|
||||
|
||||
def _make_params_with_store(self):
|
||||
params = mock.MagicMock()
|
||||
store = {}
|
||||
|
||||
def get(key, *args, **kwargs):
|
||||
return store.get(key, b"0") # b"0" -> download not cancelled
|
||||
|
||||
def put(key, value, *args, **kwargs):
|
||||
store[key] = value
|
||||
|
||||
params.get.side_effect = get
|
||||
params.put.side_effect = put
|
||||
return params, store
|
||||
|
||||
def test_download_writes_qcom_slot(self):
|
||||
"""A download resolved to the qcom source writes the qcom active bundle slot only."""
|
||||
def body():
|
||||
artifact = self.make_artifact(chunked=True)
|
||||
self._bundle.ref = "test-ref"
|
||||
self._bundle.minimumSelectorVersion = 17
|
||||
params, store = self._make_params_with_store()
|
||||
self.manager.params = params
|
||||
asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "qcom"))
|
||||
|
||||
assert "ModelManager_ActiveBundle" in store, "qcom download must write the qcom slot"
|
||||
assert "ModelManager_ActiveBundleUSBGPU" not in store, "qcom download must not touch the usbgpu slot"
|
||||
assert self.manager.selected_bundle.status == custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
assert self.manager.active_bundle is not None and self.manager.active_bundle.ref == "test-ref"
|
||||
assert self.manager.active_bundle.status == custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
chunk_names = [get_chunk_name(artifact.fileName, i, len(artifact.chunks)) for i in range(len(artifact.chunks))]
|
||||
missing = [c for c in chunk_names if not os.path.isfile(os.path.join(self.dest, c))]
|
||||
assert missing == [], f"chunks missing from the cache: {missing}"
|
||||
self.run_with_server(body)
|
||||
|
||||
def test_download_writes_usbgpu_slot(self):
|
||||
"""A download resolved to the usbgpu source writes the usbgpu active bundle slot only."""
|
||||
def body():
|
||||
self.make_artifact(chunked=True)
|
||||
self._bundle.ref = "big-ref"
|
||||
self._bundle.minimumSelectorVersion = 17
|
||||
params, store = self._make_params_with_store()
|
||||
self.manager.params = params
|
||||
asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "usbgpu"))
|
||||
|
||||
assert "ModelManager_ActiveBundleUSBGPU" in store, "usbgpu download must write the usbgpu slot"
|
||||
assert "ModelManager_ActiveBundle" not in store, "usbgpu download must not touch the qcom slot"
|
||||
assert self.manager.selected_bundle.status == custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
self.run_with_server(body)
|
||||
|
||||
|
||||
class TestManagerImports(OpenpilotTestCase):
|
||||
"""Catches undeclared dependencies. aiohttp lived only in the AGNOS venv; 19.6 dropped
|
||||
@@ -267,6 +350,292 @@ class TestManagerImports(OpenpilotTestCase):
|
||||
assert connect > 0 and read > 0, "requests defaults to no timeout; downloads would hang forever"
|
||||
|
||||
|
||||
class TestResolveBundleByRef(OpenpilotTestCase):
|
||||
"""A ref resolves to (bundle, source) across both hardware manifests. Refs are
|
||||
unique per manifest and never overlap across sources, so a ref maps to exactly
|
||||
one slot. Shared by the manager's download flow and the settings UI."""
|
||||
|
||||
@staticmethod
|
||||
def _bundle(ref: str):
|
||||
bundle = custom.ModelManagerSP.ModelBundle.new_message()
|
||||
bundle.ref = ref
|
||||
return bundle
|
||||
|
||||
def test_qcom_ref_resolves_to_qcom_slot(self):
|
||||
small = self._bundle("small")
|
||||
assert resolve_bundle_by_ref("small", {"qcom": [small], "usbgpu": []}) == (small, "qcom")
|
||||
|
||||
def test_usbgpu_ref_resolves_to_usbgpu_slot(self):
|
||||
big = self._bundle("big")
|
||||
assert resolve_bundle_by_ref("big", {"qcom": [], "usbgpu": [big]}) == (big, "usbgpu")
|
||||
|
||||
def test_unknown_ref_returns_none(self):
|
||||
source_bundles = {"qcom": [self._bundle("small")], "usbgpu": []}
|
||||
assert resolve_bundle_by_ref("nope", source_bundles) is None
|
||||
|
||||
|
||||
def manifest_bundle(short_name: str, ref: str, index: int = 0, is_big: bool = False) -> dict:
|
||||
"""Minimal manifest bundle dict, version-compatible (no chunks to avoid disk side effects).
|
||||
Big (usbgpu) bundles carry `is_big: true` in the manifest JSON."""
|
||||
return {
|
||||
"index": index,
|
||||
"short_name": short_name,
|
||||
"display_name": short_name.upper(),
|
||||
"generation": 1,
|
||||
"environment": "release",
|
||||
"runner": "tinygrad",
|
||||
"is_big": is_big,
|
||||
"minimum_selector_version": "17",
|
||||
"ref": ref,
|
||||
"models": [{
|
||||
"type": "supercombo",
|
||||
"artifact": {
|
||||
"file_name": f"{short_name}.pkl",
|
||||
"download_uri": {"url": f"https://example.com/{short_name}.pkl", "sha256": "s"},
|
||||
},
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
def fresh_sync_time() -> int:
|
||||
return int(time.monotonic() * 1e9)
|
||||
|
||||
|
||||
class TestModelFetcherSources(OpenpilotTestCase):
|
||||
"""Both manifests are always maintained: get_bundles_for_source exposes either
|
||||
source by name, and active_source picks which one matches the attached hardware."""
|
||||
|
||||
def _make_params(self, qcom_manifest, usbgpu_manifest):
|
||||
params = mock.MagicMock()
|
||||
|
||||
def get(key):
|
||||
if key == "ModelManager_ModelsCache":
|
||||
return qcom_manifest
|
||||
if key == "ModelManager_ModelsCache_USBGPU":
|
||||
return usbgpu_manifest
|
||||
if key in ("ModelManager_LastSyncTime", "ModelManager_LastSyncTime_USBGPU"):
|
||||
return fresh_sync_time()
|
||||
return None
|
||||
|
||||
params.get.side_effect = get
|
||||
return params
|
||||
|
||||
def test_active_source_follows_chestnut_presence(self):
|
||||
assert ModelFetcher.active_source(False) == "qcom"
|
||||
assert ModelFetcher.active_source(True) == "usbgpu"
|
||||
|
||||
def test_get_bundles_for_source_returns_each_source(self):
|
||||
params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]},
|
||||
{"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
assert [bundle.ref for bundle in fetcher.get_bundles_for_source("qcom")] == ["aaa"]
|
||||
assert [bundle.ref for bundle in fetcher.get_bundles_for_source("usbgpu")] == ["bbb"]
|
||||
|
||||
def test_get_bundles_for_source_unknown(self):
|
||||
assert ModelFetcher(mock.MagicMock()).get_bundles_for_source("bogus") == []
|
||||
|
||||
def test_get_cached_bundles_parses_source(self):
|
||||
params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]},
|
||||
{"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
qcom_bundles = get_cached_bundles(params, "qcom")
|
||||
usbgpu_bundles = get_cached_bundles(params, "usbgpu")
|
||||
assert [b.ref for b in qcom_bundles] == ["aaa"]
|
||||
assert [b.ref for b in usbgpu_bundles] == ["bbb"]
|
||||
assert qcom_bundles[0].displayName == "SMALL"
|
||||
|
||||
def test_get_cached_bundles_empty_when_missing(self):
|
||||
params = mock.MagicMock()
|
||||
params.get.return_value = None
|
||||
assert get_cached_bundles(params, "qcom") == []
|
||||
assert get_cached_bundles(params, "usbgpu") == []
|
||||
|
||||
def test_get_cached_bundles_unknown_source(self):
|
||||
assert get_cached_bundles(mock.MagicMock(), "bogus") == []
|
||||
|
||||
def test_active_json_has_both_urls(self):
|
||||
params = mock.MagicMock()
|
||||
ModelFetcher(params)
|
||||
active_json_calls = [call for call in params.put.call_args_list if call.args[0] == "ModelManager_ActiveJson"]
|
||||
assert active_json_calls, "expected ModelManager_ActiveJson to be written"
|
||||
assert active_json_calls[-1].args[1] == {
|
||||
"qcom": ModelFetcher.MODEL_URL,
|
||||
"usbgpu": ModelFetcher.MODEL_URL_USBGPU,
|
||||
}
|
||||
|
||||
|
||||
|
||||
class TestSourceCacheIntegrity(OpenpilotTestCase):
|
||||
"""Each source's cached manifest must contain only that source's models; the
|
||||
`is_big` flag in the JSON marks the big (usbgpu) models. A mismatched cache is
|
||||
legacy data from before the per-source split (the active manifest was cached
|
||||
under the unsuffixed key regardless of hardware) and is refetched. This
|
||||
replaces the old one-time bundle migration."""
|
||||
|
||||
def _make_params(self, qcom_manifest, usbgpu_manifest):
|
||||
params = mock.MagicMock()
|
||||
|
||||
def get(key):
|
||||
if key == "ModelManager_ModelsCache":
|
||||
return qcom_manifest
|
||||
if key == "ModelManager_ModelsCache_USBGPU":
|
||||
return usbgpu_manifest
|
||||
if key in ("ModelManager_LastSyncTime", "ModelManager_LastSyncTime_USBGPU"):
|
||||
return fresh_sync_time()
|
||||
return None
|
||||
|
||||
params.get.side_effect = get
|
||||
return params
|
||||
|
||||
def _fetched(self, *bundles):
|
||||
return ModelFetcher(mock.MagicMock()).model_parser.parse_models({"bundles": list(bundles)})
|
||||
|
||||
def test_qcom_cache_with_big_models_is_refetched(self):
|
||||
"""Legacy: the unsuffixed cache holds the big manifest. is_big confirms it is
|
||||
the wrong set for qcom, so a fresh fetch replaces it."""
|
||||
params = self._make_params({"bundles": [manifest_bundle("big", "bbb", is_big=True)]},
|
||||
{"bundles": [manifest_bundle("big2", "ccc", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
fetched = self._fetched(manifest_bundle("small", "aaa"))
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched):
|
||||
bundles = fetcher.get_bundles_for_source("qcom")
|
||||
assert [bundle.ref for bundle in bundles] == ["aaa"]
|
||||
|
||||
def test_usbgpu_cache_without_big_models_is_refetched(self):
|
||||
params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]},
|
||||
{"bundles": [manifest_bundle("big2", "ccc")]})
|
||||
fetcher = ModelFetcher(params)
|
||||
fetched = self._fetched(manifest_bundle("big", "bbb", is_big=True))
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched):
|
||||
bundles = fetcher.get_bundles_for_source("usbgpu")
|
||||
assert [bundle.ref for bundle in bundles] == ["bbb"]
|
||||
|
||||
def test_matching_caches_are_used_without_fetch(self):
|
||||
params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]},
|
||||
{"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", side_effect=AssertionError("cache should be used")):
|
||||
assert [bundle.ref for bundle in fetcher.get_bundles_for_source("qcom")] == ["aaa"]
|
||||
assert [bundle.ref for bundle in fetcher.get_bundles_for_source("usbgpu")] == ["bbb"]
|
||||
|
||||
def test_stale_version_cache_is_refetched(self):
|
||||
"""A source-matching cache whose bundles are all filtered by the selector version
|
||||
check parses to zero valid bundles; it is stale (e.g. an old manifest) and must be
|
||||
refetched instead of silently returning an empty list forever."""
|
||||
stale = manifest_bundle("small", "aaa")
|
||||
stale["minimum_selector_version"] = "16"
|
||||
params = self._make_params({"bundles": [stale]},
|
||||
{"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
fetched = self._fetched(manifest_bundle("small2", "ddd"))
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched) as fetch:
|
||||
bundles = fetcher.get_bundles_for_source("qcom")
|
||||
fetch.assert_called_once_with("qcom")
|
||||
assert [bundle.ref for bundle in bundles] == ["ddd"]
|
||||
|
||||
def test_corrupt_cache_is_refetched(self):
|
||||
"""A cache that fails to parse (e.g. truncated/foreign JSON) must trigger a
|
||||
refetch instead of raising every loop and never recovering."""
|
||||
corrupt = {"bundles": [{"short_name": "broken"}]} # missing required fields
|
||||
params = self._make_params(corrupt, {"bundles": [manifest_bundle("big", "bbb", is_big=True)]})
|
||||
fetcher = ModelFetcher(params)
|
||||
fetched = self._fetched(manifest_bundle("small", "aaa"))
|
||||
with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched) as fetch:
|
||||
bundles = fetcher.get_bundles_for_source("qcom")
|
||||
fetch.assert_called_once_with("qcom")
|
||||
assert [bundle.ref for bundle in bundles] == ["aaa"]
|
||||
|
||||
|
||||
class TestActiveBundleSelection(OpenpilotTestCase):
|
||||
"""The effective active bundle follows the hardware: the usbgpu slot wins when a GPU
|
||||
is present and compiled, otherwise the qcom slot. Each slot keeps its own selection."""
|
||||
|
||||
@staticmethod
|
||||
def _raw_bundle(ref: str) -> dict:
|
||||
bundle = custom.ModelManagerSP.ModelBundle.new_message()
|
||||
bundle.ref = ref
|
||||
bundle.minimumSelectorVersion = 17
|
||||
return bundle.to_dict()
|
||||
|
||||
def _params(self, qcom=None, usbgpu=None):
|
||||
params = mock.MagicMock()
|
||||
|
||||
def get(key, *args, **kwargs):
|
||||
if key == "ModelManager_ActiveBundle":
|
||||
return qcom
|
||||
if key == "ModelManager_ActiveBundleUSBGPU":
|
||||
return usbgpu
|
||||
return None
|
||||
|
||||
params.get.side_effect = get
|
||||
return params
|
||||
|
||||
def test_selected_bundle_is_per_slot(self):
|
||||
params = self._params(qcom=self._raw_bundle("small"), usbgpu=self._raw_bundle("big"))
|
||||
assert get_selected_bundle(params, "qcom").ref == "small"
|
||||
assert get_selected_bundle(params, "usbgpu").ref == "big"
|
||||
|
||||
def test_no_gpu_uses_qcom_slot(self):
|
||||
params = self._params(qcom=self._raw_bundle("small"), usbgpu=self._raw_bundle("big"))
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False):
|
||||
assert get_active_bundle(params).ref == "small"
|
||||
|
||||
def test_gpu_uses_usbgpu_slot(self):
|
||||
params = self._params(qcom=self._raw_bundle("small"), usbgpu=self._raw_bundle("big"))
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True):
|
||||
assert get_active_bundle(params).ref == "big"
|
||||
|
||||
def test_gpu_without_big_selection_falls_back_to_small(self):
|
||||
params = self._params(qcom=self._raw_bundle("small"), usbgpu=None)
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True):
|
||||
assert get_active_bundle(params).ref == "small"
|
||||
|
||||
|
||||
class TestEffectiveSource(OpenpilotTestCase):
|
||||
"""One gate decides the active source. With no flags it is runtime truth (GPU
|
||||
attached); display callers (mici) pass the ui_state flags, which additionally
|
||||
require the big model to be loading, active, or the device offroad. The active
|
||||
bundle is simply the selected bundle of that source."""
|
||||
|
||||
@staticmethod
|
||||
def _raw_bundle(ref: str) -> dict:
|
||||
bundle = custom.ModelManagerSP.ModelBundle.new_message()
|
||||
bundle.ref = ref
|
||||
bundle.minimumSelectorVersion = 17
|
||||
return bundle.to_dict()
|
||||
|
||||
def test_runtime_no_gpu(self):
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False):
|
||||
assert get_active_source() == "qcom"
|
||||
|
||||
def test_runtime_gpu_present(self):
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True):
|
||||
assert get_active_source() == "usbgpu"
|
||||
|
||||
def test_display_offroad_gpu_present_shows_big(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=False, usbgpu_loading=False, offroad=True) == "usbgpu"
|
||||
|
||||
def test_display_onroad_gpu_loading_shows_big(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=False, usbgpu_loading=True, offroad=False) == "usbgpu"
|
||||
|
||||
def test_display_onroad_gpu_active_shows_big(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=True, usbgpu_loading=False, offroad=False) == "usbgpu"
|
||||
|
||||
def test_display_onroad_gpu_idle_shows_small(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=False, usbgpu_loading=False, offroad=False) == "qcom"
|
||||
|
||||
def test_display_active_none_is_idle(self):
|
||||
assert get_active_source(usbgpu=True, usbgpu_active=None, usbgpu_loading=False, offroad=False) == "qcom"
|
||||
|
||||
def test_active_bundle_follows_source(self):
|
||||
params = mock.MagicMock()
|
||||
params.get.side_effect = lambda key: {"ModelManager_ActiveBundle": self._raw_bundle("small"),
|
||||
"ModelManager_ActiveBundleUSBGPU": self._raw_bundle("big")}.get(key)
|
||||
with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False):
|
||||
assert get_active_bundle(params).ref == "small"
|
||||
assert get_selected_bundle(params, get_active_source(usbgpu=True, usbgpu_active=False,
|
||||
usbgpu_loading=False, offroad=True)).ref == "big"
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get('RUN_INTEGRATION_TESTS'), 'requires external network')
|
||||
class TestLiveModelManifest(OpenpilotTestCase):
|
||||
"""Every artifact and chunk URL in the published manifest must resolve."""
|
||||
|
||||
@@ -83,9 +83,14 @@ class TestLocationdProc(OpenpilotTestCase):
|
||||
self.pm.send(msg.which(), msg)
|
||||
if msg.which() == "cameraOdometry":
|
||||
self.pm.wait_for_readers_to_update(msg.which(), timeout=1, dt=0.005)
|
||||
time.sleep(1) # wait for async params write
|
||||
for _ in range(50):
|
||||
val = self.params.get('LastGPSPositionLLK')
|
||||
if val is not None:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
|
||||
lastGPS = json.loads(self.params.get('LastGPSPositionLLK'))
|
||||
self.assertIsNotNone(val, "LastGPSPositionLLK not written within 5s")
|
||||
lastGPS = json.loads(val)
|
||||
self.assertAlmostEqual(lastGPS['latitude'], self.lat, delta=0.001)
|
||||
self.assertAlmostEqual(lastGPS['longitude'], self.lon, delta=0.001)
|
||||
self.assertAlmostEqual(lastGPS['altitude'], self.alt, delta=0.001)
|
||||
|
||||
@@ -28,7 +28,7 @@ from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutExce
|
||||
create_connection, WebSocketConnectionClosedException)
|
||||
|
||||
import openpilot.cereal.messaging as messaging
|
||||
from openpilot.sunnypilot.models.default_model import DEFAULT_MODEL
|
||||
from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL, DEFAULT_BIG_MODEL
|
||||
from openpilot.sunnypilot.selfdrive.car.sync_sunnylink_params import update_car_list_param
|
||||
from openpilot.sunnypilot.sunnylink.api import SunnylinkApi
|
||||
from openpilot.sunnypilot.sunnylink.utils import sunnylink_need_register, sunnylink_ready, get_param_as_byte, save_param_from_base64_encoded_string
|
||||
@@ -182,6 +182,8 @@ def getParamsMetadata() -> str:
|
||||
schema["capabilities"] = generate_capabilities()
|
||||
schema["capability_labels"] = CAPABILITY_LABELS
|
||||
schema["default_model"] = DEFAULT_MODEL
|
||||
schema["default_big_model"] = DEFAULT_BIG_MODEL
|
||||
schema["usbgpu_active"] = params.get_bool("UsbGpuActive")
|
||||
raw = json.dumps(schema, separators=(",", ":")).encode("utf-8")
|
||||
return base64.b64encode(gzip.compress(raw)).decode("utf-8")
|
||||
except Exception:
|
||||
|
||||
@@ -65,6 +65,7 @@ def sp_stats(end_event):
|
||||
'MadsSteeringMode',
|
||||
'MadsUnifiedEngagementMode',
|
||||
'ModelManager_ActiveBundle',
|
||||
'ModelManager_ActiveBundleUSBGPU',
|
||||
'ModelManager_Favs',
|
||||
'EnableSunnylinkUploader',
|
||||
'SunnylinkEnabled',
|
||||
|
||||
@@ -828,20 +828,22 @@ def startStream(sdp: str, enabled: bool) -> dict:
|
||||
bridge_services_in = []
|
||||
|
||||
# stale car params case taken care of by webrtcd being shut off on ignition
|
||||
cp_bytes = Params().get("CarParamsPersistent")
|
||||
cp_bytes = params.get("CarParamsPersistent")
|
||||
if cp_bytes is not None:
|
||||
with car.CarParams.from_bytes(cp_bytes) as CP:
|
||||
if CP.notCar:
|
||||
bridge_services_in.append("testJoystick")
|
||||
else:
|
||||
raise Exception("failed to get CarParamsPersistent")
|
||||
|
||||
if params.get_bool("IsOffroad"):
|
||||
# manager owns camerad/stream_encoderd/webrtcd; flip the param and let it bring them up.
|
||||
# webrtcd clears IsLiveStreaming when the session ends
|
||||
params.put_bool("IsLiveStreaming", True)
|
||||
# wait for webrtcd end points to wake up
|
||||
wait_for_webrtcd()
|
||||
try:
|
||||
wait_for_webrtcd()
|
||||
except TimeoutError:
|
||||
cloudlog.event("athena.startStream.webrtcd_offroad_start_timeout", error=True)
|
||||
raise
|
||||
|
||||
return post_stream_request(StreamRequestBody(sdp, ["wideRoad"], enabled, bridge_services_in, ["carState", "deviceState"]))
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.sunnypilot.system.statsd import statlog
|
||||
from openpilot.system.hardware.power_monitoring import PowerMonitoring
|
||||
from openpilot.system.hardware.fan_controller import FanController
|
||||
from openpilot.common.version import terms_version, training_version, get_build_metadata, terms_version_sp
|
||||
from openpilot.common.version import terms_version, training_version, get_build_metadata, terms_version_sp, CHESTNUT_BRANCHES
|
||||
|
||||
|
||||
ThermalStatus = log.DeviceState.ThermalStatus
|
||||
@@ -301,7 +301,11 @@ def hardware_thread(end_event, hw_queue) -> None:
|
||||
|
||||
set_usb_state(msg.deviceState, last_hw_state.usb_state)
|
||||
chestnut.update(started_ts is None, last_hw_state.usb_state)
|
||||
set_offroad_alert_if_changed("Offroad_ChestnutBranch", msg.deviceState.chestnutPresent and not big_model_available)
|
||||
current_channel = get_build_metadata().channel
|
||||
chestnut_target = CHESTNUT_BRANCHES.get(current_channel)
|
||||
chestnut_needs_switch = msg.deviceState.chestnutPresent and not big_model_available and chestnut_target is not None
|
||||
set_offroad_alert_if_changed("Offroad_ChestnutBranch", chestnut_needs_switch,
|
||||
extra_text=chestnut_target if chestnut_needs_switch else None)
|
||||
|
||||
# this subset is only used for offroad
|
||||
temp_sources = [
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Define the service name
|
||||
SERVICE_NAME="actions.runner.sunnypilot.$(uname -n)"
|
||||
|
||||
# Function to control the service
|
||||
control_service() {
|
||||
local action=$1 # Store the function argument in a local variable
|
||||
sudo systemctl $action ${SERVICE_NAME}
|
||||
}
|
||||
|
||||
service_exists_and_is_loaded() {
|
||||
sudo systemctl status ${SERVICE_NAME} &>/dev/null
|
||||
if [[ $? -ne 4 ]]; then
|
||||
return 0 # Service is known to systemd (i.e., loaded)
|
||||
else
|
||||
return 1 # Service is unknown to systemd (i.e., not loaded)
|
||||
fi
|
||||
}
|
||||
|
||||
# Check for required argument
|
||||
if [[ -z $1 ]] || { [[ $1 != "start" ]] && [[ $1 != "stop" ]]; }; then
|
||||
echo "Usage: $0 {start|stop}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Store the script argument in a descriptive variable
|
||||
ACTION=$1
|
||||
|
||||
# Trap EXIT signal (Ctrl+C) and stop the service
|
||||
trap 'control_service stop ; exit' SIGINT SIGKILL EXIT
|
||||
|
||||
# Enter the main loop
|
||||
while true; do
|
||||
# Check if the service is actually present on the system
|
||||
if service_exists_and_is_loaded; then
|
||||
control_service $ACTION # Call the function with the specified action
|
||||
fi
|
||||
sleep 1 # Pause before the next iteration
|
||||
done
|
||||
@@ -68,10 +68,6 @@ def only_offroad(started: bool, params: Params, CP: car.CarParams) -> bool:
|
||||
def livestream(started: bool, params: Params, CP: car.CarParams) -> bool:
|
||||
return params.get_bool("IsLiveStreaming")
|
||||
|
||||
def use_github_runner(started, params, CP: car.CarParams) -> bool:
|
||||
return not PC and params.get_bool("EnableGithubRunner") and (
|
||||
not params.get_bool("NetworkMetered") and not params.get_bool("GithubRunnerSufficientVoltage"))
|
||||
|
||||
def use_copyparty(started, params, CP: car.CarParams) -> bool:
|
||||
return bool(params.get_bool("EnableCopyparty"))
|
||||
|
||||
@@ -110,15 +106,12 @@ def or_(*fns):
|
||||
def and_(*fns):
|
||||
return lambda *args: operator.and_(*(fn(*args) for fn in fns))
|
||||
|
||||
def not_(*fns):
|
||||
return lambda *args: operator.not_(*(fn(*args) for fn in fns))
|
||||
|
||||
procs = [
|
||||
DaemonProcess("manage_athenad", "openpilot.system.athena.manage_athenad", "AthenadPid"),
|
||||
|
||||
NativeProcess("loggerd", "openpilot/system/loggerd", ["./loggerd"], logging),
|
||||
NativeProcess("encoderd", "openpilot/system/loggerd", ["./encoderd"], only_onroad),
|
||||
NativeProcess("stream_encoderd", "openpilot/system/loggerd", ["./encoderd", "--stream"], or_(and_(livestream, not_(iscar)), notcar)),
|
||||
NativeProcess("stream_encoderd", "openpilot/system/loggerd", ["./encoderd", "--stream"], or_(livestream, notcar)),
|
||||
PythonProcess("logmessaged", "openpilot.system.logmessaged", always_run),
|
||||
|
||||
NativeProcess("camerad", "openpilot/system/camerad", ["./camerad"], or_(driverview, livestream), enabled=not WEBCAM),
|
||||
@@ -163,7 +156,7 @@ procs = [
|
||||
|
||||
# debug procs
|
||||
NativeProcess("bridge", "openpilot/cereal/messaging", ["./bridge"], notcar),
|
||||
PythonProcess("webrtcd", "openpilot.system.webrtc.webrtcd", or_(and_(livestream, not_(iscar)), notcar)),
|
||||
PythonProcess("webrtcd", "openpilot.system.webrtc.webrtcd", or_(livestream, notcar)),
|
||||
PythonProcess("joystick", "openpilot.tools.joystick.joystick_control", and_(joystick, iscar)),
|
||||
|
||||
# sunnylink <3
|
||||
@@ -189,10 +182,6 @@ procs += [
|
||||
NativeProcess("locationd_llk", "openpilot/sunnypilot/selfdrive/locationd", ["./locationd"], only_onroad),
|
||||
]
|
||||
|
||||
if os.path.exists("./github_runner.sh"):
|
||||
procs += [NativeProcess("github_runner_start", "openpilot/system/manager",
|
||||
["./github_runner.sh", "start"], and_(only_offroad, use_github_runner), sigkill=False)]
|
||||
|
||||
if os.path.exists("../../sunnypilot/sunnylink/uploader.py"):
|
||||
procs += [PythonProcess("sunnylink_uploader", "openpilot.sunnypilot.sunnylink.uploader", use_sunnylink_uploader_shim)]
|
||||
|
||||
|
||||
@@ -23,9 +23,9 @@ def post_stream_request(body: StreamRequestBody) -> dict:
|
||||
ret["time"] = (t_end - t_start) * 1000
|
||||
return ret
|
||||
except requests.ConnectTimeout as e:
|
||||
raise Exception("webrtc took too long to respond.") from e
|
||||
raise Exception("device took too long to respond.") from e
|
||||
except requests.ConnectionError as e:
|
||||
raise Exception("webrtc server on device is not running.") from e
|
||||
raise Exception("turn car ignition off to use livestreaming.") from e
|
||||
|
||||
|
||||
def wait_for_webrtcd(max_retries: float = 10) -> None:
|
||||
@@ -37,4 +37,4 @@ def wait_for_webrtcd(max_retries: float = 10) -> None:
|
||||
except requests.ConnectionError:
|
||||
attempts += 1
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError("webrtcd did not initialize in time.")
|
||||
raise TimeoutError("livestreaming service did not initialize in time.")
|
||||
|
||||
@@ -21,10 +21,16 @@ from typing import Any
|
||||
from openpilot.system.webrtc.helpers import StreamRequestBody
|
||||
from openpilot.system.webrtc.schema import generate_field
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.cereal import messaging, log
|
||||
|
||||
SESSION_TIMEOUT_SECONDS = 300
|
||||
|
||||
|
||||
# ice candidate parser for logging
|
||||
def _ice_candidates(sdp: str) -> list[str]:
|
||||
return [line.removeprefix("a=") for line in sdp.splitlines() if line.startswith("a=candidate:")]
|
||||
|
||||
# socket trick: route lookup for 8.8.8.8 (nothing is sent or actually connected to)
|
||||
# return the source interfaces IP which is the default interface of the device
|
||||
def _default_route_ip() -> str | None:
|
||||
@@ -253,7 +259,7 @@ class StreamSession:
|
||||
self._cleanup_lock = asyncio.Lock()
|
||||
self._cleanup_done = False
|
||||
self.logger = logging.getLogger("webrtcd")
|
||||
self.logger.info(
|
||||
cloudlog.warning(
|
||||
"New stream session (%s), video cameras %s, video enabled %s, incoming services %s, outgoing services %s",
|
||||
self.identifier, [t.id for t in self.video_tracks], body.enabled, body.bridge_services_in, body.bridge_services_out,
|
||||
)
|
||||
@@ -329,9 +335,12 @@ class StreamSession:
|
||||
async def run(self):
|
||||
try:
|
||||
self.params.put("LivestreamRequestKeyframe", True)
|
||||
|
||||
# avoid datachannel race by adding messange_handler immediately
|
||||
self.stream.set_message_handler(self.message_handler)
|
||||
|
||||
await asyncio.wait_for(self.stream.wait_for_connection(), timeout=15)
|
||||
if self.stream.has_messaging_channel():
|
||||
self.stream.set_message_handler(self.message_handler)
|
||||
if self.incoming_bridge is not None:
|
||||
await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services)
|
||||
if self.outgoing_bridge is not None:
|
||||
@@ -341,14 +350,18 @@ class StreamSession:
|
||||
if self.bitrate_controller is not None:
|
||||
self.bitrate_controller.start()
|
||||
|
||||
self.logger.info("Stream session (%s) connected", self.identifier)
|
||||
with cloudlog.ctx(session_id=self.identifier):
|
||||
cloudlog.warning("webrtcd.session.connected")
|
||||
if self.is_body:
|
||||
await self.run_body_session()
|
||||
else:
|
||||
await self.run_normal_session()
|
||||
self.logger.info("Stream session (%s) ended", self.identifier)
|
||||
with cloudlog.ctx(session_id=self.identifier):
|
||||
cloudlog.warning("webrtcd.session.ended")
|
||||
except Exception:
|
||||
self.logger.exception("Stream session failure")
|
||||
with cloudlog.ctx(session_id=self.identifier):
|
||||
cloudlog.exception("webrtcd.session.exception")
|
||||
finally:
|
||||
await self.post_run_cleanup()
|
||||
|
||||
@@ -422,15 +435,25 @@ async def handle_get_stream(state: ServerState, raw_body: bytes, content_type: s
|
||||
stream_dict[session.identifier] = session
|
||||
try:
|
||||
answer = await asyncio.wait_for(session.get_answer(), timeout=30)
|
||||
cloudlog.event(
|
||||
"webrtcd.session.ice_candidates",
|
||||
session_id=session.identifier,
|
||||
offer_candidates=_ice_candidates(body.sdp),
|
||||
answer_candidates=_ice_candidates(answer.sdp),
|
||||
)
|
||||
except TimeoutError:
|
||||
await session.stop()
|
||||
stream_dict.pop(session.identifier, None)
|
||||
logging.getLogger("webrtcd").exception("Timed out creating stream answer")
|
||||
with cloudlog.ctx(session_id=session.identifier):
|
||||
cloudlog.warning("webrtcd.session.answer_timeout")
|
||||
raise
|
||||
except Exception:
|
||||
await session.stop()
|
||||
stream_dict.pop(session.identifier, None)
|
||||
logging.getLogger("webrtcd").exception("Failed to create stream answer")
|
||||
with cloudlog.ctx(session_id=session.identifier):
|
||||
cloudlog.exception("webrtcd.session.answer_exception")
|
||||
raise
|
||||
session.start()
|
||||
|
||||
|
||||
@@ -1,260 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
# Default values
|
||||
DEFAULT_REPO_URL="https://github.com/sunnypilot"
|
||||
START_AT_BOOT=false
|
||||
RESTORE_MODE=false
|
||||
RUNNER_VERSION="2.325.0"
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--start-at-boot)
|
||||
START_AT_BOOT=true
|
||||
shift
|
||||
;;
|
||||
--token)
|
||||
GITHUB_TOKEN="$2"
|
||||
shift 2
|
||||
;;
|
||||
--repo)
|
||||
REPO_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--restore)
|
||||
RESTORE_MODE=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
if [ -z "$GITHUB_TOKEN" ]; then
|
||||
GITHUB_TOKEN="$1"
|
||||
elif [ -z "$REPO_URL" ]; then
|
||||
REPO_URL="$1"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Determine BASE_DIR based on mount point
|
||||
if mountpoint -q /data/media; then
|
||||
BASE_DIR="/data/media/0/github"
|
||||
else
|
||||
BASE_DIR="/data/github"
|
||||
fi
|
||||
|
||||
# Constants
|
||||
RUNNER_USER="github-runner"
|
||||
USER_GROUPS="comma,gpu,gpio,sudo"
|
||||
RUNNER_DIR="${BASE_DIR}/runner"
|
||||
BUILDS_DIR="${BASE_DIR}/builds"
|
||||
LOGS_DIR="${BASE_DIR}/logs"
|
||||
CACHE_DIR="${BASE_DIR}/cache"
|
||||
OPENPILOT_DIR="${BASE_DIR}/openpilot"
|
||||
|
||||
# Basic utility functions (no dependencies)
|
||||
remount_rw() {
|
||||
sudo mount -o remount,rw /
|
||||
}
|
||||
|
||||
remount_ro() {
|
||||
sync || true # Try to sync but continue even if it fails
|
||||
sudo mount -o remount,ro / # Always try to remount as read-only
|
||||
}
|
||||
|
||||
# Always ensure we try to remount as read-only on exit
|
||||
trap remount_ro EXIT
|
||||
|
||||
setup_runner_user() {
|
||||
sudo useradd --comment 'GitHub Runner' --create-home --home-dir ${BASE_DIR} ${RUNNER_USER} --shell /bin/bash -G ${USER_GROUPS} || sudo usermod -aG ${USER_GROUPS} ${RUNNER_USER}
|
||||
}
|
||||
|
||||
create_sudoers_entry() {
|
||||
sudo grep -qxF "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" /etc/sudoers || echo "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers
|
||||
}
|
||||
|
||||
set_directory_permissions() {
|
||||
sudo chown -R ${RUNNER_USER}:comma "$BASE_DIR"
|
||||
sudo chmod -R g+rwx "$BASE_DIR"
|
||||
sudo find "$BASE_DIR" -type d -exec chmod g+s {} +
|
||||
}
|
||||
|
||||
setup_directories() {
|
||||
echo "Creating necessary directories..."
|
||||
sudo mkdir -p "$RUNNER_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR"
|
||||
mkdir -p "/data/openpilot"
|
||||
sudo chown -R comma:comma "/data/openpilot"
|
||||
sync
|
||||
}
|
||||
|
||||
wipe_bash_logout() {
|
||||
export BASE_DIR
|
||||
sudo -u ${RUNNER_USER} bash -c "touch ${BASE_DIR}/.bash_logout"
|
||||
sudo -u ${RUNNER_USER} bash -c "truncate -s 0 '${BASE_DIR}/.bash_logout'"
|
||||
}
|
||||
|
||||
# System configuration functions (depends on basic utility functions)
|
||||
setup_system_configs() {
|
||||
echo "Setting up system configurations..."
|
||||
remount_rw
|
||||
setup_runner_user
|
||||
create_sudoers_entry
|
||||
remount_ro
|
||||
set_directory_permissions
|
||||
wipe_bash_logout
|
||||
}
|
||||
|
||||
# Runner setup functions
|
||||
install_runner() {
|
||||
echo "Downloading and setting up runner..."
|
||||
cd "$RUNNER_DIR"
|
||||
curl -o actions-runner-linux-arm64-${RUNNER_VERSION}.tar.gz -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-arm64-${RUNNER_VERSION}.tar.gz
|
||||
sudo -u ${RUNNER_USER} tar -xzf ./actions-runner-linux-arm64-${RUNNER_VERSION}.tar.gz
|
||||
sudo rm ./actions-runner-linux-arm64-${RUNNER_VERSION}.tar.gz
|
||||
sudo chmod +x ./config.sh
|
||||
}
|
||||
|
||||
configure_runner() {
|
||||
remount_rw
|
||||
echo "Configuring runner..."
|
||||
cd "$RUNNER_DIR"
|
||||
sudo -u ${RUNNER_USER} ./config.sh --url "$REPO_URL" --token "$GITHUB_TOKEN" --name $(hostname) --runnergroup "tici-tizi" --labels "tici" --work "$BUILDS_DIR" --unattended
|
||||
remount_ro
|
||||
}
|
||||
|
||||
create_service_template() {
|
||||
echo "Creating service template..."
|
||||
cat <<EOL > "$RUNNER_DIR/bin/actions.runner.service.template"
|
||||
[Unit]
|
||||
Description={{Description}}
|
||||
After=network-online.target nss-lookup.target time-sync.target
|
||||
Wants=network-online.target nss-lookup.target time-sync.target
|
||||
StartLimitInterval=5
|
||||
StartLimitBurst=10
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
ExecStart=/usr/bin/unshare -m -- /bin/bash -c 'mount --bind ${OPENPILOT_DIR} /data/openpilot && setpriv --reuid={{User}} --regid={{User}} --init-groups env HOME=${BASE_DIR} USER={{User}} LOGNAME={{User}} MAIL=/var/mail/{{User}} {{RunnerRoot}}/runsvc.sh'
|
||||
WorkingDirectory={{RunnerRoot}}
|
||||
KillMode=process
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=5min
|
||||
Restart=always
|
||||
RestartSec=120
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOL
|
||||
}
|
||||
|
||||
install_service() {
|
||||
local service_name
|
||||
if [ -f "${RUNNER_DIR}/.service" ]; then
|
||||
service_name=$(cat "${RUNNER_DIR}/.service")
|
||||
else
|
||||
service_name="actions.runner.sunnypilot.$(uname -n)"
|
||||
fi
|
||||
|
||||
create_service_template
|
||||
remount_rw
|
||||
local service_path="/etc/systemd/system/${service_name}"
|
||||
echo "Installing systemd service..."
|
||||
if [ -f "${service_path}" ]; then
|
||||
echo "Service ${service_path} found in systemd, we will delete it"
|
||||
sudo rm -f "${service_path}"
|
||||
fi
|
||||
|
||||
cd "$RUNNER_DIR"
|
||||
sudo ./svc.sh install $RUNNER_USER
|
||||
|
||||
if [ "$START_AT_BOOT" = false ]; then
|
||||
sudo systemctl disable "${service_name}"
|
||||
fi
|
||||
remount_ro
|
||||
}
|
||||
|
||||
check_restore_prerequisites() {
|
||||
local can_restore=false
|
||||
local service_name=""
|
||||
|
||||
# Check if base runner directory exists
|
||||
if [ ! -d "${RUNNER_DIR}" ]; then
|
||||
echo "ERROR: Runner directory ${RUNNER_DIR} does not exist"
|
||||
echo "This directory is required for restore operations"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# First check if we have the required files for restoration
|
||||
if [ -f "${RUNNER_DIR}/.credentials" ] && [ -f "${RUNNER_DIR}/.service" ]; then
|
||||
can_restore=true
|
||||
service_name=$(cat "${RUNNER_DIR}/.service")
|
||||
echo "Found required runner configuration files"
|
||||
else
|
||||
echo "Missing required runner configuration files"
|
||||
echo "Required: .credentials and .service files in ${RUNNER_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! id "${RUNNER_USER}" &>/dev/null; then
|
||||
echo "User ${RUNNER_USER} does not exist"
|
||||
fi
|
||||
|
||||
# Only proceed if we can restore AND need to restore
|
||||
if [ "$can_restore" = true ]; then
|
||||
echo "Restoration is possible"
|
||||
return 0
|
||||
else
|
||||
echo "No restoration possible"
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
perform_restore() {
|
||||
echo "Starting runner restoration..."
|
||||
setup_directories
|
||||
setup_system_configs
|
||||
install_service
|
||||
echo "Runner restoration completed successfully"
|
||||
}
|
||||
|
||||
perform_install() {
|
||||
echo "Starting fresh installation..."
|
||||
setup_directories
|
||||
setup_system_configs
|
||||
install_runner
|
||||
set_directory_permissions
|
||||
configure_runner
|
||||
install_service
|
||||
echo "Installation completed successfully"
|
||||
}
|
||||
|
||||
main() {
|
||||
if [ "$RESTORE_MODE" = true ]; then
|
||||
echo "Running in restore mode - will only restore system configurations..."
|
||||
check_restore_prerequisites
|
||||
perform_restore
|
||||
else
|
||||
# Check required arguments for normal installation
|
||||
if [ -z "$GITHUB_TOKEN" ]; then
|
||||
echo "Usage: $0 [--start-at-boot] [--token <github_token>] [--repo <repository_url>] [--restore]"
|
||||
echo "Required argument (except for --restore): github_token"
|
||||
echo "Optional arguments:"
|
||||
echo " --start-at-boot Enable auto-start at boot (default: false)"
|
||||
echo " --repo Repository URL (default: ${DEFAULT_REPO_URL})"
|
||||
echo " --restore Restore existing runner configuration"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set repository URL if not provided
|
||||
REPO_URL="${REPO_URL:-$DEFAULT_REPO_URL}"
|
||||
perform_install
|
||||
fi
|
||||
|
||||
echo "Starting runner service..."
|
||||
cd "$RUNNER_DIR"
|
||||
sudo ./svc.sh start
|
||||
}
|
||||
|
||||
main
|
||||
@@ -53,24 +53,28 @@ def create_pkl_name(full_name: str) -> str:
|
||||
return pkl
|
||||
|
||||
|
||||
def _read_pkl_bytes(pkl_path: Path) -> bytes:
|
||||
def _hash_pkl(pkl_path: Path) -> str:
|
||||
manifest = Path(f"{pkl_path}.chunkmanifest")
|
||||
if manifest.exists():
|
||||
num_chunks = int(manifest.read_text().strip())
|
||||
parts = []
|
||||
for i in range(num_chunks):
|
||||
chunk = Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}")
|
||||
parts.append(chunk.read_bytes())
|
||||
return b''.join(parts)
|
||||
return pkl_path.read_bytes()
|
||||
paths = [Path(f"{pkl_path}.chunk{i + 1:02d}of{num_chunks:02d}") for i in range(num_chunks)]
|
||||
else:
|
||||
paths = [pkl_path]
|
||||
|
||||
digest = hashlib.sha256()
|
||||
for path in paths:
|
||||
with path.open('rb') as f:
|
||||
while block := f.read(1024 * 1024):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _find_driving_pkl(output_path: Path) -> Path | None:
|
||||
for pattern in ('driving_tinygrad.pkl', 'driving_*_tinygrad.pkl'):
|
||||
for pattern in ('*driving_tinygrad.pkl', '*driving_*_tinygrad.pkl'):
|
||||
matches = sorted(output_path.glob(pattern))
|
||||
if matches:
|
||||
return matches[0]
|
||||
for pattern in ('driving_tinygrad.pkl.chunkmanifest', 'driving_*_tinygrad.pkl.chunkmanifest'):
|
||||
for pattern in ('*driving_tinygrad.pkl.chunkmanifest', '*driving_*_tinygrad.pkl.chunkmanifest'):
|
||||
matches = sorted(output_path.glob(pattern))
|
||||
if matches:
|
||||
return Path(str(matches[0]).removesuffix('.chunkmanifest'))
|
||||
@@ -86,8 +90,20 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path:
|
||||
return old_pkl.rename(new_pkl)
|
||||
|
||||
|
||||
def _hash_onnx_files(model_dir: Path) -> str | None:
|
||||
onnx_files = sorted(model_dir.glob("*.onnx"))
|
||||
if not onnx_files:
|
||||
return None
|
||||
digest = hashlib.sha256()
|
||||
for f in onnx_files:
|
||||
with f.open('rb') as fh:
|
||||
while block := fh.read(1024 * 1024):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def generate_chunked_model(driving_pkl: Path) -> dict:
|
||||
tinygrad_hash = hashlib.sha256(_read_pkl_bytes(driving_pkl)).hexdigest()
|
||||
tinygrad_hash = _hash_pkl(driving_pkl)
|
||||
|
||||
chunks_config = []
|
||||
manifest_file = Path(f"{driving_pkl}.chunkmanifest")
|
||||
@@ -119,7 +135,8 @@ def generate_chunked_model(driving_pkl: Path) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown") -> None:
|
||||
def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown",
|
||||
onnx_sha256=None) -> None:
|
||||
bundle_json = {
|
||||
"short_name": short_name,
|
||||
"display_name": custom_name or upstream_branch,
|
||||
@@ -135,6 +152,9 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short
|
||||
"models": models,
|
||||
}
|
||||
|
||||
if onnx_sha256:
|
||||
bundle_json["onnx_sha256"] = onnx_sha256
|
||||
|
||||
# Write metadata to output_dir
|
||||
metadata_json = {
|
||||
"bundles": [bundle_json]
|
||||
@@ -174,4 +194,6 @@ if __name__ == "__main__":
|
||||
_driving_pkl = new_pkl
|
||||
|
||||
_model_metadata = generate_chunked_model(_driving_pkl)
|
||||
create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch)
|
||||
_onnx_sha256 = _hash_onnx_files(Path(args.model_dir))
|
||||
create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch,
|
||||
onnx_sha256=_onnx_sha256)
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Determine BASE_DIR based on mount point
|
||||
if mountpoint -q /data/media; then
|
||||
GITHUB_BASE_DIR="/data/media/0/github"
|
||||
else
|
||||
GITHUB_BASE_DIR="/data/github"
|
||||
fi
|
||||
|
||||
# Define directories and user
|
||||
BIN_DIR="$GITHUB_BASE_DIR/bin"
|
||||
BUILDS_DIR="$GITHUB_BASE_DIR/builds"
|
||||
OPENPILOT_DIR="$GITHUB_BASE_DIR/openpilot"
|
||||
LOGS_DIR="$GITHUB_BASE_DIR/logs"
|
||||
CACHE_DIR="$GITHUB_BASE_DIR/cache"
|
||||
RUNNER_USERNAME="github-runner"
|
||||
# Define the systemd service name
|
||||
SERVICE_NAME="github-runner"
|
||||
USER_GROUPS="comma,gpu,gpio,sudo"
|
||||
|
||||
# Function to stop and disable the systemd service
|
||||
stop_and_uninstall_service() {
|
||||
cd $GITHUB_BASE_DIR/runner
|
||||
sudo ./svc.sh stop
|
||||
sudo ./svc.sh uninstall
|
||||
}
|
||||
|
||||
# Function to remove the systemd service file
|
||||
remove_runner() {
|
||||
cd $GITHUB_BASE_DIR/runner
|
||||
sudo rm .runner
|
||||
sudo su -c './config.sh remove' github-runner
|
||||
}
|
||||
|
||||
# Function to delete the Github Runner directories
|
||||
delete_directories() {
|
||||
sudo rm -rf "$BIN_DIR/github-runner"
|
||||
sudo rm -rf "$GITHUB_BASE_DIR" "$BIN_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR"
|
||||
}
|
||||
|
||||
# Function to remove the Github Runner user
|
||||
delete_user() {
|
||||
for group in ${USER_GROUPS//,/ }
|
||||
do
|
||||
sudo gpasswd -d ${RUNNER_USERNAME} ${group}
|
||||
done
|
||||
sudo userdel -r ${RUNNER_USERNAME}
|
||||
}
|
||||
|
||||
# Function to remove sudoers entry
|
||||
remove_sudoers_entry() {
|
||||
sudo sed -i.bak "/${RUNNER_USERNAME} ALL=(ALL) NOPASSWD: ALL/d" /etc/sudoers
|
||||
}
|
||||
|
||||
# Make filesystem writable
|
||||
sudo mount -o remount rw /
|
||||
|
||||
# Ensure filesystem is remounted as read-only on script exit
|
||||
trap "sudo mount -o remount ro /" EXIT
|
||||
|
||||
# Call functions
|
||||
stop_and_uninstall_service
|
||||
remove_runner
|
||||
delete_directories
|
||||
delete_user
|
||||
remove_sudoers_entry
|
||||
# End of uninstall script
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import tempfile
|
||||
|
||||
from huggingface_hub import HfApi, hf_hub_download
|
||||
|
||||
|
||||
def hash_file(path: str) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with open(path, 'rb') as f:
|
||||
while block := f.read(1024 * 1024):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--hf-repo", required=True)
|
||||
parser.add_argument("--hf-defaults-path", required=True)
|
||||
parser.add_argument("--artifact-name", required=True)
|
||||
parser.add_argument("--model-dir", required=True)
|
||||
parser.add_argument("--onnx-path", required=True)
|
||||
parser.add_argument("--onnx-ref", required=True)
|
||||
parser.add_argument("--model-name", required=True)
|
||||
parser.add_argument("--tinygrad-ref", required=True)
|
||||
parser.add_argument("--run-number", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
api = HfApi()
|
||||
onnx_sha256 = hash_file(args.onnx_path)
|
||||
short_ref = args.onnx_ref[:8]
|
||||
folder_name = f"model-{args.model_name}-{short_ref}-{args.run_number}"
|
||||
|
||||
print(f"ONNX hash: {onnx_sha256}")
|
||||
print(f"ONNX ref: {args.onnx_ref} (short: {short_ref})")
|
||||
print(f"Folder: {folder_name}")
|
||||
|
||||
metadata_path = f"{args.model_dir}/metadata.json"
|
||||
with open(metadata_path) as f:
|
||||
metadata = json.load(f)
|
||||
|
||||
bundle = metadata['bundles'][0]
|
||||
bundle['display_name'] = args.model_name
|
||||
bundle['onnx_sha256'] = onnx_sha256
|
||||
bundle['onnx_ref'] = args.onnx_ref
|
||||
|
||||
artifact = bundle['models'][0]['artifact']
|
||||
hf_base = f"https://huggingface.co/datasets/{args.hf_repo}/resolve/main/{args.hf_defaults_path}/{folder_name}"
|
||||
artifact['download_uri']['url'] = f"{hf_base}/{artifact['file_name']}"
|
||||
for chunk in artifact.get('chunks', []):
|
||||
chunk['url'] = f"{hf_base}/{chunk['file_name']}"
|
||||
|
||||
print(f"Uploading model to {args.hf_defaults_path}/{folder_name}/")
|
||||
api.upload_folder(
|
||||
folder_path=args.model_dir,
|
||||
path_in_repo=f"{args.hf_defaults_path}/{folder_name}",
|
||||
repo_id=args.hf_repo,
|
||||
repo_type="dataset",
|
||||
)
|
||||
|
||||
json_filename = f"{args.hf_defaults_path}/default_models.json"
|
||||
try:
|
||||
local_path = hf_hub_download(repo_id=args.hf_repo, repo_type='dataset', filename=json_filename)
|
||||
with open(local_path) as f:
|
||||
defaults_json = json.load(f)
|
||||
except Exception:
|
||||
defaults_json = {"tinygrad_ref": args.tinygrad_ref, "bundles": []}
|
||||
|
||||
defaults_json['tinygrad_ref'] = args.tinygrad_ref
|
||||
|
||||
existing_idx = next((i for i, b in enumerate(defaults_json['bundles'])
|
||||
if b.get('onnx_sha256') == onnx_sha256), None)
|
||||
if existing_idx is not None:
|
||||
defaults_json['bundles'][existing_idx] = bundle
|
||||
else:
|
||||
defaults_json['bundles'].append(bundle)
|
||||
|
||||
print(json.dumps(defaults_json, indent=2))
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||||
json.dump(defaults_json, f, indent=2)
|
||||
tmp_path = f.name
|
||||
|
||||
api.upload_file(
|
||||
path_or_fileobj=tmp_path,
|
||||
path_in_repo=json_filename,
|
||||
repo_id=args.hf_repo,
|
||||
repo_type="dataset",
|
||||
)
|
||||
|
||||
print(f"Updated {json_filename}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user