mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-11-22 05:33:43 +08:00
Compare commits
23 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 85423d2b74 | |||
| 2d99b9e47d | |||
| b88039da3f | |||
| f2dd04fec2 | |||
| 0f4c20d430 | |||
| 13097515a6 | |||
| 9489b2f481 | |||
| c270c7835e | |||
| c6e564cdcd | |||
| b55c4be825 | |||
| dbba01d933 | |||
| 06604f601f | |||
| 8108740b60 | |||
| 64f2381c2e | |||
| 61af0a0512 | |||
| 066c0e2957 | |||
| 26b8501e7b | |||
| 3d8f211d4b | |||
| 02ef6e4ce0 | |||
| a1a28e467e | |||
| 932ff06d93 | |||
| 8cf6ff9a33 | |||
| a86979d2b2 |
@@ -34,6 +34,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'qcom'
|
||||
on_device:
|
||||
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
hf_repo:
|
||||
description: 'Hugging Face dataset repository (e.g. sunnypilot/sunnypilot_models_v1)'
|
||||
required: false
|
||||
@@ -107,6 +112,11 @@ on:
|
||||
options:
|
||||
- qcom
|
||||
- chestnut
|
||||
on_device:
|
||||
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
hf_repo:
|
||||
description: 'Hugging Face dataset repository'
|
||||
required: false
|
||||
@@ -130,6 +140,7 @@ jobs:
|
||||
is_20hz: ${{ inputs.is_20hz }}
|
||||
artifact_suffix: ${{ inputs.artifact_suffix }}
|
||||
target_hardware: ${{ inputs.target_hardware }}
|
||||
on_device: ${{ inputs.on_device || false }}
|
||||
secrets: inherit
|
||||
|
||||
publish_model:
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
name: Chestnut mock build parity
|
||||
description: >-
|
||||
Convert one commaai precompiled chestnut model (LS2) on this runner under tinygrad's amd mock, as a hosted chestnut build does, and require the
|
||||
exact bytes the chestnut runner built (the published artifact). Needs setup-amd-mock, numpy and zstandard; tinygrad_repo checked out.
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Key this check by everything its result depends on
|
||||
id: key
|
||||
shell: bash
|
||||
env:
|
||||
SOURCES: ${{ hashFiles('tinygrad_repo/tinygrad/**/*.py', 'tinygrad_repo/test/mockgpu/**', 'openpilot/sunnypilot/modeld_v2/*.py', '.github/workflows/chestnut-build-parity/action.yml') }}
|
||||
run: echo "key=chestnut-build-parity-$SOURCES-$(sha256sum "$LLVM_PATH" | cut -c1-16)-$(python3 -c 'import platform; print(platform.python_version())')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Passed before with all of it identical?
|
||||
id: passed
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: ~/.chestnut_build_parity.passed
|
||||
key: ${{ steps.key.outputs.key }}
|
||||
|
||||
- name: Rebuild LS2 on the mock and require the chestnut runner's artifact byte for byte
|
||||
if: steps.passed.outputs.cache-hit != 'true'
|
||||
shell: bash
|
||||
env:
|
||||
REF: b92fc4828664f8767bce5e92f1630ed319cbe509 # commaai/openpilot: precompiled big_driving_tinygrad.pkl, pins tinygrad ec91b74b5b
|
||||
PKL_OID: b4dead3ee091d66ab002a6b8e716c26f4f79e3c931e4be5f48e017ba92f6e960
|
||||
PKL_SIZE: "767228907"
|
||||
TG_SHA: ec91b74b5b6b0fc924b72d3f27689679ca2438b9
|
||||
WANT: 680a8d629785f2b1d1a2b0e40d0dfbcd0f7291c21e20c51259f1d768a22ec217 # driving_light_speed_2_november_04_2026_tinygrad.pkl, recompiled46
|
||||
run: |
|
||||
set -euo pipefail
|
||||
work="$RUNNER_TEMP/chestnut_parity" && rm -rf "$work" && mkdir -p "$work"
|
||||
href=$(curl -fsSL --retry 3 -X POST -H "Accept: application/vnd.git-lfs+json" -H "Content-Type: application/vnd.git-lfs+json" \
|
||||
https://huggingface.co/commaai/openpilot-lfs.git/info/lfs/objects/batch \
|
||||
-d "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"$PKL_OID\",\"size\":$PKL_SIZE}]}" | jq -r '.objects[0].actions.download.href')
|
||||
curl -fsSL --retry 3 -o "$work/big_driving_tinygrad.pkl" "$href"
|
||||
echo "$PKL_OID $work/big_driving_tinygrad.pkl" | sha256sum -c -
|
||||
tg="$work/tinygrad" && git init -q "$tg" && git -C "$tg" fetch -q --depth 1 https://github.com/tinygrad/tinygrad "$TG_SHA" && git -C "$tg" checkout -q FETCH_HEAD
|
||||
CHECK_OOB=0 GMMU=0 PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo" python3 tinygrad_repo/test/mockgpu/usb_server.py \
|
||||
env DEV=USB+AMD PYTHONPATH="$tg:$GITHUB_WORKSPACE" python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py "$work/big_driving_tinygrad.pkl" "$work/LS2.pkl"
|
||||
got=$(sha256sum "$work/LS2.pkl" | cut -d' ' -f1)
|
||||
if [ "$got" != "$WANT" ]; then
|
||||
echo "::error::the mock's LS2 build ($got) is not the chestnut runner's ($WANT): this runner's LLVM or the mock no longer builds what the chestnut does"
|
||||
exit 1
|
||||
fi
|
||||
echo "LS2 rebuilt on the mock: byte-identical to the chestnut runner's build"
|
||||
touch ~/.chestnut_build_parity.passed
|
||||
|
||||
- name: Remember the pass
|
||||
if: steps.passed.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: ~/.chestnut_build_parity.passed
|
||||
key: ${{ steps.key.outputs.key }}
|
||||
|
||||
- name: Already passed
|
||||
if: steps.passed.outputs.cache-hit == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
echo "LS2 rebuild skipped: it already matched the chestnut runner's build with this exact mock, converter, LLVM and python"
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
WARP_PKL="openpilot/sunnypilot/modeld_v2/models/driving_warp_${res}_tinygrad.pkl"
|
||||
echo "Compiling $WARP_PKL on QCOM"
|
||||
taskset -c 7 env DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1 python3 openpilot/sunnypilot/modeld_v2/compile_warp.py \
|
||||
--frame ${FRAME_PARAM} --warp-to ${MODEL_SIZE} --output "${WARP_PKL}"
|
||||
--frame ${FRAME_PARAM} --warp-to ${MODEL_SIZE} --split-frames --output "${WARP_PKL}"
|
||||
|
||||
BIG_WARP_PKL="openpilot/sunnypilot/modeld_v2/models/big_driving_warp_${res}_tinygrad.pkl"
|
||||
echo "Compiling $BIG_WARP_PKL on AMD"
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
name: Set up tinygrad's amd mock
|
||||
description: LLVM 20 and ROCm comgr, as tinygrad_repo/.github/actions/setup-tinygrad installs them for the mock gpu
|
||||
inputs:
|
||||
emu_cache:
|
||||
description: >-
|
||||
Keep tinygrad's cache db (the emulator's compiled kernels and gate results) between runs under this name, e.g. the model's file name.
|
||||
Empty: no cache.
|
||||
default: ''
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install LLVM 20 and ROCm comgr
|
||||
shell: bash
|
||||
run: |
|
||||
. /etc/os-release
|
||||
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
|
||||
echo "deb http://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
|
||||
sudo mkdir -p /etc/apt/keyrings
|
||||
curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
|
||||
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.1 $VERSION_CODENAME main" | sudo tee /etc/apt/sources.list.d/rocm.list
|
||||
printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/rocm-pin-600
|
||||
sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends libllvm20 clang-20 lld-20 comgr
|
||||
printf '/opt/rocm/lib\n/opt/rocm/lib64\n' | sudo tee /etc/ld.so.conf.d/rocm.conf && sudo ldconfig
|
||||
echo "/usr/lib/llvm-20/bin" >> "$GITHUB_PATH"
|
||||
llvm=$(ls /usr/lib/llvm-20/lib/libLLVM*.so* /usr/lib/*-linux-gnu/libLLVM-20.so* 2>/dev/null | head -1 || true)
|
||||
[ -n "$llvm" ] || { echo "::error::libLLVM 20 not found"; exit 1; }
|
||||
echo "LLVM_PATH=$llvm" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache the chestnut's firmware
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/tinygrad/downloads/fw
|
||||
key: chestnut-fw-${{ hashFiles('tinygrad_repo/tinygrad/runtime/autogen/am/fw.py') }}
|
||||
|
||||
- name: Fetch the chestnut's firmware (gitlab times out now and then; the mock boots from it)
|
||||
shell: bash
|
||||
run: |
|
||||
for i in 1 2 3 4 5; do
|
||||
PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo" python3 -c "
|
||||
from tinygrad.helpers import fetch_fw
|
||||
from tinygrad.runtime.autogen.am import fw
|
||||
for n, h in fw.hashes.items():
|
||||
if n.startswith(('psp_14_0_2_', 'smu_14_0_2.', 'sdma_7_0_0.', 'gc_12_0_0_')): fetch_fw('amdgpu', n, h)" && exit 0
|
||||
sleep $((i * 10))
|
||||
done
|
||||
echo "::error::could not fetch the chestnut's firmware from gitlab"; exit 1
|
||||
|
||||
- name: runner cpu
|
||||
if: inputs.emu_cache != ''
|
||||
id: cpu
|
||||
shell: bash
|
||||
run: echo "cpu=$( (grep -m1 '^model name' /proc/cpuinfo; grep -m1 '^flags' /proc/cpuinfo) | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Keep the emulator's compiled kernels between runs
|
||||
if: inputs.emu_cache != ''
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/tinygrad/cache.db
|
||||
key: emu-${{ steps.cpu.outputs.cpu }}-${{ hashFiles('tinygrad_repo/tinygrad/**/*.py', 'tinygrad_repo/test/mockgpu/**') }}-${{ inputs.emu_cache }}
|
||||
restore-keys: emu-${{ steps.cpu.outputs.cpu }}-${{ hashFiles('tinygrad_repo/tinygrad/**/*.py', 'tinygrad_repo/test/mockgpu/**') }}-
|
||||
@@ -35,6 +35,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'qcom'
|
||||
on_device:
|
||||
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
upstream_branch:
|
||||
@@ -59,6 +64,11 @@ on:
|
||||
- qcom
|
||||
- chestnut
|
||||
default: 'qcom'
|
||||
on_device:
|
||||
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
|
||||
run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
|
||||
@@ -111,7 +121,12 @@ jobs:
|
||||
grep -qs "^version https://git-lfs" "$f" || continue
|
||||
OID=$(awk -F'sha256:' '/^oid/{print $2}' "$f"); SIZE=$(awk '/^size/{print $2}' "$f")
|
||||
HREF=$(curl -fsSL --retry 3 -X POST -H "Accept: application/vnd.git-lfs+json" -H "Content-Type: application/vnd.git-lfs+json" "$LFS_URL/objects/batch" \
|
||||
-d "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"$OID\",\"size\":$SIZE}]}" | jq -r '.objects[0].actions.download.href')
|
||||
-d "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"$OID\",\"size\":$SIZE}]}" | jq -r '.objects[0].actions.download.href // empty')
|
||||
if [ -z "$HREF" ]; then
|
||||
HF_PATH=$(curl -fsSL --retry 3 "https://huggingface.co/api/models/commaai/openpilot_driving_models/tree/main?recursive=true" | jq -r --arg oid "$OID" 'map(select(.lfs.oid == $oid))[0].path // empty')
|
||||
[ -n "$HF_PATH" ] || { echo "::error::$f ($OID) is in neither openpilot's LFS nor commaai/openpilot_driving_models"; exit 1; }
|
||||
HREF="https://huggingface.co/commaai/openpilot_driving_models/resolve/main/$HF_PATH"
|
||||
fi
|
||||
aria2c -x 16 -s 16 -k 16M --max-tries=5 --retry-wait=5 --file-allocation=none --summary-interval=30 --allow-overwrite=true \
|
||||
-d "$(dirname "$f")" -o "$(basename "$f")" "$HREF"
|
||||
echo "$OID $f" | sha256sum -c -
|
||||
@@ -119,8 +134,9 @@ jobs:
|
||||
}
|
||||
UPSTREAM_PKL=selfdrive/modeld/models/big_driving_tinygrad.pkl
|
||||
if [ "${{ inputs.target_hardware }}" != "chestnut" ]; then
|
||||
git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx,**/selfdrive/modeld/models/dmonitoring_*.onnx,**/selfdrive/modeld/models/big_*.pkl,**/selfdrive/modeld/models/dmonitoring_*.pkl"
|
||||
git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx,**/selfdrive/modeld/models/dmonitoring_*.onnx,**/selfdrive/modeld/models/big_*.pkl,**/selfdrive/modeld/models/dmonitoring_*.pkl" || true
|
||||
rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx selfdrive/modeld/models/big_*.pkl selfdrive/modeld/models/dmonitoring_*.pkl
|
||||
fetch_lfs selfdrive/modeld/models/*.onnx
|
||||
elif compgen -G "selfdrive/modeld/models/big_*.onnx" > /dev/null; then
|
||||
fetch_lfs selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/big_*.pkl
|
||||
find selfdrive/modeld/models -type f \( -name "*.onnx" -o -name "*.pkl" \) ! -name "big_*.onnx" ! -name "big_*.pkl" -delete
|
||||
@@ -153,18 +169,67 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build_model:
|
||||
runs-on: [self-hosted, "${{ inputs.target_hardware == 'chestnut' && 'chestnut' || 'tici' }}"]
|
||||
runs-on: ${{ inputs.on_device && fromJSON(inputs.target_hardware == 'chestnut' && '["self-hosted","chestnut"]' || '["self-hosted","tici"]') || 'ubuntu-24.04' }}
|
||||
needs: get_model
|
||||
env:
|
||||
MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
|
||||
REF: ${{ inputs.upstream_branch }}
|
||||
MOCK_BUILD: ${{ !inputs.on_device }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up the hosted mock build
|
||||
if: env.MOCK_BUILD == 'true'
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install LLVM 20 and ROCm comgr for the amd mock
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware == 'chestnut'
|
||||
uses: ./.github/workflows/setup-amd-mock
|
||||
with:
|
||||
emu_cache: build
|
||||
|
||||
- name: Install the amd mock build dependencies
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware == 'chestnut'
|
||||
run: |
|
||||
python3 -m pip install -q numpy zstandard
|
||||
|
||||
- name: Install the qcom mock build dependencies
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut'
|
||||
id: qcom_deps
|
||||
run: |
|
||||
sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends qemu-user-static aria2
|
||||
python3 -m pip install -q numpy zstandard tinymesa==25.2.7.2
|
||||
echo "qemu=$(dpkg-query -W -f='${Version}' qemu-user-static)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Compiler check passed before with all of it identical?
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut'
|
||||
id: qcom_compiler
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: ~/.qcom_compiler_parity.passed
|
||||
key: qcom-compiler-parity-${{ steps.qcom_deps.outputs.qemu }}-${{ hashFiles('tinygrad_repo/test/device/qcom/compiler_parity.py', 'tinygrad_repo/test/mockgpu/qcom/a630_compiled.pkl.xz', 'tinygrad_repo/tinygrad/runtime/support/compiler_qcom.py') }}
|
||||
|
||||
- name: Check the off-device qcom compiler builds the a630's bytes
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut' && steps.qcom_compiler.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
# every kernel an a630 has compiled must compile to the same bytes here before a model is built here
|
||||
PYTHONPATH="$TINYGRAD_PATH" python3 "$TINYGRAD_PATH/test/device/qcom/compiler_parity.py" check "$TINYGRAD_PATH/test/mockgpu/qcom/a630_compiled.pkl.xz"
|
||||
touch ~/.qcom_compiler_parity.passed
|
||||
|
||||
- name: Remember the compiler check passed
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut' && steps.qcom_compiler.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: ~/.qcom_compiler_parity.passed
|
||||
key: ${{ steps.qcom_compiler.outputs.cache-primary-key }}
|
||||
|
||||
- name: Set environment variables
|
||||
if: env.MOCK_BUILD != 'true'
|
||||
id: set-env
|
||||
run: |
|
||||
# Set up common environment
|
||||
@@ -177,6 +242,7 @@ jobs:
|
||||
printenv >> $GITHUB_ENV
|
||||
|
||||
- name: Setup build environment
|
||||
if: env.MOCK_BUILD != 'true'
|
||||
run: |
|
||||
mkdir -p "${BUILD_DIR}/"
|
||||
sudo find $BUILD_DIR/ -mindepth 1 -delete
|
||||
@@ -198,21 +264,28 @@ jobs:
|
||||
|
||||
- name: Build Model
|
||||
run: |
|
||||
source /etc/profile
|
||||
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
||||
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
if [ "$MOCK_BUILD" = "true" ]; then
|
||||
TASKSET=""
|
||||
export CHECK_OOB=0 GMMU=0 # as test/mockgpu runs the amd mock
|
||||
else
|
||||
source /etc/profile
|
||||
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
||||
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
TASKSET="taskset -c 7"
|
||||
fi
|
||||
export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
|
||||
|
||||
COMPILE_MODELD="${{ github.workspace }}/openpilot/sunnypilot/modeld_v2/compile_modeld.py"
|
||||
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
|
||||
CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
|
||||
|
||||
TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 OPENPILOT_HACKS=1"
|
||||
MOCK=$([ "$MOCK_BUILD" = "true" ] && echo MOCK || true)
|
||||
TG_FLAGS_QCOM="DEV=${MOCK:+MOCK+}QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 OPENPILOT_HACKS=1"
|
||||
if [ "${{ inputs.target_hardware }}" == "chestnut" ]; then
|
||||
echo "CHESTNUT build"
|
||||
export CHESTNUT=1
|
||||
TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
|
||||
TG_FLAGS="DEBUG=1 DEV=${MOCK}USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
|
||||
OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
|
||||
else
|
||||
echo "QCOM build"
|
||||
@@ -227,14 +300,16 @@ jobs:
|
||||
echo "Converting commaai's precompiled model with tinygrad $TG_SHA"
|
||||
UPSTREAM_TG="${{ runner.temp }}/upstream_tinygrad"
|
||||
rm -rf "$UPSTREAM_TG" && git init -q "$UPSTREAM_TG" && git -C "$UPSTREAM_TG" fetch -q --depth 1 "$TG_URL" "$TG_SHA" && git -C "$UPSTREAM_TG" checkout -q FETCH_HEAD
|
||||
ON_MOCK=""
|
||||
if [ "$MOCK_BUILD" = "true" ]; then ON_MOCK="env PYTHONPATH=$TINYGRAD_PATH python3 $TINYGRAD_PATH/test/mockgpu/usb_server.py"; fi
|
||||
CONVERTED="${{ runner.temp }}/big_driving_tinygrad_plain.pkl"
|
||||
env DEV=USB+AMD PYTHONPATH="$UPSTREAM_TG:${{ github.workspace }}" taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py \
|
||||
$ON_MOCK env DEV=USB+AMD PYTHONPATH="$UPSTREAM_TG:${{ github.workspace }}" $TASKSET python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py \
|
||||
"${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" "$CONVERTED"
|
||||
mv "$CONVERTED" "$OUTPUT_PKL"
|
||||
elif NEW_ARCH_ONNX=$(find "${{ env.MODELS_DIR }}" -maxdepth 1 -name "${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}*driving*.onnx" -print -quit) &&
|
||||
[ -n "$NEW_ARCH_ONNX" ] && grep -qa new_img "$NEW_ARCH_ONNX"; then # the new architecture: one ONNX taking the warped frames as new_img
|
||||
echo "Compiling new architecture ONNX: $NEW_ARCH_ONNX"
|
||||
env ${TG_FLAGS} taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/compile_modeld_v2.py \
|
||||
env ${TG_FLAGS} $TASKSET python3 openpilot/sunnypilot/modeld_v2/compile_modeld_v2.py \
|
||||
"$NEW_ARCH_ONNX" "$OUTPUT_PKL" --device-input "*" --benchmark-runs 1
|
||||
else
|
||||
echo "Detected onnx architecture. Routing to native compile_modeld.py."
|
||||
@@ -284,7 +359,7 @@ jobs:
|
||||
|
||||
if [ -n "$MODEL_TYPE" ]; then
|
||||
echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL"
|
||||
env ${TG_FLAGS} taskset -c 7 python3 "$COMPILE_MODELD" \
|
||||
env ${TG_FLAGS} $TASKSET python3 "$COMPILE_MODELD" \
|
||||
--model-type $MODEL_TYPE \
|
||||
--model-size $MODEL_SIZE \
|
||||
--camera-resolutions $CAMERA_RES \
|
||||
@@ -293,6 +368,13 @@ jobs:
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: Replay its check frame on the mock and require the result bit for bit
|
||||
if: env.MOCK_BUILD == 'true'
|
||||
env:
|
||||
DEV: ${{ inputs.target_hardware == 'chestnut' && 'MOCKUSB+AMD' || 'MOCK+QCOM' }}
|
||||
PYTHONPATH: ${{ env.TINYGRAD_PATH }}:${{ github.workspace }}
|
||||
run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "${{ env.MODELS_DIR }}/${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}driving_tinygrad.pkl"
|
||||
|
||||
- name: Prepare Output
|
||||
run: |
|
||||
sudo rm -rf ${{ env.OUTPUT_DIR }}
|
||||
@@ -304,7 +386,7 @@ jobs:
|
||||
--include='/${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}driving_tinygrad.pkl' \
|
||||
--exclude='*' \
|
||||
--delete-excluded \
|
||||
--chown=comma:comma \
|
||||
$([ "$MOCK_BUILD" = "true" ] || echo --chown=comma:comma) \
|
||||
${{ env.MODELS_DIR }}/ ${{ env.OUTPUT_DIR }}/
|
||||
|
||||
python3 "${{ github.workspace }}/release/ci/model_generator.py" \
|
||||
@@ -333,7 +415,7 @@ jobs:
|
||||
path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
|
||||
|
||||
- name: Re-enable powersave
|
||||
if: always()
|
||||
if: always() && env.MOCK_BUILD != 'true'
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
|
||||
|
||||
@@ -28,24 +28,30 @@ jobs:
|
||||
env:
|
||||
DEV: CPU
|
||||
run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_plain_artifact.py
|
||||
- name: Install LLVM 20 and ROCm comgr for the amd mock, as tinygrad_repo/.github/actions/setup-tinygrad does
|
||||
run: |
|
||||
. /etc/os-release
|
||||
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
|
||||
echo "deb http://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
|
||||
sudo mkdir -p /etc/apt/keyrings
|
||||
curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
|
||||
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.1 $VERSION_CODENAME main" | sudo tee /etc/apt/sources.list.d/rocm.list
|
||||
printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/rocm-pin-600
|
||||
sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends libllvm20 clang-20 lld-20 comgr
|
||||
printf '/opt/rocm/lib\n/opt/rocm/lib64\n' | sudo tee /etc/ld.so.conf.d/rocm.conf && sudo ldconfig
|
||||
echo "/usr/lib/llvm-20/bin" >> "$GITHUB_PATH"
|
||||
- uses: ./.github/workflows/setup-amd-mock
|
||||
- name: Convert a pickled USB+AMD jit and replay it under tinygrad's amd mock
|
||||
env:
|
||||
DEV: MOCKUSB+AMD
|
||||
CHECK_OOB: 0
|
||||
GMMU: 0
|
||||
run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py
|
||||
- name: Resolve commaai's tinygrad pin, which their precompiled chestnut model is built and converted with
|
||||
id: upstream-tg
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
read -r sha url < <(gh api repos/commaai/openpilot/contents/tinygrad_repo --jq '.sha + " " + .submodule_git_url')
|
||||
echo "commaai/openpilot master pins $url@$sha, ours is $(git rev-parse HEAD:tinygrad_repo)"
|
||||
dir="$RUNNER_TEMP/upstream_tinygrad"
|
||||
git init -q "$dir" && git -C "$dir" fetch -q --depth 1 "$url" "$sha" && git -C "$dir" checkout -q FETCH_HEAD
|
||||
echo "path=$dir" >> "$GITHUB_OUTPUT"
|
||||
- name: Convert a pickled USB+AMD jit under commaai's tinygrad, as the chestnut model build does
|
||||
env:
|
||||
DEV: MOCKUSB+AMD
|
||||
CHECK_OOB: 0
|
||||
GMMU: 0
|
||||
PYTHONPATH: ${{ steps.upstream-tg.outputs.path }}:${{ github.workspace }}
|
||||
run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py
|
||||
- name: Where tinygrad moved, for the helper or failure point named above
|
||||
if: failure() && github.event_name == 'pull_request'
|
||||
run: |
|
||||
@@ -63,6 +69,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
models: ${{ steps.set-matrix.outputs.models }}
|
||||
chestnut: ${{ steps.set-matrix.outputs.chestnut }}
|
||||
qcom: ${{ steps.set-matrix.outputs.qcom }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Fetch and Parse json
|
||||
@@ -74,16 +82,22 @@ jobs:
|
||||
with open("openpilot/sunnypilot/models/fetcher.py", "r") as f:
|
||||
urls = re.findall(r"MODEL_URL(?:_CHESTNUT)?\s*=\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", f.read())
|
||||
|
||||
artifacts = []
|
||||
artifacts, chestnut, qcom = [], [], []
|
||||
for url in urls:
|
||||
data = json.loads(urllib.request.urlopen(url).read())
|
||||
for bundle in data.get("bundles", []):
|
||||
for model in bundle.get("models", []):
|
||||
if "artifact" in model:
|
||||
artifacts.append(model["artifact"])
|
||||
if "chestnut" in url:
|
||||
chestnut.append(model["artifact"])
|
||||
elif "tinygrad" in model["artifact"]["file_name"]:
|
||||
qcom.append(model["artifact"])
|
||||
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
||||
f.write(f"models={json.dumps(artifacts)}\n")
|
||||
f.write(f"chestnut={json.dumps(chestnut)}\n")
|
||||
f.write(f"qcom={json.dumps(qcom)}\n")
|
||||
'
|
||||
|
||||
test-model:
|
||||
@@ -120,3 +134,81 @@ jobs:
|
||||
export PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo:$GITHUB_WORKSPACE:$PYTHONPATH"
|
||||
python3 -m pip uninstall -y tinygrad || uv pip uninstall -y tinygrad --system || true
|
||||
python3 -m pytest openpilot/sunnypilot/modeld_v2/tests/test_models.py
|
||||
|
||||
chestnut-mock-parity:
|
||||
name: Mock parity ${{ matrix.artifact.file_name }}
|
||||
needs: generate-matrix
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
artifact: ${{ fromJson(needs.generate-matrix.outputs.chestnut) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- uses: ./.github/workflows/setup-amd-mock
|
||||
with:
|
||||
emu_cache: ${{ matrix.artifact.file_name }}
|
||||
- run: python3 -m pip install -q numpy
|
||||
- name: Download model
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
MODEL_URL: ${{ matrix.artifact.download_uri.url }}
|
||||
MODEL_SHA: ${{ matrix.artifact.download_uri.sha256 }}
|
||||
shell: bash
|
||||
run: |
|
||||
AUTH=()
|
||||
if [ -n "$HF_TOKEN" ]; then AUTH=(-H "Authorization: Bearer $HF_TOKEN"); fi
|
||||
curl --retry 3 --retry-all-errors -s -S -f -L "${AUTH[@]}" -o "$RUNNER_TEMP/model.pkl" "$MODEL_URL"
|
||||
if [ -n "$MODEL_SHA" ]; then echo "$MODEL_SHA $RUNNER_TEMP/model.pkl" | sha256sum -c -; fi
|
||||
- name: Replay its check frame on the amd mock and require the chestnut's result bit for bit
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
|
||||
run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "$RUNNER_TEMP/model.pkl"
|
||||
|
||||
qcom-mock-parity:
|
||||
name: QCOM mock parity ${{ matrix.artifact.file_name }}
|
||||
needs: generate-matrix
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
artifact: ${{ fromJson(needs.generate-matrix.outputs.qcom) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: python3 -m pip install -q numpy tinymesa==25.2.7.2
|
||||
- name: Download model
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
MODEL_URL: ${{ matrix.artifact.download_uri.url }}
|
||||
MODEL_SHA: ${{ matrix.artifact.download_uri.sha256 }}
|
||||
shell: bash
|
||||
run: |
|
||||
AUTH=()
|
||||
if [ -n "$HF_TOKEN" ]; then AUTH=(-H "Authorization: Bearer $HF_TOKEN"); fi
|
||||
curl --retry 3 --retry-all-errors -s -S -f -L "${AUTH[@]}" -o "$RUNNER_TEMP/model.pkl" "$MODEL_URL"
|
||||
if [ -n "$MODEL_SHA" ]; then echo "$MODEL_SHA $RUNNER_TEMP/model.pkl" | sha256sum -c -; fi
|
||||
- name: Replay its check frame on the qcom mock and require the comma device's result bit for bit
|
||||
env:
|
||||
DEV: MOCK+QCOM
|
||||
PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
|
||||
run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "$RUNNER_TEMP/model.pkl"
|
||||
|
||||
chestnut-mock-build-parity:
|
||||
name: Chestnut build on the amd mock = chestnut runner build
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- uses: ./.github/workflows/setup-amd-mock
|
||||
with:
|
||||
emu_cache: chestnut-build-parity
|
||||
- run: python3 -m pip install -q numpy zstandard
|
||||
- uses: ./.github/workflows/chestnut-build-parity
|
||||
|
||||
+7
-1
@@ -4,7 +4,7 @@
|
||||
|
||||
A supported vehicle is one that just works when you install a comma device. All supported cars provide a better experience than any stock system. Supported vehicles reference the US market unless otherwise specified.
|
||||
|
||||
# 351 Supported Cars
|
||||
# 357 Supported Cars
|
||||
|
||||
|Make|Model|Supported Package|ACC|No ACC accel below|No ALC below|Steering Torque|Resume from stop|<a href="##"><img width=2000></a>Hardware Needed<br> |Video|Setup Video|
|
||||
|---|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
|
||||
@@ -139,6 +139,7 @@ A supported vehicle is one that just works when you install a comma device. All
|
||||
|Hyundai|Ioniq Plug-in Hybrid 2020-22|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Ioniq Plug-in Hybrid 2020-22">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Kona 2020|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|6 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai B connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Kona 2020">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Kona 2022-23|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai O connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Kona 2022-23">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Kona (without HDA II) 2024-25|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai L connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Kona (without HDA II) 2024-25">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Kona Electric 2018-21|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai G connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Kona Electric 2018-21">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Kona Electric 2022-23|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai O connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Kona Electric 2022-23">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Kona Electric (with HDA II, Korea only) 2023|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai R connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Kona Electric (with HDA II, Korea only) 2023">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=U2fOCmcQ8hw" target="_blank"><img height="18px" src="assets/icon-youtube.svg" /></a>||
|
||||
@@ -152,19 +153,23 @@ A supported vehicle is one that just works when you install a comma device. All
|
||||
|Hyundai|Santa Fe Plug-in Hybrid 2022-23|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai L connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Santa Fe Plug-in Hybrid 2022-23">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Sonata 2018-19|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Sonata 2018-19">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Sonata 2020-23|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Sonata 2020-23">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=ix63r9kE3Fw" target="_blank"><img height="18px" src="assets/icon-youtube.svg" /></a>||
|
||||
|Hyundai|Sonata (without HDA II) 2024-26|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Sonata (without HDA II) 2024-26">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Sonata Hybrid 2020-23|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Sonata Hybrid 2020-23">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Sonata Hybrid (without HDA II) 2024-26|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Sonata Hybrid (without HDA II) 2024-26">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Staria 2023|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Staria 2023">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Tucson 2021|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|19 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai L connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Tucson 2021">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Tucson 2022|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai N connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Tucson 2022">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Tucson 2023-24|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai N connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Tucson 2023-24">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Tucson Diesel 2019|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai L connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Tucson Diesel 2019">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Tucson Hybrid 2022-24|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai N connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Tucson Hybrid 2022-24">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Tucson Hybrid (without HDA II) 2025-26|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai N connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Tucson Hybrid (without HDA II) 2025-26">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Tucson Plug-in Hybrid 2024|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai N connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Tucson Plug-in Hybrid 2024">Buy Here</a></sub></details>|||
|
||||
|Hyundai|Veloster 2019-20|Smart Cruise Control (SCC)|Stock|5 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Hyundai Veloster 2019-20">Buy Here</a></sub></details>|||
|
||||
|Jeep|Grand Cherokee 2016-18|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Jeep Grand Cherokee 2016-18">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=eLR9o2JkuRk" target="_blank"><img height="18px" src="assets/icon-youtube.svg" /></a>||
|
||||
|Jeep|Grand Cherokee 2019-21|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Jeep Grand Cherokee 2019-21">Buy Here</a></sub></details>|||
|
||||
|Kia|Carnival 2022-24|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Carnival 2022-24">Buy Here</a></sub></details>|||
|
||||
|Kia|Carnival (China only) 2023|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Carnival (China only) 2023">Buy Here</a></sub></details>|||
|
||||
|Kia|Carnival Hybrid (with HDA II) 2025-26|Highway Driving Assist II|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Carnival Hybrid (with HDA II) 2025-26">Buy Here</a></sub></details>|||
|
||||
|Kia|Ceed 2019-21|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Ceed 2019-21">Buy Here</a></sub></details>|||
|
||||
|Kia|Ceed Plug-in Hybrid 2021|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai B connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Ceed Plug-in Hybrid 2021">Buy Here</a></sub></details>|||
|
||||
|Kia|EV6 (Southeast Asia only) 2022-24|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai P connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia EV6 (Southeast Asia only) 2022-24">Buy Here</a></sub></details>|||
|
||||
@@ -190,6 +195,7 @@ A supported vehicle is one that just works when you install a comma device. All
|
||||
|Kia|Niro Plug-in Hybrid 2020|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai D connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Niro Plug-in Hybrid 2020">Buy Here</a></sub></details>|||
|
||||
|Kia|Niro Plug-in Hybrid 2021|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai D connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Niro Plug-in Hybrid 2021">Buy Here</a></sub></details>|||
|
||||
|Kia|Niro Plug-in Hybrid 2022|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai F connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Niro Plug-in Hybrid 2022">Buy Here</a></sub></details>|||
|
||||
|Kia|Niro Plus EV 2024|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Niro Plus EV 2024">Buy Here</a></sub></details>|||
|
||||
|Kia|Optima 2017|Advanced Smart Cruise Control|Stock|0 mph|32 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai B connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Optima 2017">Buy Here</a></sub></details>|||
|
||||
|Kia|Optima 2019-20|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai G connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Optima 2019-20">Buy Here</a></sub></details>|||
|
||||
|Kia|Optima Hybrid 2019|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[](##)|[](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 OBD-C cable (2 ft)<br>- 1 comma four<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br><a href="https://comma.ai/shop/comma-3x?harness=Kia Optima Hybrid 2019">Buy Here</a></sub></details>|||
|
||||
|
||||
+1
-1
Submodule opendbc_repo updated: c1ad0ee7fb...b2de53fba5
@@ -149,6 +149,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
{"Version", {PERSISTENT, STRING}},
|
||||
|
||||
// --- sunnypilot params --- //
|
||||
{"AccelBoostSP", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"ApiCache_DriveStats", {PERSISTENT, JSON}},
|
||||
{"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
@@ -199,7 +200,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
{"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"TripSummary", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"TrueVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
|
||||
// MADS params
|
||||
|
||||
@@ -28,6 +28,8 @@ class DesireHelper:
|
||||
self.alc = AutoLaneChangeController(self)
|
||||
self.lane_turn_controller = LaneTurnController(self)
|
||||
self.lane_turn_direction = TurnDirection.none
|
||||
self.turn_frames = 0
|
||||
self.turn_repulse = 98
|
||||
|
||||
@staticmethod
|
||||
def get_lane_change_direction(CS):
|
||||
@@ -95,8 +97,10 @@ class DesireHelper:
|
||||
self.prev_one_blinker = one_blinker and lateral_active
|
||||
|
||||
if self.lane_turn_direction != TurnDirection.none:
|
||||
self.desire = TURN_DESIRES[self.lane_turn_direction]
|
||||
self.turn_frames += 1
|
||||
self.desire = TURN_DESIRES[self.lane_turn_direction] if self.turn_frames % self.turn_repulse else log.Desire.none
|
||||
else:
|
||||
self.turn_frames = 0
|
||||
self.desire = log.Desire.none
|
||||
if self.lane_change_state == LaneChangeState.laneChangeStarting:
|
||||
if self.lane_change_direction == LaneChangeDirection.left:
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:65a08adc31d5c456219687d99b7bf5e44d61dae2d49ea67850e76105c7248cce
|
||||
size 60918562
|
||||
oid sha256:416c99bbd5563b01518be6de91b856986f66ffa03451ea9c451736dcc99896fd
|
||||
size 83976077
|
||||
|
||||
@@ -10,7 +10,6 @@ from openpilot.selfdrive.ui.onroad.augmented_road_view import AugmentedRoadView
|
||||
from openpilot.selfdrive.ui.ui_state import device, ui_state
|
||||
from openpilot.selfdrive.ui.layouts.onboarding import OnboardingWindow
|
||||
from openpilot.selfdrive.ui.body.layouts.onroad import BodyLayout
|
||||
from openpilot.selfdrive.ui.sunnypilot.layouts.trip_summary import TripSummary
|
||||
|
||||
if gui_app.sunnypilot_ui():
|
||||
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.settings import SettingsLayoutSP as SettingsLayout
|
||||
@@ -32,7 +31,6 @@ class MainLayout(Widget):
|
||||
self._sidebar = Sidebar()
|
||||
self._current_mode = MainState.HOME
|
||||
self._prev_onroad = False
|
||||
self._trip_summary = TripSummary()
|
||||
|
||||
# Initialize layouts
|
||||
self._home_layout = HomeLayout()
|
||||
|
||||
@@ -10,7 +10,6 @@ from openpilot.selfdrive.ui.body.layouts.onroad import BodyLayout
|
||||
from openpilot.system.ui.widgets import Widget
|
||||
from openpilot.system.ui.widgets.scroller import Scroller
|
||||
from openpilot.system.ui.lib.application import gui_app
|
||||
from openpilot.selfdrive.ui.sunnypilot.layouts.trip_summary import TripSummary
|
||||
|
||||
if gui_app.sunnypilot_ui():
|
||||
from openpilot.selfdrive.ui.sunnypilot.mici.layouts.settings import SettingsLayoutSP as SettingsLayout
|
||||
@@ -58,7 +57,6 @@ class MiciMainLayout(Scroller):
|
||||
self._setup_callbacks()
|
||||
|
||||
gui_app.add_nav_stack_tick(self._handle_transitions)
|
||||
self._trip_summary = TripSummary()
|
||||
gui_app.push_widget(self)
|
||||
|
||||
# Start onboarding if terms or training not completed, make sure to push after self
|
||||
|
||||
@@ -57,6 +57,12 @@ class CruiseLayout(Widget):
|
||||
description=tr("Use map data to estimate the appropriate speed to drive through turns ahead."),
|
||||
param="SmartCruiseControlMap")
|
||||
|
||||
self.accel_boost_toggle = toggle_item_sp(
|
||||
title=tr("Acceleration Boost"),
|
||||
description=tr("Adds a small push on top of sunnypilot's own acceleration, stronger in Aggressive and gentler in Relaxed. " +
|
||||
"Eases off in curves and when sunnypilot slows or stops. Never goes past your set speed, and lead-car following still comes first."),
|
||||
param="AccelBoostSP")
|
||||
|
||||
self.custom_acc_toggle = toggle_item_sp(
|
||||
title=tr("Custom ACC Speed Increments"),
|
||||
description="",
|
||||
@@ -86,6 +92,7 @@ class CruiseLayout(Widget):
|
||||
self.icbm_toggle,
|
||||
self.scc_v_toggle,
|
||||
self.scc_m_toggle,
|
||||
self.accel_boost_toggle,
|
||||
self.custom_acc_toggle,
|
||||
self.custom_acc_short_increment,
|
||||
self.custom_acc_long_increment,
|
||||
@@ -136,6 +143,8 @@ class CruiseLayout(Widget):
|
||||
self.icbm_toggle.set_description(new_desc)
|
||||
self.icbm_toggle.show_description(True)
|
||||
|
||||
self.accel_boost_toggle.action_item.set_enabled(has_long)
|
||||
|
||||
if has_long or has_icbm:
|
||||
self.custom_acc_toggle.action_item.set_enabled(((has_long and not ui_state.CP.pcmCruise) or has_icbm) and ui_state.is_offroad())
|
||||
self.scc_v_toggle.action_item.set_enabled(True)
|
||||
@@ -151,6 +160,7 @@ class CruiseLayout(Widget):
|
||||
else:
|
||||
has_icbm = has_long = False
|
||||
self.icbm_toggle.action_item.set_enabled(False)
|
||||
self.accel_boost_toggle.action_item.set_enabled(False)
|
||||
self.icbm_toggle.set_description(tr(ONROAD_ONLY_DESCRIPTION))
|
||||
|
||||
show_custom_acc_desc = False
|
||||
|
||||
@@ -249,7 +249,11 @@ class ModelsLayout(Widget):
|
||||
return {"name": name, "downloading": True, "progress": progress, "status_text": tr("verifying")}
|
||||
if ds.downloading in statuses:
|
||||
speed = max((p.speed for p in progresses), default=0.0)
|
||||
status_text = f"{speed / 1024 ** 2:.1f} MB/s" if speed > 0 else "--.- MB/s"
|
||||
speed_text = f"{speed / 1024 ** 2:.1f} MB/s" if speed > 0 else "--.- MB/s"
|
||||
if any(p.resuming for p in progresses):
|
||||
status_text = tr("resuming...") + f" {speed_text}"
|
||||
else:
|
||||
status_text = speed_text
|
||||
return {"name": name, "downloading": True, "progress": progress, "status_text": status_text}
|
||||
if statuses <= {ds.downloaded, ds.cached}:
|
||||
return {"name": name, "text_color": ON_COLOR, "icon": "icons/checkmark.png"}
|
||||
|
||||
@@ -72,11 +72,6 @@ class VisualsLayout(Widget):
|
||||
"It is the driver's responsibility to observe their environment and make decisions accordingly."),
|
||||
None,
|
||||
),
|
||||
"TripSummary": (
|
||||
lambda: tr("Trip Summary"),
|
||||
tr("Show a summary of distance and engagement when you shift into park after a drive."),
|
||||
None,
|
||||
),
|
||||
"TrueVEgoUI": (
|
||||
lambda: tr("Speedometer: Always Display True Speed"),
|
||||
tr("For applicable vehicles, always display the true vehicle current speed from wheel speed sensors."),
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
"""
|
||||
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
import time
|
||||
import pyray as rl
|
||||
|
||||
from opendbc.car.structs import car
|
||||
|
||||
from openpilot.common.constants import CV
|
||||
from openpilot.selfdrive.ui.onroad.augmented_road_view import BORDER_COLORS
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus
|
||||
from openpilot.system.ui.lib.application import gui_app, FontWeight
|
||||
from openpilot.system.ui.lib.text_measure import measure_text_cached
|
||||
from openpilot.system.ui.widgets import Widget
|
||||
|
||||
AUTO_DISMISS = 20.
|
||||
FADE_IN, FADE_OUT = 0.35, 0.3
|
||||
LAT = (UIStatus.ENGAGED, UIStatus.LAT_ONLY)
|
||||
LONG = (UIStatus.ENGAGED, UIStatus.LONG_ONLY)
|
||||
GEAR = car.CarState.GearShifter
|
||||
FORWARD = (GEAR.drive, GEAR.sport, GEAR.low, GEAR.brake, GEAR.eco, GEAR.manumatic)
|
||||
|
||||
|
||||
class TripTracker:
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.segments: list[list] = []
|
||||
self.seconds = 0.
|
||||
self.takeovers = 0
|
||||
self._t = None
|
||||
self._status = None
|
||||
|
||||
def update(self, driving: bool):
|
||||
now = time.monotonic()
|
||||
if not driving:
|
||||
self._t = None
|
||||
self._status = None
|
||||
return
|
||||
dt = 0. if self._t is None else min(now - self._t, 0.5)
|
||||
self._t = now
|
||||
self.seconds += dt
|
||||
if self._status not in (None, UIStatus.DISENGAGED) and ui_state.status == UIStatus.DISENGAGED:
|
||||
self.takeovers += 1
|
||||
self._status = ui_state.status
|
||||
d = max(ui_state.sm["carState"].vEgo, 0.) * dt
|
||||
if d <= 0.:
|
||||
return
|
||||
if self.segments and self.segments[-1][0] == ui_state.status:
|
||||
self.segments[-1][1] += d
|
||||
else:
|
||||
self.segments.append([ui_state.status, d])
|
||||
|
||||
def total(self, statuses=None) -> float:
|
||||
return sum(m for s, m in self.segments if statuses is None or s in statuses)
|
||||
|
||||
|
||||
class TripSummary:
|
||||
def __init__(self):
|
||||
self._trip = TripTracker()
|
||||
self._dialog: TripSummaryDialog | None = None
|
||||
self._gear = None
|
||||
self._drove = False
|
||||
gui_app.add_nav_stack_tick(self._tick)
|
||||
|
||||
def _tick(self):
|
||||
gear = ui_state.sm["carState"].gearShifter if ui_state.started else None
|
||||
self._trip.update(gear not in (None, GEAR.park))
|
||||
self._drove |= gear in FORWARD
|
||||
if gear == self._gear:
|
||||
return
|
||||
self._gear = gear
|
||||
|
||||
if gear not in (None, GEAR.park):
|
||||
self._pop()
|
||||
elif gear == GEAR.park and self._drove and ui_state.trip_summary and not ui_state.is_body and self._trip.total() > 0.:
|
||||
self._dialog = TripSummaryDialog(self._trip, gui_app.get_active_widget(), self._pop)
|
||||
gui_app.push_widget(self._dialog)
|
||||
if gear is None:
|
||||
self._trip.reset()
|
||||
if gear in (None, GEAR.park):
|
||||
self._drove = False
|
||||
|
||||
def _pop(self):
|
||||
if self._dialog is not None and gui_app.get_active_widget() is self._dialog:
|
||||
gui_app.pop_widget()
|
||||
self._dialog = None
|
||||
|
||||
|
||||
def _ease(t: float) -> float:
|
||||
t = min(max(t, 0.), 1.)
|
||||
return 1 - (1 - t) ** 3
|
||||
|
||||
|
||||
def _fade(c, a: float) -> rl.Color:
|
||||
r, g, b, al = c if isinstance(c, tuple) else (c.r, c.g, c.b, c.a)
|
||||
return rl.Color(r, g, b, int(al * a))
|
||||
|
||||
|
||||
class TripSummaryDialog(Widget):
|
||||
def __init__(self, trip: TripTracker, below: Widget | None, on_closed):
|
||||
super().__init__()
|
||||
self._below = below
|
||||
self._on_closed = on_closed
|
||||
self._segments = segments = trip.segments
|
||||
total = sum(m for _, m in segments)
|
||||
|
||||
def pct(statuses):
|
||||
return 100 * sum(m for s, m in segments if s in statuses) / total
|
||||
|
||||
dist = total * (CV.MS_TO_KPH if ui_state.is_metric else CV.MS_TO_MPH) / 3600
|
||||
dist_fmt = "{:.1f}" if dist < 9.95 else "{:.0f}"
|
||||
self._stats = [
|
||||
(dist, dist_fmt, "km" if ui_state.is_metric else "mi", "driven"),
|
||||
(pct(LAT + LONG), "{:.0f}", "%", "engaged"),
|
||||
(pct(LAT), "{:.0f}", "%", "lateral"),
|
||||
(pct(LONG), "{:.0f}", "%", "longitudinal"),
|
||||
]
|
||||
mins = round(trip.seconds / 60)
|
||||
self._details = [
|
||||
(f"{mins // 60}h {mins % 60}" if mins >= 60 else f"{mins}", "min"),
|
||||
(f"{trip.takeovers}", "takeover" if trip.takeovers == 1 else "takeovers"),
|
||||
]
|
||||
self._bold = gui_app.font(FontWeight.BOLD)
|
||||
self._semi = gui_app.font(FontWeight.SEMI_BOLD)
|
||||
self._normal = gui_app.font(FontWeight.NORMAL)
|
||||
self._big = gui_app.big_ui()
|
||||
self._shown_t = rl.get_time()
|
||||
self._closing_t: float | None = None
|
||||
self.set_click_callback(self.close)
|
||||
|
||||
def close(self):
|
||||
if self._closing_t is None:
|
||||
self._closing_t = rl.get_time()
|
||||
|
||||
def _render(self, rect: rl.Rectangle):
|
||||
t = rl.get_time() - self._shown_t
|
||||
if self._closing_t is None and t > AUTO_DISMISS:
|
||||
self.close()
|
||||
out = 0. if self._closing_t is None else (rl.get_time() - self._closing_t) / FADE_OUT
|
||||
if out >= 1.:
|
||||
self._on_closed()
|
||||
return
|
||||
bg = 1 - _ease(out)
|
||||
a = _ease(t / FADE_IN) * bg
|
||||
|
||||
if self._big and self._below is not None and bg < 1.:
|
||||
self._below.render(rect)
|
||||
rl.draw_rectangle_rec(rect, _fade(rl.BLACK, bg))
|
||||
|
||||
k = rect.height / 1080 * (1.0 if self._big else 1.22)
|
||||
pad = 120 * k
|
||||
x, w = rect.x + pad, rect.width - 2 * pad
|
||||
y = rect.y + (rect.height - 850 * k) / 2
|
||||
|
||||
rl.draw_text_ex(self._bold, "trip summary", rl.Vector2(x, y), int(150 * k), 0, _fade(rl.WHITE, a))
|
||||
y += 260 * k
|
||||
|
||||
col_w = w / len(self._stats)
|
||||
for i, (target, fmt, unit, label) in enumerate(self._stats):
|
||||
cx = x + col_w * (i + 0.5)
|
||||
v = measure_text_cached(self._bold, fmt.format(target), int(170 * k))
|
||||
u = measure_text_cached(self._semi, unit, int(80 * k))
|
||||
vx = cx - (v.x + 10 * k + u.x) / 2
|
||||
rl.draw_text_ex(self._bold, fmt.format(target), rl.Vector2(vx, y), int(170 * k), 0, _fade(rl.WHITE, a))
|
||||
rl.draw_text_ex(self._semi, unit, rl.Vector2(vx + v.x + 10 * k, y + 75 * k), int(80 * k), 0, _fade(rl.WHITE, a))
|
||||
lb = measure_text_cached(self._normal, label, int(70 * k))
|
||||
rl.draw_text_ex(self._normal, label, rl.Vector2(cx - lb.x / 2, y + 180 * k), int(70 * k), 0, _fade(rl.Color(200, 200, 200, 255), a))
|
||||
y += 330 * k
|
||||
|
||||
bar = rl.Rectangle(x, y, w, 90 * k)
|
||||
total = sum(m for _, m in self._segments)
|
||||
sx = bar.x
|
||||
for status, m in self._segments:
|
||||
sw = bar.width * m / total
|
||||
rl.draw_rectangle_rec(rl.Rectangle(sx, bar.y, min(sw + 1, bar.x + bar.width - sx), bar.height), _fade(BORDER_COLORS[status], a))
|
||||
sx += sw
|
||||
rl.draw_rectangle_rounded_lines_ex(bar, 1.0, 20, 30 * k, _fade(rl.BLACK, a))
|
||||
|
||||
y += (90 + 70) * k
|
||||
|
||||
size = int(90 * k)
|
||||
parts = []
|
||||
for i, (value, unit) in enumerate(self._details):
|
||||
if i:
|
||||
parts.append((" ", self._normal, None))
|
||||
parts += [(value, self._bold, rl.WHITE), (" " + unit, self._normal, rl.Color(170, 170, 170, 255))]
|
||||
widths = [measure_text_cached(f, t, size).x for t, f, _ in parts]
|
||||
lx = x + (w - sum(widths)) / 2
|
||||
for (text, font, color), tw in zip(parts, widths, strict=True):
|
||||
if color is None:
|
||||
rl.draw_circle(int(lx + tw / 2), int(y + size * 0.6), 12 * k, _fade(rl.Color(110, 110, 110, 255), a))
|
||||
else:
|
||||
rl.draw_text_ex(font, text, rl.Vector2(lx, y), size, 0, _fade(color, a))
|
||||
lx += tw
|
||||
@@ -267,8 +267,8 @@ class ModelsLayoutMici(NavScroller):
|
||||
self.current_model_info.info_text.set_text(info_text)
|
||||
|
||||
if manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.failed:
|
||||
self.current_model_info.info_header.set_text(tr("error") + self._download_progress)
|
||||
self.current_model_info.info_text.set_text(tr("download failed"))
|
||||
self.current_model_info.info_header.set_text(tr("error"))
|
||||
self.current_model_info.info_text.set_text(f"{manager.selectedBundle.internalName.lower()} | {tr('download failed')}")
|
||||
|
||||
elif manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.downloading:
|
||||
self.cancel_download_btn.set_visible(ui_state.params.get("ModelManager_DownloadRef") is not None)
|
||||
@@ -277,6 +277,7 @@ class ModelsLayoutMici(NavScroller):
|
||||
count = 0
|
||||
verifying = False
|
||||
canceling = False
|
||||
resuming = False
|
||||
speed = 0.0
|
||||
for model in manager.selectedBundle.models:
|
||||
count += 1
|
||||
@@ -286,12 +287,13 @@ class ModelsLayoutMici(NavScroller):
|
||||
progress += p.progress
|
||||
verifying = verifying or p.status == custom.ModelManagerSP.DownloadStatus.verifying
|
||||
canceling = canceling or p.canceling
|
||||
resuming = resuming or p.resuming
|
||||
speed = max(speed, p.speed) # only one file downloads at a time, so the non-zero speed is the live one
|
||||
elif p.status in (custom.ModelManagerSP.DownloadStatus.downloaded,
|
||||
custom.ModelManagerSP.DownloadStatus.cached):
|
||||
progress += 100.0
|
||||
|
||||
header = tr("canceling...") if canceling else tr("verifying") if verifying else tr("downloading")
|
||||
header = tr("canceling...") if canceling else tr("verifying") if verifying else tr("resuming...") if resuming else tr("downloading")
|
||||
self.current_model_info.current_model_header.set_text(header)
|
||||
self.cancel_download_btn.set_text(tr("cancel verification") if verifying else tr("cancel download"))
|
||||
self.current_model_info.current_model_header._shimmer = True
|
||||
|
||||
@@ -48,7 +48,6 @@ class UIStateSP:
|
||||
self.model_runner_tinygrad: bool = False
|
||||
self.active_runner_tinygrad: bool = False
|
||||
self.blindspot: bool = False
|
||||
self.trip_summary: bool = True
|
||||
self.chevron_metrics = None
|
||||
self.custom_interactive_timeout: int = 0
|
||||
self.developer_ui = None
|
||||
@@ -186,7 +185,6 @@ class UIStateSP:
|
||||
self.standstill_timer = self.params.get_bool("StandstillTimer")
|
||||
self.sunnylink_enabled = self.params.get_bool("SunnylinkEnabled")
|
||||
self.torque_bar = self.params.get_bool("TorqueBar")
|
||||
self.trip_summary = self.params.get_bool("TripSummary")
|
||||
self.enforce_torque_control = self.params.get_bool("EnforceTorqueControl")
|
||||
self.custom_torque_params = self.params.get_bool("CustomTorqueParams")
|
||||
self.torque_override_enabled = self.params.get_bool("TorqueParamsOverrideEnabled")
|
||||
|
||||
@@ -8,10 +8,10 @@ import numpy as np
|
||||
|
||||
from openpilot.common.filter_simple import FirstOrderFilter
|
||||
from openpilot.common.realtime import DT_MDL
|
||||
from openpilot.sunnypilot.modeld_v2.constants import ModelConstants
|
||||
from openpilot.sunnypilot.modeld_v2.constants import ModelConstants, Plan
|
||||
|
||||
LANE_WIDTH_DEFAULT = 3.6 # m
|
||||
LANE_WIDTH_CLIP = (2.8, 4.5) # m
|
||||
LANE_WIDTH_CLIP = (2.4, 4.5) # m
|
||||
LOOKAHEAD_TIME = 1.5 # s
|
||||
MIN_LOOKAHEAD = 10.0 # m
|
||||
MIN_SPEED = 5.0 # m/s
|
||||
@@ -19,6 +19,11 @@ LINE_STD = (0.2, 0.6)
|
||||
POINT_STD = 0.1
|
||||
YIELD = (0.2, 0.45)
|
||||
X_IDXS = np.array(ModelConstants.X_IDXS)
|
||||
JUMP = 0.5
|
||||
JUMP_TAU = 0.5
|
||||
JUMP_ACCEPT = 3.0
|
||||
WIDEN = 0.4
|
||||
YAW_RATE = Plan.ORIENTATION_RATE.start + 2 # matched .991 correlation to actual car
|
||||
|
||||
|
||||
class Blendful:
|
||||
@@ -36,7 +41,8 @@ class Blendful:
|
||||
self.l_prob = 0.0
|
||||
self.r_prob = 0.0
|
||||
self.model_curvature = 0.0
|
||||
self.smooth = 0.0 # smooths the correction target below .53 lat accel
|
||||
self.ref = None
|
||||
self.jumped = np.zeros(2)
|
||||
self.curvature = 0.0
|
||||
|
||||
def update(self, model_output: dict[str, np.ndarray], v_ego: float, active: bool, model_curvature: float, enabled: bool = True) -> float:
|
||||
@@ -52,8 +58,8 @@ class Blendful:
|
||||
self.usual_offset.initialized = False
|
||||
self.lane_width.initialized = False
|
||||
self.going_around = False
|
||||
self.ref = None
|
||||
target = 0.0 if fading else self._target(model_output, v_ego, model_curvature)
|
||||
target = self.smooth = self.smooth + (target - self.smooth) * DT_MDL / (DT_MDL + np.interp(abs(model_curvature) * v_ego ** 2, [0.18, 0.53], [0.5, 0.0]))
|
||||
build = 0.5 / max(v_ego, 1.0) ** 2 * DT_MDL # jerk over 1 second
|
||||
release = (0.5 if fading else 2.0) / max(v_ego, 1.0) ** 2 * DT_MDL # lets go fast, but past zero it builds like any other pull
|
||||
k = self.curvature
|
||||
@@ -63,6 +69,10 @@ class Blendful:
|
||||
def _target(self, model_output: dict[str, np.ndarray], v_ego: float, model_curvature: float) -> float:
|
||||
lookahead = max(v_ego * LOOKAHEAD_TIME, MIN_LOOKAHEAD)
|
||||
lines = model_output['lane_lines'][0, :, :, 0]
|
||||
plan = model_output['plan'][0]
|
||||
self._jumps(lines[1:3], v_ego, plan[0, YAW_RATE] if plan.shape[-1] > YAW_RATE else model_curvature * v_ego,
|
||||
np.interp(abs(model_curvature) * v_ego ** 2, [0.18, 0.53], [JUMP_TAU, 0.0]), lookahead)
|
||||
held = bool(np.any((self.jumped > 0) & (self.jumped < JUMP_ACCEPT)))
|
||||
stds = model_output['lane_lines_stds'][0, :, :, 0]
|
||||
|
||||
lone_weight = np.interp(abs(model_curvature) * v_ego ** 2, [0.18, 0.53], [0.5, 1.0]) # 0.5 on straights, 1.0 in curves
|
||||
@@ -83,6 +93,8 @@ class Blendful:
|
||||
lane_trust = min(l_prob, r_prob)
|
||||
else:
|
||||
lane_trust = l_prob + r_prob - l_prob * r_prob
|
||||
if held or (l_prob * r_prob > 0.5 and np.interp(lookahead, X_IDXS, lines[2] - lines[1]) > self.lane_width.x + WIDEN):
|
||||
lane_trust = 0.0
|
||||
if lone_weight > 0.5 and lane_trust < self.line_trust:
|
||||
lane_trust = self.line_trust + (lane_trust - self.line_trust) * DT_MDL / (lone_weight - 0.5 + DT_MDL)
|
||||
self.line_trust = lane_trust
|
||||
@@ -96,7 +108,7 @@ class Blendful:
|
||||
win = (X_IDXS >= 0.5 * lookahead) & (X_IDXS <= 1.5 * lookahead)
|
||||
x = X_IDXS[win]
|
||||
half_width = self.lane_width.x / 2
|
||||
if l_prob * r_prob > 0.5:
|
||||
if l_prob * r_prob > 0.5 and not held:
|
||||
width = np.interp(lookahead, X_IDXS, lines[2]) - np.interp(lookahead, X_IDXS, lines[1])
|
||||
self.lane_width.update(float(np.clip(width, *LANE_WIDTH_CLIP)))
|
||||
half_width = np.clip(lines[2, win] - lines[1, win], *LANE_WIDTH_CLIP) / 2
|
||||
@@ -143,3 +155,14 @@ class Blendful:
|
||||
self.trust = lane_trust if lane_trust < self.trust else self.trust + (lane_trust - self.trust) * DT_MDL / (1.5 + DT_MDL)
|
||||
self.dk = dk
|
||||
return float(self.trust * dk)
|
||||
|
||||
def _jumps(self, raw: np.ndarray, v_ego: float, yaw_rate: float, tau: float, lookahead: float) -> None:
|
||||
if self.ref is None:
|
||||
self.ref, self.jumped = raw.copy(), np.zeros(2)
|
||||
return
|
||||
ref = np.array([np.interp(X_IDXS + v_ego * DT_MDL, X_IDXS, line) for line in self.ref]) - yaw_rate * DT_MDL * X_IDXS
|
||||
off = np.abs([np.interp(lookahead, X_IDXS, d) for d in raw - ref]) > JUMP
|
||||
self.jumped = np.where(off & ~off[::-1], self.jumped + DT_MDL, 0.0)
|
||||
self.ref = ref + (raw - ref) * DT_MDL / (DT_MDL + tau)
|
||||
accept = (self.jumped >= JUMP_ACCEPT) | off.all()
|
||||
self.ref[accept] = raw[accept]
|
||||
|
||||
@@ -371,7 +371,7 @@ def _compile_warp_resolution_worker(cam_w, cam_h, model_w, model_h, benchmark_ru
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if 'USB' in os.getenv('DEV', '') or os.getenv('CHESTNUT'):
|
||||
if ('USB' in os.getenv('DEV', '') or os.getenv('CHESTNUT')) and not os.getenv('DEV', '').startswith('MOCK'):
|
||||
from openpilot.system.hardware.chestnut.flash import link_up
|
||||
for _ in range(10):
|
||||
if link_up():
|
||||
|
||||
@@ -17,16 +17,28 @@ def main():
|
||||
parser.add_argument("--frame", required=True, help="width,height,stride,y_height,uv_height,buffer_size")
|
||||
parser.add_argument("--warp-to", required=True, help="model input WxH")
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--light", action="store_true")
|
||||
parser.add_argument("--split-frames", action="store_true", help="one input per camera (frame, big_frame), bound to the camera buffers by pointer")
|
||||
args = parser.parse_args()
|
||||
|
||||
from examples.openpilot.compile_warp import make_frame_prepare, parse_frame, parse_size
|
||||
from tinygrad import Device, Tensor, TinyJit
|
||||
frame = parse_frame(args.frame)
|
||||
warp = make_frame_prepare(frame, *parse_size(args.warp_to))
|
||||
run = TinyJit(lambda input_frame, M_inv: Tensor.stack(*(warp(input_frame[i], M_inv[i]) for i in range(2))), prune=True)
|
||||
n = 1 if args.light else 2
|
||||
cams = list(range(n))
|
||||
rng = np.random.default_rng(42)
|
||||
inputs = {"input_frame": Tensor(rng.integers(0, 256, (2, frame.size), dtype=np.uint8), device=Device.DEFAULT).realize(),
|
||||
"M_inv": Tensor((rng.standard_normal((2, 3, 3)) * 8).astype(np.float32), device=Device.DEFAULT).realize()}
|
||||
frames = rng.integers(0, 256, (2, frame.size), dtype=np.uint8)
|
||||
m_inv = Tensor((rng.standard_normal((n, 3, 3)) * 8).astype(np.float32), device=Device.DEFAULT).realize()
|
||||
if args.split_frames:
|
||||
names = [("frame", "big_frame")[c] for c in cams]
|
||||
run = TinyJit(lambda M_inv, **f: Tensor.stack(*(warp(f[nm], M_inv[i]) for i, nm in enumerate(names))), prune=True)
|
||||
inputs = {nm: Tensor(frames[c], device=Device.DEFAULT).realize() for nm, c in zip(names, cams, strict=True)} | {"M_inv": m_inv}
|
||||
specs = {nm: ((frame.size,), "|u1", Device.DEFAULT) for nm in names} | {"M_inv": ((n, 3, 3), "<f4", Device.DEFAULT)} # noqa: C420
|
||||
else:
|
||||
run = TinyJit(lambda input_frame, M_inv: Tensor.stack(*(warp(input_frame[c], M_inv[i]) for i, c in enumerate(cams))), prune=True)
|
||||
inputs = {"input_frame": Tensor(frames, device=Device.DEFAULT).realize(), "M_inv": m_inv}
|
||||
specs = {"input_frame": ((2, frame.size), "|u1", Device.DEFAULT), "M_inv": ((n, 3, 3), "<f4", Device.DEFAULT)}
|
||||
with record_kernel_linear() as recorded:
|
||||
run(**inputs)
|
||||
run(**inputs)
|
||||
@@ -34,7 +46,6 @@ def main():
|
||||
assert len(recorded) == 1, f"expected one captured kernel list, got {len(recorded)}"
|
||||
part = serialize_jit(run, recorded[0], check)
|
||||
PlainJit(*part)
|
||||
specs = {"input_frame": ((2, frame.size), "|u1", Device.DEFAULT), "M_inv": ((2, 3, 3), "<f4", Device.DEFAULT)}
|
||||
write_bundle(args.output, {"run": part}, {"input_specs": specs, "metadata": {}})
|
||||
print(f"Saved warp to {args.output} ({len(part[0]['calls'])} kernels)")
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import dataclasses
|
||||
import inspect
|
||||
import itertools
|
||||
import mmap
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import tempfile
|
||||
@@ -66,47 +67,75 @@ def _patched(buf, topo) -> tuple[bytearray, dict]:
|
||||
return blob, rows
|
||||
|
||||
|
||||
def _evaluate(u, env):
|
||||
def _packed_args(program) -> dict[int, int]:
|
||||
from tinygrad.device import TinyELF
|
||||
params = getattr(program.arg, "params", None) or ()
|
||||
return {off: p.arg.slot for (off, _), p in zip(TinyELF.iter_sig(tuple((None, p.addrspace, p.dtype, ()) for p in params)), params, strict=True)}
|
||||
|
||||
|
||||
def _outer_slot(u, packed: dict[int, int]) -> int | None:
|
||||
from tinygrad.uop.ops import Ops
|
||||
if u.op is Ops.PARAM:
|
||||
return None if packed else u.arg.slot
|
||||
if u.op.name == "REFPARAM":
|
||||
return u.arg.slot
|
||||
if packed and u.op is Ops.LOAD and u.src[0].op is Ops.INDEX and _root(u.src[0].src[0]).op is Ops.PARAM:
|
||||
index = _evaluate(u.src[0].src[1], {}, packed)
|
||||
return None if index is None else packed.get(index * u.dtype.itemsize)
|
||||
return None
|
||||
|
||||
|
||||
def _evaluate(u, env, packed: dict[int, int]):
|
||||
from tinygrad.uop.ops import GroupOp, Ops, exec_alu
|
||||
if u.op in (Ops.AFTER, Ops.CAST, Ops.BITCAST):
|
||||
return _evaluate(u.src[0], env)
|
||||
return _evaluate(u.src[0], env, packed)
|
||||
if u.op is Ops.CONST:
|
||||
return u.arg
|
||||
if u.op in (Ops.PARAM, Ops.RANGE):
|
||||
return env.get(u.arg.slot if u.op is Ops.PARAM else u)
|
||||
if u.op is Ops.RANGE:
|
||||
return env.get(u)
|
||||
if (slot := _outer_slot(u, packed)) is not None:
|
||||
return env.get(slot)
|
||||
if u.op is Ops.LOAD:
|
||||
table, index = _root(u.src[0].src[0]), _evaluate(u.src[0].src[1], env)
|
||||
table, index = _root(u.src[0].src[0]), _evaluate(u.src[0].src[1], env, packed)
|
||||
return struct.unpack_from(u.dtype.fmt, table.arg, index * u.dtype.itemsize)[0] if table.op is Ops.BINARY and index is not None else None
|
||||
if u.op in GroupOp.ALU and None not in (values := [_evaluate(x, env) for x in u.src]):
|
||||
if u.op in GroupOp.ALU and None not in (values := [_evaluate(x, env, packed) for x in u.src]):
|
||||
return exec_alu(u.op, u.dtype, values, *([False] if "truncate_output" in inspect.signature(exec_alu).parameters else []))
|
||||
return None
|
||||
|
||||
|
||||
def _input_pokes(submit, kernargs) -> dict[int, tuple[int, int]]:
|
||||
from tinygrad.uop.ops import Ops
|
||||
outer = dict(enumerate(submit.src[1:]))
|
||||
outer, packed = dict(enumerate(submit.src[1:])), _packed_args(submit.src[0])
|
||||
kernargs_params = {i for i, a in outer.items() if a.op is Ops.GETADDR and _root(a.src[0]) is kernargs}
|
||||
inputs = {i: _root(a.src[0]).arg.slot for i, a in outer.items() if a.op is Ops.GETADDR and _root(a.src[0]).op is Ops.PARAM}
|
||||
pokes = {}
|
||||
for call in (u for u in submit.src[0].src[0].toposort() if u.op is Ops.CALL and len(u.src) > 3):
|
||||
_, address, value, *_ = call.src[1:]
|
||||
used = list({x.arg.slot for x in value.toposort() if x.op is Ops.PARAM} & inputs.keys())
|
||||
if len(used) != 1 or not kernargs_params & {x.arg.slot for x in address.toposort() if x.op is Ops.PARAM}:
|
||||
used = list({slot for x in value.toposort() if (slot := _outer_slot(x, packed)) is not None} & inputs.keys())
|
||||
if len(used) != 1 or not kernargs_params & {_outer_slot(x, packed) for x in address.toposort()}:
|
||||
continue
|
||||
ranges = list(dict.fromkeys(x for x in (*address.toposort(), *value.toposort()) if x.op is Ops.RANGE))
|
||||
for trip in itertools.product(*[range(int(r.vmax) + 1) for r in ranges]):
|
||||
env = dict.fromkeys(outer, 0) | dict(zip(ranges, trip, strict=True))
|
||||
if (offset := _evaluate(address, env)) % 8 == 0:
|
||||
pokes[offset] = (inputs[used[0]], _evaluate(value, env))
|
||||
if (offset := _evaluate(address, env, packed)) % 8 == 0:
|
||||
pokes[offset] = (inputs[used[0]], _evaluate(value, env, packed))
|
||||
return pokes
|
||||
|
||||
|
||||
def _elf(image: bytes, rodata: int) -> bytes:
|
||||
names, offset, data, headers = b"\0.text\0.rodata\0.shstrtab\0", 64, b"", []
|
||||
for name, kind, address, content in [(0, 0, 0, b""), (1, 1, ALIGNMENT, image[:rodata]), (7, 1, ALIGNMENT + rodata, image[rodata:]), (15, 3, 0, names)]:
|
||||
headers.append(struct.pack("<IIQQQQIIQQ", name, kind, 2 if kind == 1 else 0, address, offset, len(content), 0, 0, 1, 0))
|
||||
def _elf(image: bytes, rodata: int, entry: int) -> bytes:
|
||||
spans = [(".text", 0, rodata), (".rodata", rodata, len(image))] if entry < rodata else \
|
||||
[(".data", 0, rodata), (".rodata", rodata, entry), (".text", entry, len(image))]
|
||||
sections, names = [(0, 0, 0, 0, b"")], b"\0"
|
||||
for name, start, end in (span for span in spans if span[1] < span[2]):
|
||||
sections.append((len(names), 1, 6 if name == ".text" else 2, ALIGNMENT + start, image[start:end]))
|
||||
names += name.encode() + b"\0"
|
||||
sections.append((len(names), 3, 0, 0, names + b".shstrtab\0"))
|
||||
offset, data, headers = 64, b"", []
|
||||
for name, kind, flags, address, content in sections:
|
||||
headers.append(struct.pack("<IIQQQQIIQQ", name, kind, flags, address, offset, len(content), 0, 0, 1, 0))
|
||||
data, offset = data + content, offset + len(content)
|
||||
return b"\x7fELF\x02\x01\x01" + bytes(9) + struct.pack("<HHIQQQIHHHHHH", 1, 0xe0, 1, 0, 0, offset, 0, 64, 0, 0, 64, 4, 3) + data + b"".join(headers)
|
||||
header = struct.pack("<HHIQQQIHHHHHH", 1, 0xe0, 1, 0, 0, offset, 0, 64, 0, 0, 64, len(sections), len(sections) - 1)
|
||||
return b"\x7fELF\x02\x01\x01" + bytes(9) + header + data + b"".join(headers)
|
||||
|
||||
|
||||
def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
|
||||
@@ -155,7 +184,7 @@ def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
|
||||
if image_alloc not in programs:
|
||||
image = bytes(_patched(image_alloc, topo)[0])
|
||||
rodata = next(r for r in range(0, len(image) - 24, 64) if r + struct.unpack_from("<q", image, r + 16)[0] == entry)
|
||||
programs[image_alloc] = blob.add(_elf(image, rodata))
|
||||
programs[image_alloc] = blob.add(_elf(image, rodata, entry))
|
||||
start, size = _view(arguments.src[0])
|
||||
args = []
|
||||
for slot in range(start, start + unwrap(size), 8):
|
||||
@@ -166,7 +195,8 @@ def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
|
||||
base, at, nbytes = _address(argument_words[slot])
|
||||
args.append({"buffer": buffers.setdefault(base, len(buffers)), "offset": at, "nbytes": nbytes or base.max_numel() * base.dtype.itemsize, "dtype": u8})
|
||||
outs, ins = kernel[6]
|
||||
calls.append({"kind": "program", "name": re.sub(r"\x1b\[[0-9;]*m", "", kernel[1]), "lib": programs[image_alloc], "target": dataclasses.asdict(target),
|
||||
device_target = dataclasses.asdict(dataclasses.replace(target, interface=target.interface.removeprefix("MOCK")))
|
||||
calls.append({"kind": "program", "name": re.sub(r"\x1b\[[0-9;]*m", "", kernel[1]), "lib": programs[image_alloc], "target": device_target,
|
||||
"signature": [[f"data{i}", i, u8, [a.get("nbytes", 1)]] for i, a in enumerate(args)], "global_size": grid, "local_size": local,
|
||||
"globals": list(range(len(args))), "outs": list(outs), "ins": list(ins), "args": args})
|
||||
needs_init |= {args[i]["buffer"] for i in ins if "buffer" in args[i] and args[i]["buffer"] not in written}
|
||||
@@ -207,14 +237,16 @@ def main():
|
||||
frame, _ = capture_frame(artifact["run"], kwargs, "output_buffers")
|
||||
part = convert(artifact["run"], kwargs, frame, Device[Device.DEFAULT].renderer.target)
|
||||
|
||||
plain = PlainJit(*part) # replays the frame, within tolerance
|
||||
replay = {path[0]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 1}
|
||||
replay["output_buffers"] = {path[1]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 2}
|
||||
flatten(plain(**replay))
|
||||
for index, (got, expected) in enumerate(zip([t.numpy() for t in replay["output_buffers"].values()], frame["outputs"], strict=True)):
|
||||
assert np.array_equal(got, expected), f"output {index} of the plain artifact differs from the pickle"
|
||||
if not os.getenv("MOCKUSB_SOCKET"):
|
||||
plain = PlainJit(*part)
|
||||
replay = {path[0]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 1}
|
||||
replay["output_buffers"] = {path[1]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 2}
|
||||
flatten(plain(**replay))
|
||||
for index, (got, expected) in enumerate(zip([t.numpy() for t in replay["output_buffers"].values()], frame["outputs"], strict=True)):
|
||||
assert np.array_equal(got, expected), f"output {index} of the plain artifact differs from the pickle"
|
||||
write_bundle(args.output, {"run": part}, {key: artifact[key] for key in ("metadata", "input_specs", "output_specs")})
|
||||
print(f"converted {len(part[0]['calls'])} kernels, bit-exact with the pickle: {args.output}")
|
||||
checked = "left to mock_parity" if os.getenv("MOCKUSB_SOCKET") else "bit-exact with the pickle"
|
||||
print(f"converted {len(part[0]['calls'])} kernels, {checked}: {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -19,6 +19,13 @@ def pack_layout(shapes: dict[str, tuple], frame_size: int) -> tuple[dict[str, in
|
||||
offsets[name], offset = offset, offset + -(-math.prod(shape) * 4 // 128) * 128
|
||||
return offsets | {"frames": offset}, offset + 2 * frame_size
|
||||
|
||||
EXTRA_WARPS = {"light_img": "light_tfm"}
|
||||
SPLIT_FRAMES = ("big_frame", "frame")
|
||||
|
||||
def host_shapes(input_shapes: dict[str, tuple], states) -> dict[str, tuple]:
|
||||
shapes = {name: shape for name, shape in input_shapes.items() if name not in states and name != "new_img" and name not in EXTRA_WARPS}
|
||||
return shapes | {tfm: (1, 3, 3) for img, tfm in EXTRA_WARPS.items() if img in input_shapes}
|
||||
|
||||
|
||||
class _Blobs: # the model's and the warp's blobs as one address space, for a part whose ranges span both files
|
||||
def __init__(self, *blobs):
|
||||
@@ -30,40 +37,59 @@ class _Blobs: # the model's and the warp's blobs as one address space, for a pa
|
||||
|
||||
|
||||
def fused_with_warp(cam_w: int, cam_h: int, chestnut: bool): # load_model's extend: a native model runs upload, warp and model as one submit
|
||||
return partial(frame_part, f"{BASEDIR}/openpilot/sunnypilot/modeld_v2/models/{'big_' if chestnut else ''}driving_warp_{cam_w}x{cam_h}_tinygrad.pkl")
|
||||
models, big = f"{BASEDIR}/openpilot/sunnypilot/modeld_v2/models", 'big_' if chestnut else ''
|
||||
return partial(frame_part, f"{models}/{big}driving_warp_{cam_w}x{cam_h}_tinygrad.pkl",
|
||||
light_warp_path=f"{models}/{big}driving_lightwarp_{cam_w}x{cam_h}_tinygrad.pkl")
|
||||
|
||||
|
||||
def frame_part(warp_path: str, parts: dict, extras: dict) -> dict:
|
||||
def frame_part(warp_path: str, parts: dict, extras: dict, light_warp_path: str | None = None) -> dict:
|
||||
if "input_specs" not in extras: # a legacy model warps in its own parts
|
||||
return {}
|
||||
(model, model_blob), (warp, warp_blob) = parts["run"], read_parts(warp_path)[0]["run"]
|
||||
states = sorted(name for name in extras["input_specs"] if f"next_{name}" in extras["metadata"]["output_shapes"])
|
||||
host = {name: shape for name, (shape, _, _) in extras["input_specs"].items() if name not in states and name != "new_img"}
|
||||
frame_size = next(s["numel"] for s in warp["inputs"] if s["name"] == "input_frame") // 2
|
||||
offsets, packed = pack_layout(host, frame_size)
|
||||
shift, upload, u8 = len(model_blob), len(model["buffers"]) + len(warp["buffers"]), repr(_dtype("dtypes.uint8"))
|
||||
host = host_shapes({name: shape for name, (shape, _, _) in extras["input_specs"].items()}, states)
|
||||
paths = {"light_img": light_warp_path}
|
||||
extra = [(img, tfm, *read_parts(paths[img])[0]["run"]) for img, tfm in EXTRA_WARPS.items() if img in extras["input_specs"]]
|
||||
split = any(s["name"] in SPLIT_FRAMES for s in warp["inputs"])
|
||||
assert all(any(s["name"] in SPLIT_FRAMES for s in part["inputs"]) == split for _, _, part, _ in extra), "camera and extra warps must share a frame layout"
|
||||
offsets, packed = pack_layout(host, 0 if split else next(s["numel"] for s in warp["inputs"] if s["name"] == "input_frame") // 2)
|
||||
shift = len(model_blob)
|
||||
buf_base, blob_base = [len(model["buffers"]) + len(warp["buffers"])], [len(model_blob) + len(warp_blob)]
|
||||
for _, _, part, blob in extra:
|
||||
buf_base.append(buf_base[-1] + len(part["buffers"])); blob_base.append(blob_base[-1] + len(blob))
|
||||
upload, u8 = buf_base[-1], repr(_dtype("dtypes.uint8"))
|
||||
extra_at = {img: (part, buf_base[i], blob_base[i], tfm) for i, (img, tfm, part, _) in enumerate(extra)}
|
||||
|
||||
def view(name: str, nbytes: int, dtype: str) -> dict:
|
||||
return {"buffer": upload, "offset": offsets[name], "nbytes": nbytes, "dtype": dtype}
|
||||
outputs = sorted(s["slot"] for s in model["inputs"] if s["name"] is None)
|
||||
names = sorted(["host", *states])
|
||||
frame_specs = {s["name"]: s for _, _, part, _ in [(None, None, warp, None), *extra] for s in part["inputs"] if s["name"] in SPLIT_FRAMES}
|
||||
names = sorted(["host", *states, *frame_specs])
|
||||
slots = {s["slot"]: names.index(s["name"]) for s in model["inputs"] if s["name"] in states} | \
|
||||
{slot: len(names) + i for i, slot in enumerate(outputs)}
|
||||
returned = warp["returns"]["tensors"][0]
|
||||
def warped(part: dict, base: int) -> dict:
|
||||
returned = part["returns"]["tensors"][0]
|
||||
return {k: v for k, v in returned.items() if k != "shape"} | {"buffer": returned["buffer"] + base}
|
||||
model_inputs = {s["slot"]: {"input": slots[s["slot"]]} if s["slot"] in slots else
|
||||
{k: v for k, v in returned.items() if k != "shape"} | {"buffer": returned["buffer"] + len(model["buffers"])} if s["name"] == "new_img" else
|
||||
warped(warp, len(model["buffers"])) if s["name"] == "new_img" else
|
||||
warped(extra_at[s["name"]][0], extra_at[s["name"]][1]) if s["name"] in extra_at else
|
||||
view(s["name"], s["numel"] * _dtype(s["dtype"]).itemsize, s["dtype"]) for s in model["inputs"]}
|
||||
warp_inputs = {s["slot"]: view("frames" if s["name"] == "input_frame" else "tfm", s["numel"] * _dtype(s["dtype"]).itemsize, s["dtype"])
|
||||
for s in warp["inputs"]}
|
||||
def warp_inputs(part: dict, tfm: str) -> dict:
|
||||
return {s["slot"]: {"input": names.index(s["name"])} if s["name"] in frame_specs else
|
||||
view("frames" if s["name"] == "input_frame" else tfm, s["numel"] * _dtype(s["dtype"]).itemsize, s["dtype"]) for s in part["inputs"]}
|
||||
|
||||
def remap(calls: list, inputs: dict, buffers: int, lib: int) -> list:
|
||||
return [call | ({"lib": [call["lib"][0] + lib, call["lib"][1] + lib]} if "lib" in call else {}) |
|
||||
{"args": [dict(inputs[a["input"]]) if "input" in a else a | {"buffer": a["buffer"] + buffers} for a in call["args"]]} for call in calls]
|
||||
header = {"input_names": names, "returns": {"kind": "NoneType", "tensors": []}, "check": None,
|
||||
"inputs": [{"slot": names.index("host"), "name": "host", "dtype": u8, "numel": packed, "device": "NPY"}] +
|
||||
[s | {"slot": slots[s["slot"]]} for s in model["inputs"] if s["slot"] in slots],
|
||||
"buffers": model["buffers"] + [b | ({"init": [b["init"][0] + shift, b["init"][1] + shift]} if "init" in b else {}) for b in warp["buffers"]] +
|
||||
[s | {"slot": slots[s["slot"]]} for s in model["inputs"] if s["slot"] in slots] +
|
||||
[s | {"slot": names.index(name)} for name, s in frame_specs.items()],
|
||||
"buffers": model["buffers"] + [b | ({"init": [b["init"][0] + d, b["init"][1] + d]} if "init" in b else {})
|
||||
for part, d in ((warp, shift), *((p, bb) for p, _, bb, _ in extra_at.values())) for b in part["buffers"]] +
|
||||
[{"nbytes": packed, "device": model["inputs"][0]["device"], "dtype": u8}],
|
||||
"calls": [{"kind": "copy", "args": [{"buffer": upload, "offset": 0, "nbytes": packed, "dtype": u8}, {"input": names.index("host")}]}] +
|
||||
remap(warp["calls"], warp_inputs, len(model["buffers"]), shift) + remap(model["calls"], model_inputs, 0, 0)}
|
||||
return {"run": (header, _Blobs(model_blob, warp_blob))} # replaces the model's run: one set of buffers, no separate warp
|
||||
remap(warp["calls"], warp_inputs(warp, "tfm"), len(model["buffers"]), shift) +
|
||||
[c for p, b0, bb, tfm in extra_at.values() for c in remap(p["calls"], warp_inputs(p, tfm), b0, bb)] +
|
||||
remap(model["calls"], model_inputs, 0, 0)}
|
||||
return {"run": (header, _Blobs(model_blob, warp_blob, *(blob for *_, blob in extra)))}
|
||||
|
||||
@@ -264,21 +264,38 @@ def _program_uop(call: dict, lib: bytes):
|
||||
parameter = UOp.param(call["globals"][compact_slot], _dtype(dtype), math.prod(shape))
|
||||
image = (shape[0], shape[1]) if len(shape) == 3 and shape[2] == 4 else None
|
||||
parameters.append(parameter.replace(arg=_fields(parameter.arg, name=name, image=image)))
|
||||
info = _fields(ProgramInfo, global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), globals=tuple(call["globals"]),
|
||||
outs=tuple(call["outs"]), ins=tuple(call["ins"]), target=_fields(Target, **call["target"]))
|
||||
info = _fields(ProgramInfo, global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), params=tuple(parameters),
|
||||
globals=tuple(call["globals"]), outs=tuple(call["outs"]), ins=tuple(call["ins"]), target=_fields(Target, **call["target"]))
|
||||
return UOp(Ops.PROGRAM, src=(UOp(Ops.SINK, arg=KernelInfo(name=call["name"])), UOp(Ops.LINEAR, src=tuple(parameters)),
|
||||
UOp(Ops.SOURCE, arg=""), UOp(Ops.BINARY, arg=lib)), arg=info)
|
||||
|
||||
|
||||
def _input_tensors(kwargs: dict) -> list: # in _prepare_jit_inputs' order
|
||||
from tinygrad import Tensor
|
||||
tensors = [t for _, t in sorted(kwargs.items()) if t.__class__ is Tensor]
|
||||
for value in kwargs.values():
|
||||
tensors += [t for t in (value.values() if isinstance(value, dict) else value if isinstance(value, (tuple, list)) else ())
|
||||
if t.__class__ is Tensor and not any(t is y for y in tensors)]
|
||||
return tensors
|
||||
|
||||
|
||||
class _Jit:
|
||||
def __init__(self, captured, input_names: list, buffers: list):
|
||||
self.captured, self.input_names, self.buffers = captured, input_names, buffers
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
import tinygrad.engine.jit as jit
|
||||
from tinygrad.uop.ops import Ops
|
||||
# same names, no variables: the binding is the tensors' own uops, without _prepare_jit_inputs' graph rewrites
|
||||
if not args and self.__dict__.get("_names") == sorted(kwargs):
|
||||
tensors = _input_tensors(kwargs)
|
||||
if all(t.uop.is_realized for t in tensors):
|
||||
return self.captured([t.uop.src[0] if t.uop.op is Ops.UNSHARD else t.uop for t in tensors], {})
|
||||
input_uops, var_vals, names, _ = jit._prepare_jit_inputs(args, kwargs)
|
||||
if names != self.input_names:
|
||||
raise ValueError(f"expected inputs {self.input_names}, got {names}")
|
||||
if not args and not var_vals:
|
||||
self._names = sorted(kwargs)
|
||||
return self.captured(input_uops, var_vals)
|
||||
|
||||
|
||||
@@ -311,7 +328,7 @@ class PlainJit(_Jit):
|
||||
calls.append(_program_uop(call, bytes(blob[slice(*call["lib"])])).call(*arguments))
|
||||
else:
|
||||
calls.append(arguments[0].store_call(arguments[1]))
|
||||
# stand-ins for JIT inputs: copy staging to GPU is decided from the input buffers, the real ones bind per call
|
||||
# allocate gpu buffers
|
||||
stand_ins = {slot: UOp.from_buffer(Buffer(_device(spec["device"], device_map), spec["numel"] * _dtype(spec["dtype"]).itemsize).ensure_allocated(),
|
||||
_dtype(spec["dtype"])) for slot, spec in inputs.items()}
|
||||
input_uops = tuple(stand_ins.get(slot, next(iter(stand_ins.values()), None)) for slot in range(max(stand_ins, default=-1) + 1))
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"""
|
||||
Copyright (c) 2021-, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
import numpy as np
|
||||
|
||||
from openpilot.common.transformations.camera import get_view_frame_from_calib_frame, view_frame_from_device_frame
|
||||
from openpilot.common.transformations.orientation import rot_from_euler
|
||||
|
||||
LIGHT_INPUT_SIZE = (512, 256)
|
||||
light_intrinsics = np.array([
|
||||
[1000.0, 0.0, 256.0],
|
||||
[0.0, 1000.0, 200.0],
|
||||
[0.0, 0.0, 1.0]])
|
||||
calib_from_light = np.linalg.inv((light_intrinsics @ get_view_frame_from_calib_frame(0, 0, 0, 0))[:, :3])
|
||||
|
||||
|
||||
def get_light_warp_matrix(device_from_calib_euler: np.ndarray, intrinsics: np.ndarray) -> np.ndarray:
|
||||
camera_from_calib = intrinsics @ view_frame_from_device_frame @ rot_from_euler(device_from_calib_euler)
|
||||
return (camera_from_calib @ calib_from_light).astype(np.float32)
|
||||
@@ -0,0 +1,153 @@
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
os.environ.setdefault("DEV", "MOCKUSB+AMD")
|
||||
QCOM = "QCOM" in os.environ["DEV"]
|
||||
ON_MOCK = os.environ["DEV"].startswith("MOCK")
|
||||
ARCH, HARDWARE = ("a630", "comma 3x/4 (a630)") if QCOM else ("gfx1200", "chestnut")
|
||||
os.environ.setdefault("CHECK_OOB", "0")
|
||||
os.environ.setdefault("GMMU", "0")
|
||||
if sys.platform == "darwin" and os.environ["DEV"].startswith("MOCKUSB"): # the mock's VRAM is a memfd, which macOS lacks: an unlinked file is the same
|
||||
from tinygrad.runtime.autogen import libc
|
||||
|
||||
def _memfd_create(name, flags):
|
||||
fd, path = tempfile.mkstemp()
|
||||
os.unlink(path)
|
||||
return fd
|
||||
libc.memfd_create = _memfd_create
|
||||
|
||||
import numpy as np
|
||||
|
||||
from openpilot.sunnypilot.modeld_v2.helpers import ALIGNMENT, MAGIC, PlainJit, _device, read_parts
|
||||
|
||||
|
||||
class ParityError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _emulator():
|
||||
try:
|
||||
if QCOM:
|
||||
from test.mockgpu.qcom import emu
|
||||
else:
|
||||
from test.mockgpu.amd import emu
|
||||
except ImportError as e:
|
||||
raise ParityError(f"tinygrad's {'qcom' if QCOM else 'amd'} mock is not importable: put tinygrad_repo on PYTHONPATH") from e
|
||||
if not hasattr(emu, "hw_unverified"):
|
||||
raise ParityError(f"this tinygrad's emulator has no {ARCH} rounding model, its check frames would not match a {HARDWARE}")
|
||||
return emu
|
||||
|
||||
|
||||
def unverified(header: dict, blob: memoryview) -> dict[str, int]:
|
||||
emu, out = _emulator(), {}
|
||||
for call in header["calls"]:
|
||||
if call["kind"] != "program":
|
||||
continue
|
||||
if (arch := call["target"].get("arch", "")).split(",")[0] != ARCH: # IMAGE builds append ",IMAGE_PITCH_ALIGNMENT=64"
|
||||
out[f"arch={arch}"] = out.get(f"arch={arch}", 0) + 1
|
||||
continue
|
||||
for sig, n in emu.hw_unverified(bytes(blob[slice(*call["lib"])])).items():
|
||||
out[sig] = out.get(sig, 0) + n
|
||||
return out
|
||||
|
||||
|
||||
def _array(blob: memoryview, stored: dict) -> np.ndarray:
|
||||
return np.frombuffer(blob[slice(*stored["data"])], dtype=np.dtype(stored["dtype"])).reshape(stored["shape"]).copy()
|
||||
|
||||
|
||||
def replay(header: dict, blob: memoryview, shared: dict) -> list[np.ndarray]:
|
||||
from tinygrad import Tensor
|
||||
check = header["check"]
|
||||
jit = PlainJit(header, blob, None, verify=False, shared=shared)
|
||||
kwargs: dict = {}
|
||||
for path, stored, device in check["inputs"]:
|
||||
tensor = Tensor(_array(blob, stored), device=_device(device, None)).realize()
|
||||
kwargs[path[0]] = tensor if len(path) == 1 else {**kwargs.get(path[0], {}), path[1]: tensor}
|
||||
result = jit(**kwargs)
|
||||
outputs = kwargs[check["output_container"]].values() if check["output_container"] else result
|
||||
return [t.numpy() for t in (outputs if isinstance(outputs, (list, tuple)) or check["output_container"] else [outputs])]
|
||||
|
||||
|
||||
def _parts_with_offsets(path: str) -> tuple[dict, dict]:
|
||||
parts, _ = read_parts(path)
|
||||
with open(path, "rb") as f:
|
||||
head = f.read(len(MAGIC) + 8)
|
||||
start = len(MAGIC) + 8 + struct.unpack_from("<Q", head, len(MAGIC))[0]
|
||||
f.seek(len(MAGIC) + 8)
|
||||
entries = json.loads(f.read(start - len(MAGIC) - 8))["parts"]
|
||||
base = start + -start % ALIGNMENT
|
||||
offsets = {tuple(e["key"]) if isinstance(e["key"], list) else e["key"]: base + e["blob"][0] for e in entries}
|
||||
return parts, offsets
|
||||
|
||||
|
||||
def _gate(parts: dict) -> None:
|
||||
missing: dict[str, int] = {}
|
||||
for header, blob in parts.values():
|
||||
if "check" not in header:
|
||||
raise ParityError("artifact has no check frame")
|
||||
for sig, n in unverified(header, blob).items():
|
||||
missing[sig] = missing.get(sig, 0) + n
|
||||
if missing:
|
||||
raise ParityError(f"kernels use instructions the emulator is not verified against a {HARDWARE} for: " +
|
||||
", ".join(f"{s} ({n}x)" for s, n in sorted(missing.items(), key=lambda kv: -kv[1])))
|
||||
|
||||
|
||||
def check(path: str) -> None:
|
||||
parts, _ = read_parts(path)
|
||||
if ON_MOCK:
|
||||
_gate(parts)
|
||||
shared: dict = {}
|
||||
for key, (header, blob) in parts.items():
|
||||
got = replay(header, blob, shared)
|
||||
for i, (g, stored) in enumerate(zip(got, header["check"]["outputs"], strict=True)):
|
||||
want = _array(blob, stored)
|
||||
if g.shape != want.shape or not np.array_equal(g.view(np.uint8), want.view(np.uint8)):
|
||||
n = int(np.count_nonzero(g != want)) if g.shape == want.shape else -1
|
||||
where = "mock" if ON_MOCK else "device"
|
||||
raise ParityError(f"part {key} output {i}: the {where}'s check frame differs from the stored one in {n} of {want.size} values")
|
||||
|
||||
|
||||
def record(path: str, out: str) -> None:
|
||||
parts, offsets = _parts_with_offsets(path)
|
||||
_gate(parts)
|
||||
shared, writes = {}, []
|
||||
for key, (header, blob) in parts.items():
|
||||
for g, stored in zip(replay(header, blob, shared), header["check"]["outputs"], strict=True):
|
||||
want = np.dtype(stored["dtype"])
|
||||
if tuple(g.shape) != tuple(stored["shape"]) or g.dtype != want:
|
||||
raise ParityError(f"part {key}: replayed output {g.dtype}{g.shape} does not fit the stored {want}{tuple(stored['shape'])}")
|
||||
writes.append((offsets[key] + stored["data"][0], np.ascontiguousarray(g).tobytes()))
|
||||
shutil.copyfile(path, out)
|
||||
with open(out, "r+b") as f:
|
||||
for offset, data in writes:
|
||||
f.seek(offset)
|
||||
f.write(data)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 2 or argv[0] not in ("check", "record") or (argv[0] == "record" and len(argv) != 3):
|
||||
print("usage: mock_parity check ARTIFACT... | record ARTIFACT OUT", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
if argv[0] == "record":
|
||||
st = time.perf_counter()
|
||||
record(argv[1], argv[2])
|
||||
print(f"{argv[2]}: check frame recorded on the mock in {time.perf_counter() - st:.0f}s")
|
||||
return 0
|
||||
for path in argv[1:]:
|
||||
st = time.perf_counter()
|
||||
check(path)
|
||||
print(f"{os.path.basename(path)}: bit-identical to its {HARDWARE} check frame on the {'mock' if ON_MOCK else 'device'} ({time.perf_counter() - st:.0f}s)")
|
||||
return 0
|
||||
except ParityError as e:
|
||||
print(f"PARITY FAIL: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -11,7 +11,7 @@ import pickle
|
||||
import numpy as np
|
||||
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.sunnypilot.modeld_v2.frame_fusion import pack_layout
|
||||
from openpilot.sunnypilot.modeld_v2.frame_fusion import host_shapes, pack_layout, SPLIT_FRAMES
|
||||
from openpilot.sunnypilot.modeld_v2.helpers import load_model
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import (POLICY_INPUTS, derive_frame_skip,
|
||||
make_split_input_queues, make_supercombo_input_queues)
|
||||
@@ -169,16 +169,18 @@ class NativeTinygradAdapter(BaseModelAdapter):
|
||||
self.warp_frame_size = self.frame_copy_size
|
||||
self._init_common()
|
||||
self.run_model = self.jits['run']
|
||||
self.split = [name for name in SPLIT_FRAMES if name in self.run_model.input_names]
|
||||
self.frame_inputs: dict[str, Tensor] = {}
|
||||
|
||||
self.input_queues = {name: Tensor(np.zeros(shape, dtype=dtype), device=self.model_device).realize()
|
||||
for name, (shape, dtype) in self.input_shapes.items() if name in self.state_pairs}
|
||||
shapes = {name: shape for name, (shape, _) in self.input_shapes.items() if name not in self.state_pairs and name != 'new_img'}
|
||||
offsets, packed = pack_layout(shapes, self.warp_frame_size)
|
||||
shapes = host_shapes({name: shape for name, (shape, _) in self.input_shapes.items()}, self.state_pairs)
|
||||
offsets, packed = pack_layout(shapes, 0 if self.split else self.warp_frame_size)
|
||||
self.packed_input = np.zeros(packed, dtype=np.uint8)
|
||||
self.host = Tensor(self.packed_input, device='NPY')
|
||||
self.numpy_inputs = {name: np.ndarray(shape, dtype=np.float32, buffer=self.packed_input, offset=offsets[name])
|
||||
for name, shape in ({'tfm': (2, 3, 3)} | shapes).items()}
|
||||
self.frames = self.packed_input[offsets['frames']:].reshape(2, self.warp_frame_size)
|
||||
self.frames = self.packed_input[offsets['frames']:].reshape(2, -1)
|
||||
|
||||
self.outputs = {name: Tensor(np.zeros(shape, dtype=dtype), device=device).realize() for name, (shape, dtype, device) in self.jits['output_specs'].items()}
|
||||
for name, next_name in self.state_pairs.items():
|
||||
@@ -187,17 +189,29 @@ class NativeTinygradAdapter(BaseModelAdapter):
|
||||
|
||||
def copy_frames(self, bufs):
|
||||
for i, key in enumerate(self._vision_input_names):
|
||||
if key in bufs:
|
||||
if key not in bufs:
|
||||
continue
|
||||
if not self.split:
|
||||
data = bufs[key].data if hasattr(bufs[key], 'data') else bufs[key]
|
||||
np.copyto(self.frames[i], np.frombuffer(data, dtype=np.uint8, count=self.warp_frame_size))
|
||||
elif (name := ('frame', 'big_frame')[i]) in self.split:
|
||||
if isinstance(bufs[key], np.ndarray): # cached memory: the GPU would read stale lines in place
|
||||
self.frame_inputs[name] = Tensor(np.ascontiguousarray(bufs[key][:self.warp_frame_size]), device=self.model_device).realize()
|
||||
continue
|
||||
ptr = np.frombuffer(bufs[key].data, dtype=np.uint8).ctypes.data # visionipc buffers are coherent
|
||||
if (key, ptr) not in self._blob_cache:
|
||||
self._blob_cache[(key, ptr)] = Tensor.from_blob(ptr, (self.warp_frame_size,), dtype='uint8', device=self.model_device)
|
||||
self.frame_inputs[name] = self._blob_cache[(key, ptr)]
|
||||
|
||||
def reset_warmup_buffers(self) -> None:
|
||||
self.packed_input[:] = 0
|
||||
self.frame_inputs.clear()
|
||||
self._blob_cache.clear()
|
||||
for key in self.state_pairs:
|
||||
self.input_queues[key].assign(0).realize()
|
||||
|
||||
def run(self):
|
||||
self.run_model(host=self.host, **self.input_queues, output_buffers=self.outputs)
|
||||
self.run_model(host=self.host, **self.input_queues, **self.frame_inputs, output_buffers=self.outputs)
|
||||
return self.outputs['outputs']
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from functools import partial
|
||||
import gc
|
||||
import os
|
||||
os.environ['GMMU'] = '0'
|
||||
os.environ['QCOM_PRIORITY'] = '1'
|
||||
from tinygrad.device import Device
|
||||
import numpy as np
|
||||
import threading
|
||||
@@ -44,6 +45,7 @@ from openpilot.sunnypilot.modeld_v2.blendful import Blendful
|
||||
from openpilot.sunnypilot.modeld_v2.constants import ModelConstants, Plan
|
||||
from openpilot.sunnypilot.modeld_v2.meta_helper import load_meta_constants
|
||||
from openpilot.sunnypilot.modeld_v2.camera_offset_helper import CameraOffsetHelper
|
||||
from openpilot.sunnypilot.modeld_v2.light_frame import get_light_warp_matrix
|
||||
from openpilot.sunnypilot.livedelay.helpers import get_lat_delay
|
||||
from openpilot.sunnypilot.modeld_v2.modeld_base import ModelStateBase
|
||||
from openpilot.sunnypilot.modeld_v2.helpers import load_model
|
||||
@@ -171,7 +173,7 @@ class ModelState(ModelStateBase):
|
||||
np.where(inputs[desire_key] - self.prev_desire > .99, inputs[desire_key], 0)
|
||||
self.prev_desire[:] = inputs[desire_key]
|
||||
|
||||
for key in ('traffic_convention', 'lateral_control_params', 'action_t'):
|
||||
for key in ('traffic_convention', 'lateral_control_params', 'action_t', 'speed_limit'):
|
||||
if key in self.numpy_inputs and key in inputs:
|
||||
self.numpy_inputs[key][:] = inputs[key]
|
||||
|
||||
@@ -179,6 +181,8 @@ class ModelState(ModelStateBase):
|
||||
for i, key in enumerate(self._vision_input_names):
|
||||
if key in transforms:
|
||||
self.numpy_inputs['tfm'][i] = transforms[key].reshape(3, 3)
|
||||
if 'light_tfm' in self.numpy_inputs and 'light' in transforms:
|
||||
self.numpy_inputs['light_tfm'][0] = transforms['light'].reshape(3, 3)
|
||||
else:
|
||||
self.numpy_inputs['tfm'][:, :] = transforms[self._road_key].reshape(3, 3)
|
||||
self.numpy_inputs['big_tfm'][:, :] = transforms[self._wide_key].reshape(3, 3)
|
||||
@@ -242,7 +246,7 @@ class ModelState(ModelStateBase):
|
||||
desired_curvature = get_curvature_from_output(model_output, curvature_plan, v_ego, lat_action_t, self.mlsim)
|
||||
else:
|
||||
desired_accel = model_output['action'][0, 1]
|
||||
desired_curvature = model_output['action'][0, 0] / (max(1.0, v_ego))**2
|
||||
desired_curvature = model_output['action'][0, 0] / (max(2.5, v_ego))**2
|
||||
if blendful is not None:
|
||||
desired_curvature += blendful(desired_curvature)
|
||||
|
||||
@@ -338,7 +342,7 @@ def main(demo=False):
|
||||
pub_socks = ["modelV2", "drivingModelData", "cameraOdometry", "modelDataV2SP"] + (["chestnutGpuState"] if CHESTNUT else [])
|
||||
pm = PubMaster(pub_socks)
|
||||
sm = SubMaster(["deviceState", "carState", "narrowRoadCameraState", "extrinsicsCalibration", "driverMonitoringState",
|
||||
"carControl", "controlsState", "lateralDelay"])
|
||||
"carControl", "controlsState", "lateralDelay", "longitudinalPlanSP"])
|
||||
|
||||
publish_state = PublishState()
|
||||
chestnut_state = ChestnutGpuState(pm, model.chestnut) if CHESTNUT else None
|
||||
@@ -351,6 +355,7 @@ def main(demo=False):
|
||||
|
||||
model_transform_main = np.zeros((3, 3), dtype=np.float32)
|
||||
model_transform_extra = np.zeros((3, 3), dtype=np.float32)
|
||||
model_transform_light = np.zeros((3, 3), dtype=np.float32)
|
||||
live_calib_seen = False
|
||||
buf_main, buf_extra = None, None
|
||||
meta_main = FrameMeta()
|
||||
@@ -369,6 +374,9 @@ def main(demo=False):
|
||||
prev_action = log.ModelDataV2.Action()
|
||||
|
||||
DH = DesireHelper()
|
||||
desire_frames = [q.shape[0] for m in (model, small_model) if m is not None for name, q in m.adapter.input_queues.items() if 'desire' in name]
|
||||
if desire_frames:
|
||||
DH.turn_repulse = min(desire_frames) - 2
|
||||
meta_constants = load_meta_constants()
|
||||
RELC = RoadEdgeLaneChangeController()
|
||||
blendful = Blendful()
|
||||
@@ -425,6 +433,7 @@ def main(demo=False):
|
||||
model_transform_main = get_warp_matrix(device_from_calib_euler, main_intrinsics, False).astype(np.float32)
|
||||
model_transform_extra = get_warp_matrix(device_from_calib_euler, dc.wide_road.intrinsics, True).astype(np.float32)
|
||||
model_transform_main, model_transform_extra = camera_offset_helper.update(model_transform_main, model_transform_extra, sm, main_wide_camera)
|
||||
model_transform_light = get_light_warp_matrix(device_from_calib_euler, main_intrinsics)
|
||||
live_calib_seen = True
|
||||
|
||||
traffic_convention = np.zeros(2)
|
||||
@@ -446,6 +455,7 @@ def main(demo=False):
|
||||
|
||||
bufs = {name: buf_extra if 'big' in name else buf_main for name in model.vision_input_names}
|
||||
transforms = {name: model_transform_extra if 'big' in name else model_transform_main for name in model.vision_input_names}
|
||||
transforms['light'] = model_transform_light
|
||||
|
||||
frame_delay = DT_MDL # compensate for time passed since the frame was captured: current_time - timestamp_eof is 50ms on average
|
||||
action_delay = DT_MDL / 2 # middle of the interval between model output (current state) and next frame (expected state)
|
||||
@@ -463,6 +473,10 @@ def main(demo=False):
|
||||
if 'action_t' in model.numpy_inputs:
|
||||
inputs['action_t'] = np.array([lat_action_t, long_action_t], dtype=np.float32)
|
||||
|
||||
if 'speed_limit' in model.numpy_inputs:
|
||||
resolver = sm['longitudinalPlanSP'].speedLimit.resolver
|
||||
inputs['speed_limit'] = np.array([resolver.speedLimitFinal if resolver.speedLimitValid else 0.0], dtype=np.float32)
|
||||
|
||||
mt1 = time.perf_counter()
|
||||
try:
|
||||
send_chestnut = (chestnut_state is not None and
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1baac6f02902df5689e9e275633c122a9955b7c57de2681e0137e1724c1f32dd
|
||||
size 3463278
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0478837491c584b2212e31a91c494a908b8fb048205700f05a02a13ee7fc9b90
|
||||
size 7690094
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:580fadf61ded34b4b19f0ba1f5e4b61c0ea489ca34a0952ccf2878593e3f68c4
|
||||
size 1851239
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:44700f3533abb3e65232c8aca46b48384a59218545ed6bb505c5d4762204359b
|
||||
size 3964263
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:4b22fd6bcb3a3e8a642e6866449dd9c30c12e694ff0741cad9e34e4b00d8c5fa
|
||||
size 3698799
|
||||
oid sha256:1f5dd8855de07d0e8b9ed836934e1dcbe77aa27c3b5c77eee1f21888f974ba8b
|
||||
size 3699072
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9a17fd6acece4fab23784ef0f3f8bba28a049e9599e5e9e5ee7b36e4b1d24a6f
|
||||
size 7924847
|
||||
oid sha256:226d40112a292077b7eca26b97273c56ab3a84feee8ddf9b6d02d46c2a7b8403
|
||||
size 7924864
|
||||
|
||||
@@ -80,7 +80,8 @@ def serialize_jit(jit, kernel_linear, check: dict | None = None) -> tuple[dict,
|
||||
if ast.op is Ops.PROGRAM:
|
||||
elf, info = ast.to_elf(), ast.arg
|
||||
global_size, local_size = info.launch_dims({})
|
||||
calls.append({"kind": "program", "name": elf.name, "lib": blob.add(bytes(elf.lib)), "target": dataclasses.asdict(elf.target),
|
||||
device_target = dataclasses.asdict(dataclasses.replace(elf.target, interface=elf.target.interface.removeprefix("MOCK")))
|
||||
calls.append({"kind": "program", "name": elf.name, "lib": blob.add(bytes(elf.lib)), "target": device_target,
|
||||
"signature": kernel_signature(ast, elf),
|
||||
"global_size": [int(x) for x in global_size], "local_size": [int(x) for x in local_size],
|
||||
"globals": list(info.globals), "outs": list(info.outs), "ins": list(info.ins), "args": arguments})
|
||||
|
||||
@@ -6,6 +6,7 @@ See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -25,8 +26,8 @@ if sys.platform == "darwin" and os.environ.get("DEV", "").startswith("MOCKUSB"):
|
||||
|
||||
from openpilot.common.test import OpenpilotTestCase
|
||||
from openpilot.sunnypilot.modeld_v2 import convert_upstream_pkl
|
||||
from openpilot.sunnypilot.modeld_v2.convert_upstream_pkl import convert
|
||||
from openpilot.sunnypilot.modeld_v2.helpers import PlainJit, load_model
|
||||
from openpilot.sunnypilot.modeld_v2.convert_upstream_pkl import _elf, convert
|
||||
from openpilot.sunnypilot.modeld_v2.helpers import ALIGNMENT, PlainJit, load_model
|
||||
from openpilot.sunnypilot.modeld_v2.plain_artifact_compiler import capture_frame
|
||||
|
||||
IMG, DESIRE, STATE, OUT = (6, 8, 8), (8,), (4, 16), (1, 32)
|
||||
@@ -119,3 +120,18 @@ class TestConvertUpstreamPkl(OpenpilotTestCase):
|
||||
bundle = load_model(out, cache=False, verify=True)
|
||||
assert bundle["metadata"] == {"name": "tiny"} and bundle["input_specs"] == self.input_specs and bundle["output_specs"] == self.output_specs
|
||||
self.assert_matches_eager(bundle["run"], 6)
|
||||
|
||||
|
||||
class TestKernelElf(OpenpilotTestCase):
|
||||
def test_sections_follow_the_code(self):
|
||||
from tinygrad.runtime.support.elf import elf_loader
|
||||
code = bytes(range(1, 129))
|
||||
for rodata, entry in ((len(code), 0), (0, 256)):
|
||||
image = bytearray(code + bytes(64) if entry < rodata else bytes(256) + code)
|
||||
struct.pack_into("<q", image, rodata + 16, entry - rodata)
|
||||
loaded, sections, _ = elf_loader(_elf(bytes(image), rodata, entry))
|
||||
headers = {section.name: section.header for section in sections}
|
||||
assert bytes(loaded[ALIGNMENT:]) == bytes(image), "the uploaded program differs from the pickle's"
|
||||
assert headers[".rodata"].sh_addr == ALIGNMENT + rodata, "the loader reads the kernel descriptor at .rodata"
|
||||
text = headers[".text"]
|
||||
assert text.sh_addr <= ALIGNMENT + entry < text.sh_addr + text.sh_size, "the amd mock sizes the code from the entry to the end of .text"
|
||||
|
||||
@@ -17,4 +17,4 @@ class TestRegistryModels(unittest.TestCase):
|
||||
self.skipTest("MODEL_BASE_NAME env var not set, skipping integration test.")
|
||||
bundle = load_model(os.path.join(os.environ.get("MODEL_DIR", "/tmp/models"), base_name), {"QCOM": "CPU", "AMD": "CPU"}, verify=False, cache=False)
|
||||
jits = [value for value in bundle.values() if isinstance(value, PlainJit)]
|
||||
assert "metadata" in bundle and jits and all(len(jit.captured.linear.src) > 0 for jit in jits)
|
||||
assert "metadata" in bundle and jits and all(len(jit.captured._linear.src) > 0 for jit in jits)
|
||||
|
||||
@@ -144,6 +144,21 @@ class TestPlainArtifact(OpenpilotTestCase):
|
||||
for name in ("outputs", "next_state_feat_q"):
|
||||
assert np.array_equal(inputs["output_buffers"][name].numpy(), expected["output_buffers"][name].numpy()), name
|
||||
|
||||
split_path = os.path.join(self.tmp.name, "warp_split_tinygrad.pkl")
|
||||
with mock.patch.object(sys, "argv", ["compile_warp", "--frame", FRAME, "--warp-to", f"{MODEL_W}x{MODEL_H}", "--output", split_path, "--split-frames"]):
|
||||
compile_warp.main()
|
||||
split = load_model(path, cache=False, extend=functools.partial(frame_part, split_path))
|
||||
assert split["run"].input_names == ["big_frame", "frame", "host", "state_feat_q"]
|
||||
offsets, packed = pack_layout(HOST, 0)
|
||||
host = np.zeros(packed, np.uint8)
|
||||
for name, shape in ({"tfm": (2, 3, 3)} | HOST).items():
|
||||
np.ndarray(shape, np.float32, host, offsets[name])[...] = tfm.numpy() if name == "tfm" else inputs[name].numpy()
|
||||
out = {"outputs": Tensor.zeros(OUT).contiguous().realize(), "next_state_feat_q": Tensor.zeros(STATE).contiguous().realize()}
|
||||
split["run"](host=Tensor(host, device="NPY"), frame=Tensor(frames.numpy()[0]).realize(), big_frame=Tensor(frames.numpy()[1]).realize(),
|
||||
state_feat_q=expected["state_feat_q"], output_buffers=out)
|
||||
for name in ("outputs", "next_state_feat_q"):
|
||||
assert np.array_equal(out[name].numpy(), expected["output_buffers"][name].numpy()), name
|
||||
|
||||
def test_tinygrad_api_the_loader_reads(self):
|
||||
import tinygrad.engine.jit as jit
|
||||
from tinygrad.device import Buffer, TinyELF
|
||||
|
||||
@@ -112,7 +112,7 @@ class ModelCache:
|
||||
|
||||
class ModelFetcher:
|
||||
"""Handles fetching and caching of model data from remote source"""
|
||||
MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v25.json"
|
||||
MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v26.json"
|
||||
MODEL_URL_CHESTNUT = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_chestnut_v28.json"
|
||||
|
||||
MODEL_SOURCES = {
|
||||
|
||||
@@ -6,6 +6,7 @@ See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
@@ -28,8 +29,11 @@ DOWNLOAD_TIMEOUT = (30, 30)
|
||||
# HuggingFace throttles each connection to ~1-2 MB/s but not the connection count; 12 saturates the device link
|
||||
MAX_CONCURRENT_PIECES = 12
|
||||
PIECE_SIZE = 8 * 1024 * 1024
|
||||
PIECE_ATTEMPTS = 3
|
||||
PIECE_ATTEMPTS = 3 # per piece, before the transfer as a whole is retried
|
||||
PIECE_BACKOFF = 1.0 # seconds, growing linearly: waits 1 s then 2 s between a piece's three attempts
|
||||
DOWNLOAD_ATTEMPTS = 4 # whole-transfer attempts, each resuming from the sidecar
|
||||
RETRY_BACKOFF = 2.0 # seconds, doubling: waits 2, 4 then 8 s between the four transfer attempts
|
||||
RETRY_POLL = 0.25 # how often a backoff checks for a cancel
|
||||
TRANSIENT_HTTP_STATUS = frozenset({408, 429, 500, 502, 503, 504})
|
||||
REPORT_INTERVAL = 0.5 # seconds between progress publications
|
||||
# byte ranges count encoded bytes but iter_content yields decoded ones, so ask for an unencoded body
|
||||
@@ -49,26 +53,69 @@ def _is_transient(e: BaseException) -> bool:
|
||||
|
||||
|
||||
class _Progress:
|
||||
def __init__(self):
|
||||
self._bytes = 0
|
||||
def __init__(self, done: set[int], done_bytes: int):
|
||||
self._bytes = done_bytes
|
||||
self._done = set(done)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add_bytes(self, n: int) -> None:
|
||||
with self._lock:
|
||||
self._bytes += n
|
||||
|
||||
def snapshot(self) -> int:
|
||||
def mark_done(self, index: int) -> None:
|
||||
with self._lock:
|
||||
return self._bytes
|
||||
self._done.add(index)
|
||||
|
||||
def snapshot(self) -> tuple[int, set[int]]:
|
||||
with self._lock:
|
||||
return self._bytes, set(self._done)
|
||||
|
||||
|
||||
def _piece_ranges(total: int, piece_size: int) -> list[tuple[int, int]]:
|
||||
return [(start, min(start + piece_size, total)) for start in range(0, total, piece_size)]
|
||||
|
||||
|
||||
def _prepare_target(path: str, total: int) -> None:
|
||||
def _sidecar_path(path: str) -> str:
|
||||
return f"{path}.download"
|
||||
|
||||
|
||||
def _download_in_progress(path: str) -> bool:
|
||||
return os.path.isfile(_sidecar_path(path))
|
||||
|
||||
|
||||
def _save_resume_state(path: str, layout: dict, done: set[int]) -> None:
|
||||
tmp = _sidecar_path(path) + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump({**layout, "done": sorted(done)}, f)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, _sidecar_path(path)) # a power loss leaves the old sidecar or the new one, never a torn file
|
||||
dir_fd = os.open(os.path.dirname(path) or ".", os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(dir_fd)
|
||||
finally:
|
||||
os.close(dir_fd)
|
||||
|
||||
|
||||
def _remove_download_state(path: str) -> None:
|
||||
for stale in (_sidecar_path(path), _sidecar_path(path) + ".tmp"):
|
||||
if os.path.isfile(stale):
|
||||
os.remove(stale)
|
||||
|
||||
|
||||
def _prepare_target(path: str, layout: dict) -> set[int]:
|
||||
total = layout["total"]
|
||||
try:
|
||||
with open(_sidecar_path(path)) as f:
|
||||
state = json.load(f)
|
||||
if isinstance(state, dict) and all(state.get(k) == v for k, v in layout.items()) and os.path.getsize(path) == total:
|
||||
return {i for i in state.get("done", []) if isinstance(i, int)}
|
||||
except (OSError, ValueError, TypeError): # missing, unreadable or malformed sidecar: start over
|
||||
pass
|
||||
_save_resume_state(path, layout, set())
|
||||
with open(path, "wb") as f:
|
||||
f.truncate(total)
|
||||
return set()
|
||||
|
||||
|
||||
def _content_range_total(response: requests.Response) -> int | None:
|
||||
@@ -110,6 +157,9 @@ def _fetch_piece(url: str, path: str, index: int, start: int, end: int | None, p
|
||||
f.truncate() # unknown length: the body defines the file size
|
||||
if end is not None and written != end - start:
|
||||
raise requests.exceptions.ConnectionError(f"short read: {written} of {end - start} bytes")
|
||||
f.flush()
|
||||
os.fsync(f.fileno()) # before it is recorded, so a power loss never leaves the sidecar naming unsynced bytes
|
||||
progress.mark_done(index)
|
||||
return
|
||||
except requests.RequestException as e:
|
||||
progress.add_bytes(-written) # the piece restarts from scratch
|
||||
@@ -120,6 +170,11 @@ def _fetch_piece(url: str, path: str, index: int, start: int, end: int | None, p
|
||||
raise DownloadCancelled("Download cancelled") from None
|
||||
|
||||
|
||||
def _remove_file(path: str) -> None:
|
||||
if os.path.isfile(path):
|
||||
os.remove(path)
|
||||
|
||||
|
||||
class ModelManagerSP:
|
||||
"""Manages model downloads and status reporting"""
|
||||
|
||||
@@ -186,11 +241,12 @@ class ModelManagerSP:
|
||||
else:
|
||||
eta = self._calculate_eta(artifact.fileName, progress)
|
||||
self._set_progress(artifact, custom.ModelManagerSP.DownloadStatus.downloading, progress, eta, speed)
|
||||
artifact.downloadProgress.resuming = False # shown only for the first report, when the bar jumps to the resumed point
|
||||
|
||||
async def _report_until_done(self, tasks: list[asyncio.Future], artifact, progress: _Progress, total: int,
|
||||
started: tuple[int, float]) -> None:
|
||||
async def _report_until_done(self, tasks: list[asyncio.Future], artifact, progress: _Progress, path: str, layout: dict,
|
||||
started: tuple[int, set[int], float]) -> None:
|
||||
pending = set(tasks)
|
||||
last_bytes, last_time = started
|
||||
last_bytes, saved, last_time = started
|
||||
speed = 0.0
|
||||
while pending:
|
||||
done, pending = await asyncio.wait(pending, timeout=REPORT_INTERVAL)
|
||||
@@ -199,14 +255,17 @@ class ModelManagerSP:
|
||||
if self._download_interrupted():
|
||||
raise DownloadCancelled("Download cancelled")
|
||||
now = time.monotonic()
|
||||
done_bytes = progress.snapshot()
|
||||
done_bytes, done_pieces = progress.snapshot()
|
||||
if done_pieces != saved:
|
||||
_save_resume_state(path, layout, done_pieces)
|
||||
saved = done_pieces
|
||||
if (dt := now - last_time) > 0:
|
||||
instant = max(0.0, (done_bytes - last_bytes) / dt) # bytes/s this tick; clamped since retried pieces can lower the byte count
|
||||
speed = instant if speed == 0 else SPEED_SMOOTHING * speed + (1 - SPEED_SMOOTHING) * instant
|
||||
if speed < SPEED_FLOOR: # a stall decays toward 0 without reaching it, and a near-0 speed overflows the eta
|
||||
speed = 0.0
|
||||
last_time, last_bytes = now, done_bytes
|
||||
self._publish_progress(artifact, done_bytes, total, speed)
|
||||
self._publish_progress(artifact, done_bytes, layout["total"], speed)
|
||||
|
||||
async def _download_file(self, url: str, path: str, artifact) -> None:
|
||||
self._download_start_times[artifact.fileName] = time.monotonic()
|
||||
@@ -214,16 +273,18 @@ class ModelManagerSP:
|
||||
|
||||
total, ranged = await loop.run_in_executor(None, _probe_size, url)
|
||||
pieces: list[tuple[int, int | None]] = list(_piece_ranges(total, PIECE_SIZE)) if ranged else [(0, None)]
|
||||
await loop.run_in_executor(None, _prepare_target, path, total)
|
||||
progress = _Progress()
|
||||
layout = {"total": total, "piece_size": PIECE_SIZE, "ranged": ranged, "sha256": artifact.downloadUri.sha256}
|
||||
done = await loop.run_in_executor(None, _prepare_target, path, layout)
|
||||
resumed_bytes = sum(end - start for i, (start, end) in enumerate(pieces) if i in done and end is not None)
|
||||
progress = _Progress(done, resumed_bytes) # the bar and eta start from the resumed pieces
|
||||
cancel = threading.Event()
|
||||
started = progress.snapshot(), time.monotonic() # before any worker can land bytes
|
||||
started = *progress.snapshot(), time.monotonic() # before any worker can land bytes
|
||||
|
||||
with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_PIECES) as pool:
|
||||
tasks = [loop.run_in_executor(pool, _fetch_piece, url, path, i, start, end, progress, cancel, self._block_size)
|
||||
for i, (start, end) in enumerate(pieces)]
|
||||
for i, (start, end) in enumerate(pieces) if i not in done]
|
||||
try:
|
||||
await self._report_until_done(tasks, artifact, progress, total, started)
|
||||
await self._report_until_done(tasks, artifact, progress, path, layout, started)
|
||||
except BaseException as e:
|
||||
cancel.set()
|
||||
if isinstance(e, DownloadCancelled):
|
||||
@@ -232,9 +293,32 @@ class ModelManagerSP:
|
||||
for task in tasks:
|
||||
task.cancel() # drops queued pieces; running ones see the event at their next block
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
await loop.run_in_executor(None, pool.shutdown) # let running pieces stop so none lands after the save
|
||||
_save_resume_state(path, layout, progress.snapshot()[1])
|
||||
raise
|
||||
|
||||
_remove_download_state(path)
|
||||
artifact.downloadProgress.resuming = False
|
||||
del self._download_start_times[artifact.fileName]
|
||||
|
||||
async def _download_with_retries(self, url: str, path: str, artifact) -> None:
|
||||
for attempt in range(DOWNLOAD_ATTEMPTS):
|
||||
try:
|
||||
await self._download_file(url, path, artifact)
|
||||
return
|
||||
except Exception as e:
|
||||
if not _is_transient(e) or attempt == DOWNLOAD_ATTEMPTS - 1:
|
||||
raise
|
||||
delay = RETRY_BACKOFF * 2 ** attempt
|
||||
cloudlog.warning(f"Retrying {artifact.fileName} in {delay:g}s after {type(e).__name__}: {e}")
|
||||
progress = artifact.downloadProgress
|
||||
self._set_progress(artifact, progress.status, progress.progress, progress.eta, 0.0) # stalled: no speed
|
||||
deadline = time.monotonic() + delay
|
||||
while (remaining := deadline - time.monotonic()) > 0:
|
||||
if self._download_interrupted():
|
||||
raise DownloadCancelled("Download cancelled") from e
|
||||
await asyncio.sleep(min(RETRY_POLL, remaining))
|
||||
|
||||
async def _process_artifact(self, artifact, destination_path: str) -> None:
|
||||
if not artifact.downloadUri.uri:
|
||||
return None
|
||||
@@ -246,37 +330,39 @@ class ModelManagerSP:
|
||||
filename = artifact.fileName
|
||||
full_path = os.path.join(destination_path, filename)
|
||||
status = custom.ModelManagerSP.DownloadStatus
|
||||
artifact.downloadProgress.resuming = False
|
||||
artifact.downloadProgress.canceling = False
|
||||
|
||||
try:
|
||||
self._set_progress(artifact, status.verifying, 0)
|
||||
if await verify_file(full_path, expected_hash):
|
||||
self._set_progress(artifact, status.cached, 100)
|
||||
return
|
||||
if _download_in_progress(full_path):
|
||||
cloudlog.info(f"Resuming interrupted download of {filename}")
|
||||
artifact.downloadProgress.resuming = True
|
||||
else:
|
||||
self._set_progress(artifact, status.verifying, 0)
|
||||
if await verify_file(full_path, expected_hash):
|
||||
self._set_progress(artifact, status.cached, 100)
|
||||
return
|
||||
|
||||
self._set_progress(artifact, status.downloading, 0)
|
||||
await self._download_file(url, full_path, artifact)
|
||||
await self._download_with_retries(url, full_path, artifact)
|
||||
self._set_progress(artifact, status.verifying, 99)
|
||||
if not await verify_file(full_path, expected_hash):
|
||||
_remove_file(full_path)
|
||||
_remove_download_state(full_path)
|
||||
raise ValueError(f"Hash validation failed for {filename}")
|
||||
|
||||
self._set_progress(artifact, status.downloaded, 100)
|
||||
|
||||
except DownloadCancelled:
|
||||
except DownloadCancelled: # not a failure: the request handler clears the row
|
||||
self._download_start_times.pop(artifact.fileName, None)
|
||||
artifact.downloadProgress.status = status.failed
|
||||
artifact.downloadProgress.eta = 0
|
||||
artifact.downloadProgress.speed = 0
|
||||
self._sync_artifact_progress(artifact)
|
||||
if self.selected_bundle:
|
||||
self.selected_bundle.status = status.failed
|
||||
self._report_status()
|
||||
raise
|
||||
|
||||
except Exception as e:
|
||||
cloudlog.error(f"Error downloading {filename}: {str(e)}")
|
||||
if os.path.isfile(full_path): # noqa: ASYNC240
|
||||
os.remove(full_path)
|
||||
artifact.downloadProgress.status = status.failed
|
||||
artifact.downloadProgress.eta = 0
|
||||
artifact.downloadProgress.speed = 0
|
||||
@@ -333,6 +419,8 @@ class ModelManagerSP:
|
||||
self.params.put(ACTIVE_BUNDLE_KEYS[source], model_bundle.to_dict(), block=True)
|
||||
self.active_bundle = get_active_bundle(self.params, chestnut=self.chestnut_present)
|
||||
|
||||
except DownloadCancelled:
|
||||
raise # not a failure
|
||||
except Exception:
|
||||
if self.selected_bundle is not None:
|
||||
self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed
|
||||
@@ -360,11 +448,14 @@ class ModelManagerSP:
|
||||
self._download_ref = ref_to_download
|
||||
try:
|
||||
self.download(model_to_download, Paths.model_root(), source)
|
||||
except DownloadCancelled:
|
||||
self.selected_bundle = None
|
||||
except Exception as e:
|
||||
cloudlog.exception(e)
|
||||
cloudlog.exception(e) # a failure stays on the row until the next request
|
||||
else:
|
||||
self.selected_bundle = None
|
||||
finally:
|
||||
self._release_download_ref()
|
||||
self.selected_bundle = None
|
||||
|
||||
def main_thread(self) -> None:
|
||||
"""Main thread for model management"""
|
||||
|
||||
@@ -11,11 +11,13 @@ import contextlib
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import http.server
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import requests
|
||||
import stat
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
@@ -55,6 +57,8 @@ class ServerState:
|
||||
fail_ranges: dict[tuple[int, int], int] = dataclasses.field(default_factory=dict) # range -> HTTP status, every time
|
||||
fail_times: dict[tuple[int, int], int] = dataclasses.field(default_factory=dict) # range -> how many more requests 503
|
||||
short_once: set[tuple[int, int]] = dataclasses.field(default_factory=set) # ranges whose first answer is cut short
|
||||
fail_after_others: bool = False # hold each failure until every other piece has been sent
|
||||
sent: set[tuple[int, int]] = dataclasses.field(default_factory=set) # ranges sent in full
|
||||
stall_ranges: set[tuple[int, int]] = dataclasses.field(default_factory=set)
|
||||
stall_event: threading.Event | None = None
|
||||
stall_released_by_event: bool | None = None
|
||||
@@ -88,6 +92,10 @@ class DownloadHandler(http.server.BaseHTTPRequestHandler):
|
||||
state.fail_times[rng] -= 1
|
||||
status = 503
|
||||
if status is not None:
|
||||
others = {piece_range(i) for i in range(NUM_PIECES)} - {rng}
|
||||
deadline = time.monotonic() + 5
|
||||
while state.fail_after_others and not others <= state.sent and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self.send_response(status)
|
||||
self.end_headers()
|
||||
return
|
||||
@@ -115,6 +123,8 @@ class DownloadHandler(http.server.BaseHTTPRequestHandler):
|
||||
self.wfile.write(payload[100:])
|
||||
else:
|
||||
self.wfile.write(payload)
|
||||
if rng is not None:
|
||||
state.sent.add(rng)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
@@ -133,7 +143,10 @@ def threaded_server(handler):
|
||||
class InlineExecutor(concurrent.futures.Executor):
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
future: concurrent.futures.Future = concurrent.futures.Future()
|
||||
future.set_result(fn(*args, **kwargs)) # the piece lands before the reporter starts
|
||||
try:
|
||||
future.set_result(fn(*args, **kwargs)) # the piece lands before the reporter starts
|
||||
except BaseException as e:
|
||||
future.set_exception(e)
|
||||
return future
|
||||
|
||||
|
||||
@@ -147,11 +160,12 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.dest = self._tmp.name
|
||||
|
||||
for name, value in (('PIECE_SIZE', PIECE_SIZE), ('REPORT_INTERVAL', 0.05), ('PIECE_BACKOFF', 0.01), ('SPEED_FLOOR', 100)):
|
||||
for name, value in (('PIECE_SIZE', PIECE_SIZE), ('REPORT_INTERVAL', 0.05), ('PIECE_BACKOFF', 0.01), ('SPEED_FLOOR', 100), ('RETRY_BACKOFF', 0.01)):
|
||||
self.enterContext(mock.patch.object(manager_module, name, value))
|
||||
|
||||
self.reported: list[float] = []
|
||||
self.reported_statuses: list[int] = []
|
||||
self.reported_resuming: list[bool] = []
|
||||
|
||||
self.manager = ModelManagerSP.__new__(ModelManagerSP)
|
||||
self.params = self.manager.params = mock.MagicMock()
|
||||
@@ -172,6 +186,7 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
|
||||
self.reported.append(float(artifact.downloadProgress.progress))
|
||||
status = artifact.downloadProgress.status
|
||||
self.reported_statuses.append(getattr(status, 'raw', status)) # .raw: _DynamicEnum equals its int but does not hash like it
|
||||
self.reported_resuming.append(bool(artifact.downloadProgress.resuming))
|
||||
|
||||
def make_artifact(self):
|
||||
bundle = custom.ModelManagerSP.ModelBundle.new_message()
|
||||
@@ -188,6 +203,24 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
|
||||
def path(self) -> str:
|
||||
return os.path.join(self.dest, FILE_NAME)
|
||||
|
||||
@property
|
||||
def sidecar(self) -> str:
|
||||
return manager_module._sidecar_path(self.path)
|
||||
|
||||
def resume_state(self) -> dict:
|
||||
with open(self.sidecar) as f:
|
||||
return json.load(f)
|
||||
|
||||
def write_resume_state(self, done: list[int], total: int = len(WHOLE_BODY), file_size: int | None = None,
|
||||
file_sha256: str = sha256(WHOLE_BODY)) -> None:
|
||||
with open(self.path, 'wb') as f:
|
||||
f.truncate(len(WHOLE_BODY) if file_size is None else file_size)
|
||||
for i in done:
|
||||
f.seek(i * PIECE_SIZE)
|
||||
f.write(WHOLE_BODY[i * PIECE_SIZE:(i + 1) * PIECE_SIZE])
|
||||
with open(self.sidecar, 'w') as f:
|
||||
json.dump({"total": total, "piece_size": PIECE_SIZE, "ranged": True, "sha256": file_sha256, "done": done}, f)
|
||||
|
||||
def piece_requests(self) -> list[tuple[int, int]]:
|
||||
return [r for r in self.server.request_ranges if r is not None and r != (0, 0)]
|
||||
|
||||
@@ -209,6 +242,17 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
|
||||
def download_bundle(self, source: str):
|
||||
asyncio.run(self.manager._download_bundle(self._bundle, self.dest, source))
|
||||
|
||||
def request_download(self) -> dict:
|
||||
self.make_artifact()
|
||||
self._bundle.ref = "ref"
|
||||
self._bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
|
||||
self.manager.source_models = {"qcom": [self._bundle], "chestnut": []}
|
||||
self.manager.params, store = self._make_params_with_store()
|
||||
store["ModelManager_DownloadRef"] = "ref"
|
||||
with mock.patch.object(manager_module.Paths, 'model_root', return_value=self.dest):
|
||||
self.manager._process_download_requests()
|
||||
return store
|
||||
|
||||
def read_path(self) -> bytes:
|
||||
with open(self.path, 'rb') as f:
|
||||
return f.read()
|
||||
@@ -225,6 +269,7 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
|
||||
|
||||
params.get.side_effect = get
|
||||
params.put.side_effect = put
|
||||
params.remove.side_effect = lambda key: put(key, None)
|
||||
return params, store
|
||||
|
||||
|
||||
@@ -238,6 +283,7 @@ class TestManagerDownload(ManagerDownloadTestBase):
|
||||
assert max(self.reported) <= 99.0, "progress holds below 100 until the hash check"
|
||||
assert artifact.fileName not in self.manager._download_start_times
|
||||
assert artifact.downloadProgress.speed > 0, "a running download shows its speed"
|
||||
assert not os.path.exists(self.sidecar), "the resume state goes once every piece has landed"
|
||||
|
||||
def test_speed_counts_bytes_landed_before_the_first_report(self):
|
||||
self.enterContext(mock.patch.object(manager_module, 'ThreadPoolExecutor', lambda max_workers: InlineExecutor()))
|
||||
@@ -296,9 +342,42 @@ class TestManagerDownload(ManagerDownloadTestBase):
|
||||
|
||||
def test_permanent_failure_is_not_retried(self):
|
||||
self.server.fail_ranges = {piece_range(1): 404}
|
||||
self.enterContext(mock.patch.object(manager_module, 'ThreadPoolExecutor', lambda max_workers: InlineExecutor()))
|
||||
with self.assertRaises(requests.exceptions.HTTPError):
|
||||
self.download_file()
|
||||
self.process_artifact()
|
||||
assert self.piece_requests().count(piece_range(1)) == 1
|
||||
assert os.path.isfile(self.path)
|
||||
assert self.resume_state()["done"] == [i for i in range(NUM_PIECES) if i != 1]
|
||||
|
||||
def test_failed_transfer_resumes_on_retry(self):
|
||||
self.server.fail_times = {piece_range(1): manager_module.PIECE_ATTEMPTS} # the first transfer gives up on piece 1
|
||||
self.server.fail_after_others = True # by then every other piece is saved
|
||||
self.process_artifact()
|
||||
assert self.read_path() == WHOLE_BODY
|
||||
assert self.piece_requests().count(piece_range(1)) == manager_module.PIECE_ATTEMPTS + 1, "the retry fetched piece 1"
|
||||
others = [i for i in range(NUM_PIECES) if i != 1]
|
||||
assert all(self.piece_requests().count(piece_range(i)) == 1 for i in others), "the retry skipped every saved piece"
|
||||
|
||||
def test_lasting_transient_failure_gives_up(self):
|
||||
self.server.fail_ranges = {piece_range(1): 503}
|
||||
with self.assertRaises(requests.exceptions.HTTPError):
|
||||
self.process_artifact()
|
||||
assert self.piece_requests().count(piece_range(1)) == manager_module.PIECE_ATTEMPTS * manager_module.DOWNLOAD_ATTEMPTS
|
||||
|
||||
def test_cancel_ends_retry_backoff(self):
|
||||
self.enterContext(mock.patch.object(manager_module, 'RETRY_BACKOFF', 30.0))
|
||||
self.manager._download_ref = b"ref"
|
||||
|
||||
def fail_then_cancel(*args):
|
||||
self.params.get.return_value = None # the user cancels while the manager waits to retry
|
||||
raise requests.exceptions.ConnectionError("dropped")
|
||||
transfer = self.enterContext(mock.patch.object(self.manager, '_download_file', side_effect=fail_then_cancel))
|
||||
started = time.monotonic()
|
||||
with self.assertRaises(DownloadCancelled):
|
||||
self.process_artifact()
|
||||
assert time.monotonic() - started < 5, "the wait must end at the cancel, not after 30 s"
|
||||
assert transfer.call_count == 1, "no second transfer after a cancel"
|
||||
assert self.artifact.downloadProgress.status != custom.ModelManagerSP.DownloadStatus.failed
|
||||
|
||||
def test_removing_download_ref_cancels(self):
|
||||
stall_event = self.stall(3)
|
||||
@@ -316,6 +395,51 @@ class TestManagerDownload(ManagerDownloadTestBase):
|
||||
assert self.server.stall_released_by_event is True, "the cancel must have caught the piece mid-transfer"
|
||||
end = piece_range(3)[1] + 1
|
||||
assert self.read_path()[end - 100:end] == bytes(100), "the cancelled piece must stop before its last bytes land"
|
||||
done = self.resume_state()["done"]
|
||||
assert done and 3 not in done, "a cancel records the finished pieces, and only those, for resume"
|
||||
|
||||
def test_resume_skips_complete_pieces(self):
|
||||
self.write_resume_state(done=[0, 4])
|
||||
self.download_file()
|
||||
assert piece_range(0) not in self.piece_requests() and piece_range(4) not in self.piece_requests()
|
||||
assert self.read_path() == WHOLE_BODY
|
||||
assert min(self.reported) >= 2 * PIECE_SIZE / len(WHOLE_BODY) * 100 - 1, "progress must not restart below the resumed share"
|
||||
|
||||
def test_unusable_resume_state_starts_over(self):
|
||||
def malformed(content: bytes):
|
||||
self.write_resume_state(done=[0])
|
||||
with open(self.sidecar, 'wb') as f:
|
||||
f.write(content)
|
||||
|
||||
layout = {"total": len(WHOLE_BODY), "piece_size": PIECE_SIZE, "ranged": True, "sha256": sha256(WHOLE_BODY)}
|
||||
cases = {
|
||||
"other size": lambda: self.write_resume_state(done=[0], total=len(WHOLE_BODY) + 1),
|
||||
"other bytes": lambda: self.write_resume_state(done=[0], file_sha256="0" * 64),
|
||||
"truncated file": lambda: self.write_resume_state(done=[0], file_size=PIECE_SIZE),
|
||||
"not an object": lambda: malformed(b'[]'),
|
||||
"cut-off json": lambda: malformed(b'{"total": '),
|
||||
"done not a list": lambda: malformed(json.dumps({**layout, "done": 3}).encode()),
|
||||
"done not ints": lambda: malformed(json.dumps({**layout, "done": ["0", None]}).encode()),
|
||||
}
|
||||
for name, write_state in cases.items():
|
||||
with self.subTest(name):
|
||||
self.server.request_ranges = []
|
||||
write_state()
|
||||
self.download_file()
|
||||
assert sorted(self.piece_requests()) == [piece_range(i) for i in range(NUM_PIECES)], "every piece is fetched again"
|
||||
assert self.read_path() == WHOLE_BODY
|
||||
|
||||
def test_pieces_and_resume_state_are_synced_to_disk(self):
|
||||
real_fsync, synced = os.fsync, []
|
||||
|
||||
def record_sync(fd):
|
||||
st = os.fstat(fd)
|
||||
synced.append("dir" if stat.S_ISDIR(st.st_mode) else "model" if os.path.exists(self.path) and st.st_ino == os.stat(self.path).st_ino else "sidecar")
|
||||
real_fsync(fd)
|
||||
with mock.patch.object(manager_module.os, 'fsync', side_effect=record_sync):
|
||||
self.download_file()
|
||||
assert synced.count("model") == NUM_PIECES, f"expected one fsync per piece, got {synced.count('model')}"
|
||||
assert "sidecar" in synced and "dir" in synced, "sidecar file and directory must be fsync'd"
|
||||
|
||||
def test_replaced_download_ref_queues_instead_of_canceling(self):
|
||||
self.params.get.return_value = b"other-ref"
|
||||
@@ -345,9 +469,26 @@ class TestProcessArtifact(ManagerDownloadTestBase):
|
||||
self.server.corrupt = True
|
||||
with self.assertRaises(ValueError):
|
||||
self.process_artifact()
|
||||
assert not os.path.isfile(self.path)
|
||||
assert not os.path.isfile(self.path) and not os.path.exists(self.sidecar), "a corrupt file must not be resumed from"
|
||||
assert self.artifact.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.failed
|
||||
|
||||
def test_interrupted_download_resumes_without_reverifying(self):
|
||||
self.write_resume_state(done=[0, 1, 2])
|
||||
artifact = self.process_artifact()
|
||||
ds = custom.ModelManagerSP.DownloadStatus
|
||||
assert self.reported_statuses[0] == ds.downloading, "an in-progress download is not verified first"
|
||||
assert piece_range(0) not in self.piece_requests()
|
||||
assert artifact.downloadProgress.status == ds.downloaded
|
||||
assert any(self.reported_resuming), "a resumed download is flagged resuming until its first progress report"
|
||||
assert not artifact.downloadProgress.resuming
|
||||
|
||||
def test_resume_with_every_piece_done_clears_resuming(self):
|
||||
self.write_resume_state(done=list(range(NUM_PIECES)))
|
||||
artifact = self.process_artifact()
|
||||
assert self.piece_requests() == []
|
||||
assert artifact.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.downloaded
|
||||
assert not artifact.downloadProgress.resuming
|
||||
|
||||
def test_probe_failure_moves_past_verifying(self):
|
||||
self.server.fail_ranges = {(0, 0): 404} # the size probe
|
||||
with self.assertRaises(requests.exceptions.HTTPError):
|
||||
@@ -374,6 +515,23 @@ class TestProcessArtifact(ManagerDownloadTestBase):
|
||||
self.download_bundle("qcom")
|
||||
assert ACTIVE_BUNDLE_KEYS["qcom"] not in store
|
||||
assert os.path.isfile(self.path), "cancel must not delete a cached model"
|
||||
assert self.manager.selected_bundle.status != custom.ModelManagerSP.DownloadStatus.failed, "a cancel is not a failure"
|
||||
|
||||
def test_failed_download_stays_on_the_row(self):
|
||||
self.server.fail_ranges = {piece_range(1): 404}
|
||||
store = self.request_download()
|
||||
selected = self.manager.selected_bundle
|
||||
assert selected is not None and selected.status == custom.ModelManagerSP.DownloadStatus.failed, "a failure stays on the row"
|
||||
assert store["ModelManager_DownloadRef"] is None, "the failed request is released"
|
||||
|
||||
def test_finished_download_clears_the_row(self):
|
||||
self.request_download()
|
||||
assert self.manager.selected_bundle is None
|
||||
|
||||
def test_cancelled_download_clears_the_row(self):
|
||||
self.pm.send.side_effect = lambda *args: self.manager.params.remove("ModelManager_DownloadRef") # cancel at the first report
|
||||
self.request_download()
|
||||
assert self.manager.selected_bundle is None
|
||||
|
||||
def test_download_ref_is_released_only_if_unchanged(self):
|
||||
for current, removed in ((b"new-ref", False), (b"ref", True)):
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""
|
||||
Copyright (c) 2021-, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
import numpy as np
|
||||
|
||||
from openpilot.cereal import log
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.realtime import DT_MDL
|
||||
from openpilot.sunnypilot import PARAMS_UPDATE_PERIOD
|
||||
|
||||
ACCEL_BOOST_MIN_SPEED = 1.0
|
||||
ACCEL_BOOST_MAX = (0.4, 0.25, 0.1) # aggressive, standard, relaxed, m/s^2
|
||||
ACCEL_BOOST_BUILD, ACCEL_BOOST_RELEASE = 0.2, 1.0 # m/s^3
|
||||
BRAKE_PROB = (0.1, 0.3) # the model's brake press in 4 s: it wants to slow
|
||||
GAS_PROB = (0.6, 0.8) # predicts a human gas press at this hypothesis at about 23%
|
||||
CURVE_LAT_ACCEL = (1.5, 2.5)
|
||||
|
||||
|
||||
def get_boost_max(personality=log.LongitudinalPersonality.standard):
|
||||
if personality == log.LongitudinalPersonality.relaxed:
|
||||
return ACCEL_BOOST_MAX[2]
|
||||
elif personality == log.LongitudinalPersonality.standard:
|
||||
return ACCEL_BOOST_MAX[1]
|
||||
elif personality == log.LongitudinalPersonality.aggressive:
|
||||
return ACCEL_BOOST_MAX[0]
|
||||
else:
|
||||
raise NotImplementedError("Longitudinal personality not supported")
|
||||
|
||||
|
||||
class AccelBoostSP:
|
||||
def __init__(self):
|
||||
self.params = Params()
|
||||
self.frame = -1
|
||||
self.enabled = False
|
||||
self.boost = 0.0
|
||||
self.total_boost = 0.0
|
||||
|
||||
def update_params(self):
|
||||
self.frame += 1
|
||||
if self.frame % int(PARAMS_UPDATE_PERIOD / DT_MDL) != 0:
|
||||
return False
|
||||
enabled = self.params.get_bool("AccelBoostSP")
|
||||
if enabled == self.enabled:
|
||||
return False
|
||||
self.enabled = enabled
|
||||
self.boost = self.total_boost = 0.0
|
||||
return True
|
||||
|
||||
def update(self, sm, output_a_target_e2e, output_a_target_mpc, a_cruise):
|
||||
model = sm['modelV2']
|
||||
brake_probs = model.meta.disengagePredictions.brakePressProbs
|
||||
gas_probs = model.meta.disengagePredictions.gasPressProbs
|
||||
if not sm['selfdriveState'].enabled or len(brake_probs) < 2 or len(gas_probs) < 2:
|
||||
self.boost = self.total_boost = 0.0
|
||||
return output_a_target_e2e
|
||||
|
||||
want = max(float(np.interp(brake_probs[1], BRAKE_PROB, [1.0, 0.0])), float(np.interp(gas_probs[1], GAS_PROB, [0.0, 1.0])))
|
||||
want = 0.0 if model.action.shouldStop else want
|
||||
want *= float(np.interp(abs(model.action.desiredCurvature) * sm['carState'].vEgo ** 2, CURVE_LAT_ACCEL, [1.0, 0.0]))
|
||||
target = want * get_boost_max(sm['selfdriveState'].personality)
|
||||
self.boost = float(np.clip(target, self.boost - ACCEL_BOOST_RELEASE * DT_MDL, self.boost + ACCEL_BOOST_BUILD * DT_MDL))
|
||||
speed_scale = np.interp(sm['carState'].vEgo, [ACCEL_BOOST_MIN_SPEED, 2 * ACCEL_BOOST_MIN_SPEED], [0.0, 1.0])
|
||||
self.total_boost = self.boost * speed_scale * np.interp(output_a_target_e2e, [-0.7, -0.2, 5.0], [0.0, 1.0, 1.0], right=0.0)
|
||||
return output_a_target_e2e + self.total_boost
|
||||
@@ -9,6 +9,8 @@ from openpilot.cereal import messaging, custom
|
||||
from opendbc.car import structs
|
||||
from openpilot.common.constants import CV
|
||||
from openpilot.selfdrive.car.cruise import V_CRUISE_MAX
|
||||
from openpilot.selfdrive.controls.lib.accel_boost import AccelBoost
|
||||
from openpilot.sunnypilot.selfdrive.controls.lib.accel_boost import AccelBoostSP
|
||||
from openpilot.sunnypilot.selfdrive.controls.lib.e2e_alerts_helper import E2EAlertsHelper
|
||||
from openpilot.sunnypilot.selfdrive.controls.lib.smart_cruise_control.smart_cruise_control import SmartCruiseControl
|
||||
from openpilot.sunnypilot.selfdrive.controls.lib.speed_limit.speed_limit_assist import SpeedLimitAssist
|
||||
@@ -29,6 +31,8 @@ class LongitudinalPlannerSP:
|
||||
self.generation = int(model_bundle.generation) if (model_bundle := get_active_bundle()) else None
|
||||
self.source = LongitudinalPlanSource.cruise
|
||||
self.e2e_alerts_helper = E2EAlertsHelper()
|
||||
self.accel_boost_sp = AccelBoostSP()
|
||||
self.openpilot_longitudinal = CP.openpilotLongitudinalControl
|
||||
|
||||
self.output_v_target = 0.
|
||||
self.output_a_target = 0.
|
||||
@@ -64,6 +68,9 @@ class LongitudinalPlannerSP:
|
||||
return self.output_v_target, self.output_a_target
|
||||
|
||||
def update(self, sm: messaging.SubMaster) -> None:
|
||||
if self.accel_boost_sp.update_params():
|
||||
self.accel_boost = self.accel_boost_sp if self.accel_boost_sp.enabled and self.openpilot_longitudinal else AccelBoost()
|
||||
|
||||
self.events_sp.clear()
|
||||
self.e2e_alerts_helper.update(sm, self.events_sp)
|
||||
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""
|
||||
Copyright (c) 2021-, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from openpilot.common.parameterized import parameterized
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.realtime import DT_MDL
|
||||
from openpilot.common.test import OpenpilotTestCase
|
||||
|
||||
from openpilot.sunnypilot import PARAMS_UPDATE_PERIOD
|
||||
from openpilot.sunnypilot.selfdrive.controls.lib.accel_boost import AccelBoostSP, ACCEL_BOOST_MAX
|
||||
|
||||
|
||||
def make_sm(brake_prob=0.05, gas_prob=0.3, lat_accel=0.0, v_ego=30.0, personality=0, enabled=True, should_stop=False):
|
||||
meta = SimpleNamespace(disengagePredictions=SimpleNamespace(brakePressProbs=[0.0, brake_prob, 0.0, 0.0, 0.0, 0.0],
|
||||
gasPressProbs=[0.0, gas_prob, 0.0, 0.0, 0.0, 0.0]))
|
||||
action = SimpleNamespace(shouldStop=should_stop, desiredCurvature=lat_accel / v_ego ** 2)
|
||||
return {'modelV2': SimpleNamespace(meta=meta, action=action), 'carState': SimpleNamespace(vEgo=v_ego),
|
||||
'selfdriveState': SimpleNamespace(enabled=enabled, personality=personality)}
|
||||
|
||||
|
||||
def settle(sm, frames=100):
|
||||
boost = AccelBoostSP()
|
||||
for _ in range(frames):
|
||||
boost.update(sm, 0.0, 9.0, 9.0)
|
||||
return boost.total_boost
|
||||
|
||||
|
||||
class TestAccelBoostSP(OpenpilotTestCase):
|
||||
@parameterized.expand([(0,), (1,), (2,)])
|
||||
def test_personality_max(self, personality):
|
||||
assert abs(settle(make_sm(personality=personality)) - ACCEL_BOOST_MAX[personality]) < 1e-6
|
||||
|
||||
def test_brake_gate_closes(self):
|
||||
assert settle(make_sm(brake_prob=0.5)) == 0.0
|
||||
|
||||
def test_gas_reopens(self):
|
||||
assert abs(settle(make_sm(brake_prob=0.5, gas_prob=0.9)) - ACCEL_BOOST_MAX[0]) < 1e-6
|
||||
|
||||
def test_should_stop(self):
|
||||
assert settle(make_sm(gas_prob=0.9, should_stop=True)) == 0.0
|
||||
|
||||
@parameterized.expand([(1.5, 1.0), (2.0, 0.5), (2.5, 0.0), (-2.0, 0.5)])
|
||||
def test_curve_fade(self, lat_accel, scale):
|
||||
assert abs(settle(make_sm(lat_accel=lat_accel)) - scale * ACCEL_BOOST_MAX[0]) < 1e-6
|
||||
|
||||
def test_low_speed(self):
|
||||
assert settle(make_sm(v_ego=0.5)) == 0.0
|
||||
|
||||
def test_disabled_resets(self):
|
||||
boost = AccelBoostSP()
|
||||
for _ in range(100):
|
||||
boost.update(make_sm(), 0.0, 9.0, 9.0)
|
||||
assert boost.update(make_sm(enabled=False), 0.0, 9.0, 9.0) == 0.0
|
||||
assert boost.boost == 0.0 and boost.total_boost == 0.0
|
||||
|
||||
def test_build_and_release_rates(self):
|
||||
boost = AccelBoostSP()
|
||||
boost.update(make_sm(), 0.0, 9.0, 9.0)
|
||||
assert abs(boost.total_boost - 0.01) < 1e-6
|
||||
for _ in range(100):
|
||||
boost.update(make_sm(), 0.0, 9.0, 9.0)
|
||||
boost.update(make_sm(brake_prob=0.5), 0.0, 9.0, 9.0)
|
||||
assert abs(boost.total_boost - (ACCEL_BOOST_MAX[0] - 0.05)) < 1e-6
|
||||
|
||||
def test_param_switch(self):
|
||||
params = Params()
|
||||
boost = AccelBoostSP()
|
||||
params.put_bool("AccelBoostSP", True, block=True)
|
||||
assert boost.update_params() and boost.enabled
|
||||
for _ in range(100):
|
||||
boost.update(make_sm(), 0.0, 9.0, 9.0)
|
||||
params.put_bool("AccelBoostSP", False, block=True)
|
||||
for _ in range(int(PARAMS_UPDATE_PERIOD / DT_MDL) - 1):
|
||||
assert not boost.update_params()
|
||||
assert boost.update_params() and not boost.enabled
|
||||
assert boost.boost == 0.0 and boost.total_boost == 0.0
|
||||
@@ -620,6 +620,19 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "AccelBoostSP",
|
||||
"widget": "toggle",
|
||||
"title": "Acceleration Boost",
|
||||
"description": "Adds a small push on top of sunnypilot's own acceleration, stronger in Aggressive and gentler in Relaxed. Eases off in curves and when sunnypilot slows or stops. Never goes past your set speed, and lead-car following still comes first.",
|
||||
"enablement": [
|
||||
{
|
||||
"type": "capability",
|
||||
"field": "has_longitudinal_control",
|
||||
"equals": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "IntelligentCruiseButtonManagement",
|
||||
"widget": "toggle",
|
||||
@@ -1349,7 +1362,17 @@
|
||||
"key": "TorqueBar",
|
||||
"widget": "toggle",
|
||||
"title": "Steering Arc",
|
||||
"description": "Display steering arc on the driving screen when lateral control is enabled."
|
||||
"description": "Display steering arc on the driving screen when lateral control is enabled.",
|
||||
"visibility": [
|
||||
{
|
||||
"type": "not",
|
||||
"condition": {
|
||||
"type": "capability",
|
||||
"field": "device_type",
|
||||
"equals": "mici"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "ShowTurnSignals",
|
||||
@@ -1389,12 +1412,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "TripSummary",
|
||||
"widget": "toggle",
|
||||
"title": "Trip Summary",
|
||||
"description": "Show a summary of distance and engagement when you shift into park after a drive."
|
||||
},
|
||||
{
|
||||
"key": "RocketFuel",
|
||||
"widget": "toggle",
|
||||
|
||||
@@ -30,6 +30,12 @@ sections:
|
||||
label: Relaxed
|
||||
enablement:
|
||||
- $ref: '#/macros/longitudinal'
|
||||
- key: AccelBoostSP
|
||||
widget: toggle
|
||||
title: Acceleration Boost
|
||||
description: "Adds a small push on top of sunnypilot's own acceleration, stronger in Aggressive and gentler in Relaxed. Eases off in curves and when sunnypilot slows or stops. Never goes past your set speed, and lead-car following still comes first."
|
||||
enablement:
|
||||
- $ref: '#/macros/longitudinal'
|
||||
- key: IntelligentCruiseButtonManagement
|
||||
widget: toggle
|
||||
title: Intelligent Cruise Button Management (ICBM) (Alpha)
|
||||
|
||||
@@ -20,6 +20,8 @@ sections:
|
||||
widget: toggle
|
||||
title: Steering Arc
|
||||
description: Display steering arc on the driving screen when lateral control is enabled.
|
||||
visibility:
|
||||
- $ref: '#/macros/hide_on_mici'
|
||||
- key: ShowTurnSignals
|
||||
widget: toggle
|
||||
title: Display Turn Signals
|
||||
@@ -37,10 +39,6 @@ sections:
|
||||
description: Show a timer on the HUD when the car is at a standstill.
|
||||
visibility:
|
||||
- $ref: '#/macros/hide_on_mici'
|
||||
- key: TripSummary
|
||||
widget: toggle
|
||||
title: Trip Summary
|
||||
description: Show a summary of distance and engagement when you shift into park after a drive.
|
||||
- key: RocketFuel
|
||||
widget: toggle
|
||||
title: Real-time Acceleration Bar
|
||||
|
||||
@@ -19,10 +19,9 @@ from openpilot.tools.lib.route import Route
|
||||
REPO_ID = os.environ.get("SUNNYPILOT_CAR_SEGMENTS_REPO_ID", "sunnypilot/sunnypilotCarSegments")
|
||||
|
||||
|
||||
def upload_route(route_name: str, dry_run: bool = False) -> None:
|
||||
route = Route(route_name)
|
||||
log_paths = route.log_paths()
|
||||
valid_segments = [(i, url) for i, url in enumerate(log_paths) if url is not None]
|
||||
def upload_route(route_name: str, dry_run: bool = False, local_dir: str | None = None) -> None:
|
||||
log_paths = Route(route_name, data_dir=local_dir).log_paths()
|
||||
valid_segments = [(i, path) for i, path in enumerate(log_paths) if path is not None]
|
||||
|
||||
print(f"Route: {route_name}")
|
||||
print(f"Segments: {len(valid_segments)}/{len(log_paths)}")
|
||||
@@ -34,14 +33,17 @@ def upload_route(route_name: str, dry_run: bool = False) -> None:
|
||||
api = HfApi()
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
for seg_idx, url in tqdm(valid_segments, desc="Uploading"):
|
||||
filename = url.split("?")[0].rsplit("/", 1)[-1]
|
||||
local_path = os.path.join(tmpdir, f"{seg_idx}_{filename}")
|
||||
resp = requests.get(url, stream=True)
|
||||
resp.raise_for_status()
|
||||
with open(local_path, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
for seg_idx, src in tqdm(valid_segments, desc="Uploading"):
|
||||
filename = src.split("?")[0].rsplit("/", 1)[-1]
|
||||
if local_dir is not None:
|
||||
local_path = src
|
||||
else:
|
||||
local_path = os.path.join(tmpdir, f"{seg_idx}_{filename}")
|
||||
resp = requests.get(src, stream=True)
|
||||
resp.raise_for_status()
|
||||
with open(local_path, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
repo_path = f"segments/{route_name.replace('|', '/')}/{seg_idx}/{filename}"
|
||||
|
||||
@@ -63,6 +65,7 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Upload route rlogs to sunnypilot HuggingFace dataset")
|
||||
parser.add_argument("route", help="Route ID (e.g. 5beb9b58bd12b691/0000010a--a51155e496)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Download and show sizes without uploading")
|
||||
parser.add_argument("--local", metavar="DIR", help="Upload rlogs from DIR (<dongle id>|<log id>--<segment>/) instead of the comma API")
|
||||
args = parser.parse_args()
|
||||
|
||||
upload_route(args.route, dry_run=args.dry_run)
|
||||
upload_route(args.route, dry_run=args.dry_run, local_dir=args.local)
|
||||
|
||||
@@ -13,26 +13,41 @@ from pathlib import Path
|
||||
from datetime import datetime, UTC
|
||||
|
||||
|
||||
_VERSION_REGEX = re.compile(r'^(?:\d+[a-zA-Z]+|\d+[vVbB]\d+|[vVbB]\d+|\d{4})$')
|
||||
_DECIMAL_VERSION_REGEX = re.compile(r'^([vVbB]?)(\d+(?:\.\d+)+)$')
|
||||
|
||||
|
||||
def _version(word: str) -> str | None:
|
||||
match = _DECIMAL_VERSION_REGEX.match(word.strip('()[]{}.,;:'))
|
||||
if match:
|
||||
return match.group(1).upper() + match.group(2)
|
||||
stripped = re.sub(r'[^a-zA-Z0-9]', '', word)
|
||||
return stripped.upper() if _VERSION_REGEX.match(stripped) else None
|
||||
|
||||
|
||||
def _truncate(short_name: str) -> str:
|
||||
return short_name[:8].rstrip('.')
|
||||
|
||||
|
||||
def create_short_name(full_name: str) -> str:
|
||||
# Remove parentheses and extract alphanumeric words
|
||||
# Remove parentheses and extract alphanumeric words, keeping decimals for versioning
|
||||
clean_name = re.sub(r'\([^)]*\)', '', full_name)
|
||||
words = [re.sub(r'[^a-zA-Z0-9]', '', word) for word in clean_name.split() if re.sub(r'[^a-zA-Z0-9]', '', word)]
|
||||
words = [re.sub(r'[^a-zA-Z0-9.]', '', word) for word in clean_name.split()]
|
||||
words = [word for word in words if word]
|
||||
|
||||
if len(words) == 1:
|
||||
return words[0][:8].upper()
|
||||
return _truncate((_version(words[0]) or words[0]).upper())
|
||||
|
||||
# Handle special case: Name + Version (e.g., "Word A1" -> "WordA1")
|
||||
if len(words) == 2 and re.match(r'^[A-Za-z]\d+$', words[1]):
|
||||
return (words[0] + words[1])[:8].upper()
|
||||
# Handle special case: Name + Version (e.g., "Word A1" -> "WordA1", "Model v1.1" -> "ModelV1.1")
|
||||
if len(words) == 2 and re.match(r'^[A-Za-z]?\d+$', words[1]):
|
||||
return _truncate((words[0] + (_version(words[1]) or words[1])).upper())
|
||||
|
||||
result = ""
|
||||
for word in words:
|
||||
# Version or number patterns
|
||||
if (re.match(r'^\d+[a-zA-Z]+$', word) or
|
||||
re.match(r'^\d+[vVbB]\d+$', word) or
|
||||
re.match(r'^[vVbB]\d+$', word) or
|
||||
re.match(r'^\d{4}$', word)):
|
||||
result += word.upper()
|
||||
version = _version(word)
|
||||
if version:
|
||||
result += version
|
||||
# All uppercase abbreviations (2-3 letters)
|
||||
elif re.match(r'^[A-Z]{2,3}$', word):
|
||||
result += word
|
||||
@@ -45,7 +60,7 @@ def create_short_name(full_name: str) -> str:
|
||||
result += word
|
||||
else:
|
||||
result += word[0].upper()
|
||||
return result[:8]
|
||||
return _truncate(result)
|
||||
|
||||
|
||||
def create_pkl_name(full_name: str) -> str:
|
||||
|
||||
+1
-1
Submodule tinygrad_repo updated: 92b742f711...301a0d1ded
Reference in New Issue
Block a user