Compare commits

..

15 Commits

Author SHA1 Message Date
rav4kumar 7feeda0ca4 ramp after stop 2026-11-08 14:10:02 -07:00
rav4kumar 6443f0f266 fede with speed 2026-11-07 10:49:54 -07:00
rav4kumar 4629ac0a8e ref 2026-11-05 13:31:28 -07:00
rav4kumar 13838673e2 rate based final stop 2026-11-05 13:31:28 -07:00
rav4kumar 696c223033 trip-summary 2026-11-05 11:28:47 -07:00
rav4kumar 1b0e466e2f boooooost-ui 2026-11-05 11:28:31 -07:00
Kumar 5958b2d25b Merge branch 'master' into toyota-bsm 2026-11-05 11:26:28 -07:00
Kumar 73c8b30fa9 Merge branch 'master' into toyota-bsm 2026-11-04 12:14:55 -07:00
rav4kumar 8bdb2f54d9 Merge branch 'master' into toyota-bsm 2026-11-03 10:37:16 -07:00
rav4kumar 9c1bdde65b toyota: enable the enhanced BSM toggle for TSS-P cars with a smartDSU 2026-11-03 10:31:07 -07:00
Kumar 914aaefac1 Merge branch 'master' into toyota-bsm 2026-10-26 11:35:14 -07:00
rav4kumar d05649c0a4 toyota: bump opendbc and fix BSM wiring 2026-10-26 11:32:51 -07:00
rav4kumar 3521140920 ref 2026-10-25 11:56:26 -07:00
rav4kumar 56628fc770 ref 2026-10-25 11:55:52 -07:00
rav4kumar 60a9cc8657 toyota: add Enhanced Blind Spot Monitoring setting 2026-10-25 11:55:32 -07:00
82 changed files with 873 additions and 4283 deletions
-5
View File
@@ -14,8 +14,3 @@ openpilot/sunnypilot/modeld_v2/models/*.pkl filter=lfs diff=lfs merge=lfs -text
openpilot/selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
openpilot/common/hardware/comma/updater filter=lfs diff=lfs merge=lfs -text
# Achievements icons are kept out of LFS on this dev branch (no push access to the LFS store).
# Move them into LFS before this goes upstream.
openpilot/sunnypilot/selfdrive/assets/icons_mici/achievements.png -filter binary
openpilot/sunnypilot/selfdrive/assets/offroad/icon_achievements.png -filter binary
@@ -34,11 +34,6 @@ on:
required: false
type: string
default: 'qcom'
on_device:
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
required: false
type: boolean
default: false
hf_repo:
description: 'Hugging Face dataset repository (e.g. sunnypilot/sunnypilot_models_v1)'
required: false
@@ -112,11 +107,6 @@ on:
options:
- qcom
- chestnut
on_device:
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
required: false
type: boolean
default: false
hf_repo:
description: 'Hugging Face dataset repository'
required: false
@@ -140,7 +130,6 @@ jobs:
is_20hz: ${{ inputs.is_20hz }}
artifact_suffix: ${{ inputs.artifact_suffix }}
target_hardware: ${{ inputs.target_hardware }}
on_device: ${{ inputs.on_device || false }}
secrets: inherit
publish_model:
@@ -1,61 +0,0 @@
name: Chestnut mock build parity
description: >-
Convert one commaai precompiled chestnut model (LS2) on this runner under tinygrad's amd mock, as a hosted chestnut build does, and require the
exact bytes the chestnut runner built (the published artifact). Needs setup-amd-mock, numpy and zstandard; tinygrad_repo checked out.
runs:
using: composite
steps:
- name: Key this check by everything its result depends on
id: key
shell: bash
env:
SOURCES: ${{ hashFiles('tinygrad_repo/tinygrad/**/*.py', 'tinygrad_repo/test/mockgpu/**', 'openpilot/sunnypilot/modeld_v2/*.py', '.github/workflows/chestnut-build-parity/action.yml') }}
run: echo "key=chestnut-build-parity-$SOURCES-$(sha256sum "$LLVM_PATH" | cut -c1-16)-$(python3 -c 'import platform; print(platform.python_version())')" >> "$GITHUB_OUTPUT"
- name: Passed before with all of it identical?
id: passed
uses: actions/cache/restore@v4
with:
path: ~/.chestnut_build_parity.passed
key: ${{ steps.key.outputs.key }}
- name: Rebuild LS2 on the mock and require the chestnut runner's artifact byte for byte
if: steps.passed.outputs.cache-hit != 'true'
shell: bash
env:
REF: b92fc4828664f8767bce5e92f1630ed319cbe509 # commaai/openpilot: precompiled big_driving_tinygrad.pkl, pins tinygrad ec91b74b5b
PKL_OID: b4dead3ee091d66ab002a6b8e716c26f4f79e3c931e4be5f48e017ba92f6e960
PKL_SIZE: "767228907"
TG_SHA: ec91b74b5b6b0fc924b72d3f27689679ca2438b9
WANT: 680a8d629785f2b1d1a2b0e40d0dfbcd0f7291c21e20c51259f1d768a22ec217 # driving_light_speed_2_november_04_2026_tinygrad.pkl, recompiled46
run: |
set -euo pipefail
work="$RUNNER_TEMP/chestnut_parity" && rm -rf "$work" && mkdir -p "$work"
href=$(curl -fsSL --retry 3 -X POST -H "Accept: application/vnd.git-lfs+json" -H "Content-Type: application/vnd.git-lfs+json" \
https://huggingface.co/commaai/openpilot-lfs.git/info/lfs/objects/batch \
-d "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"$PKL_OID\",\"size\":$PKL_SIZE}]}" | jq -r '.objects[0].actions.download.href')
curl -fsSL --retry 3 -o "$work/big_driving_tinygrad.pkl" "$href"
echo "$PKL_OID $work/big_driving_tinygrad.pkl" | sha256sum -c -
tg="$work/tinygrad" && git init -q "$tg" && git -C "$tg" fetch -q --depth 1 https://github.com/tinygrad/tinygrad "$TG_SHA" && git -C "$tg" checkout -q FETCH_HEAD
CHECK_OOB=0 GMMU=0 PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo" python3 tinygrad_repo/test/mockgpu/usb_server.py \
env DEV=USB+AMD PYTHONPATH="$tg:$GITHUB_WORKSPACE" python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py "$work/big_driving_tinygrad.pkl" "$work/LS2.pkl"
got=$(sha256sum "$work/LS2.pkl" | cut -d' ' -f1)
if [ "$got" != "$WANT" ]; then
echo "::error::the mock's LS2 build ($got) is not the chestnut runner's ($WANT): this runner's LLVM or the mock no longer builds what the chestnut does"
exit 1
fi
echo "LS2 rebuilt on the mock: byte-identical to the chestnut runner's build"
touch ~/.chestnut_build_parity.passed
- name: Remember the pass
if: steps.passed.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: ~/.chestnut_build_parity.passed
key: ${{ steps.key.outputs.key }}
- name: Already passed
if: steps.passed.outputs.cache-hit == 'true'
shell: bash
run: |
echo "LS2 rebuild skipped: it already matched the chestnut runner's build with this exact mock, converter, LLVM and python"
@@ -1,60 +0,0 @@
name: Set up tinygrad's amd mock
description: LLVM 20 and ROCm comgr, as tinygrad_repo/.github/actions/setup-tinygrad installs them for the mock gpu
inputs:
emu_cache:
description: >-
Keep tinygrad's cache db (the emulator's compiled kernels and gate results) between runs under this name, e.g. the model's file name.
Empty: no cache.
default: ''
runs:
using: composite
steps:
- name: Install LLVM 20 and ROCm comgr
shell: bash
run: |
. /etc/os-release
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
echo "deb http://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.1 $VERSION_CODENAME main" | sudo tee /etc/apt/sources.list.d/rocm.list
printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/rocm-pin-600
sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends libllvm20 clang-20 lld-20 comgr
printf '/opt/rocm/lib\n/opt/rocm/lib64\n' | sudo tee /etc/ld.so.conf.d/rocm.conf && sudo ldconfig
echo "/usr/lib/llvm-20/bin" >> "$GITHUB_PATH"
llvm=$(ls /usr/lib/llvm-20/lib/libLLVM*.so* /usr/lib/*-linux-gnu/libLLVM-20.so* 2>/dev/null | head -1 || true)
[ -n "$llvm" ] || { echo "::error::libLLVM 20 not found"; exit 1; }
echo "LLVM_PATH=$llvm" >> "$GITHUB_ENV"
- name: Cache the chestnut's firmware
uses: actions/cache@v4
with:
path: ~/.cache/tinygrad/downloads/fw
key: chestnut-fw-${{ hashFiles('tinygrad_repo/tinygrad/runtime/autogen/am/fw.py') }}
- name: Fetch the chestnut's firmware (gitlab times out now and then; the mock boots from it)
shell: bash
run: |
for i in 1 2 3 4 5; do
PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo" python3 -c "
from tinygrad.helpers import fetch_fw
from tinygrad.runtime.autogen.am import fw
for n, h in fw.hashes.items():
if n.startswith(('psp_14_0_2_', 'smu_14_0_2.', 'sdma_7_0_0.', 'gc_12_0_0_')): fetch_fw('amdgpu', n, h)" && exit 0
sleep $((i * 10))
done
echo "::error::could not fetch the chestnut's firmware from gitlab"; exit 1
- name: runner cpu
if: inputs.emu_cache != ''
id: cpu
shell: bash
run: echo "cpu=$( (grep -m1 '^model name' /proc/cpuinfo; grep -m1 '^flags' /proc/cpuinfo) | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
- name: Keep the emulator's compiled kernels between runs
if: inputs.emu_cache != ''
uses: actions/cache@v4
with:
path: ~/.cache/tinygrad/cache.db
key: emu-${{ steps.cpu.outputs.cpu }}-${{ hashFiles('tinygrad_repo/tinygrad/**/*.py', 'tinygrad_repo/test/mockgpu/**') }}-${{ inputs.emu_cache }}
restore-keys: emu-${{ steps.cpu.outputs.cpu }}-${{ hashFiles('tinygrad_repo/tinygrad/**/*.py', 'tinygrad_repo/test/mockgpu/**') }}-
+12 -88
View File
@@ -35,11 +35,6 @@ on:
required: false
type: string
default: 'qcom'
on_device:
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
required: false
type: boolean
default: false
workflow_dispatch:
inputs:
upstream_branch:
@@ -64,11 +59,6 @@ on:
- qcom
- chestnut
default: 'qcom'
on_device:
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
required: false
type: boolean
default: false
run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
@@ -163,67 +153,18 @@ jobs:
if-no-files-found: error
build_model:
runs-on: ${{ inputs.on_device && fromJSON(inputs.target_hardware == 'chestnut' && '["self-hosted","chestnut"]' || '["self-hosted","tici"]') || 'ubuntu-24.04' }}
runs-on: [self-hosted, "${{ inputs.target_hardware == 'chestnut' && 'chestnut' || 'tici' }}"]
needs: get_model
env:
MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
REF: ${{ inputs.upstream_branch }}
MOCK_BUILD: ${{ !inputs.on_device }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1
submodules: recursive
- name: Set up the hosted mock build
if: env.MOCK_BUILD == 'true'
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install LLVM 20 and ROCm comgr for the amd mock
if: env.MOCK_BUILD == 'true' && inputs.target_hardware == 'chestnut'
uses: ./.github/workflows/setup-amd-mock
with:
emu_cache: build
- name: Install the amd mock build dependencies
if: env.MOCK_BUILD == 'true' && inputs.target_hardware == 'chestnut'
run: |
python3 -m pip install -q numpy zstandard
- name: Install the qcom mock build dependencies
if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut'
id: qcom_deps
run: |
sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends qemu-user-static aria2
python3 -m pip install -q numpy zstandard tinymesa==25.2.7.2
echo "qemu=$(dpkg-query -W -f='${Version}' qemu-user-static)" >> "$GITHUB_OUTPUT"
- name: Compiler check passed before with all of it identical?
if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut'
id: qcom_compiler
uses: actions/cache/restore@v4
with:
path: ~/.qcom_compiler_parity.passed
key: qcom-compiler-parity-${{ steps.qcom_deps.outputs.qemu }}-${{ hashFiles('tinygrad_repo/test/device/qcom/compiler_parity.py', 'tinygrad_repo/test/mockgpu/qcom/a630_compiled.pkl.xz', 'tinygrad_repo/tinygrad/runtime/support/compiler_qcom.py') }}
- name: Check the off-device qcom compiler builds the a630's bytes
if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut' && steps.qcom_compiler.outputs.cache-hit != 'true'
run: |
# every kernel an a630 has compiled must compile to the same bytes here before a model is built here
PYTHONPATH="$TINYGRAD_PATH" python3 "$TINYGRAD_PATH/test/device/qcom/compiler_parity.py" check "$TINYGRAD_PATH/test/mockgpu/qcom/a630_compiled.pkl.xz"
touch ~/.qcom_compiler_parity.passed
- name: Remember the compiler check passed
if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut' && steps.qcom_compiler.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: ~/.qcom_compiler_parity.passed
key: ${{ steps.qcom_compiler.outputs.cache-primary-key }}
- name: Set environment variables
if: env.MOCK_BUILD != 'true'
id: set-env
run: |
# Set up common environment
@@ -236,7 +177,6 @@ jobs:
printenv >> $GITHUB_ENV
- name: Setup build environment
if: env.MOCK_BUILD != 'true'
run: |
mkdir -p "${BUILD_DIR}/"
sudo find $BUILD_DIR/ -mindepth 1 -delete
@@ -258,28 +198,21 @@ jobs:
- name: Build Model
run: |
if [ "$MOCK_BUILD" = "true" ]; then
TASKSET=""
export CHECK_OOB=0 GMMU=0 # as test/mockgpu runs the amd mock
else
source /etc/profile
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
TASKSET="taskset -c 7"
fi
source /etc/profile
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
COMPILE_MODELD="${{ github.workspace }}/openpilot/sunnypilot/modeld_v2/compile_modeld.py"
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
MOCK=$([ "$MOCK_BUILD" = "true" ] && echo MOCK || true)
TG_FLAGS_QCOM="DEV=${MOCK:+MOCK+}QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 OPENPILOT_HACKS=1"
TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 OPENPILOT_HACKS=1"
if [ "${{ inputs.target_hardware }}" == "chestnut" ]; then
echo "CHESTNUT build"
export CHESTNUT=1
TG_FLAGS="DEBUG=1 DEV=${MOCK}USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
else
echo "QCOM build"
@@ -294,16 +227,14 @@ jobs:
echo "Converting commaai's precompiled model with tinygrad $TG_SHA"
UPSTREAM_TG="${{ runner.temp }}/upstream_tinygrad"
rm -rf "$UPSTREAM_TG" && git init -q "$UPSTREAM_TG" && git -C "$UPSTREAM_TG" fetch -q --depth 1 "$TG_URL" "$TG_SHA" && git -C "$UPSTREAM_TG" checkout -q FETCH_HEAD
ON_MOCK=""
if [ "$MOCK_BUILD" = "true" ]; then ON_MOCK="env PYTHONPATH=$TINYGRAD_PATH python3 $TINYGRAD_PATH/test/mockgpu/usb_server.py"; fi
CONVERTED="${{ runner.temp }}/big_driving_tinygrad_plain.pkl"
$ON_MOCK env DEV=USB+AMD PYTHONPATH="$UPSTREAM_TG:${{ github.workspace }}" $TASKSET python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py \
env DEV=USB+AMD PYTHONPATH="$UPSTREAM_TG:${{ github.workspace }}" taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py \
"${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" "$CONVERTED"
mv "$CONVERTED" "$OUTPUT_PKL"
elif NEW_ARCH_ONNX=$(find "${{ env.MODELS_DIR }}" -maxdepth 1 -name "${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}*driving*.onnx" -print -quit) &&
[ -n "$NEW_ARCH_ONNX" ] && grep -qa new_img "$NEW_ARCH_ONNX"; then # the new architecture: one ONNX taking the warped frames as new_img
echo "Compiling new architecture ONNX: $NEW_ARCH_ONNX"
env ${TG_FLAGS} $TASKSET python3 openpilot/sunnypilot/modeld_v2/compile_modeld_v2.py \
env ${TG_FLAGS} taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/compile_modeld_v2.py \
"$NEW_ARCH_ONNX" "$OUTPUT_PKL" --device-input "*" --benchmark-runs 1
else
echo "Detected onnx architecture. Routing to native compile_modeld.py."
@@ -353,7 +284,7 @@ jobs:
if [ -n "$MODEL_TYPE" ]; then
echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL"
env ${TG_FLAGS} $TASKSET python3 "$COMPILE_MODELD" \
env ${TG_FLAGS} taskset -c 7 python3 "$COMPILE_MODELD" \
--model-type $MODEL_TYPE \
--model-size $MODEL_SIZE \
--camera-resolutions $CAMERA_RES \
@@ -362,13 +293,6 @@ jobs:
fi
fi
- name: Replay its check frame on the mock and require the result bit for bit
if: env.MOCK_BUILD == 'true'
env:
DEV: ${{ inputs.target_hardware == 'chestnut' && 'MOCKUSB+AMD' || 'MOCK+QCOM' }}
PYTHONPATH: ${{ env.TINYGRAD_PATH }}:${{ github.workspace }}
run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "${{ env.MODELS_DIR }}/${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}driving_tinygrad.pkl"
- name: Prepare Output
run: |
sudo rm -rf ${{ env.OUTPUT_DIR }}
@@ -380,7 +304,7 @@ jobs:
--include='/${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}driving_tinygrad.pkl' \
--exclude='*' \
--delete-excluded \
$([ "$MOCK_BUILD" = "true" ] || echo --chown=comma:comma) \
--chown=comma:comma \
${{ env.MODELS_DIR }}/ ${{ env.OUTPUT_DIR }}/
python3 "${{ github.workspace }}/release/ci/model_generator.py" \
@@ -409,7 +333,7 @@ jobs:
path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
- name: Re-enable powersave
if: always() && env.MOCK_BUILD != 'true'
if: always()
run: |
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
+13 -105
View File
@@ -28,30 +28,24 @@ jobs:
env:
DEV: CPU
run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_plain_artifact.py
- uses: ./.github/workflows/setup-amd-mock
- name: Install LLVM 20 and ROCm comgr for the amd mock, as tinygrad_repo/.github/actions/setup-tinygrad does
run: |
. /etc/os-release
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
echo "deb http://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.1 $VERSION_CODENAME main" | sudo tee /etc/apt/sources.list.d/rocm.list
printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/rocm-pin-600
sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends libllvm20 clang-20 lld-20 comgr
printf '/opt/rocm/lib\n/opt/rocm/lib64\n' | sudo tee /etc/ld.so.conf.d/rocm.conf && sudo ldconfig
echo "/usr/lib/llvm-20/bin" >> "$GITHUB_PATH"
- name: Convert a pickled USB+AMD jit and replay it under tinygrad's amd mock
env:
DEV: MOCKUSB+AMD
CHECK_OOB: 0
GMMU: 0
run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py
- name: Resolve commaai's tinygrad pin, which their precompiled chestnut model is built and converted with
id: upstream-tg
env:
GH_TOKEN: ${{ github.token }}
run: |
read -r sha url < <(gh api repos/commaai/openpilot/contents/tinygrad_repo --jq '.sha + " " + .submodule_git_url')
echo "commaai/openpilot master pins $url@$sha, ours is $(git rev-parse HEAD:tinygrad_repo)"
dir="$RUNNER_TEMP/upstream_tinygrad"
git init -q "$dir" && git -C "$dir" fetch -q --depth 1 "$url" "$sha" && git -C "$dir" checkout -q FETCH_HEAD
echo "path=$dir" >> "$GITHUB_OUTPUT"
- name: Convert a pickled USB+AMD jit under commaai's tinygrad, as the chestnut model build does
env:
DEV: MOCKUSB+AMD
CHECK_OOB: 0
GMMU: 0
PYTHONPATH: ${{ steps.upstream-tg.outputs.path }}:${{ github.workspace }}
run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py
- name: Where tinygrad moved, for the helper or failure point named above
if: failure() && github.event_name == 'pull_request'
run: |
@@ -69,8 +63,6 @@ jobs:
runs-on: ubuntu-latest
outputs:
models: ${{ steps.set-matrix.outputs.models }}
chestnut: ${{ steps.set-matrix.outputs.chestnut }}
qcom: ${{ steps.set-matrix.outputs.qcom }}
steps:
- uses: actions/checkout@v4
- name: Fetch and Parse json
@@ -82,22 +74,16 @@ jobs:
with open("openpilot/sunnypilot/models/fetcher.py", "r") as f:
urls = re.findall(r"MODEL_URL(?:_CHESTNUT)?\s*=\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", f.read())
artifacts, chestnut, qcom = [], [], []
artifacts = []
for url in urls:
data = json.loads(urllib.request.urlopen(url).read())
for bundle in data.get("bundles", []):
for model in bundle.get("models", []):
if "artifact" in model:
artifacts.append(model["artifact"])
if "chestnut" in url:
chestnut.append(model["artifact"])
elif "tinygrad" in model["artifact"]["file_name"]:
qcom.append(model["artifact"])
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"models={json.dumps(artifacts)}\n")
f.write(f"chestnut={json.dumps(chestnut)}\n")
f.write(f"qcom={json.dumps(qcom)}\n")
'
test-model:
@@ -134,81 +120,3 @@ jobs:
export PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo:$GITHUB_WORKSPACE:$PYTHONPATH"
python3 -m pip uninstall -y tinygrad || uv pip uninstall -y tinygrad --system || true
python3 -m pytest openpilot/sunnypilot/modeld_v2/tests/test_models.py
chestnut-mock-parity:
name: Mock parity ${{ matrix.artifact.file_name }}
needs: generate-matrix
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
artifact: ${{ fromJson(needs.generate-matrix.outputs.chestnut) }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-amd-mock
with:
emu_cache: ${{ matrix.artifact.file_name }}
- run: python3 -m pip install -q numpy
- name: Download model
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
MODEL_URL: ${{ matrix.artifact.download_uri.url }}
MODEL_SHA: ${{ matrix.artifact.download_uri.sha256 }}
shell: bash
run: |
AUTH=()
if [ -n "$HF_TOKEN" ]; then AUTH=(-H "Authorization: Bearer $HF_TOKEN"); fi
curl --retry 3 --retry-all-errors -s -S -f -L "${AUTH[@]}" -o "$RUNNER_TEMP/model.pkl" "$MODEL_URL"
if [ -n "$MODEL_SHA" ]; then echo "$MODEL_SHA $RUNNER_TEMP/model.pkl" | sha256sum -c -; fi
- name: Replay its check frame on the amd mock and require the chestnut's result bit for bit
env:
PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "$RUNNER_TEMP/model.pkl"
qcom-mock-parity:
name: QCOM mock parity ${{ matrix.artifact.file_name }}
needs: generate-matrix
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
artifact: ${{ fromJson(needs.generate-matrix.outputs.qcom) }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- run: python3 -m pip install -q numpy tinymesa==25.2.7.2
- name: Download model
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
MODEL_URL: ${{ matrix.artifact.download_uri.url }}
MODEL_SHA: ${{ matrix.artifact.download_uri.sha256 }}
shell: bash
run: |
AUTH=()
if [ -n "$HF_TOKEN" ]; then AUTH=(-H "Authorization: Bearer $HF_TOKEN"); fi
curl --retry 3 --retry-all-errors -s -S -f -L "${AUTH[@]}" -o "$RUNNER_TEMP/model.pkl" "$MODEL_URL"
if [ -n "$MODEL_SHA" ]; then echo "$MODEL_SHA $RUNNER_TEMP/model.pkl" | sha256sum -c -; fi
- name: Replay its check frame on the qcom mock and require the comma device's result bit for bit
env:
DEV: MOCK+QCOM
PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "$RUNNER_TEMP/model.pkl"
chestnut-mock-build-parity:
name: Chestnut build on the amd mock = chestnut runner build
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-amd-mock
with:
emu_cache: chestnut-build-parity
- run: python3 -m pip install -q numpy zstandard
- uses: ./.github/workflows/chestnut-build-parity
+1
View File
@@ -4,6 +4,7 @@
[submodule "opendbc"]
path = opendbc_repo
url = https://github.com/sunnypilot/opendbc.git
branch = toyotabsmhack
[submodule "msgq"]
path = msgq_repo
url = https://github.com/sunnypilot/msgq.git
@@ -1,95 +0,0 @@
# Achievements: phase 1, on the device
## Status: built, tested off-device, not yet run on a car
Phase 1 of the sunnypilot community plan. The device counts driving stats and unlocks achievements, and shows them on
an Achievements screen. Nothing leaves the device yet; sharing, the community site, leaderboards, groups, the map and
contests come in later phases on a separate backend.
## Where it fits
```mermaid
flowchart LR
subgraph device[On the device: this phase]
cereal[cereal: carState, selfdriveState,<br/>selfdriveStateSP, modelV2,<br/>driverMonitoringState, GPS] --> d[achievementsd<br/>10 Hz, read-only]
d --> f[(state.json<br/>/data/media/0/achievements)]
f --> ui[Achievements screen<br/>comma four and comma 3X]
ui --> seen[(seen.json)]
end
f -. later phase: opt-in .-> backend[Achievements backend] --> site[Community site] & forum[Forum badges]
```
## How a drive is counted
```mermaid
flowchart TD
s[Sample every 0.1 s] --> g{gap over 1 s or<br/>speed outside 0-70 m/s?}
g -- yes --> skip[count nothing]
g -- no --> e{engaged?<br/>selfdriveState.active<br/>or MADS active}
e -- no --> lc
e -- yes --> add[add distance, time,<br/>day, model, area, MADS-only km]
add --> lc{lane change reached<br/>finishing?}
lc --> u[check every tier,<br/>record new unlocks]
```
- **Engaged** means `selfdriveState.active` (including driver override) or MADS lateral active. MADS-only distance
is MADS active without `selfdriveState.active`.
- **Days** (Regular, Streak, Road Trip, Four Seasons) need a valid wall clock. Before the clock is set, distance still
counts but no day does. The calendar day comes from GPS longitude (15 degrees per hour), since the device clock runs
in UTC; without GPS it falls back to the system zone. Southern-hemisphere fixes flip the seasons.
- **Explorer areas** are geohash cells of about 5 km, salted with a random per-device salt and SHA-256 hashed before
they are stored. No coordinates are written anywhere.
- **Attentive** is the longest engaged stretch without a red (level three) driver monitoring alert.
- **Dev Branch Brave** counts on any channel that is not a release channel, plus staging channels.
## Catalog
`openpilot/sunnypilot/achievements/catalog.json`, 14 achievements and 32 tiers. Unlock ids are `odometer.silver`, or
just `first_light` for single-tier ones, so a later backend can map them one-to-one to badges.
| Achievement | Measures | Tiers |
| --- | --- | --- |
| First Light | engaged km | 0.1 |
| Odometer | engaged km | 100 / 1,000 / 10,000 / 50,000 |
| Captain's Hours | engaged hours | 10 / 100 / 1,000 |
| Regular | days engaged | 7 / 30 / 100 / 365 |
| Streak | days in a row | 7 / 30 |
| Steady Hands | MADS-only km | 100 / 1,000 |
| Smooth Merger | assisted lane changes | 10 / 100 / 1,000 |
| Road Trip | km in one day | 500 |
| Explorer | 5 km areas | 5 / 25 / 100 |
| Model Hopper | models with 50+ km | 3 / 5 / 10 |
| Dev Branch Brave | km on dev or staging | 100 |
| Attentive | hours without a red DM alert | 10 / 50 / 100 |
| Four Seasons | seasons driven | 4 |
| Linked Up | sunnylink on and registered | yes |
Safety rules for anything added here: never reward speed, hard driving, night or fatigue driving, or fewer
takeovers, and never show anything while driving.
## Why a state file, not Params
This is a prebuilt branch: a new Params key means rebuilding the ARM64 `libparams_c.so` (see
`openpilot/tools/params/README.md`). Progress lives in `/data/media/0/achievements/state.json` instead, written
atomically every 30 s and when the drive ends. It survives updates. The UI keeps its own `seen.json`, so the daemon and
the UI never write the same file. A corrupt state file is moved to `state.corrupt.json` instead of being overwritten.
## Files
| File | What it is |
| --- | --- |
| `sunnypilot/achievements/catalog.json`, `catalog.py` | the goals, and display text for both UIs |
| `sunnypilot/achievements/engine.py` | pure Python counting and unlocks, no cereal or Params |
| `sunnypilot/achievements/achievementsd.py` | onroad process that feeds the engine from cereal |
| `system/manager/process_config.py` | registers `achievementsd` as `only_onroad` |
| `selfdrive/ui/sunnypilot/achievements_store.py` | shared reader for both screens |
| `selfdrive/ui/sunnypilot/mici/layouts/achievements.py` | comma four screen, plus the settings button with "N new" |
| `selfdrive/ui/sunnypilot/layouts/settings/achievements.py` | comma 3X settings panel |
## Testing
`pytest openpilot/sunnypilot/achievements/tests openpilot/selfdrive/ui/sunnypilot/tests/test_achievements_store.py`
(26 tests, no device libraries needed). The screens and achievementsd were syntax- and lint-checked only: the UI and
cereal libraries on this branch are ARM64 builds, so neither has run yet. First drive to check: the process starts
onroad, `state.json` appears within 30 s of engaging, and the comma four settings show "achievements" with
"2 new" (First Light, and Linked Up if sunnylink is on) after the drive.
+1
View File
@@ -369,6 +369,7 @@ struct CarParamsSP @0x80ae746ee2596b11 {
pcmCruiseSpeed @3 :Bool;
intelligentCruiseButtonManagementAvailable @4 :Bool;
enableGasInterceptor @5 :Bool;
smoothStopDecelRate @6 :Float32;
neuralNetworkLateralControl @2 :NeuralNetworkLateralControl;
+3
View File
@@ -150,6 +150,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
// --- sunnypilot params --- //
{"AccelBoostSP", {PERSISTENT | BACKUP, BOOL, "1"}},
{"SmoothStopSP", {PERSISTENT | BACKUP, BOOL, "1"}},
{"ApiCache_DriveStats", {PERSISTENT, JSON}},
{"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
{"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
@@ -200,6 +201,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
{"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TripSummary", {PERSISTENT | BACKUP, BOOL, "1"}},
{"TrueVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
// MADS params
@@ -245,6 +247,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TeslaMadsScreenButton", {PERSISTENT | BACKUP, INT, "0"}},
{"ToyotaEnhancedBsm", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ToyotaTSS2Smooth", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -57,6 +57,7 @@ class Controls(ControlsExt):
self.calibrated_pose: Pose | None = None
self.LoC = LongControl(self.CP, self.CP_SP)
self.LoC = ControlsExt.initialize_longitudinal_control(self, self.LoC)
self.VM = VehicleModel(self.CP)
self.LaC: LatControl
if self.CP.steerControlType == car.CarParams.SteerControlType.angle:
@@ -28,8 +28,6 @@ class DesireHelper:
self.alc = AutoLaneChangeController(self)
self.lane_turn_controller = LaneTurnController(self)
self.lane_turn_direction = TurnDirection.none
self.turn_frames = 0
self.turn_repulse = 98
@staticmethod
def get_lane_change_direction(CS):
@@ -97,10 +95,8 @@ class DesireHelper:
self.prev_one_blinker = one_blinker and lateral_active
if self.lane_turn_direction != TurnDirection.none:
self.turn_frames += 1
self.desire = TURN_DESIRES[self.lane_turn_direction] if self.turn_frames % self.turn_repulse else log.Desire.none
self.desire = TURN_DESIRES[self.lane_turn_direction]
else:
self.turn_frames = 0
self.desire = log.Desire.none
if self.lane_change_state == LaneChangeState.laneChangeStarting:
if self.lane_change_direction == LaneChangeDirection.left:
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b043d3bbc556e3a872c0edde4081f99b5ba92cbfd98eaea2ee2cbb87e9f277ec
size 82539558
oid sha256:65a08adc31d5c456219687d99b7bf5e44d61dae2d49ea67850e76105c7248cce
size 60918562
+2
View File
@@ -10,6 +10,7 @@ from openpilot.selfdrive.ui.onroad.augmented_road_view import AugmentedRoadView
from openpilot.selfdrive.ui.ui_state import device, ui_state
from openpilot.selfdrive.ui.layouts.onboarding import OnboardingWindow
from openpilot.selfdrive.ui.body.layouts.onroad import BodyLayout
from openpilot.selfdrive.ui.sunnypilot.layouts.trip_summary import TripSummary
if gui_app.sunnypilot_ui():
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.settings import SettingsLayoutSP as SettingsLayout
@@ -31,6 +32,7 @@ class MainLayout(Widget):
self._sidebar = Sidebar()
self._current_mode = MainState.HOME
self._prev_onroad = False
self._trip_summary = TripSummary()
# Initialize layouts
self._home_layout = HomeLayout()
@@ -10,6 +10,7 @@ from openpilot.selfdrive.ui.body.layouts.onroad import BodyLayout
from openpilot.system.ui.widgets import Widget
from openpilot.system.ui.widgets.scroller import Scroller
from openpilot.system.ui.lib.application import gui_app
from openpilot.selfdrive.ui.sunnypilot.layouts.trip_summary import TripSummary
if gui_app.sunnypilot_ui():
from openpilot.selfdrive.ui.sunnypilot.mici.layouts.settings import SettingsLayoutSP as SettingsLayout
@@ -57,6 +58,7 @@ class MiciMainLayout(Scroller):
self._setup_callbacks()
gui_app.add_nav_stack_tick(self._handle_transitions)
self._trip_summary = TripSummary()
gui_app.push_widget(self)
# Start onboarding if terms or training not completed, make sure to push after self
@@ -22,6 +22,7 @@ from openpilot.common.transformations.orientation import rot_from_euler
from enum import IntEnum
if gui_app.sunnypilot_ui():
from openpilot.selfdrive.ui.sunnypilot.mici.onroad.augmented_road_view import AugmentedRoadViewSP
from openpilot.selfdrive.ui.sunnypilot.mici.onroad.hud_renderer import HudRendererSP as HudRenderer
from openpilot.selfdrive.ui.sunnypilot.ui_state import OnroadTimerStatus
@@ -156,9 +157,10 @@ class BookmarkIcon(Widget):
rl.draw_texture_ex(icon, rl.Vector2(icon_x, icon_y), 0.0, 1.0, rl.Color(255, 255, 255, round(255 * alpha)))
class AugmentedRoadView(CameraView):
class AugmentedRoadView(CameraView, AugmentedRoadViewSP):
def __init__(self, bookmark_callback=None, stream_type: VisionStreamType = VisionStreamType.VISION_STREAM_NARROW_ROAD):
super().__init__("camerad", stream_type)
AugmentedRoadViewSP.__init__(self)
self._bookmark_callback = bookmark_callback
self._set_placeholder_color(rl.BLACK)
@@ -275,7 +277,7 @@ class AugmentedRoadView(CameraView):
if show_navigation:
self._navigation.render(rl.Rectangle(self._content_rect.x + self._content_rect.width, self.rect.y, PANEL_WIDTH, self.rect.height))
else:
self._confidence_ball.render(self.rect)
AugmentedRoadViewSP.render_side_panel(self, self.rect)
self._bookmark_icon.render(self.rect)
@@ -1,97 +0,0 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
What both Achievements screens and the settings buttons read: the state file
achievementsd writes, plus the UI's own list of unlocks already seen.
No pyray here, so it can be tested without a display.
"""
import time
from pathlib import Path
from openpilot.sunnypilot.achievements.catalog import AchievementStatus, Catalog, format_amount, load_catalog, statuses
from openpilot.sunnypilot.achievements.engine import State, load_seen, load_state, save_seen, seen_path, state_path
REFRESH_INTERVAL_S = 5.0
class AchievementsStore:
def __init__(self, state_file: Path | None = None, seen_file: Path | None = None, catalog: Catalog | None = None):
self._state_file = state_file
self._seen_file = seen_file
self.catalog = catalog or load_catalog()
self.state = State()
self._seen: set[str] = set()
self._mtime: float | None = None
self._last_check = 0.0
self.version = 0 # bumps on every reload or seen change, so several screens can share one store
def _state_path(self) -> Path:
return self._state_file or state_path()
def _seen_path(self) -> Path:
return self._seen_file or seen_path()
def refresh(self, force: bool = False) -> bool:
"""Reload if the state file changed. Cheap enough to call every frame. Returns True when reloaded."""
now = time.monotonic()
if not force and now - self._last_check < REFRESH_INTERVAL_S:
return False
self._last_check = now
try:
mtime = self._state_path().stat().st_mtime
except OSError:
mtime = None
if not force and mtime == self._mtime:
return False
self._mtime = mtime
try:
self.state = load_state(self._state_path()) if mtime is not None else State()
except OSError:
return False
self._seen = load_seen(self._seen_path())
self.version += 1
return True
def statuses(self) -> list[AchievementStatus]:
return statuses(self.catalog, self.state.metrics(), self.state.unlocked)
def status(self, achievement_id: str) -> AchievementStatus | None:
return next((s for s in self.statuses() if s.achievement.id == achievement_id), None)
def unlock_ids(self, achievement_id: str) -> list[str]:
a = next((a for a in self.catalog.achievements if a.id == achievement_id), None)
return [a.unlock_id(t) for t in a.tiers] if a else []
def unlocked_count(self) -> tuple[int, int]:
all_ids = self.catalog.all_unlock_ids()
return sum(1 for uid in all_ids if uid in self.state.unlocked), len(all_ids)
def new_ids(self) -> set[str]:
return {uid for uid in self.state.unlocked if uid not in self._seen and uid in set(self.catalog.all_unlock_ids())}
def mark_all_seen(self) -> None:
new = self.new_ids()
if not new:
return
self._seen |= new
self.version += 1
try:
save_seen(self._seen, self._seen_path())
except OSError:
pass
def totals_text(self, is_metric: bool = True) -> str:
m = self.state.metrics()
unlocked, total = self.unlocked_count()
return "\n".join([
f"{format_amount(m['engaged_km'], 'km', is_metric)} engaged",
f"{format_amount(m['engaged_h'], 'h')} engaged",
f"{int(m['active_days'])} days, best streak {int(m['best_streak_days'])}",
f"{unlocked} of {total} unlocked",
])
achievements_store = AchievementsStore()
@@ -1,52 +0,0 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
comma 3X Achievements panel: a totals row, then one row per achievement.
Tap a row to read what it is for. Opening the panel marks new unlocks as seen.
"""
from openpilot.selfdrive.ui.sunnypilot.achievements_store import AchievementsStore, achievements_store
from openpilot.selfdrive.ui.ui_state import ui_state
from openpilot.system.ui.lib.multilang import tr
from openpilot.system.ui.widgets import Widget
from openpilot.system.ui.widgets.list_view import text_item
from openpilot.system.ui.widgets.scroller_tici import Scroller
class AchievementsLayout(Widget):
def __init__(self, store: AchievementsStore = achievements_store):
super().__init__()
self._store = store
self._new: set[str] = set()
unlocked = text_item(tr("Unlocked"), self._unlocked_text, description=lambda: self._store.totals_text(ui_state.is_metric))
items = [unlocked]
for a in self._store.catalog.achievements:
items.append(text_item(a.name, lambda aid=a.id: self._row_value(aid), description=a.description))
self._scroller = Scroller(items, line_separator=True, spacing=0)
def _unlocked_text(self) -> str:
unlocked, total = self._store.unlocked_count()
return f"{unlocked} / {total}"
def _row_value(self, achievement_id: str) -> str:
status = self._store.status(achievement_id)
if status is None:
return ""
is_new = any(uid in self._new for uid in self._store.unlock_ids(achievement_id))
return (tr("NEW") + " · " if is_new else "") + status.summary(ui_state.is_metric)
def show_event(self):
self._scroller.show_event()
self._store.refresh(force=True)
# remembered for this visit, so the NEW labels stay up while the panel is open
self._new = self._store.new_ids()
self._store.mark_all_seen()
def _update_state(self):
self._store.refresh()
def _render(self, rect):
self._scroller.render(rect)
@@ -63,6 +63,13 @@ class CruiseLayout(Widget):
"Eases off in curves and when sunnypilot slows or stops. Never goes past your set speed, and lead-car following still comes first."),
param="AccelBoostSP")
self.smooth_stop_toggle = toggle_item_sp(
title=tr("Smooth Stop"),
description=tr("Brakes gently as sunnypilot comes to a final stop to reduce the lurch at the end. " +
"Applies to all vehicles with sunnypilot longitudinal control."),
param="SmoothStopSP",
callback=lambda _: ui_state.params.put_bool("OnroadCycleRequested", True))
self.custom_acc_toggle = toggle_item_sp(
title=tr("Custom ACC Speed Increments"),
description="",
@@ -93,6 +100,7 @@ class CruiseLayout(Widget):
self.scc_v_toggle,
self.scc_m_toggle,
self.accel_boost_toggle,
self.smooth_stop_toggle,
self.custom_acc_toggle,
self.custom_acc_short_increment,
self.custom_acc_long_increment,
@@ -144,6 +152,7 @@ class CruiseLayout(Widget):
self.icbm_toggle.show_description(True)
self.accel_boost_toggle.action_item.set_enabled(has_long)
self.smooth_stop_toggle.action_item.set_enabled(has_long and not ui_state.engaged)
if has_long or has_icbm:
self.custom_acc_toggle.action_item.set_enabled(((has_long and not ui_state.CP.pcmCruise) or has_icbm) and ui_state.is_offroad())
@@ -161,6 +170,7 @@ class CruiseLayout(Widget):
has_icbm = has_long = False
self.icbm_toggle.action_item.set_enabled(False)
self.accel_boost_toggle.action_item.set_enabled(False)
self.smooth_stop_toggle.action_item.set_enabled(False)
self.icbm_toggle.set_description(tr(ONROAD_ONLY_DESCRIPTION))
show_custom_acc_desc = False
@@ -11,7 +11,6 @@ import pyray as rl
from openpilot.selfdrive.ui.layouts.settings import settings as OP
from openpilot.selfdrive.ui.layouts.settings.firehose import FirehoseLayout
from openpilot.selfdrive.ui.layouts.settings.toggles import TogglesLayout
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.achievements import AchievementsLayout
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.cruise import CruiseLayout
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.developer import DeveloperLayoutSP
from openpilot.selfdrive.ui.sunnypilot.layouts.settings.device import DeviceLayoutSP
@@ -51,7 +50,6 @@ OP.PanelType = IntEnum( # type: ignore[assignment] # ty: ignore[invalid-assignm
"NAVIGATION",
"TRIPS",
"VEHICLE",
"ACHIEVEMENTS",
],
start=0,
)
@@ -124,8 +122,6 @@ class SettingsLayoutSP(OP.SettingsLayout):
OP.PanelType.OSM: PanelInfo(tr_noop("OSM"), OSMLayout(), icon="../../sunnypilot/selfdrive/assets/offroad/icon_map.png"),
# OP.PanelType.NAVIGATION: PanelInfo(tr_noop("Navigation"), NavigationLayout(), icon="../../sunnypilot/selfdrive/assets/offroad/icon_map.png"),
OP.PanelType.TRIPS: PanelInfo(tr_noop("Trips"), TripsLayout(), icon="../../sunnypilot/selfdrive/assets/offroad/icon_trips.png"),
OP.PanelType.ACHIEVEMENTS: PanelInfo(tr_noop("Achievements"), AchievementsLayout(),
icon="../../sunnypilot/selfdrive/assets/offroad/icon_achievements.png"),
OP.PanelType.VEHICLE: PanelInfo(tr_noop("Vehicle"), VehicleLayout(), icon="../../sunnypilot/selfdrive/assets/offroad/icon_vehicle.png"),
OP.PanelType.FIREHOSE: PanelInfo(tr_noop("Firehose"), FirehoseLayout(), icon="../../sunnypilot/selfdrive/assets/offroad/icon_firehose.png"),
OP.PanelType.DEVELOPER: PanelInfo(tr_noop("Developer"), DeveloperLayoutSP(), icon="icons/shell.png"),
@@ -12,10 +12,16 @@ from openpilot.system.ui.widgets import DialogResult
from openpilot.system.ui.widgets.confirm_dialog import ConfirmDialog
from openpilot.system.ui.sunnypilot.widgets.list_view import toggle_item_sp
from opendbc.car.toyota.values import ToyotaFlags
from opendbc.sunnypilot.car.toyota.values import ToyotaFlagsSP
ONROAD_ONLY_DESCRIPTION = tr_noop("Start the vehicle to check vehicle compatibility.")
SNG_HACK_UNAVAILABLE = tr_noop("sunnypilot Longitudinal Control must be available and enabled for your vehicle to use this feature.")
BSM_UNAVAILABLE = tr_noop("Enhanced Blind Spot Monitoring requires a TSS2 Toyota/Lexus without SecOC, " +
"or a smartDSU with sunnypilot longitudinal control.")
BSM_STOCK_LONG_UNAVAILABLE = tr_noop("Enhanced Blind Spot Monitoring is not available while Enforce Factory Longitudinal Control is enabled.")
BSM_REQUIRED = tr_noop("This vehicle does not report blind spot status over the standard CAN signals. " +
"Enhanced Blind Spot Monitoring is required for blind spot detection.")
PLATFORM_UNAVAILABLE = tr_noop("This feature is currently not available on this platform.")
DESCRIPTIONS = {
@@ -26,6 +32,10 @@ DESCRIPTIONS = {
'sunnypilot will allow some Toyota/Lexus cars to auto resume during stop and go traffic. ' +
'This feature is only applicable to certain models that are able to use longitudinal control. This is an alpha feature. Use at your own risk.'
),
'enhanced_bsm': tr_noop(
'sunnypilot will poll the blind spot monitor diagnostics directly from the radar. Use this on vehicles whose blind spot monitors ' +
'do not report blind spot status over the standard CAN signals. This is an alpha feature. Use at your own risk.'
),
'tss2_smooth': tr_noop(
'Enable smoother longitudinal control for TSS2 vehicles. ' +
'Uses a custom speed-dependent integral gain curve for smoother traffic driving.'
@@ -53,6 +63,14 @@ class ToyotaSettings(BrandSettings):
enabled=lambda: not ui_state.engaged,
)
self.enhanced_bsm = toggle_item_sp(
lambda: tr("Enhanced Blind Spot Monitoring (Alpha)"),
description=lambda: tr(DESCRIPTIONS["enhanced_bsm"]),
initial_state=ui_state.params.get_bool("ToyotaEnhancedBsm"),
callback=self._on_enable_enhanced_bsm,
enabled=lambda: not ui_state.engaged,
)
self.tss2_smooth = toggle_item_sp(
lambda: tr("TSS2 Smooth Longitudinal (Alpha)"),
description=lambda: tr(DESCRIPTIONS["tss2_smooth"]),
@@ -64,6 +82,7 @@ class ToyotaSettings(BrandSettings):
self.items = [
self.enforce_stock_longitudinal,
self.stop_and_go_hack,
self.enhanced_bsm,
self.tss2_smooth,
]
@@ -130,6 +149,25 @@ class ToyotaSettings(BrandSettings):
ui_state.params.put_bool("ToyotaStopAndGoHack", False)
ui_state.params.put_bool("OnroadCycleRequested", True)
def _on_enable_enhanced_bsm(self, state: bool):
if state:
def confirm_callback(result: int):
if result == DialogResult.CONFIRM:
ui_state.params.put_bool("ToyotaEnhancedBsm", True)
ui_state.params.put_bool("OnroadCycleRequested", True)
else:
self.enhanced_bsm.action_item.set_state(False)
content = (f"<h1>{self.enhanced_bsm.title}</h1><br>" +
f"<p>{self.enhanced_bsm.description}</p>")
dlg = ConfirmDialog(content, tr("Enable"), rich=True, callback=confirm_callback)
gui_app.push_widget(dlg)
else:
ui_state.params.put_bool("ToyotaEnhancedBsm", False)
ui_state.params.put_bool("OnroadCycleRequested", True)
def update_settings(self):
if ui_state.CP is not None:
longitudinal = ui_state.CP.openpilotLongitudinalControl
@@ -150,6 +188,31 @@ class ToyotaSettings(BrandSettings):
if show_desc:
self.stop_and_go_hack.show_description(True)
# Enhanced BSM needs the radar diagnostic address, which is only in the TX allowlist for
# TSS2 cars without SecOC, or for TSS-P cars whose smartDSU lets sunnypilot control
# longitudinal. Enforcing factory longitudinal control rules both of those out.
sdsu_bsm_supported = (longitudinal and ui_state.CP_SP is not None and
bool(ui_state.CP_SP.flags & ToyotaFlagsSP.SMART_DSU))
bsm_supported = (not bool(ui_state.CP.flags & ToyotaFlags.SECOC) and
(bool(ui_state.CP.flags & ToyotaFlags.TSS2) or sdsu_bsm_supported))
bsm_available = bsm_supported and not enforce_stock
self.enhanced_bsm.action_item.set_enabled(bsm_available and not ui_state.engaged)
bsm_desc = tr(DESCRIPTIONS["enhanced_bsm"])
if not bsm_supported:
self.enhanced_bsm.action_item.set_state(False)
bsm_desc = "<b>" + tr(BSM_UNAVAILABLE) + "</b>\n\n" + bsm_desc
elif enforce_stock:
self.enhanced_bsm.action_item.set_state(False)
bsm_desc = "<b>" + tr(BSM_STOCK_LONG_UNAVAILABLE) + "</b>\n\n" + bsm_desc
elif ui_state.CP_SP is not None and ui_state.CP_SP.flags & ToyotaFlagsSP.SP_NEED_DEBUG_BSM:
bsm_desc = "<b>" + tr(BSM_REQUIRED) + "</b>\n\n" + bsm_desc
if self.enhanced_bsm.description != bsm_desc:
self.enhanced_bsm.set_description(bsm_desc)
if not bsm_available:
self.enhanced_bsm.show_description(True)
tss2 = bool(ui_state.CP.flags & ToyotaFlags.TSS2)
if longitudinal and not enforce_stock and tss2:
self.tss2_smooth.action_item.set_enabled(not ui_state.engaged)
@@ -173,6 +236,12 @@ class ToyotaSettings(BrandSettings):
self.stop_and_go_hack.set_description(new_desc)
self.stop_and_go_hack.show_description(True)
self.enhanced_bsm.action_item.set_enabled(False)
bsm_desc = "<b>" + tr(ONROAD_ONLY_DESCRIPTION) + "</b>\n\n" + tr(DESCRIPTIONS["enhanced_bsm"])
if self.enhanced_bsm.description != bsm_desc:
self.enhanced_bsm.set_description(bsm_desc)
self.enhanced_bsm.show_description(True)
self.tss2_smooth.action_item.set_enabled(False)
new_desc = "<b>" + tr(ONROAD_ONLY_DESCRIPTION) + "</b>\n\n" + tr(DESCRIPTIONS["tss2_smooth"])
if self.tss2_smooth.description != new_desc:
@@ -72,6 +72,11 @@ class VisualsLayout(Widget):
"It is the driver's responsibility to observe their environment and make decisions accordingly."),
None,
),
"TripSummary": (
lambda: tr("Trip Summary"),
tr("Show a summary of distance and engagement when you shift into park after a drive."),
None,
),
"TrueVEgoUI": (
lambda: tr("Speedometer: Always Display True Speed"),
tr("For applicable vehicles, always display the true vehicle current speed from wheel speed sensors."),
@@ -0,0 +1,199 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
"""
import time
import pyray as rl
from opendbc.car.structs import car
from openpilot.common.constants import CV
from openpilot.selfdrive.ui.onroad.augmented_road_view import BORDER_COLORS
from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus
from openpilot.system.ui.lib.application import gui_app, FontWeight
from openpilot.system.ui.lib.text_measure import measure_text_cached
from openpilot.system.ui.widgets import Widget
AUTO_DISMISS = 20.
FADE_IN, FADE_OUT = 0.35, 0.3
LAT = (UIStatus.ENGAGED, UIStatus.LAT_ONLY)
LONG = (UIStatus.ENGAGED, UIStatus.LONG_ONLY)
GEAR = car.CarState.GearShifter
FORWARD = (GEAR.drive, GEAR.sport, GEAR.low, GEAR.brake, GEAR.eco, GEAR.manumatic)
class TripTracker:
def __init__(self):
self.reset()
def reset(self):
self.segments: list[list] = []
self.seconds = 0.
self.takeovers = 0
self._t = None
self._status = None
def update(self, driving: bool):
now = time.monotonic()
if not driving:
self._t = None
self._status = None
return
dt = 0. if self._t is None else min(now - self._t, 0.5)
self._t = now
self.seconds += dt
if self._status not in (None, UIStatus.DISENGAGED) and ui_state.status == UIStatus.DISENGAGED:
self.takeovers += 1
self._status = ui_state.status
d = max(ui_state.sm["carState"].vEgo, 0.) * dt
if d <= 0.:
return
if self.segments and self.segments[-1][0] == ui_state.status:
self.segments[-1][1] += d
else:
self.segments.append([ui_state.status, d])
def total(self, statuses=None) -> float:
return sum(m for s, m in self.segments if statuses is None or s in statuses)
class TripSummary:
def __init__(self):
self._trip = TripTracker()
self._dialog: TripSummaryDialog | None = None
self._gear = None
self._drove = False
gui_app.add_nav_stack_tick(self._tick)
def _tick(self):
gear = ui_state.sm["carState"].gearShifter if ui_state.started else None
self._trip.update(gear not in (None, GEAR.park))
self._drove |= gear in FORWARD
if gear == self._gear:
return
self._gear = gear
if gear not in (None, GEAR.park):
self._pop()
elif gear == GEAR.park and self._drove and ui_state.trip_summary and not ui_state.is_body and self._trip.total() > 0.:
self._dialog = TripSummaryDialog(self._trip, gui_app.get_active_widget(), self._pop)
gui_app.push_widget(self._dialog)
if gear is None:
self._trip.reset()
if gear in (None, GEAR.park):
self._drove = False
def _pop(self):
if self._dialog is not None and gui_app.get_active_widget() is self._dialog:
gui_app.pop_widget()
self._dialog = None
def _ease(t: float) -> float:
t = min(max(t, 0.), 1.)
return 1 - (1 - t) ** 3
def _fade(c, a: float) -> rl.Color:
r, g, b, al = c if isinstance(c, tuple) else (c.r, c.g, c.b, c.a)
return rl.Color(r, g, b, int(al * a))
class TripSummaryDialog(Widget):
def __init__(self, trip: TripTracker, below: Widget | None, on_closed):
super().__init__()
self._below = below
self._on_closed = on_closed
self._segments = segments = trip.segments
total = sum(m for _, m in segments)
def pct(statuses):
return 100 * sum(m for s, m in segments if s in statuses) / total
dist = total * (CV.MS_TO_KPH if ui_state.is_metric else CV.MS_TO_MPH) / 3600
dist_fmt = "{:.1f}" if dist < 9.95 else "{:.0f}"
self._stats = [
(dist, dist_fmt, "km" if ui_state.is_metric else "mi", "driven"),
(pct(LAT + LONG), "{:.0f}", "%", "engaged"),
(pct(LAT), "{:.0f}", "%", "lateral"),
(pct(LONG), "{:.0f}", "%", "longitudinal"),
]
mins = round(trip.seconds / 60)
self._details = [
(f"{mins // 60}h {mins % 60}" if mins >= 60 else f"{mins}", "min"),
(f"{trip.takeovers}", "takeover" if trip.takeovers == 1 else "takeovers"),
]
self._bold = gui_app.font(FontWeight.BOLD)
self._semi = gui_app.font(FontWeight.SEMI_BOLD)
self._normal = gui_app.font(FontWeight.NORMAL)
self._big = gui_app.big_ui()
self._shown_t = rl.get_time()
self._closing_t: float | None = None
self.set_click_callback(self.close)
def close(self):
if self._closing_t is None:
self._closing_t = rl.get_time()
def _render(self, rect: rl.Rectangle):
t = rl.get_time() - self._shown_t
if self._closing_t is None and t > AUTO_DISMISS:
self.close()
out = 0. if self._closing_t is None else (rl.get_time() - self._closing_t) / FADE_OUT
if out >= 1.:
self._on_closed()
return
bg = 1 - _ease(out)
a = _ease(t / FADE_IN) * bg
if self._big and self._below is not None and bg < 1.:
self._below.render(rect)
rl.draw_rectangle_rec(rect, _fade(rl.BLACK, bg))
k = rect.height / 1080 * (1.0 if self._big else 1.22)
pad = 120 * k
x, w = rect.x + pad, rect.width - 2 * pad
y = rect.y + (rect.height - 850 * k) / 2
rl.draw_text_ex(self._bold, "trip summary", rl.Vector2(x, y), int(150 * k), 0, _fade(rl.WHITE, a))
y += 260 * k
col_w = w / len(self._stats)
for i, (target, fmt, unit, label) in enumerate(self._stats):
cx = x + col_w * (i + 0.5)
v = measure_text_cached(self._bold, fmt.format(target), int(170 * k))
u = measure_text_cached(self._semi, unit, int(80 * k))
vx = cx - (v.x + 10 * k + u.x) / 2
rl.draw_text_ex(self._bold, fmt.format(target), rl.Vector2(vx, y), int(170 * k), 0, _fade(rl.WHITE, a))
rl.draw_text_ex(self._semi, unit, rl.Vector2(vx + v.x + 10 * k, y + 75 * k), int(80 * k), 0, _fade(rl.WHITE, a))
lb = measure_text_cached(self._normal, label, int(70 * k))
rl.draw_text_ex(self._normal, label, rl.Vector2(cx - lb.x / 2, y + 180 * k), int(70 * k), 0, _fade(rl.Color(200, 200, 200, 255), a))
y += 330 * k
bar = rl.Rectangle(x, y, w, 90 * k)
total = sum(m for _, m in self._segments)
sx = bar.x
for status, m in self._segments:
sw = bar.width * m / total
rl.draw_rectangle_rec(rl.Rectangle(sx, bar.y, min(sw + 1, bar.x + bar.width - sx), bar.height), _fade(BORDER_COLORS[status], a))
sx += sw
rl.draw_rectangle_rounded_lines_ex(bar, 1.0, 20, 30 * k, _fade(rl.BLACK, a))
y += (90 + 70) * k
size = int(90 * k)
parts = []
for i, (value, unit) in enumerate(self._details):
if i:
parts.append((" ", self._normal, None))
parts += [(value, self._bold, rl.WHITE), (" " + unit, self._normal, rl.Color(170, 170, 170, 255))]
widths = [measure_text_cached(f, t, size).x for t, f, _ in parts]
lx = x + (w - sum(widths)) / 2
for (text, font, color), tw in zip(parts, widths, strict=True):
if color is None:
rl.draw_circle(int(lx + tw / 2), int(y + size * 0.6), 12 * k, _fade(rl.Color(110, 110, 110, 255), a))
else:
rl.draw_text_ex(font, text, rl.Vector2(lx, y), size, 0, _fade(color, a))
lx += tw
@@ -1,69 +0,0 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
comma four Achievements screen: a summary card, then one card per achievement.
Reads the state file achievementsd writes; opening the screen marks new unlocks as seen.
"""
from openpilot.selfdrive.ui.mici.widgets.button import BigButton
from openpilot.selfdrive.ui.mici.widgets.dialog import BigDialog
from openpilot.selfdrive.ui.sunnypilot.achievements_store import AchievementsStore, achievements_store
from openpilot.selfdrive.ui.ui_state import ui_state
from openpilot.system.ui.lib.application import gui_app
from openpilot.system.ui.lib.multilang import tr
from openpilot.system.ui.widgets.scroller import NavScroller
class AchievementsLayoutMici(NavScroller):
def __init__(self, store: AchievementsStore = achievements_store):
super().__init__()
self._store = store
self._shown_version = -1
self._summary_btn = BigButton(tr("achievements"), "")
self._summary_btn.set_click_callback(self._show_summary)
self._cards: dict[str, BigButton] = {}
widgets = [self._summary_btn]
for a in self._store.catalog.achievements:
card = BigButton(a.name.lower(), "")
card.set_click_callback(lambda aid=a.id: self._show_details(aid))
self._cards[a.id] = card
widgets.append(card)
self._scroller.add_widgets(widgets)
def show_event(self):
super().show_event()
self._store.refresh(force=True)
self._refresh_cards()
# marked seen after drawing, so the "new" labels stay up while this screen is open
self._store.mark_all_seen()
self._shown_version = self._store.version
def _update_state(self):
super()._update_state()
self._store.refresh()
if self._store.version != self._shown_version:
self._refresh_cards()
self._shown_version = self._store.version
def _refresh_cards(self):
is_metric = ui_state.is_metric
new = self._store.new_ids()
unlocked, total = self._store.unlocked_count()
self._summary_btn.set_value(f"{unlocked} / {total} unlocked")
for status in self._store.statuses():
a = status.achievement
card = self._cards[a.id]
is_new = any(uid in new for uid in self._store.unlock_ids(a.id))
card.set_value(("new · " if is_new else "") + status.summary(is_metric))
def _show_summary(self):
gui_app.push_widget(BigDialog("", self._store.totals_text(ui_state.is_metric)))
def _show_details(self, achievement_id: str):
status = self._store.status(achievement_id)
if status is None:
return
gui_app.push_widget(BigDialog(status.achievement.name.lower(), f"{status.achievement.description}\n{status.summary(ui_state.is_metric)}"))
@@ -9,8 +9,6 @@ from openpilot.selfdrive.ui.mici.layouts.settings.settings import SettingsBigBut
from openpilot.selfdrive.ui.mici.layouts.settings.device.device_layout import DeviceLayoutMici
from openpilot.selfdrive.ui.mici.widgets.button import BigCircleButton
from openpilot.selfdrive.ui.mici.widgets.dialog import BigConfirmationDialog, BigDialog
from openpilot.selfdrive.ui.sunnypilot.achievements_store import achievements_store
from openpilot.selfdrive.ui.sunnypilot.mici.layouts.achievements import AchievementsLayoutMici
from openpilot.selfdrive.ui.sunnypilot.mici.layouts.sunnylink import SunnylinkLayoutMici
from openpilot.selfdrive.ui.sunnypilot.mici.layouts.models import ModelsLayoutMici
from openpilot.selfdrive.ui.ui_state import ui_state
@@ -52,11 +50,6 @@ class SettingsLayoutSP(OP.SettingsLayout):
models_btn = SettingsBigButton(tr("models"), "", gui_app.texture("../../sunnypilot/selfdrive/assets/offroad/icon_models.png", ICON_SIZE, ICON_SIZE))
models_btn.set_click_callback(lambda: gui_app.push_widget(models_panel))
achievements_panel = AchievementsLayoutMici()
self._achievements_btn = SettingsBigButton(tr("achievements"), "",
gui_app.texture("../../sunnypilot/selfdrive/assets/icons_mici/achievements.png", ICON_SIZE, ICON_SIZE))
self._achievements_btn.set_click_callback(lambda: gui_app.push_widget(achievements_panel))
# onroad: enable button sits at the front (left of toggles)
self._enable_offroad_btn_onroad = BigCircleButton(self.icon_offroad_enable, red=True)
self._enable_offroad_btn_onroad.set_click_callback(lambda: self._handle_always_offroad(True))
@@ -79,7 +72,6 @@ class SettingsLayoutSP(OP.SettingsLayout):
# front slots (only one ever visible at a time): exit-always-offroad, then enable-onroad
items.insert(0, self._enable_offroad_btn_onroad)
items.insert(0, self._disable_offroad_btn)
items.append(self._achievements_btn)
# end slot: enable-offroad (right of developer)
items.append(self._enable_offroad_btn_offroad)
@@ -89,11 +81,6 @@ class SettingsLayoutSP(OP.SettingsLayout):
def _update_state(self):
super()._update_state()
achievements_store.refresh()
new = len(achievements_store.new_ids())
value = tr("{} new").format(new) if new else ""
if value != self._achievements_btn.get_value():
self._achievements_btn.set_value(value)
def _handle_always_offroad(self, enable: bool):
@@ -0,0 +1,95 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
"""
import math
import pyray as rl
from openpilot.cereal import log
from openpilot.common.filter_simple import FirstOrderFilter
from openpilot.selfdrive.ui.mici.onroad import SIDE_PANEL_WIDTH
from openpilot.selfdrive.ui.ui_state import ui_state
from openpilot.sunnypilot.selfdrive.controls.lib.accel_boost import get_boost_max
from openpilot.system.ui.lib.application import gui_app
from openpilot.system.ui.widgets import Widget
BAR_WIDTH = 26
TRACK_TOP_RATIO = 0.08
TRACK_BOTTOM_RATIO = 0.92
CAP_BLEND_START = 0.45
MAX_BOOST_BLEND = 0.55
FADE_THRESHOLD = 0.01
COLOR_TRACK = rl.Color(255, 255, 255, 28)
COLOR_FILL = rl.Color(0, 230, 120, 255)
def get_boost_fraction(boost: float, max_boost: float) -> float:
if max_boost <= 0.0:
return 0.0
return min(max(boost / max_boost, 0.0), 1.0)
def lighten(color: rl.Color, amount: float) -> rl.Color:
amount = min(max(amount, 0.0), 1.0)
return rl.Color(int(color.r + (255 - color.r) * amount),
int(color.g + (255 - color.g) * amount),
int(color.b + (255 - color.b) * amount), color.a)
def get_boost_color(fraction: float) -> rl.Color:
blend = max(0.0, (fraction - CAP_BLEND_START) / (1.0 - CAP_BLEND_START))
return lighten(COLOR_FILL, MAX_BOOST_BLEND * blend)
class AccelBoostIndicator(Widget):
def __init__(self, panel_width: float = SIDE_PANEL_WIDTH, bar_width: float = BAR_WIDTH):
super().__init__()
self._panel_width = panel_width
self._bar_width = bar_width
fps = max(gui_app.target_fps, 1)
self._boost_filter = FirstOrderFilter(0.0, 0.15, 1 / fps)
self._alpha_filter = FirstOrderFilter(0.0, 0.1, 1 / fps)
self._max_boost_filter = FirstOrderFilter(get_boost_max(), 0.25, 1 / fps)
@property
def holding_panel(self) -> bool:
return self._alpha_filter.x > FADE_THRESHOLD
def _update_state(self):
sm = ui_state.sm
visible = (ui_state.accel_boost and ui_state.has_longitudinal_control and sm.alive['selfdriveState'] and
sm.alive['longitudinalPlan'] and sm['selfdriveState'].enabled)
plan = sm['longitudinalPlan']
applied = visible and plan.longitudinalPlanSource == log.LongitudinalPlan.LongitudinalPlanSource.e2e
self._boost_filter.update(float(plan.accelBoost) if applied else 0.0)
self._alpha_filter.update(1.0 if visible else 0.0)
if sm.alive['selfdriveState']:
self._max_boost_filter.update(get_boost_max(sm['selfdriveState'].personality))
def _render(self, _):
alpha = self._alpha_filter.x
if alpha < FADE_THRESHOLD:
return
panel_x = max(self.rect.x, self.rect.x + self.rect.width - self._panel_width)
bar_x = panel_x + (self._panel_width - self._bar_width) / 2
track_top = self.rect.y + self.rect.height * TRACK_TOP_RATIO
track = rl.Rectangle(bar_x, track_top, self._bar_width,
max(1.0, self.rect.height * (TRACK_BOTTOM_RATIO - TRACK_TOP_RATIO)))
rl.draw_rectangle_rounded(track, 0.5, 12,
rl.Color(COLOR_TRACK.r, COLOR_TRACK.g, COLOR_TRACK.b, int(COLOR_TRACK.a * alpha)))
fraction = get_boost_fraction(self._boost_filter.x, self._max_boost_filter.x)
fill_h = fraction * track.height
if fill_h < 1.0:
return
fill_color = get_boost_color(fraction)
tip = track.y + track.height - fill_h
rl.begin_scissor_mode(int(track.x), int(tip), int(math.ceil(track.width)) + 1, int(math.ceil(fill_h)))
rl.draw_rectangle_rounded(track, 0.5, 12, rl.Color(fill_color.r, fill_color.g, fill_color.b, int(235 * alpha)))
rl.end_scissor_mode()
@@ -0,0 +1,19 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
"""
import pyray as rl
from openpilot.selfdrive.ui.sunnypilot.mici.onroad.accel_boost_indicator import AccelBoostIndicator
class AugmentedRoadViewSP:
def __init__(self):
self._accel_boost_indicator = AccelBoostIndicator()
def render_side_panel(self, rect: rl.Rectangle) -> None:
self._accel_boost_indicator.render(rect)
if not self._accel_boost_indicator.holding_panel:
self._confidence_ball.render(rect)
@@ -0,0 +1,36 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
"""
from openpilot.cereal import log
from openpilot.common.test import OpenpilotTestCase
from openpilot.selfdrive.ui.sunnypilot.mici.onroad.accel_boost_indicator import (
CAP_BLEND_START, COLOR_FILL, get_boost_color, get_boost_fraction,
)
from openpilot.sunnypilot.selfdrive.controls.lib.accel_boost import ACCEL_BOOST_MAX, get_boost_max
class TestAccelBoostIndicator(OpenpilotTestCase):
def test_bar_fills_at_the_active_profile_cap(self):
for personality in (log.LongitudinalPersonality.aggressive, log.LongitudinalPersonality.standard,
log.LongitudinalPersonality.relaxed):
cap = get_boost_max(personality)
assert get_boost_fraction(0.0, cap) == 0.0
assert get_boost_fraction(cap * 0.5, cap) == 0.5
assert get_boost_fraction(cap, cap) == 1.0
assert get_boost_fraction(max(ACCEL_BOOST_MAX), min(ACCEL_BOOST_MAX)) == 1.0
assert get_boost_fraction(0.0, 0.0) == 0.0
assert get_boost_fraction(-1.0, get_boost_max()) == 0.0
def test_fill_lightens_as_boost_builds(self):
base = get_boost_color(0.0)
assert (base.r, base.g, base.b) == (COLOR_FILL.r, COLOR_FILL.g, COLOR_FILL.b)
assert (get_boost_color(CAP_BLEND_START).r, get_boost_color(CAP_BLEND_START).b) == (COLOR_FILL.r, COLOR_FILL.b)
full = get_boost_color(1.0)
assert full.r > base.r
assert full.b > base.b
assert full.g > full.r and full.g > full.b
@@ -1,57 +0,0 @@
from openpilot.selfdrive.ui.sunnypilot.achievements_store import AchievementsStore
from openpilot.sunnypilot.achievements.engine import AchievementEngine, State, save_state
T0 = 1_780_000_000.0
def make_store(tmp_path):
return AchievementsStore(state_file=tmp_path / "state.json", seen_file=tmp_path / "seen.json")
def write_state(tmp_path, engaged_km):
e = AchievementEngine(State(engaged_m=engaged_km * 1000.0))
e.check_unlocks(T0)
save_state(e.state, tmp_path / "state.json")
def test_no_state_file_shows_everything_locked(tmp_path):
store = make_store(tmp_path)
assert store.refresh(force=True)
unlocked, total = store.unlocked_count()
assert unlocked == 0 and total > 10
assert store.new_ids() == set()
def test_new_unlocks_until_seen(tmp_path):
write_state(tmp_path, 150)
store = make_store(tmp_path)
store.refresh(force=True)
assert store.new_ids() == {"first_light", "odometer.bronze"}
v = store.version
store.mark_all_seen()
assert store.new_ids() == set() and store.version > v
# a second store (the other screen, or after a reboot) sees them as seen too
other = make_store(tmp_path)
other.refresh(force=True)
assert other.new_ids() == set()
def test_refresh_picks_up_new_unlocks(tmp_path):
write_state(tmp_path, 150)
store = make_store(tmp_path)
store.refresh(force=True)
store.mark_all_seen()
write_state(tmp_path, 1500)
assert store.refresh(force=True)
assert store.new_ids() == {"odometer.silver"}
def test_totals_and_status(tmp_path):
write_state(tmp_path, 1240)
store = make_store(tmp_path)
store.refresh(force=True)
assert store.status("odometer").summary() == "silver · 1,240 / 10,000 km"
assert "1,240 km engaged" in store.totals_text()
assert store.unlock_ids("odometer") == ["odometer.bronze", "odometer.silver", "odometer.gold", "odometer.platinum"]
assert store.status("nope") is None
@@ -44,10 +44,12 @@ class UIStateSP:
self.screensaver = ScreenSaverSP(params=self.params)
self.screensaver_enabled: bool = False
self.accel_boost: bool = False
self.active_bundle = None
self.model_runner_tinygrad: bool = False
self.active_runner_tinygrad: bool = False
self.blindspot: bool = False
self.trip_summary: bool = True
self.chevron_metrics = None
self.custom_interactive_timeout: int = 0
self.developer_ui = None
@@ -171,6 +173,7 @@ class UIStateSP:
# stock only counts the default big model's compiled pkl. a downloaded big bundle runs on the
# chestnut just the same, so ChestnutState has to see it as available too.
self.chestnut_compiled = self.chestnut_compiled or self.model_runner_tinygrad
self.accel_boost = self.params.get_bool("AccelBoostSP")
self.blindspot = self.params.get_bool("BlindSpot")
self.chevron_metrics = self.params.get("ChevronInfo")
self.custom_interactive_timeout = self.params.get("InteractivityTimeout", return_default=True)
@@ -185,6 +188,7 @@ class UIStateSP:
self.standstill_timer = self.params.get_bool("StandstillTimer")
self.sunnylink_enabled = self.params.get_bool("SunnylinkEnabled")
self.torque_bar = self.params.get_bool("TorqueBar")
self.trip_summary = self.params.get_bool("TripSummary")
self.enforce_torque_control = self.params.get_bool("EnforceTorqueControl")
self.custom_torque_params = self.params.get_bool("CustomTorqueParams")
self.torque_override_enabled = self.params.get_bool("TorqueParamsOverrideEnabled")
@@ -1,55 +0,0 @@
# Achievements and the community log
Everything here runs on the car. Nothing is uploaded yet: the uploads are a dry run that writes files.
| File | What it does |
| --- | --- |
| `achievementsd.py` | Onroad process. Feeds car messages to the engine and the community log. |
| `engine.py`, `catalog.py`, `catalog.json` | Driving stats and achievement unlocks (Phase 1). |
| `community.py` | Community log in `community.db`: 6 km tiles, drive start/end tiles, daily totals, region per day. Also the privacy filter and the report builders. |
| `preview.py` | Writes what the car would send, plus `map.html`, to `preview/`. |
| `h3lite.py` | The H3 functions we need, in pure Python (sunnypilot doesn't add packages). Same cell ids as Uber's H3. |
| `regions.py`, `regions.json.gz` | Tile to home region (`US-OH`, `CA-ON`, `GB`). Built by `tools/build_regions.py`. |
| `replay.py` | Runs everything over drives already saved on the car. |
| `simulate.py` | Made-up drives for tests and a demo. |
## Privacy rules (from the build plan)
- Positions only become H3 resolution 6 tiles (about 6 km). No coordinates, times of day or routes are stored.
- A tile where drives start or end on 3+ days is a frequent stop. It and the 6 tiles around it never leave the car.
- Other start and end tiles wait 14 days.
- Hidden tiles never leave the car.
- The tile batch is sorted and has no dates. It is separate from the daily report.
- Others only ever see resolution 4 blocks (about 45 km).
- Home region: where you drove most on most days in the last 90 days. A detected change is sent at most every 30 days; a change you make goes right away.
## Files on the car
All under `/data/media/0/achievements/`:
- `state.json`: achievement progress.
- `community.db`: the community log.
- `community_settings.json` (optional): `{"level": 0-3, "region": "auto" | "country" | "hidden" | "US-OH"}`. Level 0 (off) is the default.
- `preview/`: `daily-YYYY-MM-DD.json`, `tiles.json`, `delete.json`, `map.html`. Rewritten after every drive.
`map.html` shows your frequent stops. It is for you only.
## See it on your own drives
```sh
ssh comma@<car ip>
cd /data/openpilot
python -m openpilot.sunnypilot.achievements.replay # qlogs in /data/media/0/realdata; add --days 30 or --rlog
# on your computer:
scp comma@<car ip>:/data/media/0/achievements/replay/preview/map.html .
```
The replay writes to `achievements/replay/` and doesn't touch live progress.
## Tests
```sh
python -m pytest openpilot/sunnypilot/achievements/tests
```
With `h3` installed (`pip install h3`), the H3 tests also compare random points against the real library.
@@ -1,190 +0,0 @@
#!/usr/bin/env python3
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
achievementsd: counts driving stats and unlocks achievements while onroad.
Read-only toward the car: it only subscribes to messages, never publishes
controls. Progress is saved to a JSON file on the device (see engine.state_path)
every SAVE_INTERVAL_S and when the drive ends. The community log (map tiles,
stops, daily totals; see community.py) is saved next to it, and when the drive
ends a dry-run preview of what would be shared is written to preview/.
Nothing leaves the device.
"""
import datetime
import time
from openpilot.cereal import messaging
from openpilot.common.params import Params
from openpilot.common.realtime import Ratekeeper
from openpilot.common.swaglog import cloudlog
from openpilot.common.time_helpers import system_time_valid
from openpilot.sunnypilot.achievements.community import CommunityLog, Settings
from openpilot.sunnypilot.achievements.engine import (AchievementEngine, MAX_GPS_ACCURACY_M, Sample, load_state,
save_state)
from openpilot.sunnypilot.achievements.preview import write_preview
RATE_HZ = 10.
SAVE_INTERVAL_S = 30.
SLOW_POLL_S = 30. # model, build and sunnylink checks
GPS_MAX_AGE_S = 2.
UNREGISTERED_SUNNYLINK_DONGLE_ID = "UnregisteredDevice"
SERVICES = ['carState', 'selfdriveState', 'selfdriveStateSP', 'modelV2', 'driverMonitoringState',
'gpsLocation', 'gpsLocationExternal']
def _dev_build() -> bool:
try:
from openpilot.common.version import get_build_metadata
bm = get_build_metadata()
return "staging" in bm.channel or not (bm.release_channel or bm.release_sp_channel)
except Exception:
return False
def _active_model() -> tuple[str, str]:
try:
from openpilot.sunnypilot.models.helpers import get_active_bundle
bundle = get_active_bundle()
if bundle is None:
return "default", "Default"
return bundle.ref or bundle.internalName, bundle.displayName or bundle.internalName
except Exception:
cloudlog.exception("achievementsd: could not read the active model")
return "", ""
def _sunnylink_linked(params: Params) -> bool:
try:
dongle = params.get("SunnylinkDongleId")
return bool(params.get_bool("SunnylinkEnabled")) and dongle not in (None, "", UNREGISTERED_SUNNYLINK_DONGLE_ID)
except Exception:
return False
def _gps_fix(sm: messaging.SubMaster, now: float) -> tuple[float | None, float | None]:
for svc in ('gpsLocation', 'gpsLocationExternal'):
if sm.seen[svc] and now - sm.recv_time[svc] < GPS_MAX_AGE_S:
g = sm[svc]
if g.hasFix and 0 < g.horizontalAccuracy < MAX_GPS_ACCURACY_M:
return g.latitude, g.longitude
return None, None
def _car_platform(params: Params) -> str | None:
try:
from opendbc.car.structs import car
cp_bytes = params.get("CarParams")
if not cp_bytes:
return None
return messaging.log_from_bytes(cp_bytes, car.CarParams).carFingerprint or None
except Exception:
return None
def _open_community() -> CommunityLog | None:
try:
community = CommunityLog()
_ = community.regions # load the region table now, not mid-drive
return community
except Exception:
cloudlog.exception("achievementsd: could not open the community log")
return None
def _community_step(community: CommunityLog, sample: Sample, engine: AchievementEngine, save: bool) -> CommunityLog | None:
"""Feed one sample. On any error the community log is off until the next drive; achievements keep working."""
try:
community.update(sample, engine.last_step)
if save:
community.flush()
return community
except Exception:
cloudlog.exception("achievementsd: community log failed, off until the next drive")
community.close()
return None
def _finish_community(community: CommunityLog, engine: AchievementEngine, params: Params) -> None:
"""Save where the drive ended and write the dry-run preview. Must stay well under the manager's 5 s stop timeout."""
try:
community.end_drive(engine.utc_offset_h)
if system_time_valid():
today = engine.local_date(datetime.datetime.now(datetime.UTC).timestamp())
write_preview(community, engine.state, today, _car_platform(params), Settings.load())
except Exception:
cloudlog.exception("achievementsd: community log or preview failed")
finally:
community.close()
def _wall_time(valid: bool) -> float | None:
return datetime.datetime.now(datetime.UTC).timestamp() if valid else None
def main():
params = Params()
sm = messaging.SubMaster(SERVICES, ignore_alive=['gpsLocation', 'gpsLocationExternal'],
ignore_valid=['gpsLocation', 'gpsLocationExternal'])
engine = AchievementEngine(load_state())
community = _open_community()
rk = Ratekeeper(RATE_HZ, print_delay_threshold=None)
dev_build = _dev_build()
model_ref, model_name = _active_model()
last_save = last_slow = time.monotonic()
clock_ok = system_time_valid()
engine.set_sunnylink_linked(_sunnylink_linked(params), _wall_time(clock_ok))
try:
while True:
sm.update(0)
now = time.monotonic()
if now - last_slow > SLOW_POLL_S:
last_slow = now
model_ref, model_name = _active_model()
clock_ok = system_time_valid()
for uid in engine.set_sunnylink_linked(_sunnylink_linked(params), _wall_time(clock_ok)):
cloudlog.event("achievement unlocked", id=uid)
lat, lon = _gps_fix(sm, now)
sample = Sample(
t=now,
v_ego=sm['carState'].vEgo,
op_active=sm['selfdriveState'].active,
mads_active=sm['selfdriveStateSP'].mads.active,
lane_change_state=sm['modelV2'].meta.laneChangeState.raw,
dm_alert_level=sm['driverMonitoringState'].alertLevel.raw,
lat=lat,
lon=lon,
model_ref=model_ref,
model_name=model_name,
dev_build=dev_build,
wall=_wall_time(clock_ok),
)
for uid in engine.update(sample):
cloudlog.event("achievement unlocked", id=uid)
save = now - last_save > SAVE_INTERVAL_S
if save:
last_save = now
save_state(engine.state)
if community is not None:
community = _community_step(community, sample, engine, save)
rk.keep_time()
finally:
# the manager stops us with SIGINT when the drive ends
engine.end_drive()
save_state(engine.state)
if community is not None:
_finish_community(community, engine, params)
if __name__ == "__main__":
main()
@@ -1,153 +0,0 @@
{
"version": 1,
"achievements": [
{
"id": "first_light",
"name": "First Light",
"description": "Your first engaged drive with sunnypilot.",
"metric": "engaged_km",
"unit": "km",
"tiers": [{"tier": "single", "threshold": 0.1}]
},
{
"id": "odometer",
"name": "Odometer",
"description": "Distance driven with sunnypilot engaged.",
"metric": "engaged_km",
"unit": "km",
"tiers": [
{"tier": "bronze", "threshold": 100},
{"tier": "silver", "threshold": 1000},
{"tier": "gold", "threshold": 10000},
{"tier": "platinum", "threshold": 50000}
]
},
{
"id": "captains_hours",
"name": "Captain's Hours",
"description": "Time spent with sunnypilot engaged.",
"metric": "engaged_h",
"unit": "h",
"tiers": [
{"tier": "bronze", "threshold": 10},
{"tier": "silver", "threshold": 100},
{"tier": "gold", "threshold": 1000}
]
},
{
"id": "regular",
"name": "Regular",
"description": "Different days you drove with sunnypilot engaged.",
"metric": "active_days",
"unit": "days",
"tiers": [
{"tier": "bronze", "threshold": 7},
{"tier": "silver", "threshold": 30},
{"tier": "gold", "threshold": 100},
{"tier": "platinum", "threshold": 365}
]
},
{
"id": "streak",
"name": "Streak",
"description": "Engaged on this many days in a row.",
"metric": "best_streak_days",
"unit": "days",
"tiers": [
{"tier": "bronze", "threshold": 7},
{"tier": "silver", "threshold": 30}
]
},
{
"id": "steady_hands",
"name": "Steady Hands",
"description": "Distance with MADS steering on its own, without sunnypilot's cruise.",
"metric": "mads_km",
"unit": "km",
"tiers": [
{"tier": "bronze", "threshold": 100},
{"tier": "silver", "threshold": 1000}
]
},
{
"id": "smooth_merger",
"name": "Smooth Merger",
"description": "Assisted lane changes completed.",
"metric": "lane_changes",
"unit": "changes",
"tiers": [
{"tier": "bronze", "threshold": 10},
{"tier": "silver", "threshold": 100},
{"tier": "gold", "threshold": 1000}
]
},
{
"id": "road_trip",
"name": "Road Trip",
"description": "Engaged distance in a single day. Take breaks along the way.",
"metric": "best_day_km",
"unit": "km",
"tiers": [{"tier": "single", "threshold": 500}]
},
{
"id": "explorer",
"name": "Explorer",
"description": "Different areas (about 5 km across) driven engaged. Areas are kept as anonymous hashes on this device.",
"metric": "areas",
"unit": "areas",
"tiers": [
{"tier": "bronze", "threshold": 5},
{"tier": "silver", "threshold": 25},
{"tier": "gold", "threshold": 100}
]
},
{
"id": "model_hopper",
"name": "Model Hopper",
"description": "Driving models you have driven at least 50 km engaged.",
"metric": "models_50km",
"unit": "models",
"tiers": [
{"tier": "bronze", "threshold": 3},
{"tier": "silver", "threshold": 5},
{"tier": "gold", "threshold": 10}
]
},
{
"id": "dev_branch_brave",
"name": "Dev Branch Brave",
"description": "Engaged distance on a development or staging build.",
"metric": "dev_km",
"unit": "km",
"tiers": [{"tier": "single", "threshold": 100}]
},
{
"id": "attentive",
"name": "Attentive",
"description": "Engaged time in a row without a red driver monitoring alert.",
"metric": "attentive_best_h",
"unit": "h",
"tiers": [
{"tier": "bronze", "threshold": 10},
{"tier": "silver", "threshold": 50},
{"tier": "gold", "threshold": 100}
]
},
{
"id": "seasons",
"name": "Four Seasons",
"description": "Engaged drives in spring, summer, fall and winter.",
"metric": "seasons",
"unit": "seasons",
"tiers": [{"tier": "single", "threshold": 4}]
},
{
"id": "linked_up",
"name": "Linked Up",
"description": "Turn on sunnylink on this device.",
"metric": "sunnylink_linked",
"unit": "",
"tiers": [{"tier": "single", "threshold": 1}]
}
]
}
@@ -1,137 +0,0 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
The achievement catalog. Pure Python on purpose: the UI, achievementsd and the
tests all load it, and none of them should need cereal or the Params library to
read a list of goals.
"""
from __future__ import annotations
import json
from dataclasses import dataclass
from functools import cache
from pathlib import Path
CATALOG_PATH = Path(__file__).with_name("catalog.json")
SINGLE = "single"
KM_TO_MI = 0.621371
@dataclass(frozen=True)
class Tier:
tier: str
threshold: float
@dataclass(frozen=True)
class Achievement:
id: str
name: str
description: str
metric: str
unit: str
tiers: tuple[Tier, ...]
def unlock_id(self, tier: Tier) -> str:
"""The id an unlock is stored and shared under: `odometer.silver`, or just `first_light`."""
return self.id if tier.tier == SINGLE else f"{self.id}.{tier.tier}"
@property
def tiered(self) -> bool:
return not (len(self.tiers) == 1 and self.tiers[0].tier == SINGLE)
@dataclass(frozen=True)
class Catalog:
version: int
achievements: tuple[Achievement, ...]
@property
def metrics(self) -> set[str]:
return {a.metric for a in self.achievements}
def all_unlock_ids(self) -> list[str]:
return [a.unlock_id(t) for a in self.achievements for t in a.tiers]
def parse_catalog(data: dict) -> Catalog:
achievements = []
seen: set[str] = set()
for raw in data["achievements"]:
tiers = tuple(Tier(str(t["tier"]), float(t["threshold"])) for t in raw["tiers"])
if not tiers:
raise ValueError(f"achievement {raw['id']} has no tiers")
if any(b.threshold <= a.threshold for a, b in zip(tiers, tiers[1:], strict=False)):
raise ValueError(f"achievement {raw['id']} tiers must increase")
if raw["id"] in seen:
raise ValueError(f"duplicate achievement id {raw['id']}")
seen.add(raw["id"])
achievements.append(Achievement(raw["id"], raw["name"], raw["description"], raw["metric"], raw.get("unit", ""), tiers))
return Catalog(int(data["version"]), tuple(achievements))
@cache
def load_catalog(path: str = str(CATALOG_PATH)) -> Catalog:
with open(path) as f:
return parse_catalog(json.load(f))
# --- display helpers, shared by both device UIs ---
def _fmt_number(value: float) -> str:
if value >= 100 or value == int(value):
return f"{int(value):,}"
return f"{value:.1f}"
def format_amount(value: float, unit: str, is_metric: bool = True) -> str:
if unit == "km" and not is_metric:
value, unit = value * KM_TO_MI, "mi"
if not unit:
return _fmt_number(value)
return f"{_fmt_number(value)} {unit}"
@dataclass(frozen=True)
class AchievementStatus:
achievement: Achievement
reached: Tier | None # highest tier unlocked, None if locked
next: Tier | None # next tier to unlock, None when all are done
value: float
@property
def done(self) -> bool:
return self.next is None
def summary(self, is_metric: bool = True) -> str:
"""One short line: `silver · 1,240 / 10,000 km`, `locked · 3 / 7 days`, `gold · done`."""
a = self.achievement
if self.next is None:
return f"{self.reached.tier} · done" if a.tiered and self.reached else "done"
label = self.reached.tier if self.reached else "locked"
if not a.unit: # a yes/no achievement, like turning on sunnylink
return label
return f"{label} · {format_progress(self.value, self.next.threshold, a.unit, is_metric)}"
def format_progress(value: float, goal: float, unit: str, is_metric: bool = True) -> str:
if unit == "km" and not is_metric:
value, goal, unit = value * KM_TO_MI, goal * KM_TO_MI, "mi"
return f"{_fmt_number(min(value, goal))} / {_fmt_number(goal)} {unit}"
def statuses(catalog: Catalog, metrics: dict[str, float], unlocked: dict[str, str]) -> list[AchievementStatus]:
out = []
for a in catalog.achievements:
reached = None
nxt = None
for t in a.tiers:
if a.unlock_id(t) in unlocked:
reached = t
elif nxt is None:
nxt = t
out.append(AchievementStatus(a, reached, nxt, float(metrics.get(a.metric, 0.0))))
return out
@@ -1,370 +0,0 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
The community log and the privacy filter.
While driving, achievementsd hands every sample to CommunityLog, which keeps in
community.db on the car:
- the 6 km map tiles (H3 resolution 6) you drove through, with the day each was
first seen,
- the tile where each drive started and ended, per day,
- engaged distance and time per day,
- engaged distance per region per day.
Positions only ever become tiles: no coordinates, times of day or routes are
stored here.
The privacy filter (tile_view, daily_report, tile_batch, profile_update) then
decides what could leave the car, following the Privacy spec in the build plan:
- A tile holding a drive's start or end on 3+ different days is a frequent stop
(home, work, school). It and the 6 tiles around it never leave the car.
- Any other start or end tile waits 14 days.
- Tiles the driver hid never leave the car.
- Tiles go out as a sorted list with no dates, apart from the daily report.
- The home region is where you drove most on most days over 90 days. It is sent
only when it changes, and an automatic change at most every 30 days.
Nothing in this file sends anything.
"""
from __future__ import annotations
import collections
import datetime as dt
import json
import sqlite3
from dataclasses import dataclass, field
from pathlib import Path
from openpilot.sunnypilot.achievements import h3lite
from openpilot.sunnypilot.achievements.engine import Sample, State, Step, achievements_root
from openpilot.sunnypilot.achievements.regions import RegionTable, country_of
from openpilot.sunnypilot.achievements.regions import table as region_table
REPORT_VERSION = 1
TILE_RES = 6 # about 6 km across; only the owner sees these
PUBLIC_RES = 4 # about 45 km across; what crews and the public see
STOP_HOLD_DAYS = 14
FREQUENT_STOP_DAYS = 3
REGION_WINDOW_DAYS = 90
REGION_RESEND_DAYS = 30
MOVING_MS = 1.0
METERS_PER_MILE = 1609.344
UNLINKED_DEVICE = "c_" + "0" * 16 # the real id comes from the community key made at linking
LEVEL_OFF, LEVEL_ACHIEVEMENTS, LEVEL_DAILY, LEVEL_MAP = range(4)
SCHEMA = """
CREATE TABLE IF NOT EXISTS tiles (cell TEXT PRIMARY KEY, first_day TEXT NOT NULL, sent INTEGER NOT NULL DEFAULT 0);
CREATE TABLE IF NOT EXISTS stops (cell TEXT NOT NULL, day TEXT NOT NULL, PRIMARY KEY (cell, day));
CREATE TABLE IF NOT EXISTS days (day TEXT PRIMARY KEY, engaged_m REAL NOT NULL, engaged_s REAL NOT NULL);
CREATE TABLE IF NOT EXISTS day_regions (day TEXT NOT NULL, region TEXT NOT NULL, engaged_m REAL NOT NULL,
PRIMARY KEY (day, region));
CREATE TABLE IF NOT EXISTS hidden (cell TEXT PRIMARY KEY);
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
"""
def db_path() -> Path:
return achievements_root() / "community.db"
def settings_path() -> Path:
return achievements_root() / "community_settings.json"
@dataclass
class Settings:
level: int = LEVEL_OFF # 0 off (default), 1 achievements, 2 daily stats, 3 map
region: str = "auto" # "auto", "country", "hidden", or a region code the driver picked
@classmethod
def load(cls, path: Path | None = None) -> Settings:
try:
with open(path or settings_path()) as f:
d = json.load(f)
return cls(level=int(d.get("level", LEVEL_OFF)), region=str(d.get("region", "auto")))
except (OSError, ValueError, TypeError, AttributeError):
return cls()
def round_to(x: float, step: int) -> int:
return int(round(x / step)) * step
def tile_of(lat: float, lon: float) -> str:
return h3lite.cell_to_str(h3lite.latlng_to_cell(lat, lon, TILE_RES))
def ring_of(tile: str) -> set[str]:
"""The tile and the tiles touching it: an area about 18 km across."""
return {h3lite.cell_to_str(c) for c in h3lite.grid_disk1(h3lite.str_to_cell(tile))}
def public_block(tile: str) -> str:
return h3lite.cell_to_str(h3lite.cell_to_parent(h3lite.str_to_cell(tile), PUBLIC_RES))
@dataclass
class TileView:
"""Every driven tile, sorted by what the privacy filter does with it."""
driven: set[str] = field(default_factory=set)
frequent: set[str] = field(default_factory=set) # home, work, school...
ring: set[str] = field(default_factory=set) # frequent stops and the tiles around them: never sent
held: dict[str, str] = field(default_factory=dict) # recent stop tile -> first day it may be sent
hidden: set[str] = field(default_factory=set)
shareable: set[str] = field(default_factory=set)
sent: set[str] = field(default_factory=set)
@property
def to_send(self) -> set[str]:
return self.shareable - self.sent
@property
def to_delete(self) -> set[str]:
"""Tiles already sent that later became private (moved house, new job, hidden)."""
return self.sent & (self.ring | self.hidden)
class CommunityLog:
def __init__(self, path: Path | None = None, regions: RegionTable | None = None):
self.path = path or db_path()
self.path.parent.mkdir(parents=True, exist_ok=True)
self.db = sqlite3.connect(self.path)
self.db.executescript(SCHEMA)
self._regions = regions
self._known: set[str] = {r[0] for r in self.db.execute("SELECT cell FROM tiles")}
self._region_of: dict[str, str | None] = {}
self._utc_offset_h: float | None = None
self._reset_pending()
self._reset_drive()
def close(self) -> None:
self.db.close()
@property
def regions(self) -> RegionTable:
if self._regions is None:
self._regions = region_table()
return self._regions
def _reset_pending(self) -> None:
self._new_tiles: dict[str, str] = {}
self._stops: set[tuple[str, str]] = set()
self._days: dict[str, list[float]] = collections.defaultdict(lambda: [0.0, 0.0])
self._day_regions: dict[tuple[str, str], float] = collections.defaultdict(float)
def _reset_drive(self) -> None:
self._started = False
self._last_fix: tuple[str, str] | None = None
# --- while driving ---
def update(self, s: Sample, step: Step) -> None:
day = step.day
if day is None:
return
if s.engaged and step.dt_s > 0.0:
d = self._days[day]
d[0] += step.engaged_m
d[1] += step.dt_s
if s.lat is None or s.lon is None:
return
tile = tile_of(s.lat, s.lon)
if not self._started:
self._started = True
self._stops.add((tile, day))
self._last_fix = (tile, day)
if s.v_ego > MOVING_MS and tile not in self._known and tile not in self._new_tiles:
self._new_tiles[tile] = day
if step.engaged_m > 0.0:
if tile not in self._region_of:
self._region_of[tile] = self.regions.lookup_cell(h3lite.str_to_cell(tile))
region = self._region_of[tile]
if region is not None:
self._day_regions[(day, region)] += step.engaged_m
def end_drive(self, utc_offset_h: float | None = None) -> None:
"""Record where the drive ended and save. Safe to call more than once."""
if self._last_fix is not None:
self._stops.add(self._last_fix)
if utc_offset_h is not None:
self._utc_offset_h = utc_offset_h
self._reset_drive()
self.flush()
def flush(self) -> None:
with self.db:
self.db.executemany("INSERT OR IGNORE INTO tiles (cell, first_day) VALUES (?, ?)", self._new_tiles.items())
self.db.executemany("INSERT OR IGNORE INTO stops (cell, day) VALUES (?, ?)", self._stops)
self.db.executemany("""INSERT INTO days (day, engaged_m, engaged_s) VALUES (?, ?, ?)
ON CONFLICT(day) DO UPDATE SET engaged_m = engaged_m + excluded.engaged_m,
engaged_s = engaged_s + excluded.engaged_s""",
[(day, m, s) for day, (m, s) in self._days.items()])
self.db.executemany("""INSERT INTO day_regions (day, region, engaged_m) VALUES (?, ?, ?)
ON CONFLICT(day, region) DO UPDATE SET engaged_m = engaged_m + excluded.engaged_m""",
[(day, region, m) for (day, region), m in self._day_regions.items()])
if self._utc_offset_h is not None:
self._set_meta("utc_offset_h", self._utc_offset_h)
self._known.update(self._new_tiles)
self._reset_pending()
# --- settings kept with the data ---
def _get_meta(self, key: str, default=None):
row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
return json.loads(row[0]) if row else default
def _set_meta(self, key: str, value) -> None:
self.db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, json.dumps(value)))
def hide_tile(self, tile: str) -> None:
with self.db:
self.db.execute("INSERT OR IGNORE INTO hidden (cell) VALUES (?)", (tile,))
def unhide_tile(self, tile: str) -> None:
with self.db:
self.db.execute("DELETE FROM hidden WHERE cell = ?", (tile,))
def mark_sent(self, tiles: set[str], sent: bool = True) -> None:
"""For the uploader: tiles the server now has (or, with sent=False, has deleted)."""
with self.db:
self.db.executemany("UPDATE tiles SET sent = ? WHERE cell = ?", [(int(sent), t) for t in tiles])
# --- the privacy filter ---
def tile_view(self, today: dt.date) -> TileView:
v = TileView()
for cell, sent in self.db.execute("SELECT cell, sent FROM tiles"):
v.driven.add(cell)
if sent:
v.sent.add(cell)
stop_days: dict[str, set[str]] = collections.defaultdict(set)
for cell, day in self.db.execute("SELECT cell, day FROM stops"):
stop_days[cell].add(day)
v.frequent = {c for c, days in stop_days.items() if len(days) >= FREQUENT_STOP_DAYS}
for c in v.frequent:
v.ring |= ring_of(c)
for c, days in stop_days.items():
if c in v.ring:
continue
release = dt.date.fromisoformat(max(days)) + dt.timedelta(days=STOP_HOLD_DAYS)
if release > today:
v.held[c] = release.isoformat()
v.hidden = {r[0] for r in self.db.execute("SELECT cell FROM hidden")}
v.shareable = v.driven - v.ring - set(v.held) - v.hidden
return v
def driven_days(self, since: dt.date | None = None) -> list[str]:
rows = self.db.execute("SELECT day FROM days WHERE day >= ? ORDER BY day", ((since or dt.date.min).isoformat(),))
return [r[0] for r in rows]
def day_stats(self, day: str) -> tuple[float, float, int]:
"""Engaged meters, engaged seconds and new tiles on a day."""
row = self.db.execute("SELECT engaged_m, engaged_s FROM days WHERE day = ?", (day,)).fetchone()
new = self.db.execute("SELECT COUNT(*) FROM tiles WHERE first_day = ?", (day,)).fetchone()[0]
return (row[0], row[1], new) if row else (0.0, 0.0, new)
def region_days(self, today: dt.date) -> collections.Counter:
"""How many of the last 90 days each region was where you drove most, engaged."""
since = (today - dt.timedelta(days=REGION_WINDOW_DAYS)).isoformat()
best: dict[str, tuple[float, str]] = {}
for day, region, m in self.db.execute("SELECT day, region, engaged_m FROM day_regions WHERE day > ? ORDER BY region",
(since,)):
if day not in best or m > best[day][0]:
best[day] = (m, region)
return collections.Counter(region for _, region in best.values())
def home_region(self, today: dt.date) -> str | None:
tally = self.region_days(today)
if not tally:
return None
return max(sorted(tally), key=lambda r: tally[r])
def shared_region(self, today: dt.date, settings: Settings) -> str | None:
if settings.region == "hidden":
return None
if settings.region not in ("auto", "country"):
return settings.region
home = self.home_region(today)
if home is None:
return None
return country_of(home) if settings.region == "country" else home
def profile_update(self, today: dt.date, car: str | None, settings: Settings) -> dict | None:
"""Car model and home region, only when they changed since they were last sent."""
last = self._get_meta("profile_sent", {})
out: dict = {}
if car and car != last.get("car"):
out["car"] = car
region = self.shared_region(today, settings)
if region != last.get("region") and (region is not None or "region" in last):
# the driver changing the setting goes right away; a change the car detected waits
chosen = settings.region != last.get("choice", "auto")
last_day = last.get("region_day")
if chosen or last_day is None or (today - dt.date.fromisoformat(last_day)).days >= REGION_RESEND_DAYS:
out["region"] = region
return out or None
def mark_profile_sent(self, today: dt.date, profile: dict, settings: Settings) -> None:
last = self._get_meta("profile_sent", {})
if "car" in profile:
last["car"] = profile["car"]
if "region" in profile:
last["region"] = profile["region"]
last["region_day"] = today.isoformat()
last["choice"] = settings.region
with self.db:
self._set_meta("profile_sent", last)
# --- uploads ---
def unlock_day(self, unix: float) -> str | None:
if not unix:
return None # unlocked before the clock was valid
offset = self._utc_offset_h if self._utc_offset_h is not None else self._get_meta("utc_offset_h", 0)
return (dt.datetime.fromtimestamp(unix, dt.UTC) + dt.timedelta(hours=offset)).date().isoformat()
def daily_report(self, day: str, state: State, level: int, profile: dict | None = None,
include_undated: bool = False, device: str = UNLINKED_DEVICE) -> dict | None:
"""The daily report for one day, with only the parts the sharing level allows."""
if level < LEVEL_ACHIEVEMENTS:
return None
unlocks = []
for uid, unix in sorted(state.unlocked.items()):
d = self.unlock_day(unix)
if d == day or (d is None and include_undated):
unlocks.append({"id": uid, "day": day})
report: dict = {
"v": REPORT_VERSION,
"device": device,
"day": day,
"unlocks": unlocks,
"totals": {"engaged_mi": round_to(state.engaged_m / METERS_PER_MILE, 10),
"engaged_h": round_to(state.engaged_s / 3600.0, 10)},
}
if level >= LEVEL_DAILY:
m, s, new = self.day_stats(day)
report["daily"] = {"engaged_mi": round(m / METERS_PER_MILE), "engaged_min": round(s / 60.0), "new_places": new}
if profile:
report["profile"] = profile
return report
def tile_batch(self, today: dt.date, device: str = UNLINKED_DEVICE) -> dict | None:
"""The weekly tile batch: new shareable tiles, sorted, with no dates."""
tiles = self.tile_view(today).to_send
if not tiles:
return None
return {"v": REPORT_VERSION, "device": device, "tiles": sorted(tiles)}
def tile_deletes(self, today: dt.date, device: str = UNLINKED_DEVICE) -> dict | None:
tiles = self.tile_view(today).to_delete
if not tiles:
return None
return {"v": REPORT_VERSION, "device": device, "delete_tiles": sorted(tiles)}
-347
View File
@@ -1,347 +0,0 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
The achievements engine: running totals and unlocks, fed one Sample at a time.
Pure Python with no cereal or Params imports, so it can be tested and replayed
anywhere. achievementsd turns live messages into Samples; the UI only reads the
saved state file.
Everything stays on the device. Explorer areas are stored as salted hashes of
~5 km geohash cells, never as coordinates.
"""
from __future__ import annotations
import datetime as dt
import hashlib
import json
import os
import secrets
import tempfile
from dataclasses import asdict, dataclass, field
from pathlib import Path
from openpilot.sunnypilot.achievements.catalog import Catalog, load_catalog
STATE_VERSION = 1
# plausibility limits for a single update
MAX_DT_S = 1.0 # a longer gap (process stall, suspend) counts nothing
MAX_SPEED_MS = 70.0 # ~250 km/h; above this the reading is not trusted
MIN_SPEED_MS = 0.0
# GPS
GEOHASH_PRECISION = 5 # cells of about 4.9 x 4.9 km
MAX_GPS_ACCURACY_M = 100.0
MODEL_KM_GOAL = 50.0 # Model Hopper: km on a model before it counts
# driver monitoring alertLevel.three is the red, take-control-now alert
DM_ALERT_RED = 3
# modelV2.meta.laneChangeState
LANE_CHANGE_STARTING = 2
LANE_CHANGE_FINISHING = 3
_GEOHASH_BASE32 = "0123456789bcdefghjkmnpqrstuvwxyz"
def achievements_root() -> Path:
"""/data/media/0/achievements on the device (survives updates), ~/.comma/... on PC."""
if "ACHIEVEMENTS_ROOT" in os.environ:
return Path(os.environ["ACHIEVEMENTS_ROOT"])
if os.path.isfile("/TICI"):
return Path("/data/media/0/achievements")
return Path.home() / (".comma" + os.environ.get("OPENPILOT_PREFIX", "")) / "media" / "0" / "achievements"
def state_path() -> Path:
return achievements_root() / "state.json"
def seen_path() -> Path:
"""Unlocks the driver has already looked at. Written only by the UI."""
return achievements_root() / "seen.json"
def geohash(lat: float, lon: float, precision: int = GEOHASH_PRECISION) -> str:
lat_rng, lon_rng = [-90.0, 90.0], [-180.0, 180.0]
bits, bit, ch, even = [16, 8, 4, 2, 1], 0, 0, True
out = []
while len(out) < precision:
rng, val = (lon_rng, lon) if even else (lat_rng, lat)
mid = (rng[0] + rng[1]) / 2
if val >= mid:
ch |= bits[bit]
rng[0] = mid
else:
rng[1] = mid
even = not even
if bit < 4:
bit += 1
else:
out.append(_GEOHASH_BASE32[ch])
bit, ch = 0, 0
return "".join(out)
def season(day: dt.date, southern: bool = False) -> str:
s = ("winter", "winter", "spring", "spring", "spring", "summer", "summer", "summer", "fall", "fall", "fall", "winter")[day.month - 1]
if southern:
s = {"winter": "summer", "summer": "winter", "spring": "fall", "fall": "spring"}[s]
return s
@dataclass
class Sample:
"""One reading of the car, built by achievementsd from cereal."""
t: float # monotonic seconds, for time between samples
v_ego: float # m/s
op_active: bool # selfdriveState.active (sunnypilot driving, incl. driver override)
mads_active: bool = False # selfdriveStateSP.mads.active (lateral)
lane_change_state: int = 0 # modelV2.meta.laneChangeState
dm_alert_level: int = 0 # driverMonitoringState.alertLevel
lat: float | None = None # only when a good fix is present
lon: float | None = None
model_ref: str = ""
model_name: str = ""
dev_build: bool = False
wall: float | None = None # unix time; None until the device clock is valid
@property
def engaged(self) -> bool:
return self.op_active or self.mads_active
@property
def mads_only(self) -> bool:
return self.mads_active and not self.op_active
@dataclass
class Step:
"""What the last update counted, for the community log (see community.py)."""
dt_s: float = 0.0 # trusted seconds since the previous sample (0 after a gap or a bad reading)
engaged_m: float = 0.0 # meters driven engaged in this step
day: str | None = None # local calendar day; None until the clock is valid
@dataclass
class State:
version: int = STATE_VERSION
salt: str = field(default_factory=lambda: secrets.token_hex(16))
engaged_m: float = 0.0
engaged_s: float = 0.0
mads_m: float = 0.0
dev_m: float = 0.0
lane_changes: int = 0
days: list[str] = field(default_factory=list) # ISO dates with engaged driving, sorted
best_streak_days: int = 0
today: str = ""
today_m: float = 0.0
best_day_m: float = 0.0
areas: list[str] = field(default_factory=list) # salted hashes of geohash cells, sorted
model_m: dict[str, float] = field(default_factory=dict)
model_names: dict[str, str] = field(default_factory=dict)
attentive_run_s: float = 0.0
attentive_best_s: float = 0.0
seasons: list[str] = field(default_factory=list)
sunnylink_linked: bool = False
unlocked: dict[str, float] = field(default_factory=dict) # unlock id -> unix time
@classmethod
def from_dict(cls, d: dict) -> State:
known = {k: v for k, v in d.items() if k in cls.__dataclass_fields__}
return cls(**known)
def metrics(self) -> dict[str, float]:
return {
"engaged_km": self.engaged_m / 1000.0,
"engaged_h": self.engaged_s / 3600.0,
"active_days": float(len(self.days)),
"best_streak_days": float(self.best_streak_days),
"mads_km": self.mads_m / 1000.0,
"lane_changes": float(self.lane_changes),
"best_day_km": max(self.best_day_m, self.today_m) / 1000.0,
"areas": float(len(self.areas)),
"models_50km": float(sum(1 for m in self.model_m.values() if m >= MODEL_KM_GOAL * 1000.0)),
"dev_km": self.dev_m / 1000.0,
"attentive_best_h": max(self.attentive_best_s, self.attentive_run_s) / 3600.0,
"seasons": float(len(self.seasons)),
"sunnylink_linked": 1.0 if self.sunnylink_linked else 0.0,
}
def _streak_ending(days: list[str], last: str) -> int:
have = set(days)
d = dt.date.fromisoformat(last)
n = 0
while d.isoformat() in have:
n += 1
d -= dt.timedelta(days=1)
return n
class AchievementEngine:
def __init__(self, state: State | None = None, catalog: Catalog | None = None):
self.state = state or State()
self.catalog = catalog or load_catalog()
self._last_t: float | None = None
self._last_lane_change = 0
self._utc_offset_h: float | None = None # from GPS longitude, see local_date
self._southern = False
self.last_step = Step()
# --- time ---
@property
def utc_offset_h(self) -> float | None:
return self._utc_offset_h
def local_date(self, wall: float) -> dt.date:
"""Calendar day for streaks and Road Trip.
Comma devices keep their clock in UTC, which would split an evening drive in
the Americas across two days. With a GPS fix, use the solar offset from
longitude (15 degrees per hour); without one, fall back to the system zone.
"""
if self._utc_offset_h is not None:
return (dt.datetime.fromtimestamp(wall, dt.UTC) + dt.timedelta(hours=self._utc_offset_h)).date()
return dt.datetime.fromtimestamp(wall).date()
# --- updates ---
def update(self, s: Sample) -> list[str]:
"""Feed one sample. Returns unlock ids that were newly unlocked by it."""
st = self.state
dt_s = 0.0 if self._last_t is None else s.t - self._last_t
self._last_t = s.t
if not (0.0 < dt_s <= MAX_DT_S) or not (MIN_SPEED_MS <= s.v_ego <= MAX_SPEED_MS):
dt_s = 0.0
if s.lat is not None and s.lon is not None:
self._utc_offset_h = round(s.lon / 15.0)
self._southern = s.lat < 0
day = self.local_date(s.wall).isoformat() if s.wall is not None else None
self.last_step = Step(dt_s=dt_s, engaged_m=s.v_ego * dt_s if s.engaged else 0.0, day=day)
if s.engaged and dt_s > 0.0:
dist = s.v_ego * dt_s
if day is not None and day != st.today:
st.best_day_m = max(st.best_day_m, st.today_m)
st.today, st.today_m = day, 0.0
st.engaged_m += dist
st.engaged_s += dt_s
if day is not None:
st.today_m += dist
if s.mads_only:
st.mads_m += dist
if s.dev_build:
st.dev_m += dist
if s.model_ref:
st.model_m[s.model_ref] = st.model_m.get(s.model_ref, 0.0) + dist
if s.model_name:
st.model_names[s.model_ref] = s.model_name
if day is not None and day not in st.days and s.v_ego > 1.0:
st.days.append(day)
st.days.sort()
st.best_streak_days = max(st.best_streak_days, _streak_ending(st.days, day))
sn = season(dt.date.fromisoformat(day), self._southern)
if sn not in st.seasons:
st.seasons.append(sn)
if s.dm_alert_level >= DM_ALERT_RED:
st.attentive_best_s = max(st.attentive_best_s, st.attentive_run_s)
st.attentive_run_s = 0.0
else:
st.attentive_run_s += dt_s
if s.lat is not None and s.lon is not None:
area = hashlib.sha256((st.salt + geohash(s.lat, s.lon)).encode()).hexdigest()[:16]
if area not in st.areas:
st.areas.append(area)
st.areas.sort()
# a lane change counts once, when it reaches its finishing phase while engaged
if s.engaged and s.lane_change_state == LANE_CHANGE_FINISHING and self._last_lane_change == LANE_CHANGE_STARTING:
st.lane_changes += 1
self._last_lane_change = s.lane_change_state
return self.check_unlocks(s.wall)
def set_sunnylink_linked(self, linked: bool, wall: float | None) -> list[str]:
self.state.sunnylink_linked = self.state.sunnylink_linked or linked
return self.check_unlocks(wall)
def end_drive(self) -> None:
"""Close out the attentive run so the best is kept, and forget timing between drives."""
st = self.state
st.attentive_best_s = max(st.attentive_best_s, st.attentive_run_s)
self._last_t = None
self._last_lane_change = 0
def check_unlocks(self, wall: float | None) -> list[str]:
"""Unlock every tier whose goal is met. The unlock time is 0 when the clock isn't valid yet."""
metrics = self.state.metrics()
new = []
for a in self.catalog.achievements:
value = metrics.get(a.metric, 0.0)
for tier in a.tiers:
uid = a.unlock_id(tier)
if value >= tier.threshold and uid not in self.state.unlocked:
self.state.unlocked[uid] = round(wall) if wall is not None else 0
new.append(uid)
return new
# --- persistence ---
def load_state(path: Path | None = None) -> State:
path = path or state_path()
try:
with open(path) as f:
return State.from_dict(json.load(f))
except FileNotFoundError:
return State()
except (ValueError, TypeError):
# unreadable JSON: keep it aside for debugging instead of silently overwriting it.
# Other OSErrors propagate, so a transient read failure can never be saved over real progress.
os.replace(path, path.with_suffix(".corrupt.json"))
return State()
def save_json(path: Path, data: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=path.name, suffix=".tmp")
try:
with os.fdopen(fd, "w") as f:
json.dump(data, f, separators=(",", ":"), sort_keys=True)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
except BaseException:
try:
os.unlink(tmp)
except OSError:
pass
raise
def save_state(state: State, path: Path | None = None) -> None:
save_json(path or state_path(), asdict(state))
def load_seen(path: Path | None = None) -> set[str]:
try:
with open(path or seen_path()) as f:
return set(json.load(f).get("seen", []))
except (OSError, ValueError, AttributeError):
return set()
def save_seen(seen: set[str], path: Path | None = None) -> None:
save_json(path or seen_path(), {"seen": sorted(seen)})
-629
View File
@@ -1,629 +0,0 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
h3lite: the few H3 functions the car needs, in pure Python.
sunnypilot doesn't add Python packages, so instead of the h3 package this is a
line-by-line port of the parts of Uber's H3 v4.1.2 C library we use:
latLngToCell, cellToLatLng, cellToParent and a one-ring disk. Cell ids match
the real library exactly (tests/test_h3lite.py checks this against h3-py when
it's installed, and against fixed vectors when it isn't), so the site and the
server can keep using h3-js and h3-pg on the same ids.
H3 is Copyright Uber Technologies, Inc., licensed under the Apache License 2.0
(https://github.com/uber/h3). The tables below are copied from its
faceijk.c and baseCells.c.
"""
from __future__ import annotations
import math
MAX_RES = 15
NUM_FACES = 20
NUM_BASE_CELLS = 122
_EPSILON = 1e-16
_SQRT3_2 = 0.8660254037844386467637231707529361834714
_SQRT7 = 2.6457513110645905905016157536392604257102
_AP7_ROT_RADS = 0.333473172251832115336090755351601070065900389
_RES0_U_GNOMONIC = 0.38196601125010500003
_2PI = 2.0 * math.pi
# Directions (index digits)
_CENTER, _K, _J, _JK, _I, _IK, _IJ = range(7)
_UNIT_VECS = ((0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 0, 0), (1, 0, 1), (1, 1, 0))
_ROT_CCW = (0, _IK, _JK, _K, _IJ, _I, _J) # digit -> digit rotated 60 degrees ccw
_ROT_CW = (0, _JK, _IJ, _J, _IK, _K, _I) # digit -> digit rotated 60 degrees cw
# face neighbor quadrants
_IJ_Q, _KI_Q, _JK_Q = 1, 2, 3
# bit layout of an H3 index
_MODE_OFFSET = 59
_RES_OFFSET = 52
_BC_OFFSET = 45
_H3_INIT = (1 << 45) - 1 # every digit set to 7
_CELL_MODE = 1
def _max_dim(res: int) -> int: # Class II resolutions only
return 2 * 7 ** (res // 2)
def _unit_scale(res: int) -> int: # Class II resolutions only
return 7 ** (res // 2)
# --- tables copied from H3 v4.1.2 faceijk.c and baseCells.c ---
FACE_CENTER_GEO = (
(0.80358264971899, 1.2483974196173961), (1.3077478834556382, 2.5369450098779214),
(1.054751253523952, -1.3475173589003966), (0.6001915955381868, -0.45060390946975576),
(0.49171542819877384, 0.40198820291130694), (0.1727453274156187, 1.6781468852804338),
(0.6059293215713507, 2.9539233298124117), (0.42737051832897965, -1.8888762003362853),
(-0.07906611854921283, -0.7334295133808677), (-0.23096164445538364, 0.506495587332349),
(0.07906611854921283, 2.4081631402089254), (0.23096164445538364, -2.635097066257444),
(-0.1727453274156187, -1.4634457683093596), (-0.6059293215713507, -0.18766932377738163),
(-0.42737051832897965, 1.2527164532535078), (-0.6001915955381868, 2.6909887441200375),
(-0.49171542819877384, -2.7396044506784865), (-0.80358264971899, -1.8931952339723972),
(-1.3077478834556382, -0.6046476437118721), (-1.054751253523952, 1.7940752946893965),
)
FACE_CENTER_POINT = (
(0.2199307791404606, 0.6583691780274996, 0.7198475378926182),
(-0.2139234834501421, 0.1478171829550703, 0.9656017935214205),
(0.1092625278784797, -0.481195157287321, 0.8697775121287253),
(0.7428567301586791, -0.3593941678278028, 0.5648005936517033),
(0.8112534709140969, 0.3448953237639384, 0.472138773641393),
(-0.1055498149613921, 0.9794457296411413, 0.1718874610009365),
(-0.8075407579970092, 0.1533552485898818, 0.5695261994882688),
(-0.2846148069787907, -0.8644080972654206, 0.4144792552473539),
(0.7405621473854482, -0.6673299564565524, -0.0789837646326737),
(0.8512303986474293, 0.4722343788582681, -0.2289137388687808),
(-0.7405621473854481, 0.6673299564565524, 0.0789837646326737),
(-0.8512303986474292, -0.4722343788582682, 0.2289137388687808),
(0.1055498149613919, -0.9794457296411413, -0.1718874610009365),
(0.8075407579970092, -0.1533552485898819, -0.5695261994882688),
(0.2846148069787908, 0.8644080972654204, -0.4144792552473539),
(-0.7428567301586791, 0.3593941678278027, -0.5648005936517033),
(-0.811253470914097, -0.3448953237639382, -0.472138773641393),
(-0.2199307791404607, -0.6583691780274996, -0.7198475378926182),
(0.213923483450142, -0.1478171829550704, -0.9656017935214205),
(-0.1092625278784796, 0.481195157287321, -0.8697775121287253),
)
FACE_AXIS_AZ_CII = (
5.6199582685239395, 5.7603390817141875, 0.78021365439343, 0.4304693639799999,
6.130269123335111, 2.692877706530643, 2.982963003477244, 3.532912002790141,
3.494305004259568, 3.0032141694995382, 5.930472956509812, 0.13837848409025486,
0.4487149470591504, 0.15862965011254937, 5.891865957979238, 2.711123289609793,
3.294508837434268, 3.80481969224544, 3.6644388790551923, 2.361378999196363,
)
# per face: (central, ij, ki, jk) neighbors as (face, (i, j, k) translate, ccw 60 rotations)
FACE_NEIGHBORS = (
((0, (0, 0, 0), 0), (4, (2, 0, 2), 1), (1, (2, 2, 0), 5), (5, (0, 2, 2), 3)),
((1, (0, 0, 0), 0), (0, (2, 0, 2), 1), (2, (2, 2, 0), 5), (6, (0, 2, 2), 3)),
((2, (0, 0, 0), 0), (1, (2, 0, 2), 1), (3, (2, 2, 0), 5), (7, (0, 2, 2), 3)),
((3, (0, 0, 0), 0), (2, (2, 0, 2), 1), (4, (2, 2, 0), 5), (8, (0, 2, 2), 3)),
((4, (0, 0, 0), 0), (3, (2, 0, 2), 1), (0, (2, 2, 0), 5), (9, (0, 2, 2), 3)),
((5, (0, 0, 0), 0), (10, (2, 2, 0), 3), (14, (2, 0, 2), 3), (0, (0, 2, 2), 3)),
((6, (0, 0, 0), 0), (11, (2, 2, 0), 3), (10, (2, 0, 2), 3), (1, (0, 2, 2), 3)),
((7, (0, 0, 0), 0), (12, (2, 2, 0), 3), (11, (2, 0, 2), 3), (2, (0, 2, 2), 3)),
((8, (0, 0, 0), 0), (13, (2, 2, 0), 3), (12, (2, 0, 2), 3), (3, (0, 2, 2), 3)),
((9, (0, 0, 0), 0), (14, (2, 2, 0), 3), (13, (2, 0, 2), 3), (4, (0, 2, 2), 3)),
((10, (0, 0, 0), 0), (5, (2, 2, 0), 3), (6, (2, 0, 2), 3), (15, (0, 2, 2), 3)),
((11, (0, 0, 0), 0), (6, (2, 2, 0), 3), (7, (2, 0, 2), 3), (16, (0, 2, 2), 3)),
((12, (0, 0, 0), 0), (7, (2, 2, 0), 3), (8, (2, 0, 2), 3), (17, (0, 2, 2), 3)),
((13, (0, 0, 0), 0), (8, (2, 2, 0), 3), (9, (2, 0, 2), 3), (18, (0, 2, 2), 3)),
((14, (0, 0, 0), 0), (9, (2, 2, 0), 3), (5, (2, 0, 2), 3), (19, (0, 2, 2), 3)),
((15, (0, 0, 0), 0), (16, (2, 0, 2), 1), (19, (2, 2, 0), 5), (10, (0, 2, 2), 3)),
((16, (0, 0, 0), 0), (17, (2, 0, 2), 1), (15, (2, 2, 0), 5), (11, (0, 2, 2), 3)),
((17, (0, 0, 0), 0), (18, (2, 0, 2), 1), (16, (2, 2, 0), 5), (12, (0, 2, 2), 3)),
((18, (0, 0, 0), 0), (19, (2, 0, 2), 1), (17, (2, 2, 0), 5), (13, (0, 2, 2), 3)),
((19, (0, 0, 0), 0), (15, (2, 0, 2), 1), (18, (2, 2, 0), 5), (14, (0, 2, 2), 3)),
)
# [face][i][j][k] -> (base cell, ccw 60 rotations), flattened as face*27 + i*9 + j*3 + k
FACE_IJK_BASE_CELLS = (
(16,0), (18,0), (24,0), (33,0), (30,0), (32,3), (49,1), (48,3), (50,3),
(8,0), (5,5), (10,5), (22,0), (16,0), (18,0), (41,1), (33,0), (30,0),
(4,0), (0,5), (2,5), (15,1), (8,0), (5,5), (31,1), (22,0), (16,0),
(2,0), (6,0), (14,0), (10,0), (11,0), (17,3), (24,1), (23,3), (25,3),
(0,0), (1,5), (9,5), (5,0), (2,0), (6,0), (18,1), (10,0), (11,0),
(4,1), (3,5), (7,5), (8,1), (0,0), (1,5), (16,1), (5,0), (2,0),
(7,0), (21,0), (38,0), (9,0), (19,0), (34,3), (14,1), (20,3), (36,3),
(3,0), (13,5), (29,5), (1,0), (7,0), (21,0), (6,1), (9,0), (19,0),
(4,2), (12,5), (26,5), (0,1), (3,0), (13,5), (2,1), (1,0), (7,0),
(26,0), (42,0), (58,0), (29,0), (43,0), (62,3), (38,1), (47,3), (64,3),
(12,0), (28,5), (44,5), (13,0), (26,0), (42,0), (21,1), (29,0), (43,0),
(4,3), (15,5), (31,5), (3,1), (12,0), (28,5), (7,1), (13,0), (26,0),
(31,0), (41,0), (49,0), (44,0), (53,0), (61,3), (58,1), (65,3), (75,3),
(15,0), (22,5), (33,5), (28,0), (31,0), (41,0), (42,1), (44,0), (53,0),
(4,4), (8,5), (16,5), (12,1), (15,0), (22,5), (26,1), (28,0), (31,0),
(50,0), (48,0), (49,3), (32,0), (30,3), (33,3), (24,3), (18,3), (16,3),
(70,0), (67,0), (66,3), (52,3), (50,0), (48,0), (37,3), (32,0), (30,3),
(83,0), (87,3), (85,3), (74,3), (70,0), (67,0), (57,1), (52,3), (50,0),
(25,0), (23,0), (24,3), (17,0), (11,3), (10,3), (14,3), (6,3), (2,3),
(45,0), (39,0), (37,3), (35,3), (25,0), (23,0), (27,3), (17,0), (11,3),
(63,0), (59,3), (57,3), (56,3), (45,0), (39,0), (46,3), (35,3), (25,0),
(36,0), (20,0), (14,3), (34,0), (19,3), (9,3), (38,3), (21,3), (7,3),
(55,0), (40,0), (27,3), (54,3), (36,0), (20,0), (51,3), (34,0), (19,3),
(72,0), (60,3), (46,3), (73,3), (55,0), (40,0), (71,3), (54,3), (36,0),
(64,0), (47,0), (38,3), (62,0), (43,3), (29,3), (58,3), (42,3), (26,3),
(84,0), (69,0), (51,3), (82,3), (64,0), (47,0), (76,3), (62,0), (43,3),
(97,0), (89,3), (71,3), (98,3), (84,0), (69,0), (96,3), (82,3), (64,0),
(75,0), (65,0), (58,3), (61,0), (53,3), (44,3), (49,3), (41,3), (31,3),
(94,0), (86,0), (76,3), (81,3), (75,0), (65,0), (66,3), (61,0), (53,3),
(107,0), (104,3), (96,3), (101,3), (94,0), (86,0), (85,3), (81,3), (75,0),
(57,0), (59,0), (63,3), (74,0), (78,3), (79,3), (83,3), (92,3), (95,3),
(37,0), (39,3), (45,3), (52,0), (57,0), (59,0), (70,3), (74,0), (78,3),
(24,0), (23,3), (25,3), (32,3), (37,0), (39,3), (50,3), (52,0), (57,0),
(46,0), (60,0), (72,3), (56,0), (68,3), (80,3), (63,3), (77,3), (90,3),
(27,0), (40,3), (55,3), (35,0), (46,0), (60,0), (45,3), (56,0), (68,3),
(14,0), (20,3), (36,3), (17,3), (27,0), (40,3), (25,3), (35,0), (46,0),
(71,0), (89,0), (97,3), (73,0), (91,3), (103,3), (72,3), (88,3), (105,3),
(51,0), (69,3), (84,3), (54,0), (71,0), (89,0), (55,3), (73,0), (91,3),
(38,0), (47,3), (64,3), (34,3), (51,0), (69,3), (36,3), (54,0), (71,0),
(96,0), (104,0), (107,3), (98,0), (110,3), (115,3), (97,3), (111,3), (119,3),
(76,0), (86,3), (94,3), (82,0), (96,0), (104,0), (84,3), (98,0), (110,3),
(58,0), (65,3), (75,3), (62,3), (76,0), (86,3), (64,3), (82,0), (96,0),
(85,0), (87,0), (83,3), (101,0), (102,3), (100,3), (107,3), (112,3), (114,3),
(66,0), (67,3), (70,3), (81,0), (85,0), (87,0), (94,3), (101,0), (102,3),
(49,0), (48,3), (50,3), (61,3), (66,0), (67,3), (75,3), (81,0), (85,0),
(95,0), (92,0), (83,0), (79,0), (78,0), (74,3), (63,1), (59,3), (57,3),
(109,0), (108,0), (100,5), (93,1), (95,0), (92,0), (77,1), (79,0), (78,0),
(117,4), (118,5), (114,5), (106,1), (109,0), (108,0), (90,1), (93,1), (95,0),
(90,0), (77,0), (63,0), (80,0), (68,0), (56,3), (72,1), (60,3), (46,3),
(106,0), (93,0), (79,5), (99,1), (90,0), (77,0), (88,1), (80,0), (68,0),
(117,3), (109,5), (95,5), (113,1), (106,0), (93,0), (105,1), (99,1), (90,0),
(105,0), (88,0), (72,0), (103,0), (91,0), (73,3), (97,1), (89,3), (71,3),
(113,0), (99,0), (80,5), (116,1), (105,0), (88,0), (111,1), (103,0), (91,0),
(117,2), (106,5), (90,5), (121,1), (113,0), (99,0), (119,1), (116,1), (105,0),
(119,0), (111,0), (97,0), (115,0), (110,0), (98,3), (107,1), (104,3), (96,3),
(121,0), (116,0), (103,5), (120,1), (119,0), (111,0), (112,1), (115,0), (110,0),
(117,1), (113,5), (105,5), (118,1), (121,0), (116,0), (114,1), (120,1), (119,0),
(114,0), (112,0), (107,0), (100,0), (102,0), (101,3), (83,1), (87,3), (85,3),
(118,0), (120,0), (115,5), (108,1), (114,0), (112,0), (92,1), (100,0), (102,0),
(117,0), (121,5), (119,5), (109,1), (118,0), (120,0), (95,1), (108,1), (114,0),
)
# base cell -> (home face, i, j, k, is pentagon, cw offset face 1, cw offset face 2)
BASE_CELL_DATA = (
(1,1,0,0,0,0,0), (2,1,1,0,0,0,0), (1,0,0,0,0,0,0), (2,1,0,0,0,0,0),
(0,2,0,0,1,-1,-1), (1,1,1,0,0,0,0), (1,0,0,1,0,0,0), (2,0,0,0,0,0,0),
(0,1,0,0,0,0,0), (2,0,1,0,0,0,0), (1,0,1,0,0,0,0), (1,0,1,1,0,0,0),
(3,1,0,0,0,0,0), (3,1,1,0,0,0,0), (11,2,0,0,1,2,6), (4,1,0,0,0,0,0),
(0,0,0,0,0,0,0), (6,0,1,0,0,0,0), (0,0,0,1,0,0,0), (2,0,1,1,0,0,0),
(7,0,0,1,0,0,0), (2,0,0,1,0,0,0), (0,1,1,0,0,0,0), (6,0,0,1,0,0,0),
(10,2,0,0,1,1,5), (6,0,0,0,0,0,0), (3,0,0,0,0,0,0), (11,1,0,0,0,0,0),
(4,1,1,0,0,0,0), (3,0,1,0,0,0,0), (0,0,1,1,0,0,0), (4,0,0,0,0,0,0),
(5,0,1,0,0,0,0), (0,0,1,0,0,0,0), (7,0,1,0,0,0,0), (11,1,1,0,0,0,0),
(7,0,0,0,0,0,0), (10,1,0,0,0,0,0), (12,2,0,0,1,3,7), (6,1,0,1,0,0,0),
(7,1,0,1,0,0,0), (4,0,0,1,0,0,0), (3,0,0,1,0,0,0), (3,0,1,1,0,0,0),
(4,0,1,0,0,0,0), (6,1,0,0,0,0,0), (11,0,0,0,0,0,0), (8,0,0,1,0,0,0),
(5,0,0,1,0,0,0), (14,2,0,0,1,0,9), (5,0,0,0,0,0,0), (12,1,0,0,0,0,0),
(10,1,1,0,0,0,0), (4,0,1,1,0,0,0), (12,1,1,0,0,0,0), (7,1,0,0,0,0,0),
(11,0,1,0,0,0,0), (10,0,0,0,0,0,0), (13,2,0,0,1,4,8), (10,0,0,1,0,0,0),
(11,0,0,1,0,0,0), (9,0,1,0,0,0,0), (8,0,1,0,0,0,0), (6,2,0,0,1,11,15),
(8,0,0,0,0,0,0), (9,0,0,1,0,0,0), (14,1,0,0,0,0,0), (5,1,0,1,0,0,0),
(16,0,1,1,0,0,0), (8,1,0,1,0,0,0), (5,1,0,0,0,0,0), (12,0,0,0,0,0,0),
(7,2,0,0,1,12,16), (12,0,1,0,0,0,0), (10,0,1,0,0,0,0), (9,0,0,0,0,0,0),
(13,1,0,0,0,0,0), (16,0,0,1,0,0,0), (15,0,1,1,0,0,0), (15,0,1,0,0,0,0),
(16,0,1,0,0,0,0), (14,1,1,0,0,0,0), (13,1,1,0,0,0,0), (5,2,0,0,1,10,19),
(8,1,0,0,0,0,0), (14,0,0,0,0,0,0), (9,1,0,1,0,0,0), (14,0,0,1,0,0,0),
(17,0,0,1,0,0,0), (12,0,0,1,0,0,0), (16,0,0,0,0,0,0), (17,0,1,1,0,0,0),
(15,0,0,1,0,0,0), (16,1,0,1,0,0,0), (9,1,0,0,0,0,0), (15,0,0,0,0,0,0),
(13,0,0,0,0,0,0), (8,2,0,0,1,13,17), (13,0,1,0,0,0,0), (17,1,0,1,0,0,0),
(19,0,1,0,0,0,0), (14,0,1,0,0,0,0), (19,0,1,1,0,0,0), (17,0,1,0,0,0,0),
(13,0,0,1,0,0,0), (17,0,0,0,0,0,0), (16,1,0,0,0,0,0), (9,2,0,0,1,14,18),
(15,1,0,1,0,0,0), (15,1,0,0,0,0,0), (18,0,1,1,0,0,0), (18,0,0,1,0,0,0),
(19,0,0,1,0,0,0), (17,1,0,0,0,0,0), (19,0,0,0,0,0,0), (18,0,1,0,0,0,0),
(18,1,0,1,0,0,0), (19,2,0,0,1,-1,-1), (19,1,0,0,0,0,0), (18,0,0,0,0,0,0),
(19,1,0,1,0,0,0), (18,1,0,0,0,0,0),
)
# --- ijk coordinates ---
def _lround(x: float) -> int:
"""C lroundl: halves round away from zero."""
return int(math.floor(x + 0.5)) if x >= 0 else -int(math.floor(-x + 0.5))
def _normalize(i: int, j: int, k: int) -> tuple[int, int, int]:
if i < 0:
j -= i
k -= i
i = 0
if j < 0:
i -= j
k -= j
j = 0
if k < 0:
i -= k
j -= k
k = 0
m = min(i, j, k)
if m > 0:
i -= m
j -= m
k -= m
return i, j, k
def _add(a: tuple[int, int, int], b: tuple[int, int, int]) -> tuple[int, int, int]:
return _normalize(a[0] + b[0], a[1] + b[1], a[2] + b[2])
def _scaled_sum(c: tuple[int, int, int], iv, jv, kv) -> tuple[int, int, int]:
i, j, k = c
return _normalize(iv[0] * i + jv[0] * j + kv[0] * k,
iv[1] * i + jv[1] * j + kv[1] * k,
iv[2] * i + jv[2] * j + kv[2] * k)
def _up_ap7(c):
i, j = c[0] - c[2], c[1] - c[2]
return _normalize(_lround((3 * i - j) / 7.0), _lround((i + 2 * j) / 7.0), 0)
def _up_ap7r(c):
i, j = c[0] - c[2], c[1] - c[2]
return _normalize(_lround((2 * i + j) / 7.0), _lround((3 * j - i) / 7.0), 0)
def _down_ap7(c):
return _scaled_sum(c, (3, 0, 1), (1, 3, 0), (0, 1, 3))
def _down_ap7r(c):
return _scaled_sum(c, (3, 1, 0), (0, 3, 1), (1, 0, 3))
def _rotate60ccw(c):
return _scaled_sum(c, (1, 1, 0), (0, 1, 1), (1, 0, 1))
def _rotate60cw(c):
return _scaled_sum(c, (1, 0, 1), (1, 1, 0), (0, 1, 1))
def _unit_to_digit(c) -> int:
c = _normalize(*c)
for d, u in enumerate(_UNIT_VECS):
if c == u:
return d
return 7
def _hex2d_to_ijk(x: float, y: float) -> tuple[int, int, int]:
a1, a2 = abs(x), abs(y)
x2 = a2 / _SQRT3_2
x1 = a1 + x2 / 2.0
m1, m2 = int(x1), int(x2)
r1, r2 = x1 - m1, x2 - m2
if r1 < 0.5:
if r1 < 1.0 / 3.0:
i = m1
j = m2 if r2 < (1.0 + r1) / 2.0 else m2 + 1
else:
j = m2 if r2 < (1.0 - r1) else m2 + 1
i = m1 + 1 if (1.0 - r1) <= r2 < (2.0 * r1) else m1
else:
if r1 < 2.0 / 3.0:
j = m2 if r2 < (1.0 - r1) else m2 + 1
i = m1 if (2.0 * r1 - 1.0) < r2 < (1.0 - r1) else m1 + 1
else:
i = m1 + 1
j = m2 if r2 < (r1 / 2.0) else m2 + 1
# fold across the axes if necessary
if x < 0.0:
if j % 2 == 0:
i = i - 2 * (i - j // 2)
else:
i = i - (2 * (i - (j + 1) // 2) + 1)
if y < 0.0:
i = i - (2 * j + 1) // 2
j = -j
return _normalize(i, j, 0)
def _ijk_to_hex2d(c) -> tuple[float, float]:
i, j = c[0] - c[2], c[1] - c[2]
return i - 0.5 * j, j * _SQRT3_2
# --- sphere <-> face ---
def _pos_angle(rads: float) -> float:
tmp = rads + _2PI if rads < 0.0 else rads
if rads >= _2PI:
tmp -= _2PI
return tmp
def _constrain_lng(lng: float) -> float:
while lng > math.pi:
lng -= _2PI
while lng < -math.pi:
lng += _2PI
return lng
def _azimuth(lat1: float, lng1: float, lat2: float, lng2: float) -> float:
return math.atan2(math.cos(lat2) * math.sin(lng2 - lng1),
math.cos(lat1) * math.sin(lat2) - math.sin(lat1) * math.cos(lat2) * math.cos(lng2 - lng1))
def _az_distance(lat1: float, lng1: float, az: float, distance: float) -> tuple[float, float]:
if distance < _EPSILON:
return lat1, lng1
az = _pos_angle(az)
if az < _EPSILON or abs(az - math.pi) < _EPSILON:
lat2 = lat1 + distance if az < _EPSILON else lat1 - distance
if abs(lat2 - math.pi / 2) < _EPSILON:
return math.pi / 2, 0.0
if abs(lat2 + math.pi / 2) < _EPSILON:
return -math.pi / 2, 0.0
return lat2, _constrain_lng(lng1)
sinlat = math.sin(lat1) * math.cos(distance) + math.cos(lat1) * math.sin(distance) * math.cos(az)
sinlat = max(-1.0, min(1.0, sinlat))
lat2 = math.asin(sinlat)
if abs(lat2 - math.pi / 2) < _EPSILON:
return math.pi / 2, 0.0
if abs(lat2 + math.pi / 2) < _EPSILON:
return -math.pi / 2, 0.0
sinlng = math.sin(az) * math.sin(distance) / math.cos(lat2)
coslng = (math.cos(distance) - math.sin(lat1) * math.sin(lat2)) / math.cos(lat1) / math.cos(lat2)
sinlng = max(-1.0, min(1.0, sinlng))
coslng = max(-1.0, min(1.0, coslng))
return lat2, _constrain_lng(lng1 + math.atan2(sinlng, coslng))
def _closest_face(lat: float, lng: float) -> tuple[int, float]:
r = math.cos(lat)
z, x, y = math.sin(lat), math.cos(lng) * r, math.sin(lng) * r
face, sqd = 0, 5.0
for f, (fx, fy, fz) in enumerate(FACE_CENTER_POINT):
d = (fx - x) ** 2 + (fy - y) ** 2 + (fz - z) ** 2
if d < sqd:
face, sqd = f, d
return face, sqd
def _geo_to_hex2d(lat: float, lng: float, res: int) -> tuple[int, float, float]:
face, sqd = _closest_face(lat, lng)
r = math.acos(1 - sqd / 2)
if r < _EPSILON:
return face, 0.0, 0.0
flat, flng = FACE_CENTER_GEO[face]
theta = _pos_angle(FACE_AXIS_AZ_CII[face] - _pos_angle(_azimuth(flat, flng, lat, lng)))
if res % 2:
theta = _pos_angle(theta - _AP7_ROT_RADS)
r = math.tan(r) / _RES0_U_GNOMONIC
for _ in range(res):
r *= _SQRT7
return face, r * math.cos(theta), r * math.sin(theta)
def _hex2d_to_geo(x: float, y: float, face: int, res: int) -> tuple[float, float]:
r = math.hypot(x, y)
if r < _EPSILON:
return FACE_CENTER_GEO[face]
theta = math.atan2(y, x)
for _ in range(res):
r /= _SQRT7
r = math.atan(r * _RES0_U_GNOMONIC)
if res % 2:
theta = _pos_angle(theta + _AP7_ROT_RADS)
theta = _pos_angle(FACE_AXIS_AZ_CII[face] - theta)
flat, flng = FACE_CENTER_GEO[face]
return _az_distance(flat, flng, theta, r)
# --- H3 index bits ---
def get_resolution(h: int) -> int:
return (h >> _RES_OFFSET) & 15
def get_base_cell(h: int) -> int:
return (h >> _BC_OFFSET) & 127
def _digit(h: int, r: int) -> int:
return (h >> ((MAX_RES - r) * 3)) & 7
def _set_digit(h: int, r: int, d: int) -> int:
off = (MAX_RES - r) * 3
return (h & ~(7 << off)) | (d << off)
def _leading_digit(h: int) -> int:
for r in range(1, get_resolution(h) + 1):
d = _digit(h, r)
if d:
return d
return 0
def _rotate_index60ccw(h: int) -> int:
for r in range(1, get_resolution(h) + 1):
h = _set_digit(h, r, _ROT_CCW[_digit(h, r)])
return h
def _rotate_index60cw(h: int) -> int:
for r in range(1, get_resolution(h) + 1):
h = _set_digit(h, r, _ROT_CW[_digit(h, r)])
return h
def _rotate_pent60ccw(h: int) -> int:
found = False
for r in range(1, get_resolution(h) + 1):
h = _set_digit(h, r, _ROT_CCW[_digit(h, r)])
if not found and _digit(h, r) != 0:
found = True
if _leading_digit(h) == _K:
h = _rotate_index60ccw(h)
return h
def _is_pentagon_base(bc: int) -> bool:
return 0 <= bc < NUM_BASE_CELLS and bool(BASE_CELL_DATA[bc][4])
def _face_ijk_base(face: int, c) -> tuple[int, int]:
return FACE_IJK_BASE_CELLS[face * 27 + c[0] * 9 + c[1] * 3 + c[2]]
def _face_ijk_to_h3(face: int, c, res: int) -> int:
h = _H3_INIT | (_CELL_MODE << _MODE_OFFSET) | (res << _RES_OFFSET)
for r in range(res - 1, -1, -1):
last = c
if (r + 1) % 2:
c = _up_ap7(c)
center = _down_ap7(c)
else:
c = _up_ap7r(c)
center = _down_ap7r(c)
diff = _normalize(last[0] - center[0], last[1] - center[1], last[2] - center[2])
h = _set_digit(h, r + 1, _unit_to_digit(diff))
if max(c) > 2:
raise ValueError("coordinates off the base cell's face")
bc, rots = _face_ijk_base(face, c)
h = (h & ~(127 << _BC_OFFSET)) | (bc << _BC_OFFSET)
if _is_pentagon_base(bc):
if _leading_digit(h) == _K:
data = BASE_CELL_DATA[bc]
h = _rotate_index60cw(h) if face in (data[5], data[6]) else _rotate_index60ccw(h)
for _ in range(rots):
h = _rotate_pent60ccw(h)
else:
for _ in range(rots):
h = _rotate_index60ccw(h)
return h
def _adjust_overage_class2(face: int, c, res: int, pent_leading4: bool) -> tuple[int, tuple[int, int, int], bool]:
"""Move coordinates that fell off the face onto the neighboring face. Returns (face, ijk, moved)."""
max_dim = _max_dim(res)
if sum(c) <= max_dim:
return face, c, False
if c[2] > 0:
if c[1] > 0:
orient = FACE_NEIGHBORS[face][_JK_Q]
else:
orient = FACE_NEIGHBORS[face][_KI_Q]
if pent_leading4:
# rotate around the pentagon's center for its missing sequence
tmp = _rotate60cw((c[0] - max_dim, c[1], c[2]))
c = _normalize(tmp[0] + max_dim, tmp[1], tmp[2])
else:
orient = FACE_NEIGHBORS[face][_IJ_Q]
new_face, (ti, tj, tk), ccw = orient
for _ in range(ccw):
c = _rotate60ccw(c)
s = _unit_scale(res)
c = _normalize(c[0] + ti * s, c[1] + tj * s, c[2] + tk * s)
return new_face, c, True
def _h3_to_face_ijk(h: int) -> tuple[int, tuple[int, int, int]]:
bc = get_base_cell(h)
if not 0 <= bc < NUM_BASE_CELLS:
raise ValueError("invalid base cell")
if _is_pentagon_base(bc) and _leading_digit(h) == _IK:
h = _rotate_index60cw(h)
data = BASE_CELL_DATA[bc]
face, c = data[0], (data[1], data[2], data[3])
res = get_resolution(h)
possible_overage = _is_pentagon_base(bc) or not (res == 0 or c == (0, 0, 0))
for r in range(1, res + 1):
c = _down_ap7(c) if r % 2 else _down_ap7r(c)
d = _digit(h, r)
if 0 < d < 7:
c = _add(c, _UNIT_VECS[d])
if not possible_overage:
return face, c
orig = c
adj_res = res
if adj_res % 2:
c = _down_ap7r(c)
adj_res += 1
pent_leading4 = _is_pentagon_base(bc) and _leading_digit(h) == _I
face, c, moved = _adjust_overage_class2(face, c, adj_res, pent_leading4)
if moved:
if _is_pentagon_base(bc):
while moved:
face, c, moved = _adjust_overage_class2(face, c, adj_res, False)
if adj_res != res:
c = _up_ap7r(c)
elif adj_res != res:
c = orig
return face, c
# --- public API ---
def latlng_to_cell(lat: float, lng: float, res: int) -> int:
"""Cell containing a point given in degrees, like h3.latlng_to_cell."""
if not 0 <= res <= MAX_RES:
raise ValueError("resolution out of range")
if not (math.isfinite(lat) and math.isfinite(lng)):
raise ValueError("coordinates must be finite")
face, x, y = _geo_to_hex2d(math.radians(lat), math.radians(lng), res)
return _face_ijk_to_h3(face, _hex2d_to_ijk(x, y), res)
def cell_to_latlng(h: int) -> tuple[float, float]:
"""Center of a cell in degrees."""
face, c = _h3_to_face_ijk(h)
lat, lng = _hex2d_to_geo(*_ijk_to_hex2d(c), face, get_resolution(h))
return math.degrees(lat), math.degrees(lng)
def cell_to_parent(h: int, res: int) -> int:
child_res = get_resolution(h)
if not 0 <= res <= child_res:
raise ValueError("parent resolution must be between 0 and the cell's resolution")
h = (h & ~(15 << _RES_OFFSET)) | (res << _RES_OFFSET)
for r in range(res + 1, child_res + 1):
h = _set_digit(h, r, 7)
return h
def grid_disk1(h: int) -> set[int]:
"""The cell and the cells touching it (h3.grid_disk(h, 1)).
Steps one cell along each of the six ijk axes on the cell's face and maps
that center back to a cell, so it also works across face edges.
"""
face, c = _h3_to_face_ijk(h)
res = get_resolution(h)
out = {h}
for u in _UNIT_VECS[1:]:
x, y = _ijk_to_hex2d(_add(c, u))
lat, lng = _hex2d_to_geo(x, y, face, res)
out.add(latlng_to_cell(math.degrees(lat), math.degrees(lng), res))
return out
def cell_to_str(h: int) -> str:
return format(h, "x")
def str_to_cell(s: str) -> int:
return int(s, 16)
@@ -1,236 +0,0 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
Dry run of the community uploads: writes exactly what the car would send, and
a map page that shows what would be shared and what stays on the car, into
<achievements root>/preview/. Nothing is sent anywhere.
preview/daily-YYYY-MM-DD.json daily report per driven day (last 14 days), at level 2
preview/tiles.json the next weekly tile batch (level 3)
preview/delete.json tiles the car would ask the server to delete, if any
preview/map.html open in a browser (copy it off the car first)
map.html shows your frequent stops, so it is for you only: don't share it.
"""
from __future__ import annotations
import datetime as dt
import json
from pathlib import Path
from openpilot.sunnypilot.achievements.community import (LEVEL_DAILY, STOP_HOLD_DAYS, CommunityLog, Settings,
public_block)
from openpilot.sunnypilot.achievements.engine import State, achievements_root
PREVIEW_DAYS = 14
def preview_dir() -> Path:
return achievements_root() / "preview"
def _write(path: Path, data) -> None:
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(data if isinstance(data, str) else json.dumps(data, indent=2) + "\n")
tmp.replace(path)
def build_preview(log: CommunityLog, state: State, today: dt.date, car: str | None, settings: Settings) -> dict:
"""Everything the preview shows, as plain data."""
view = log.tile_view(today)
profile = log.profile_update(today, car, settings)
days = log.driven_days(since=today - dt.timedelta(days=PREVIEW_DAYS - 1))
daily = [log.daily_report(d, state, LEVEL_DAILY, profile=profile if d == days[-1] else None,
include_undated=d == days[-1]) for d in days]
region_days = log.region_days(today)
return {
"today": today.isoformat(),
"level": settings.level,
"daily": daily,
"tiles": log.tile_batch(today),
"delete": log.tile_deletes(today),
"region": {"home": log.home_region(today), "shared": log.shared_region(today, settings),
"choice": settings.region, "days": dict(region_days.most_common())},
"map": {
"shareable": sorted(view.shareable),
"blocks": sorted({public_block(t) for t in view.shareable}),
"held": view.held,
"ring": sorted(view.ring),
"frequent": sorted(view.frequent),
"hidden": sorted(view.hidden & view.driven),
"driven": len(view.driven),
},
}
def write_preview(log: CommunityLog, state: State, today: dt.date, car: str | None, settings: Settings,
out: Path | None = None) -> Path:
out = out or preview_dir()
out.mkdir(parents=True, exist_ok=True)
data = build_preview(log, state, today, car, settings)
for f in out.glob("daily-*.json"):
f.unlink()
for report in data["daily"]:
_write(out / f"daily-{report['day']}.json", report)
for name in ("tiles", "delete"):
path = out / f"{name}.json"
if data[name]:
_write(path, data[name])
elif path.exists():
path.unlink()
_write(out / "map.html", render_html(data))
return out
def render_html(data: dict) -> str:
blob = json.dumps(data, separators=(",", ":")).replace("</", "<\\/")
return HTML.replace("__DATA__", blob).replace("__HOLD_DAYS__", str(STOP_HOLD_DAYS))
HTML = """<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>What your car would share</title>
<link rel="stylesheet" href="https://unpkg.com/maplibre-gl@4.7.1/dist/maplibre-gl.css">
<script src="https://unpkg.com/maplibre-gl@4.7.1/dist/maplibre-gl.js"></script>
<script src="https://unpkg.com/h3-js@4.1.0/dist/h3-js.umd.js"></script>
<style>
:root { --ink:#1f2328; --muted:#59636e; --line:#d1d9e0; --bg:#f6f8fa; --send:#2563eb; --held:#9ca3af;
--never:#dc2626; --hidden:#7c3aed; }
* { box-sizing: border-box; }
body { margin:0; font:15px/1.5 Inter, system-ui, sans-serif; color:var(--ink); background:#fff; }
header { padding:20px 24px 8px; }
h1 { font-size:22px; margin:0 0 4px; }
.warn { background:#fff8c5; border:1px solid #d4a72c66; border-radius:8px; padding:8px 12px; margin:8px 0 0; color:#3b2300; }
.stats { display:flex; flex-wrap:wrap; gap:8px; padding:8px 24px; }
.stat { border:1px solid var(--line); border-radius:8px; padding:8px 12px; min-width:150px; }
.stat b { display:block; font-size:20px; }
.dot { display:inline-block; width:10px; height:10px; border-radius:2px; margin-right:6px; vertical-align:middle; }
.bar { display:flex; flex-wrap:wrap; gap:8px; align-items:center; padding:8px 24px; }
button { font:inherit; border:1px solid var(--line); background:#fff; border-radius:6px; padding:6px 12px; cursor:pointer; }
button.on { background:var(--ink); color:#fff; border-color:var(--ink); }
#map { height:62vh; min-height:360px; margin:0 24px; border:1px solid var(--line); border-radius:8px; }
main { padding:8px 24px 40px; display:grid; align-items:start; grid-template-columns:repeat(auto-fit, minmax(320px, 1fr)); gap:16px; }
section { border:1px solid var(--line); border-radius:8px; padding:12px 16px; }
h2 { font-size:16px; margin:0 0 6px; }
p { margin:4px 0; color:var(--muted); }
pre { background:var(--bg); border-radius:6px; padding:10px; overflow:auto; font-size:12px; max-height:340px; }
</style>
</head>
<body>
<header>
<h1>What your car would share</h1>
<div id="sub"></div>
<div class="warn">Dry run: nothing has been sent. This page marks your frequent stops, so keep it to yourself.</div>
</header>
<div class="stats" id="stats"></div>
<div class="bar">
<button id="mine" class="on">Your map (6 km tiles)</button>
<button id="others">What others see (45 km blocks)</button>
<label><input type="checkbox" id="kept" checked> Show what stays on the car</label>
</div>
<div id="map"></div>
<main>
<section><h2>Home region</h2><div id="region"></div></section>
<section><h2>Latest daily report</h2>
<p>Level 1 sends <code>unlocks</code> and <code>totals</code>. Level 2 adds <code>daily</code> and <code>profile</code>.</p>
<pre id="daily"></pre></section>
<section><h2>Next weekly tile batch (level 3)</h2>
<p>Sorted tile ids, no dates, sent separately from the daily report.</p><pre id="tiles"></pre></section>
<section><h2>Delete requests</h2>
<p>Tiles already sent that became private later (moved house, new job, hidden).</p><pre id="delete"></pre></section>
</main>
<script>
const D = __DATA__;
const M = D.map;
const held = Object.keys(M.held);
const never = M.ring;
document.getElementById('sub').textContent =
`As of ${D.today}. Sharing level on this car: ${D.level} (0 = off). The preview always shows level 3.`;
const stat = (color, n, label) =>
`<div class="stat"><b>${n}</b><span class="dot" style="background:${color}"></span>${label}</div>`;
document.getElementById('stats').innerHTML =
stat('#6e7781', M.driven, 'tiles driven') +
stat('var(--send)', M.shareable.length, 'would be shared') +
stat('var(--held)', held.length, 'drive start/end tiles waiting __HOLD_DAYS__ days') +
stat('var(--never)', never.length, `never leave the car (${M.frequent.length} frequent stops + the tiles around them)`) +
stat('var(--hidden)', M.hidden.length, 'hidden by you') +
stat('var(--send)', M.blocks.length, '45 km blocks others would see');
const r = D.region;
const days = Object.entries(r.days).map(([k, v]) => `${k}: ${v} day${v === 1 ? '' : 's'}`).join(', ') || 'no engaged days yet';
document.getElementById('region').innerHTML =
`<p>Detected: <b>${r.home || 'none yet'}</b> (${days}, last 90 days).</p>` +
`<p>Would be shared as: <b>${r.shared || 'nothing'}</b> (setting: ${r.choice}).</p>`;
const latest = D.daily.length ? D.daily[D.daily.length - 1] : null;
document.getElementById('daily').textContent = latest ? JSON.stringify(latest, null, 2) : 'No driving yet.';
const t = D.tiles;
document.getElementById('tiles').textContent = t ?
JSON.stringify({...t, tiles: t.tiles.slice(0, 12).concat(t.tiles.length > 12 ? [`... ${t.tiles.length - 12} more`] : [])}, null, 2) :
'Nothing new to send.';
document.getElementById('delete').textContent = D.delete ? JSON.stringify(D.delete, null, 2) : 'None.';
const fc = (cells, kind) => ({type: 'FeatureCollection', features: cells.map(c => {
const ring = h3.cellToBoundary(c, true); ring.push(ring[0]);
return {type: 'Feature', properties: {kind, cell: c, until: M.held[c] || ''}, geometry: {type: 'Polygon', coordinates: [ring]}};
})});
const all = [...M.shareable, ...held, ...never, ...M.hidden];
const map = new maplibregl.Map({
container: 'map',
style: {version: 8, sources: {osm: {type: 'raster', tiles: ['https://tile.openstreetmap.org/{z}/{x}/{y}.png'],
tileSize: 256, attribution: '&copy; OpenStreetMap contributors'}},
layers: [{id: 'osm', type: 'raster', source: 'osm', paint: {'raster-saturation': -0.8, 'raster-opacity': 0.8}}]},
center: [-83, 40], zoom: 5,
});
map.on('load', () => {
const add = (id, data, color, fill, dash) => {
map.addSource(id, {type: 'geojson', data});
map.addLayer({id: id + '-fill', type: 'fill', source: id, paint: {'fill-color': color, 'fill-opacity': fill}});
map.addLayer({id: id + '-line', type: 'line', source: id,
paint: {'line-color': color, 'line-width': 1.2, ...(dash ? {'line-dasharray': [2, 2]} : {})}});
};
add('blocks', fc(M.blocks, 'block'), '#2563eb', 0.35);
add('mine', fc(M.shareable, 'shared'), '#2563eb', 0.35);
add('held', fc(held, 'held'), '#9ca3af', 0.45, true);
add('never', fc(never, 'never'), '#dc2626', 0.25, true);
add('hidden', fc(M.hidden, 'hidden'), '#7c3aed', 0.3, true);
const kept = ['held', 'never', 'hidden'];
const show = (id, on) => ['-fill', '-line'].forEach(s => map.setLayoutProperty(id + s, 'visibility', on ? 'visible' : 'none'));
const fit = cells => {
if (!cells.length) return;
const pts = cells.flatMap(c => h3.cellToBoundary(c));
const lats = pts.map(p => p[0]), lngs = pts.map(p => p[1]);
map.fitBounds([[Math.min(...lngs), Math.min(...lats)], [Math.max(...lngs), Math.max(...lats)]], {padding: 40, duration: 0});
};
let mode = 'mine';
const refresh = () => {
show('mine', mode === 'mine'); show('blocks', mode === 'others');
const k = document.getElementById('kept').checked && mode === 'mine';
kept.forEach(id => show(id, k));
document.getElementById('mine').classList.toggle('on', mode === 'mine');
document.getElementById('others').classList.toggle('on', mode === 'others');
};
document.getElementById('mine').onclick = () => { mode = 'mine'; refresh(); fit(all); };
document.getElementById('others').onclick = () => { mode = 'others'; refresh(); fit(M.blocks); };
document.getElementById('kept').onchange = refresh;
refresh();
fit(all);
const label = {shared: 'Would be shared', held: 'Waiting until', never: 'Never leaves the car', hidden: 'Hidden by you',
block: 'What others see'};
map.on('click', e => {
const f = map.queryRenderedFeatures(e.point).find(x => x.properties.kind);
if (!f) return;
new maplibregl.Popup().setLngLat(e.lngLat)
.setHTML(`${label[f.properties.kind]} ${f.properties.until}<br><code>${f.properties.cell}</code>`).addTo(map);
});
});
</script>
</body>
</html>
"""
Binary file not shown.
@@ -1,74 +0,0 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
Home region lookup: map tile -> region code (US-OH, CA-ON, or a country code
like GB), from the table in regions.json.gz. Works offline on the car; no
geocoding service is ever called. See tools/build_regions.py for how the table
is made.
"""
from __future__ import annotations
import gzip
import json
from functools import lru_cache
from pathlib import Path
import numpy as np
from openpilot.sunnypilot.achievements import h3lite
TABLE_PATH = Path(__file__).parent / "regions.json.gz"
class RegionTable:
def __init__(self, path: Path = TABLE_PATH):
with gzip.open(path, "rt") as f:
data = json.load(f)
self.tile_res: int = data["tile_res"]
self.base_res: int = data["base_res"]
self.codes: list[str] = sorted(set(data["regions"]) | set(data["exceptions"]))
index = {c: i for i, c in enumerate(self.codes)}
self._base = self._arrays(data["regions"], index)
self._exceptions = self._arrays(data["exceptions"], index)
@staticmethod
def _arrays(groups: dict[str, list[str]], index: dict[str, int]) -> tuple[np.ndarray, np.ndarray]:
pairs = sorted((int(cell, 16), index[code]) for code, cells in groups.items() for cell in cells)
keys = np.array([p[0] for p in pairs], dtype=np.int64)
vals = np.array([p[1] for p in pairs], dtype=np.uint16)
return keys, vals
@staticmethod
def _find(arrays: tuple[np.ndarray, np.ndarray], cell: int) -> int | None:
keys, vals = arrays
i = int(np.searchsorted(keys, cell))
if i < len(keys) and int(keys[i]) == cell:
return int(vals[i])
return None
def lookup_cell(self, cell: int) -> str | None:
"""Region for a tile at tile_res (or finer). None at sea or off the map."""
res = h3lite.get_resolution(cell)
if res > self.tile_res:
cell, res = h3lite.cell_to_parent(cell, self.tile_res), self.tile_res
for table, top in ((self._exceptions, res), (self._base, min(res, self.base_res))):
for r in range(top, -1, -1):
hit = self._find(table, h3lite.cell_to_parent(cell, r))
if hit is not None:
return self.codes[hit]
return None
def lookup(self, lat: float, lng: float) -> str | None:
return self.lookup_cell(h3lite.latlng_to_cell(lat, lng, self.tile_res))
@lru_cache(maxsize=1)
def table() -> RegionTable:
return RegionTable()
def country_of(region: str) -> str:
return region.split("-", 1)[0]
-162
View File
@@ -1,162 +0,0 @@
#!/usr/bin/env python3
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
Runs achievements and the privacy filter over the drives already saved on the
car, then writes the dry-run preview. It uses its own folder, so live progress
is not touched, and it sends nothing.
On the car (over SSH):
cd /data/openpilot && python -m openpilot.sunnypilot.achievements.replay
Then on your computer:
scp comma@<car ip>:/data/media/0/achievements/replay/preview/map.html .
Reads qlogs from /data/media/0/realdata by default (--rlog for full logs, which
also count lane changes; much slower). Each route counts as one drive.
"""
from __future__ import annotations
import argparse
import collections
import datetime
import re
import shutil
import sys
import time
from pathlib import Path
from openpilot.sunnypilot.achievements.community import CommunityLog, Settings
from openpilot.sunnypilot.achievements.engine import (MAX_GPS_ACCURACY_M, AchievementEngine, Sample, State,
achievements_root)
from openpilot.sunnypilot.achievements.preview import write_preview
SEGMENT = re.compile(r"^(?P<route>.+)--(?P<seg>\d+)$")
GPS_MAX_AGE_S = 2.0
MIN_VALID_UNIX = 1.6e9 # anything earlier is a clock that wasn't set yet
def _now() -> float:
return datetime.datetime.now(datetime.UTC).timestamp()
def find_routes(log_root: Path, rlog: bool, days: int | None) -> list[list[Path]]:
names = ("rlog.zst", "rlog.bz2", "rlog") if rlog else ("qlog.zst", "qlog.bz2", "qlog")
cutoff = _now() - days * 86400 if days else 0
routes: dict[str, list[tuple[int, float, Path]]] = collections.defaultdict(list)
for d in log_root.iterdir():
m = SEGMENT.match(d.name)
if not m or not d.is_dir():
continue
f = next((d / n for n in names if (d / n).exists()), None)
if f is None or f.stat().st_mtime < cutoff:
continue
routes[m["route"]].append((int(m["seg"]), f.stat().st_mtime, f))
ordered = sorted(routes.values(), key=lambda segs: min(s[1] for s in segs))
return [[f for _, _, f in sorted(segs)] for segs in ordered]
class RouteFeeder:
"""Turns one route's log messages into Samples, the way achievementsd does live."""
def __init__(self, engine: AchievementEngine, log: CommunityLog):
self.engine, self.log = engine, log
self.op_active = self.mads_active = False
self.lane_change = self.dm_alert = 0
self.gps: tuple[float, float, int] | None = None
self.clock: tuple[int, float] | None = None # (logMonoTime, unix seconds)
self.car: str | None = None
self.samples = 0
def feed(self, ev) -> None:
w = ev.which()
if w == "carState":
self._sample(ev.logMonoTime, ev.carState.vEgo)
elif w == "selfdriveState":
self.op_active = ev.selfdriveState.active
elif w == "selfdriveStateSP":
self.mads_active = ev.selfdriveStateSP.mads.active
elif w == "modelV2":
self.lane_change = ev.modelV2.meta.laneChangeState.raw
elif w == "driverMonitoringState":
self.dm_alert = ev.driverMonitoringState.alertLevel.raw
elif w in ("gpsLocationExternal", "gpsLocation"):
g = getattr(ev, w)
if g.hasFix and 0 < g.horizontalAccuracy < MAX_GPS_ACCURACY_M:
self.gps = (g.latitude, g.longitude, ev.logMonoTime)
if g.unixTimestampMillis / 1000.0 > MIN_VALID_UNIX:
self.clock = (ev.logMonoTime, g.unixTimestampMillis / 1000.0)
elif w == "initData" and self.clock is None:
if ev.initData.wallTimeNanos / 1e9 > MIN_VALID_UNIX:
self.clock = (ev.logMonoTime, ev.initData.wallTimeNanos / 1e9)
elif w == "carParams" and not self.car:
self.car = ev.carParams.carFingerprint or None
def _sample(self, mono: int, v_ego: float) -> None:
fix = self.gps if self.gps and (mono - self.gps[2]) / 1e9 < GPS_MAX_AGE_S else None
wall = self.clock[1] + (mono - self.clock[0]) / 1e9 if self.clock else None
s = Sample(t=mono / 1e9, v_ego=v_ego, op_active=self.op_active, mads_active=self.mads_active,
lane_change_state=self.lane_change, dm_alert_level=self.dm_alert,
lat=fix[0] if fix else None, lon=fix[1] if fix else None, wall=wall)
self.engine.update(s)
self.log.update(s, self.engine.last_step)
self.samples += 1
def main(argv: list[str] | None = None) -> int:
from openpilot.common.hardware.hw import Paths
from openpilot.tools.lib.logreader import LogReader
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[1], formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--logs", type=Path, default=Path(Paths.log_root()), help="folder with route segments")
ap.add_argument("--out", type=Path, default=achievements_root() / "replay", help="where to write (wiped first)")
ap.add_argument("--days", type=int, default=None, help="only routes from the last N days")
ap.add_argument("--rlog", action="store_true", help="read full rlogs instead of qlogs")
args = ap.parse_args(argv)
routes = find_routes(args.logs, args.rlog, args.days)
if not routes:
print(f"no {'rlogs' if args.rlog else 'qlogs'} found in {args.logs}")
return 1
if args.out.exists():
shutil.rmtree(args.out)
args.out.mkdir(parents=True)
engine = AchievementEngine(State())
log = CommunityLog(args.out / "community.db")
car = None
n_segs = sum(len(r) for r in routes)
done = 0
start = time.monotonic()
for segments in routes:
feeder = RouteFeeder(engine, log)
for path in segments:
try:
for ev in LogReader(str(path), sort_by_time=True):
feeder.feed(ev)
except Exception as e: # one bad segment shouldn't stop the rest
print(f"skipping {path}: {e}", file=sys.stderr)
done += 1
print(f"\r{done}/{n_segs} segments", end="", file=sys.stderr, flush=True)
engine.end_drive()
log.end_drive(engine.utc_offset_h)
car = feeder.car or car
print(file=sys.stderr)
today = engine.local_date(_now())
out = write_preview(log, engine.state, today, car, Settings.load(), args.out / "preview")
view = log.tile_view(today)
print(f"{len(routes)} drives, {n_segs} segments in {time.monotonic() - start:.0f} s")
print(f"engaged: {engine.state.engaged_m / 1609.344:.0f} mi over {len(engine.state.days)} days")
kept = len(view.ring & view.driven)
print(f"tiles: {len(view.driven)} driven, {len(view.shareable)} would be shared, {len(view.held)} waiting, {kept} kept on the car")
print(f"frequent stops (never sent, with the tiles around them): {len(view.frequent)}")
print(f"home region: {log.home_region(today)}")
print(f"preview: {out / 'map.html'}")
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -1,113 +0,0 @@
#!/usr/bin/env python3
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
Made-up drives for the tests and for a demo preview without a car:
python -m openpilot.sunnypilot.achievements.simulate /tmp/demo
writes /tmp/demo/preview/map.html for two weeks of a commute from Hilliard to
Fairborn, Ohio, plus two weekend trips.
"""
from __future__ import annotations
import datetime as dt
import math
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from openpilot.sunnypilot.achievements.community import CommunityLog
from openpilot.sunnypilot.achievements.engine import AchievementEngine, Sample, State
HOME = (40.0334, -83.1588) # Hilliard
WORK = (39.8209, -84.0194) # Fairborn
SPRINGFIELD = (39.9242, -83.8088)
CINCINNATI = (39.1031, -84.5120)
DAYTON = (39.7589, -84.1916)
CLEVELAND = (41.4993, -81.6944)
MANSFIELD = (40.7584, -82.5154)
COLUMBUS = (39.9612, -82.9988)
@dataclass
class Drive:
day: dt.date
route: list[tuple[float, float]]
start_utc_h: float = 13.0 # 8 am in Ohio
engaged: bool = True
def _km(a: tuple[float, float], b: tuple[float, float]) -> float:
la1, lo1, la2, lo2 = map(math.radians, (*a, *b))
h = math.sin((la2 - la1) / 2) ** 2 + math.cos(la1) * math.cos(la2) * math.sin((lo2 - lo1) / 2) ** 2
return 2 * 6371.0 * math.asin(math.sqrt(h))
def drive_samples(drive: Drive, t0: float, speed_ms: float = 30.0):
"""1 Hz samples along straight lines between the route's points."""
wall = dt.datetime(drive.day.year, drive.day.month, drive.day.day, tzinfo=dt.UTC).timestamp() + drive.start_utc_h * 3600
t = t0
for a, b in zip(drive.route, drive.route[1:], strict=False):
n = max(1, int(_km(a, b) * 1000 / speed_ms))
for i in range(n):
f = i / n
lat, lon = a[0] + (b[0] - a[0]) * f, a[1] + (b[1] - a[1]) * f
yield Sample(t=t, v_ego=speed_ms, op_active=drive.engaged, lat=lat, lon=lon, wall=wall)
t += 1.0
wall += 1.0
end = drive.route[-1]
yield Sample(t=t, v_ego=0.0, op_active=False, lat=end[0], lon=end[1], wall=wall)
def run(drives: list[Drive], engine: AchievementEngine, log: CommunityLog, speed_ms: float = 30.0) -> None:
t = 1000.0
for d in drives:
for s in drive_samples(d, t, speed_ms):
engine.update(s)
log.update(s, engine.last_step)
t = s.t
engine.end_drive()
log.end_drive(engine.utc_offset_h)
t += 3600.0
def commute_weeks(first_monday: dt.date, weeks: int = 2) -> list[Drive]:
drives = []
for w in range(weeks):
monday = first_monday + dt.timedelta(weeks=w)
for i in range(5):
day = monday + dt.timedelta(days=i)
drives.append(Drive(day, [HOME, SPRINGFIELD, WORK], 12.5))
drives.append(Drive(day, [WORK, SPRINGFIELD, HOME], 21.5))
saturday = monday + dt.timedelta(days=5)
trip = [HOME, DAYTON, CINCINNATI] if w % 2 == 0 else [HOME, COLUMBUS, MANSFIELD, CLEVELAND]
drives.append(Drive(saturday, trip, 14.0))
drives.append(Drive(saturday, list(reversed(trip)), 20.0))
return drives
def main(out: str) -> None:
from openpilot.sunnypilot.achievements.community import Settings
from openpilot.sunnypilot.achievements.preview import write_preview
root = Path(out)
root.mkdir(parents=True, exist_ok=True)
db = root / "community.db"
if db.exists():
os.remove(db)
first_monday = dt.date(2026, 10, 26)
engine = AchievementEngine(State())
log = CommunityLog(db)
run(commute_weeks(first_monday), engine, log)
today = first_monday + dt.timedelta(days=15)
path = write_preview(log, engine.state, today, "HYUNDAI_IONIQ_5", Settings(level=0), root / "preview")
print(f"wrote {path / 'map.html'}")
if __name__ == "__main__":
main(sys.argv[1] if len(sys.argv) > 1 else "/tmp/achievements_demo")
@@ -1,195 +0,0 @@
"""
The privacy filter, on made-up drives: two weeks of a Hilliard -> Fairborn
commute plus two Saturday trips (see simulate.py).
"""
import datetime as dt
import re
import sqlite3
import pytest
from openpilot.sunnypilot.achievements import h3lite
from openpilot.sunnypilot.achievements.community import (LEVEL_ACHIEVEMENTS, LEVEL_DAILY, LEVEL_MAP, LEVEL_OFF,
CommunityLog, Settings, public_block, ring_of, tile_of)
from openpilot.sunnypilot.achievements.engine import AchievementEngine, State
from openpilot.sunnypilot.achievements.preview import build_preview, write_preview
from openpilot.sunnypilot.achievements.regions import table
from openpilot.sunnypilot.achievements.simulate import (CINCINNATI, CLEVELAND, HOME, WORK, Drive, commute_weeks,
run)
FIRST_MONDAY = dt.date(2026, 10, 26)
TODAY = FIRST_MONDAY + dt.timedelta(days=15) # Tuesday after the second week
HEX = re.compile(r"^86[0-9a-f]{13}$")
@pytest.fixture(scope="module")
def world(tmp_path_factory):
engine = AchievementEngine(State())
log = CommunityLog(tmp_path_factory.mktemp("community") / "community.db")
run(commute_weeks(FIRST_MONDAY), engine, log, speed_ms=60.0)
return engine, log
def test_home_and_work_never_leave(world):
_, log = world
view = log.tile_view(TODAY)
home, work = tile_of(*HOME), tile_of(*WORK)
assert view.frequent == {home, work}
assert ring_of(home) | ring_of(work) <= view.ring
assert len(ring_of(home)) == 7
assert not view.shareable & view.ring
batch = log.tile_batch(TODAY)
assert not set(batch["tiles"]) & view.ring
def test_trip_ends_wait_14_days(world):
_, log = world
cincinnati = tile_of(*CINCINNATI) # Saturday Oct 31
cleveland = tile_of(*CLEVELAND) # Saturday Nov 7
view = log.tile_view(TODAY)
assert view.held == {cincinnati: "2026-11-14", cleveland: "2026-11-21"}
assert cincinnati not in view.shareable
later = log.tile_view(dt.date(2026, 11, 14))
assert cincinnati in later.shareable
assert cleveland in later.held
def test_tile_batch_has_no_dates(world):
_, log = world
batch = log.tile_batch(TODAY)
assert set(batch) == {"v", "device", "tiles"}
assert batch["tiles"] == sorted(batch["tiles"])
assert all(HEX.match(t) for t in batch["tiles"])
assert len(batch["tiles"]) == len(log.tile_view(TODAY).shareable)
def test_nothing_finer_than_a_tile_is_stored(world):
_, log = world
db = sqlite3.connect(log.path)
assert all(HEX.match(c) for (c,) in db.execute("SELECT cell FROM tiles UNION SELECT cell FROM stops"))
days = {d for (d,) in db.execute("SELECT first_day FROM tiles UNION SELECT day FROM stops UNION SELECT day FROM days")}
assert all(re.fullmatch(r"\d{4}-\d\d-\d\d", d) for d in days)
def test_others_see_45km_blocks(world):
_, log = world
data = build_preview(log, world[0].state, TODAY, None, Settings())
shareable = log.tile_view(TODAY).shareable
assert set(data["map"]["blocks"]) == {public_block(t) for t in shareable}
assert all(h3lite.get_resolution(h3lite.str_to_cell(b)) == 4 for b in data["map"]["blocks"])
# the commute fills in the blocks around home and work, so no hole points at them
home_block, work_block = public_block(tile_of(*HOME)), public_block(tile_of(*WORK))
assert {home_block, work_block} <= set(data["map"]["blocks"])
def test_daily_report_levels(world):
engine, log = world
day = "2026-11-06"
assert log.daily_report(day, engine.state, LEVEL_OFF) is None
r1 = log.daily_report(day, engine.state, LEVEL_ACHIEVEMENTS)
assert set(r1) == {"v", "device", "day", "unlocks", "totals"}
assert r1["totals"]["engaged_mi"] % 10 == 0 and r1["totals"]["engaged_h"] % 10 == 0
r2 = log.daily_report(day, engine.state, LEVEL_DAILY, profile={"region": "US-OH"})
assert set(r2) == {"v", "device", "day", "unlocks", "totals", "daily", "profile"}
assert set(r2["daily"]) == {"engaged_mi", "engaged_min", "new_places"}
assert all(isinstance(v, int) for v in r2["daily"].values())
assert 80 < r2["daily"]["engaged_mi"] < 110 # two ~47 mile legs
assert LEVEL_MAP > LEVEL_DAILY
def test_unlocks_go_in_the_report_for_their_day(world):
engine, log = world
days = {}
for uid in engine.state.unlocked:
for day in log.driven_days():
if any(u["id"] == uid for u in log.daily_report(day, engine.state, LEVEL_ACHIEVEMENTS)["unlocks"]):
days.setdefault(uid, []).append(day)
assert days and all(len(v) == 1 for v in days.values())
def test_home_region(world):
_, log = world
assert log.home_region(TODAY) == "US-OH"
assert log.shared_region(TODAY, Settings(region="country")) == "US"
assert log.shared_region(TODAY, Settings(region="hidden")) is None
assert log.shared_region(TODAY, Settings(region="US-KY")) == "US-KY"
def test_profile_only_when_changed(tmp_path):
engine, log = AchievementEngine(State()), CommunityLog(tmp_path / "c.db")
run([Drive(dt.date(2026, 11, 2), [HOME, WORK])], engine, log, speed_ms=60.0)
today = dt.date(2026, 11, 3)
auto = Settings()
first = log.profile_update(today, "HYUNDAI_IONIQ_5", auto)
assert first == {"car": "HYUNDAI_IONIQ_5", "region": "US-OH"}
log.mark_profile_sent(today, first, auto)
assert log.profile_update(today, "HYUNDAI_IONIQ_5", auto) is None
# a road trip that moves the automatic home region waits 30 days after the last send
for i in range(3):
run([Drive(dt.date(2026, 11, 4 + i), [CINCINNATI, (38.25, -85.76)])], engine, log, speed_ms=60.0) # into Kentucky
assert log.home_region(dt.date(2026, 11, 7)) != "US-OH"
assert log.profile_update(dt.date(2026, 11, 7), "HYUNDAI_IONIQ_5", auto) is None
assert "region" in log.profile_update(dt.date(2026, 12, 3), "HYUNDAI_IONIQ_5", auto)
# a region the driver picks goes right away; hiding sends an empty region
assert log.profile_update(today, "HYUNDAI_IONIQ_5", Settings(region="US-MI")) == {"region": "US-MI"}
assert log.profile_update(today, "HYUNDAI_IONIQ_5", Settings(region="hidden")) == {"region": None}
def test_late_catches_and_hidden_tiles_are_deleted(tmp_path):
engine, log = AchievementEngine(State()), CommunityLog(tmp_path / "c.db")
friend = (39.95, -82.85) # east Columbus
run([Drive(dt.date(2026, 10, 1), [HOME, friend])], engine, log, speed_ms=60.0)
today = dt.date(2026, 10, 20)
view = log.tile_view(today)
assert tile_of(*friend) in view.shareable # one visit, held 14 days, then shared
log.mark_sent(view.to_send)
assert log.tile_batch(today) is None
# three more days ending there: it's a regular stop now, so ask the server to delete it
run([Drive(dt.date(2026, 10, 21 + i), [HOME, friend]) for i in range(3)], engine, log, speed_ms=60.0)
deletes = log.tile_deletes(dt.date(2026, 10, 25))
assert tile_of(*friend) in deletes["delete_tiles"]
assert set(deletes["delete_tiles"]) <= ring_of(tile_of(*friend)) | ring_of(tile_of(*HOME))
middle = sorted(log.tile_view(dt.date(2026, 10, 25)).shareable)[0]
log.hide_tile(middle)
view = log.tile_view(dt.date(2026, 10, 25))
assert middle not in view.shareable and middle in view.to_delete
def test_no_gps_no_tiles(tmp_path):
engine, log = AchievementEngine(State()), CommunityLog(tmp_path / "c.db")
drive = Drive(dt.date(2026, 11, 2), [HOME, WORK])
from openpilot.sunnypilot.achievements.simulate import drive_samples
for s in drive_samples(drive, 0.0, 60.0):
s.lat = s.lon = None
engine.update(s)
log.update(s, engine.last_step)
log.end_drive()
assert log.tile_view(dt.date(2026, 11, 3)).driven == set()
m, secs, _ = log.day_stats("2026-11-02")
assert m > 50_000 and secs > 0 # engaged miles still count
def test_write_preview(world, tmp_path):
engine, log = world
out = write_preview(log, engine.state, TODAY, "HYUNDAI_IONIQ_5", Settings(), tmp_path / "preview")
names = sorted(p.name for p in out.iterdir())
assert "map.html" in names and "tiles.json" in names
assert "daily-2026-11-07.json" in names
html = (out / "map.html").read_text()
assert "What your car would share" in html and tile_of(*HOME) in html
def test_region_table():
t = table()
assert t.lookup(*HOME) == "US-OH"
assert t.lookup(43.65, -79.38) == "CA-ON" # Toronto
assert t.lookup(40.76, -73.98) == "US-NY" # Manhattan, next to New Jersey
assert t.lookup(38.90, -77.04) == "US-DC"
assert t.lookup(51.50, -0.12) == "GB" # countries without subdivisions in the table
assert t.lookup(35.0, -40.0) is None # mid-Atlantic
@@ -1,219 +0,0 @@
import datetime as dt
import json
import pytest
from openpilot.sunnypilot.achievements.catalog import CATALOG_PATH, load_catalog, parse_catalog, statuses
from openpilot.sunnypilot.achievements.engine import (AchievementEngine, LANE_CHANGE_FINISHING, LANE_CHANGE_STARTING, Sample, State,
geohash, load_seen, load_state, save_seen, save_state, season)
T0 = dt.datetime(2026, 6, 1, 16, 0, tzinfo=dt.UTC).timestamp() # noon in New York
def drive(engine, seconds, v=20.0, t0=T0, hz=10, **kw):
"""Feed `seconds` of driving at `v` m/s; returns every unlock id along the way."""
new = []
n = int(seconds * hz)
for i in range(n + 1):
extra = {k: val for k, val in kw.items() if k != "op_active"}
new += engine.update(Sample(t=t0 + i / hz, wall=t0 + i / hz, v_ego=v, op_active=kw.get("op_active", True), **extra))
return new
def test_catalog_loads_and_ids_are_unique():
cat = load_catalog()
ids = cat.all_unlock_ids()
assert len(ids) == len(set(ids))
assert "odometer.silver" in ids and "first_light" in ids
engine_metrics = set(State().metrics())
assert cat.metrics <= engine_metrics, f"catalog uses metrics the engine doesn't compute: {cat.metrics - engine_metrics}"
def test_catalog_rejects_decreasing_tiers():
data = json.loads(CATALOG_PATH.read_text())
data["achievements"][1]["tiers"][1]["threshold"] = 1
with pytest.raises(ValueError):
parse_catalog(data)
def test_engaged_distance_and_first_light():
e = AchievementEngine()
new = drive(e, 10, v=20.0)
assert e.state.engaged_m == pytest.approx(200.0, rel=1e-6)
assert e.state.engaged_s == pytest.approx(10.0, rel=1e-6)
assert new == ["first_light"]
def test_not_engaged_counts_nothing():
e = AchievementEngine()
drive(e, 10, op_active=False)
assert e.state.engaged_m == 0.0
assert e.state.unlocked == {}
def test_gaps_and_bad_speeds_are_ignored():
e = AchievementEngine()
e.update(Sample(t=T0, v_ego=20.0, op_active=True))
e.update(Sample(t=T0 + 30.0, v_ego=20.0, op_active=True)) # 30 s gap: process stalled
e.update(Sample(t=T0 + 30.1, v_ego=500.0, op_active=True)) # implausible speed
assert e.state.engaged_m == 0.0
def test_odometer_tiers_unlock_in_order():
e = AchievementEngine()
e.state.engaged_m = 99_999.0
new = drive(e, 1, v=10.0)
assert "odometer.bronze" in new and "odometer.silver" not in new
assert set(e.state.unlocked) >= {"first_light", "odometer.bronze"}
def test_unlocks_only_once():
e = AchievementEngine()
assert drive(e, 5) == ["first_light"]
assert drive(e, 5, t0=T0 + 100) == []
def test_mads_only_distance():
e = AchievementEngine()
drive(e, 10, v=10.0, op_active=False, mads_active=True)
drive(e, 10, v=10.0, t0=T0 + 60, op_active=True, mads_active=True)
assert e.state.mads_m == pytest.approx(100.0, rel=1e-6)
assert e.state.engaged_m == pytest.approx(200.0, rel=1e-6)
def test_lane_change_counted_once_when_engaged():
e = AchievementEngine()
seq = [0, 1, 2, 2, 3, 3, 0, 0, 2, 3, 0]
for i, lc in enumerate(seq):
e.update(Sample(t=T0 + i * 0.1, v_ego=25.0, op_active=True, lane_change_state=lc))
assert e.state.lane_changes == 2
e.update(Sample(t=T0 + 5, v_ego=25.0, op_active=False, lane_change_state=LANE_CHANGE_STARTING))
e.update(Sample(t=T0 + 5.1, v_ego=25.0, op_active=False, lane_change_state=LANE_CHANGE_FINISHING))
assert e.state.lane_changes == 2
def test_days_streak_and_best_day():
e = AchievementEngine()
for d in range(8):
drive(e, 2, t0=T0 + d * 86400)
assert len(e.state.days) == 8
assert e.state.best_streak_days == 8
assert "streak.bronze" in e.state.unlocked and "regular.bronze" in e.state.unlocked
# a gap day resets the run but keeps the best
drive(e, 2, t0=T0 + 10 * 86400)
assert e.state.best_streak_days == 8
def test_local_date_follows_gps_longitude():
e = AchievementEngine()
late = dt.datetime(2026, 6, 2, 2, 0, tzinfo=dt.UTC).timestamp() # 10 pm on June 1 in New York
e.update(Sample(t=late, wall=late, v_ego=10.0, op_active=True, lat=40.7, lon=-74.0))
e.update(Sample(t=late + 0.1, wall=late + 0.1, v_ego=10.0, op_active=True, lat=40.7, lon=-74.0))
assert e.state.days == ["2026-06-01"]
def test_invalid_clock_counts_distance_but_not_days():
e = AchievementEngine()
for i in range(11):
e.update(Sample(t=i / 10, v_ego=10.0, op_active=True)) # wall=None: clock not set yet
assert e.state.engaged_m == pytest.approx(10.0, rel=1e-6)
assert e.state.days == [] and e.state.today_m == 0.0
assert e.state.unlocked == {}
def test_attentive_run_resets_on_red_alert():
e = AchievementEngine()
drive(e, 100)
drive(e, 1, t0=T0 + 200, dm_alert_level=3)
assert e.state.attentive_run_s < 1.0
assert e.state.attentive_best_s == pytest.approx(100.0, rel=1e-3)
def test_areas_are_hashed_and_counted():
e = AchievementEngine()
drive(e, 1, lat=40.70, lon=-74.00)
drive(e, 1, t0=T0 + 10, lat=40.701, lon=-74.001) # same ~5 km cell
drive(e, 1, t0=T0 + 20, lat=41.50, lon=-73.00) # different cell
assert len(e.state.areas) == 2
raw = json.dumps(e.state.__dict__)
assert "40.7" not in raw and geohash(40.70, -74.00) not in raw
def test_model_hopper_counts_models_over_50km():
e = AchievementEngine()
e.state.model_m = {"a": 50_000.0, "b": 50_000.0, "c": 10.0}
drive(e, 1, model_ref="c", model_name="Model C")
assert e.state.metrics()["models_50km"] == 2
e.state.model_m["c"] = 50_000.0
assert "model_hopper.bronze" in drive(e, 1, t0=T0 + 10, model_ref="c")
assert e.state.model_names["c"] == "Model C"
def test_dev_build_and_seasons():
e = AchievementEngine()
drive(e, 1, dev_build=True)
assert e.state.dev_m > 0
assert season(dt.date(2026, 1, 5)) == "winter"
assert season(dt.date(2026, 1, 5), southern=True) == "summer"
def test_sunnylink_flag_sticks():
e = AchievementEngine()
assert e.set_sunnylink_linked(True, T0) == ["linked_up"]
e.set_sunnylink_linked(False, T0 + 1)
assert e.state.sunnylink_linked
def test_state_round_trip(tmp_path):
e = AchievementEngine()
drive(e, 10, lat=40.7, lon=-74.0, model_ref="x")
path = tmp_path / "state.json"
save_state(e.state, path)
loaded = load_state(path)
assert loaded == e.state
def test_corrupt_state_is_moved_aside(tmp_path):
path = tmp_path / "state.json"
path.write_text("{not json")
assert load_state(path).engaged_m == 0.0
assert (tmp_path / "state.corrupt.json").exists()
assert not path.exists()
def test_unknown_fields_are_ignored(tmp_path):
path = tmp_path / "state.json"
path.write_text(json.dumps({"engaged_m": 5.0, "from_the_future": 1}))
assert load_state(path).engaged_m == 5.0
def test_seen_round_trip(tmp_path):
path = tmp_path / "seen.json"
assert load_seen(path) == set()
save_seen({"a", "b"}, path)
assert load_seen(path) == {"a", "b"}
def test_status_summaries():
cat = load_catalog()
st = State(engaged_m=1_240_000.0)
e = AchievementEngine(st)
e.check_unlocks(T0)
by_id = {s.achievement.id: s for s in statuses(cat, st.metrics(), st.unlocked)}
assert by_id["odometer"].summary() == "silver · 1,240 / 10,000 km"
assert by_id["odometer"].summary(is_metric=False) == "silver · 770 / 6,213 mi"
assert by_id["first_light"].summary() == "done"
assert by_id["regular"].summary() == "locked · 0 / 7 days"
assert by_id["linked_up"].summary() == "locked"
def test_last_step_for_the_community_log():
e = AchievementEngine(State())
wall = 1_762_000_000.0
e.update(Sample(t=0.0, v_ego=20.0, op_active=True, wall=wall))
e.update(Sample(t=0.5, v_ego=20.0, op_active=True, wall=wall + 0.5))
assert e.last_step.dt_s == 0.5 and e.last_step.engaged_m == 10.0 and e.last_step.day is not None
e.update(Sample(t=1.0, v_ego=20.0, op_active=False, wall=wall + 1.0))
assert e.last_step.dt_s == 0.5 and e.last_step.engaged_m == 0.0
e.update(Sample(t=9.0, v_ego=20.0, op_active=True, wall=None)) # a gap and no clock
assert e.last_step.dt_s == 0.0 and e.last_step.day is None
@@ -1,125 +0,0 @@
"""
h3lite must give the same cell ids as Uber's H3, because the site and the
server use h3-js and h3-pg on the ids the car sends. The vectors below come
from h3-py 4.1.2; with h3 installed, the last test also compares random points.
"""
import random
import pytest
from openpilot.sunnypilot.achievements import h3lite
RESOLUTIONS = (0, 4, 6, 9, 15)
CELLS = [
(40.0334, -83.1588, ("802bfffffffffff", "842a90bffffffff", "862a90b4fffffff", "892a90b4917ffff", "8f2a90b4915d388")),
(37.7749, -122.4194, ("8029fffffffffff", "8428309ffffffff", "86283082fffffff", "89283082803ffff", "8f283082800b390")),
(51.5007, -0.1246, ("8019fffffffffff", "84194adffffffff", "86194ad17ffffff", "89194ad14c3ffff", "8f194ad14c254d3")),
(-33.8688, 151.2093, ("80bffffffffffff", "84be0e3ffffffff", "86be0e35fffffff", "89be0e35cbbffff", "8fbe0e35cbad0a8")),
(35.6762, 139.6503, ("802ffffffffffff", "842f5a3ffffffff", "862f5a367ffffff", "892f5a363bbffff", "8f2f5a363ba005a")),
(64.8378, -147.7164, ("800dfffffffffff", "840d5a9ffffffff", "860d5a9afffffff", "890d5a9add3ffff", "8f0d5a9add25781")),
(-54.8019, -68.303, ("80dffffffffffff", "84df451ffffffff", "86df45177ffffff", "89df4517557ffff", "8fdf45175542186")),
(0.0, 0.0, ("8075fffffffffff", "84754a9ffffffff", "86754e64fffffff", "89754e64993ffff", "8f754e64992d6d8")),
(89.9, 45.0, ("8001fffffffffff", "8403263ffffffff", "86032630fffffff", "89032630ac3ffff", "8f032630ac0311a")),
(-89.9, -45.0, ("80f3fffffffffff", "84f2939ffffffff", "86f2938efffffff", "89f2938ec7bffff", "8ff2938ec78c80e")),
(52.0, 179.999, ("8017fffffffffff", "841659bffffffff", "861659aefffffff", "891659acc9bffff", "8f1659acc9ad193")),
(-16.5, -179.999, ("809bfffffffffff", "849b5ddffffffff", "869b5dc57ffffff", "899b5dc5597ffff", "8f9b5dc5596a500")),
(-28.18676, -125.6943, ("80b1fffffffffff", "84b1713ffffffff", "86b171247ffffff", "89b1712423bffff", "8fb171242396a80")),
(24.14952, -153.92294, ("805dfffffffffff", "845d35bffffffff", "865d35b0fffffff", "895d35b0b23ffff", "8f5d35b0b200c24")),
(5.74112, -48.35199, ("805ffffffffffff", "845fa25ffffffff", "865fa248fffffff", "895fa248b23ffff", "8f5fa248b218936")),
(-70.72017, 2.67686, ("80e7fffffffffff", "84e749dffffffff", "86e749c47ffffff", "89e749c44c3ffff", "8fe749c44c216ec")),
(-74.00069, -23.88755, ("80effffffffffff", "84ee139ffffffff", "86ee13967ffffff", "89ee1396293ffff", "8fee139629208d2")),
(-68.82313, -147.34332, ("80e3fffffffffff", "84e334dffffffff", "86e334c9fffffff", "89e334c983bffff", "8fe334c9839a3b1")),
(-12.07693, 117.66676, ("8095fffffffffff", "849581dffffffff", "869581d97ffffff", "899581d9487ffff", "8f9581d9486a96e")),
(-60.19169, -99.63397, ("80e9fffffffffff", "84e95c1ffffffff", "86e95c107ffffff", "89e95c1000bffff", "8fe95c1000b639c")),
]
PARENTS = [
("8f2a90b4915d388", ("802bfffffffffff", "842a90bffffffff", "862a90b4fffffff", "892a90b4917ffff", "8f2a90b4915d388")),
("8f283082800b390", ("8029fffffffffff", "8428309ffffffff", "86283082fffffff", "89283082803ffff", "8f283082800b390")),
("8f194ad14c254d3", ("8019fffffffffff", "84194adffffffff", "86194ad17ffffff", "89194ad14c3ffff", "8f194ad14c254d3")),
("8fbe0e35cbad0a8", ("80bffffffffffff", "84be0e3ffffffff", "86be0e35fffffff", "89be0e35cbbffff", "8fbe0e35cbad0a8")),
("8f2f5a363ba005a", ("802ffffffffffff", "842f5a3ffffffff", "862f5a367ffffff", "892f5a363bbffff", "8f2f5a363ba005a")),
("8f0d5a9add25781", ("800dfffffffffff", "840d5a9ffffffff", "860d5a9afffffff", "890d5a9add3ffff", "8f0d5a9add25781")),
("8fdf45175542186", ("80dffffffffffff", "84df451ffffffff", "86df45177ffffff", "89df4517557ffff", "8fdf45175542186")),
("8f754e64992d6d8", ("8075fffffffffff", "84754e7ffffffff", "86754e64fffffff", "89754e64993ffff", "8f754e64992d6d8")),
("8f032630ac0311a", ("8003fffffffffff", "8403263ffffffff", "86032630fffffff", "89032630ac3ffff", "8f032630ac0311a")),
("8ff2938ec78c80e", ("80f3fffffffffff", "84f2939ffffffff", "86f2938efffffff", "89f2938ec7bffff", "8ff2938ec78c80e")),
("8f1659acc9ad193", ("8017fffffffffff", "841659bffffffff", "861659acfffffff", "891659acc9bffff", "8f1659acc9ad193")),
("8f9b5dc5596a500", ("809bfffffffffff", "849b5ddffffffff", "869b5dc57ffffff", "899b5dc5597ffff", "8f9b5dc5596a500")),
("8fb171242396a80", ("80b1fffffffffff", "84b1713ffffffff", "86b171247ffffff", "89b1712423bffff", "8fb171242396a80")),
("8f5d35b0b200c24", ("805dfffffffffff", "845d35bffffffff", "865d35b0fffffff", "895d35b0b23ffff", "8f5d35b0b200c24")),
("8f5fa248b218936", ("805ffffffffffff", "845fa25ffffffff", "865fa248fffffff", "895fa248b23ffff", "8f5fa248b218936")),
("8fe749c44c216ec", ("80e7fffffffffff", "84e749dffffffff", "86e749c47ffffff", "89e749c44c3ffff", "8fe749c44c216ec")),
("8fee139629208d2", ("80effffffffffff", "84ee139ffffffff", "86ee13967ffffff", "89ee1396293ffff", "8fee139629208d2")),
("8fe334c9839a3b1", ("80e3fffffffffff", "84e334dffffffff", "86e334c9fffffff", "89e334c983bffff", "8fe334c9839a3b1")),
("8f9581d9486a96e", ("8095fffffffffff", "849581dffffffff", "869581d97ffffff", "899581d9487ffff", "8f9581d9486a96e")),
("8fe95c1000b639c", ("80e9fffffffffff", "84e95c1ffffffff", "86e95c107ffffff", "89e95c1000bffff", "8fe95c1000b639c")),
("8f2a90b4915d388", ("802bfffffffffff", "842a90bffffffff", "862a90b4fffffff", "892a90b4917ffff", "8f2a90b4915d388")),
]
CENTERS = [
("862a90b4fffffff", 40.05041861648768, -83.181315523919),
("842a90bffffffff", 40.08105106496825, -83.41180170584032),
("864c00007ffffff", 23.71792527122296, -67.13232636643565),
("8f2a90b4915d388", 40.03339893697043, -83.1588004749386),
("8001fffffffffff", 79.24239850975903, 38.02340700796989),
]
DISKS = {
"862a90b4fffffff": ['862a90a67ffffff', '862a90b47ffffff', '862a90b4fffffff', '862a90b5fffffff', '862a90b6fffffff', '862a95697ffffff', '862a956b7ffffff'],
# a pentagon: 5 neighbors
"864c00007ffffff": [
'864c00007ffffff',
'864c00017ffffff',
'864c0001fffffff',
'864c00027ffffff',
'864c0002fffffff',
'864c00037ffffff',
],
}
@pytest.mark.parametrize("lat,lng,expected", CELLS)
def test_latlng_to_cell(lat, lng, expected):
for res, want in zip(RESOLUTIONS, expected, strict=True):
assert h3lite.cell_to_str(h3lite.latlng_to_cell(lat, lng, res)) == want
@pytest.mark.parametrize("cell,expected", PARENTS)
def test_cell_to_parent(cell, expected):
# a parent isn't always the cell that holds the same point: H3 children stick out a little
for res, want in zip(RESOLUTIONS, expected, strict=True):
assert h3lite.cell_to_str(h3lite.cell_to_parent(h3lite.str_to_cell(cell), res)) == want
assert h3lite.get_resolution(h3lite.str_to_cell(want)) == res
@pytest.mark.parametrize("cell,lat,lng", CENTERS)
def test_cell_to_latlng(cell, lat, lng):
got = h3lite.cell_to_latlng(h3lite.str_to_cell(cell))
assert got == pytest.approx((lat, lng), abs=1e-9)
@pytest.mark.parametrize("cell,expected", DISKS.items())
def test_grid_disk1(cell, expected):
got = {h3lite.cell_to_str(c) for c in h3lite.grid_disk1(h3lite.str_to_cell(cell))}
assert got == set(expected)
def test_bad_input():
with pytest.raises(ValueError):
h3lite.latlng_to_cell(40.0, -83.0, 16)
with pytest.raises(ValueError):
h3lite.latlng_to_cell(float("nan"), -83.0, 6)
with pytest.raises(ValueError):
h3lite.cell_to_parent(h3lite.str_to_cell("862a90b4fffffff"), 7)
def test_matches_h3_on_random_points():
h3 = pytest.importorskip("h3")
rnd = random.Random(0)
for _ in range(1500):
lat, lng = rnd.uniform(-90, 90), rnd.uniform(-180, 180)
for res in range(16):
want = h3.latlng_to_cell(lat, lng, res)
cell = h3lite.latlng_to_cell(lat, lng, res)
assert h3lite.cell_to_str(cell) == want
assert h3lite.cell_to_latlng(cell) == pytest.approx(h3.cell_to_latlng(want), abs=1e-9)
want_disk = {h3.str_to_int(c) for c in h3.grid_disk(h3.latlng_to_cell(lat, lng, 6), 1)}
assert h3lite.grid_disk1(h3lite.latlng_to_cell(lat, lng, 6)) == want_disk
@@ -1,68 +0,0 @@
"""The replay tool, fed fake log messages (the real LogReader needs the full openpilot build)."""
import datetime as dt
import os
from types import SimpleNamespace as NS
from openpilot.sunnypilot.achievements.community import CommunityLog, tile_of
from openpilot.sunnypilot.achievements.engine import AchievementEngine, State
from openpilot.sunnypilot.achievements.replay import RouteFeeder, find_routes
from openpilot.sunnypilot.achievements.simulate import HOME, WORK
T0 = int(dt.datetime(2026, 11, 2, 13, 0, tzinfo=dt.UTC).timestamp() * 1e3)
class Ev:
def __init__(self, kind: str, mono_s: float, **fields):
self._kind, self.logMonoTime = kind, int(mono_s * 1e9)
setattr(self, kind, NS(**fields))
def which(self):
return self._kind
def fake_route(start, end, seconds=600):
yield Ev("carParams", 0.0, carFingerprint="HYUNDAI_IONIQ_5")
yield Ev("selfdriveState", 0.0, active=True)
yield Ev("selfdriveStateSP", 0.0, mads=NS(active=False))
yield Ev("driverMonitoringState", 0.0, alertLevel=NS(raw=0))
for i in range(seconds * 10): # carState at 10 Hz, GPS at 1 Hz, like a qlog
t = i / 10
if i % 10 == 0:
f = t / seconds
yield Ev("gpsLocationExternal", t, hasFix=True, horizontalAccuracy=5.0,
latitude=start[0] + (end[0] - start[0]) * f, longitude=start[1] + (end[1] - start[1]) * f,
unixTimestampMillis=T0 + int(t * 1000))
yield Ev("carState", t, vEgo=30.0)
def test_route_feeder_counts_a_drive(tmp_path):
engine, log = AchievementEngine(State()), CommunityLog(tmp_path / "c.db")
feeder = RouteFeeder(engine, log)
for ev in fake_route(HOME, WORK):
feeder.feed(ev)
engine.end_drive()
log.end_drive(engine.utc_offset_h)
assert feeder.car == "HYUNDAI_IONIQ_5"
assert 17_000 < engine.state.engaged_m < 18_500 # 600 s at 30 m/s
assert engine.state.days == ["2026-11-02"]
view = log.tile_view(dt.date(2026, 11, 3))
assert tile_of(*HOME) in view.held # where the drive started
assert len(view.driven) > 3
assert log.home_region(dt.date(2026, 11, 3)) == "US-OH"
def test_find_routes(tmp_path):
for name, mtime in (("00000002--bbbb--0", 200), ("00000002--bbbb--1", 260), ("00000001--aaaa--0", 100),
("00000001--aaaa--10", 160), ("00000001--aaaa--2", 130), ("boot", 0)):
d = tmp_path / name
d.mkdir()
if name != "boot":
(d / "qlog.zst").write_bytes(b"")
os.utime(d / "qlog.zst", (mtime, mtime))
routes = find_routes(tmp_path, rlog=False, days=None)
assert [[p.parent.name for p in r] for r in routes] == [
["00000001--aaaa--0", "00000001--aaaa--2", "00000001--aaaa--10"],
["00000002--bbbb--0", "00000002--bbbb--1"],
]
assert find_routes(tmp_path, rlog=True, days=None) == []
@@ -1,136 +0,0 @@
#!/usr/bin/env python3
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
Builds regions.json.gz, the table the car uses to turn a map tile into a home
region code. Dev tool only: it is not run on the car.
pip install h3 shapely
curl -LO https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_1_states_provinces.geojson
python openpilot/sunnypilot/achievements/tools/build_regions.py \
ne_10m_admin_1_states_provinces.geojson openpilot/sunnypilot/achievements/regions.json.gz
Rules:
- Region codes are ISO 3166-2 (US-OH) in the countries in SUBDIVISIONS, and the
ISO 3166-1 country code (GB) everywhere else.
- A resolution-6 tile (about 6 km) belongs to the region its center is in.
Coastal tiles whose center is in the water go to the region whose coastline
passes through them most.
- Stored as the majority region of each resolution-4 cell ("regions") plus the
resolution-6 tiles that differ from it ("exceptions"), each compacted. That
keeps 6 km accuracy at borders in about 200 KB.
Boundaries: Natural Earth 1:10m admin 1, public domain.
"""
import collections
import gzip
import json
import re
import sys
import h3
from shapely.geometry import box, shape
SUBDIVISIONS = {"US", "CA", "MX", "BR", "AU", "DE"}
TILE_RES = 6
BASE_RES = 4
CUT_DEG = 10.0 # H3 can't fill huge polygons at resolution 6, so cut them into squares first
ISO_3166_2 = re.compile(r"^[A-Z]{2}-[A-Z0-9]{1,3}$")
def region_code(props: dict) -> str | None:
country = props.get("iso_a2") or ""
if not re.fullmatch(r"[A-Z]{2}", country):
return None
code = props.get("iso_3166_2") or ""
if country in SUBDIVISIONS and ISO_3166_2.match(code):
return code
return country
def pieces(geom: dict):
"""Polygon rings, as (lng, lat) lists, of the geometry cut into CUT_DEG squares."""
g = shape(geom)
if not g.is_valid:
g = g.buffer(0)
minx, miny, maxx, maxy = g.bounds
x = minx
while x < maxx:
y = miny
while y < maxy:
part = g.intersection(box(x, y, x + CUT_DEG, y + CUT_DEG))
for poly in getattr(part, "geoms", [part]):
if poly.geom_type == "Polygon" and not poly.is_empty:
yield [list(poly.exterior.coords)] + [list(r.coords) for r in poly.interiors]
y += CUT_DEG
x += CUT_DEG
def outer_rings(geom: dict):
polys = [geom["coordinates"]] if geom["type"] == "Polygon" else geom["coordinates"]
for rings in polys:
yield rings[0]
def winner(counter: collections.Counter) -> str:
return max(sorted(counter), key=lambda k: counter[k])
def main(src: str, dst: str) -> None:
features = json.load(open(src))["features"]
by_center: dict[str, str] = {}
by_coast: dict[str, collections.Counter] = collections.defaultdict(collections.Counter)
for n, f in enumerate(features):
code = region_code(f["properties"])
if code is None or f["geometry"] is None:
continue
for rings in pieces(f["geometry"]):
outer, holes = rings[0], rings[1:]
poly = h3.LatLngPoly([(lat, lng) for lng, lat in outer], *[[(lat, lng) for lng, lat in r] for r in holes])
for cell in h3.polygon_to_cells(poly, TILE_RES):
by_center[cell] = code
for ring in outer_rings(f["geometry"]):
for lng, lat in ring:
by_coast[h3.latlng_to_cell(lat, lng, TILE_RES)][code] += 1
if n % 500 == 0:
print(f"{n}/{len(features)} boundaries", file=sys.stderr)
tiles = dict(by_center)
for cell, votes in by_coast.items():
tiles.setdefault(cell, winner(votes))
base_votes: dict[str, collections.Counter] = collections.defaultdict(collections.Counter)
for cell, code in tiles.items():
base_votes[h3.cell_to_parent(cell, BASE_RES)][code] += 1
base = {cell: winner(votes) for cell, votes in base_votes.items()}
regions: dict[str, list[str]] = collections.defaultdict(list)
for cell, code in base.items():
regions[code].append(cell)
exceptions: dict[str, list[str]] = collections.defaultdict(list)
for cell, code in tiles.items():
if base[h3.cell_to_parent(cell, BASE_RES)] != code:
exceptions[code].append(cell)
data = {
"v": 1,
"source": "Natural Earth 1:10m admin 1 (public domain); built by tools/build_regions.py",
"subdivisions": sorted(SUBDIVISIONS),
"tile_res": TILE_RES,
"base_res": BASE_RES,
"regions": {k: sorted(h3.compact_cells(v)) for k, v in sorted(regions.items())},
"exceptions": {k: sorted(h3.compact_cells(v)) for k, v in sorted(exceptions.items())},
}
raw = json.dumps(data, separators=(",", ":"), sort_keys=True).encode()
with gzip.GzipFile(dst, "wb", mtime=0) as out:
out.write(raw)
n_regions = sum(map(len, data["regions"].values()))
n_exceptions = sum(map(len, data["exceptions"].values()))
print(f"{len(tiles)} tiles, {len(data['regions'])} regions, {n_regions} base cells, {n_exceptions} exceptions", file=sys.stderr)
if __name__ == "__main__":
main(sys.argv[1], sys.argv[2])
@@ -371,7 +371,7 @@ def _compile_warp_resolution_worker(cam_w, cam_h, model_w, model_h, benchmark_ru
if __name__ == "__main__":
if ('USB' in os.getenv('DEV', '') or os.getenv('CHESTNUT')) and not os.getenv('DEV', '').startswith('MOCK'):
if 'USB' in os.getenv('DEV', '') or os.getenv('CHESTNUT'):
from openpilot.system.hardware.chestnut.flash import link_up
for _ in range(10):
if link_up():
@@ -17,18 +17,16 @@ def main():
parser.add_argument("--frame", required=True, help="width,height,stride,y_height,uv_height,buffer_size")
parser.add_argument("--warp-to", required=True, help="model input WxH")
parser.add_argument("--output", required=True)
parser.add_argument("--light", action="store_true")
args = parser.parse_args()
from examples.openpilot.compile_warp import make_frame_prepare, parse_frame, parse_size
from tinygrad import Device, Tensor, TinyJit
frame = parse_frame(args.frame)
warp = make_frame_prepare(frame, *parse_size(args.warp_to))
n = 1 if args.light else 2
run = TinyJit(lambda input_frame, M_inv: Tensor.stack(*(warp(input_frame[i], M_inv[i]) for i in range(n))), prune=True)
run = TinyJit(lambda input_frame, M_inv: Tensor.stack(*(warp(input_frame[i], M_inv[i]) for i in range(2))), prune=True)
rng = np.random.default_rng(42)
inputs = {"input_frame": Tensor(rng.integers(0, 256, (2, frame.size), dtype=np.uint8), device=Device.DEFAULT).realize(),
"M_inv": Tensor((rng.standard_normal((n, 3, 3)) * 8).astype(np.float32), device=Device.DEFAULT).realize()}
"M_inv": Tensor((rng.standard_normal((2, 3, 3)) * 8).astype(np.float32), device=Device.DEFAULT).realize()}
with record_kernel_linear() as recorded:
run(**inputs)
run(**inputs)
@@ -36,7 +34,7 @@ def main():
assert len(recorded) == 1, f"expected one captured kernel list, got {len(recorded)}"
part = serialize_jit(run, recorded[0], check)
PlainJit(*part)
specs = {"input_frame": ((2, frame.size), "|u1", Device.DEFAULT), "M_inv": ((n, 3, 3), "<f4", Device.DEFAULT)}
specs = {"input_frame": ((2, frame.size), "|u1", Device.DEFAULT), "M_inv": ((2, 3, 3), "<f4", Device.DEFAULT)}
write_bundle(args.output, {"run": part}, {"input_specs": specs, "metadata": {}})
print(f"Saved warp to {args.output} ({len(part[0]['calls'])} kernels)")
@@ -10,7 +10,6 @@ import dataclasses
import inspect
import itertools
import mmap
import os
import re
import struct
import tempfile
@@ -67,75 +66,47 @@ def _patched(buf, topo) -> tuple[bytearray, dict]:
return blob, rows
def _packed_args(program) -> dict[int, int]:
from tinygrad.device import TinyELF
params = getattr(program.arg, "params", None) or ()
return {off: p.arg.slot for (off, _), p in zip(TinyELF.iter_sig(tuple((None, p.addrspace, p.dtype, ()) for p in params)), params, strict=True)}
def _outer_slot(u, packed: dict[int, int]) -> int | None:
from tinygrad.uop.ops import Ops
if u.op is Ops.PARAM:
return None if packed else u.arg.slot
if u.op.name == "REFPARAM":
return u.arg.slot
if packed and u.op is Ops.LOAD and u.src[0].op is Ops.INDEX and _root(u.src[0].src[0]).op is Ops.PARAM:
index = _evaluate(u.src[0].src[1], {}, packed)
return None if index is None else packed.get(index * u.dtype.itemsize)
return None
def _evaluate(u, env, packed: dict[int, int]):
def _evaluate(u, env):
from tinygrad.uop.ops import GroupOp, Ops, exec_alu
if u.op in (Ops.AFTER, Ops.CAST, Ops.BITCAST):
return _evaluate(u.src[0], env, packed)
return _evaluate(u.src[0], env)
if u.op is Ops.CONST:
return u.arg
if u.op is Ops.RANGE:
return env.get(u)
if (slot := _outer_slot(u, packed)) is not None:
return env.get(slot)
if u.op in (Ops.PARAM, Ops.RANGE):
return env.get(u.arg.slot if u.op is Ops.PARAM else u)
if u.op is Ops.LOAD:
table, index = _root(u.src[0].src[0]), _evaluate(u.src[0].src[1], env, packed)
table, index = _root(u.src[0].src[0]), _evaluate(u.src[0].src[1], env)
return struct.unpack_from(u.dtype.fmt, table.arg, index * u.dtype.itemsize)[0] if table.op is Ops.BINARY and index is not None else None
if u.op in GroupOp.ALU and None not in (values := [_evaluate(x, env, packed) for x in u.src]):
if u.op in GroupOp.ALU and None not in (values := [_evaluate(x, env) for x in u.src]):
return exec_alu(u.op, u.dtype, values, *([False] if "truncate_output" in inspect.signature(exec_alu).parameters else []))
return None
def _input_pokes(submit, kernargs) -> dict[int, tuple[int, int]]:
from tinygrad.uop.ops import Ops
outer, packed = dict(enumerate(submit.src[1:])), _packed_args(submit.src[0])
outer = dict(enumerate(submit.src[1:]))
kernargs_params = {i for i, a in outer.items() if a.op is Ops.GETADDR and _root(a.src[0]) is kernargs}
inputs = {i: _root(a.src[0]).arg.slot for i, a in outer.items() if a.op is Ops.GETADDR and _root(a.src[0]).op is Ops.PARAM}
pokes = {}
for call in (u for u in submit.src[0].src[0].toposort() if u.op is Ops.CALL and len(u.src) > 3):
_, address, value, *_ = call.src[1:]
used = list({slot for x in value.toposort() if (slot := _outer_slot(x, packed)) is not None} & inputs.keys())
if len(used) != 1 or not kernargs_params & {_outer_slot(x, packed) for x in address.toposort()}:
used = list({x.arg.slot for x in value.toposort() if x.op is Ops.PARAM} & inputs.keys())
if len(used) != 1 or not kernargs_params & {x.arg.slot for x in address.toposort() if x.op is Ops.PARAM}:
continue
ranges = list(dict.fromkeys(x for x in (*address.toposort(), *value.toposort()) if x.op is Ops.RANGE))
for trip in itertools.product(*[range(int(r.vmax) + 1) for r in ranges]):
env = dict.fromkeys(outer, 0) | dict(zip(ranges, trip, strict=True))
if (offset := _evaluate(address, env, packed)) % 8 == 0:
pokes[offset] = (inputs[used[0]], _evaluate(value, env, packed))
if (offset := _evaluate(address, env)) % 8 == 0:
pokes[offset] = (inputs[used[0]], _evaluate(value, env))
return pokes
def _elf(image: bytes, rodata: int, entry: int) -> bytes:
spans = [(".text", 0, rodata), (".rodata", rodata, len(image))] if entry < rodata else \
[(".data", 0, rodata), (".rodata", rodata, entry), (".text", entry, len(image))]
sections, names = [(0, 0, 0, 0, b"")], b"\0"
for name, start, end in (span for span in spans if span[1] < span[2]):
sections.append((len(names), 1, 6 if name == ".text" else 2, ALIGNMENT + start, image[start:end]))
names += name.encode() + b"\0"
sections.append((len(names), 3, 0, 0, names + b".shstrtab\0"))
offset, data, headers = 64, b"", []
for name, kind, flags, address, content in sections:
headers.append(struct.pack("<IIQQQQIIQQ", name, kind, flags, address, offset, len(content), 0, 0, 1, 0))
def _elf(image: bytes, rodata: int) -> bytes:
names, offset, data, headers = b"\0.text\0.rodata\0.shstrtab\0", 64, b"", []
for name, kind, address, content in [(0, 0, 0, b""), (1, 1, ALIGNMENT, image[:rodata]), (7, 1, ALIGNMENT + rodata, image[rodata:]), (15, 3, 0, names)]:
headers.append(struct.pack("<IIQQQQIIQQ", name, kind, 2 if kind == 1 else 0, address, offset, len(content), 0, 0, 1, 0))
data, offset = data + content, offset + len(content)
header = struct.pack("<HHIQQQIHHHHHH", 1, 0xe0, 1, 0, 0, offset, 0, 64, 0, 0, 64, len(sections), len(sections) - 1)
return b"\x7fELF\x02\x01\x01" + bytes(9) + header + data + b"".join(headers)
return b"\x7fELF\x02\x01\x01" + bytes(9) + struct.pack("<HHIQQQIHHHHHH", 1, 0xe0, 1, 0, 0, offset, 0, 64, 0, 0, 64, 4, 3) + data + b"".join(headers)
def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
@@ -184,7 +155,7 @@ def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
if image_alloc not in programs:
image = bytes(_patched(image_alloc, topo)[0])
rodata = next(r for r in range(0, len(image) - 24, 64) if r + struct.unpack_from("<q", image, r + 16)[0] == entry)
programs[image_alloc] = blob.add(_elf(image, rodata, entry))
programs[image_alloc] = blob.add(_elf(image, rodata))
start, size = _view(arguments.src[0])
args = []
for slot in range(start, start + unwrap(size), 8):
@@ -195,8 +166,7 @@ def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
base, at, nbytes = _address(argument_words[slot])
args.append({"buffer": buffers.setdefault(base, len(buffers)), "offset": at, "nbytes": nbytes or base.max_numel() * base.dtype.itemsize, "dtype": u8})
outs, ins = kernel[6]
device_target = dataclasses.asdict(dataclasses.replace(target, interface=target.interface.removeprefix("MOCK")))
calls.append({"kind": "program", "name": re.sub(r"\x1b\[[0-9;]*m", "", kernel[1]), "lib": programs[image_alloc], "target": device_target,
calls.append({"kind": "program", "name": re.sub(r"\x1b\[[0-9;]*m", "", kernel[1]), "lib": programs[image_alloc], "target": dataclasses.asdict(target),
"signature": [[f"data{i}", i, u8, [a.get("nbytes", 1)]] for i, a in enumerate(args)], "global_size": grid, "local_size": local,
"globals": list(range(len(args))), "outs": list(outs), "ins": list(ins), "args": args})
needs_init |= {args[i]["buffer"] for i in ins if "buffer" in args[i] and args[i]["buffer"] not in written}
@@ -237,16 +207,14 @@ def main():
frame, _ = capture_frame(artifact["run"], kwargs, "output_buffers")
part = convert(artifact["run"], kwargs, frame, Device[Device.DEFAULT].renderer.target)
if not os.getenv("MOCKUSB_SOCKET"):
plain = PlainJit(*part)
replay = {path[0]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 1}
replay["output_buffers"] = {path[1]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 2}
flatten(plain(**replay))
for index, (got, expected) in enumerate(zip([t.numpy() for t in replay["output_buffers"].values()], frame["outputs"], strict=True)):
assert np.array_equal(got, expected), f"output {index} of the plain artifact differs from the pickle"
plain = PlainJit(*part) # replays the frame, within tolerance
replay = {path[0]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 1}
replay["output_buffers"] = {path[1]: Tensor(array, device=device).realize() for path, array, device in frame["inputs"] if len(path) == 2}
flatten(plain(**replay))
for index, (got, expected) in enumerate(zip([t.numpy() for t in replay["output_buffers"].values()], frame["outputs"], strict=True)):
assert np.array_equal(got, expected), f"output {index} of the plain artifact differs from the pickle"
write_bundle(args.output, {"run": part}, {key: artifact[key] for key in ("metadata", "input_specs", "output_specs")})
checked = "left to mock_parity" if os.getenv("MOCKUSB_SOCKET") else "bit-exact with the pickle"
print(f"converted {len(part[0]['calls'])} kernels, {checked}: {args.output}")
print(f"converted {len(part[0]['calls'])} kernels, bit-exact with the pickle: {args.output}")
if __name__ == "__main__":
+11 -27
View File
@@ -20,11 +20,6 @@ def pack_layout(shapes: dict[str, tuple], frame_size: int) -> tuple[dict[str, in
return offsets | {"frames": offset}, offset + 2 * frame_size
def host_shapes(input_shapes: dict[str, tuple], states) -> dict[str, tuple]:
shapes = {name: shape for name, shape in input_shapes.items() if name not in states and name not in ("new_img", "light_img")}
return shapes | ({"light_tfm": (1, 3, 3)} if "light_img" in input_shapes else {})
class _Blobs: # the model's and the warp's blobs as one address space, for a part whose ranges span both files
def __init__(self, *blobs):
self.blobs, self.starts = blobs, list(itertools.accumulate([0] + [len(blob) for blob in blobs[:-1]]))
@@ -35,23 +30,18 @@ class _Blobs: # the model's and the warp's blobs as one address space, for a pa
def fused_with_warp(cam_w: int, cam_h: int, chestnut: bool): # load_model's extend: a native model runs upload, warp and model as one submit
models = f"{BASEDIR}/openpilot/sunnypilot/modeld_v2/models"
return partial(frame_part, f"{models}/{'big_' if chestnut else ''}driving_warp_{cam_w}x{cam_h}_tinygrad.pkl",
light_warp_path=f"{models}/driving_lightwarp_{cam_w}x{cam_h}_tinygrad.pkl")
return partial(frame_part, f"{BASEDIR}/openpilot/sunnypilot/modeld_v2/models/{'big_' if chestnut else ''}driving_warp_{cam_w}x{cam_h}_tinygrad.pkl")
def frame_part(warp_path: str, parts: dict, extras: dict, light_warp_path: str | None = None) -> dict:
def frame_part(warp_path: str, parts: dict, extras: dict) -> dict:
if "input_specs" not in extras: # a legacy model warps in its own parts
return {}
(model, model_blob), (warp, warp_blob) = parts["run"], read_parts(warp_path)[0]["run"]
states = sorted(name for name in extras["input_specs"] if f"next_{name}" in extras["metadata"]["output_shapes"])
host = host_shapes({name: shape for name, (shape, _, _) in extras["input_specs"].items()}, states)
light = "light_img" in extras["input_specs"]
light_warp, light_blob = read_parts(light_warp_path)[0]["run"] if light else ({"buffers": [], "calls": [], "inputs": []}, memoryview(b""))
host = {name: shape for name, (shape, _, _) in extras["input_specs"].items() if name not in states and name != "new_img"}
frame_size = next(s["numel"] for s in warp["inputs"] if s["name"] == "input_frame") // 2
offsets, packed = pack_layout(host, frame_size)
shift, light_shift = len(model_blob), len(model_blob) + len(warp_blob)
upload, u8 = len(model["buffers"]) + len(warp["buffers"]) + len(light_warp["buffers"]), repr(_dtype("dtypes.uint8"))
shift, upload, u8 = len(model_blob), len(model["buffers"]) + len(warp["buffers"]), repr(_dtype("dtypes.uint8"))
def view(name: str, nbytes: int, dtype: str) -> dict:
return {"buffer": upload, "offset": offsets[name], "nbytes": nbytes, "dtype": dtype}
@@ -59,15 +49,12 @@ def frame_part(warp_path: str, parts: dict, extras: dict, light_warp_path: str |
names = sorted(["host", *states])
slots = {s["slot"]: names.index(s["name"]) for s in model["inputs"] if s["name"] in states} | \
{slot: len(names) + i for i, slot in enumerate(outputs)}
def warped(part: dict, base: int) -> dict:
returned = part["returns"]["tensors"][0]
return {k: v for k, v in returned.items() if k != "shape"} | {"buffer": returned["buffer"] + base}
returned = warp["returns"]["tensors"][0]
model_inputs = {s["slot"]: {"input": slots[s["slot"]]} if s["slot"] in slots else
warped(warp, len(model["buffers"])) if s["name"] == "new_img" else
warped(light_warp, len(model["buffers"]) + len(warp["buffers"])) if s["name"] == "light_img" else
{k: v for k, v in returned.items() if k != "shape"} | {"buffer": returned["buffer"] + len(model["buffers"])} if s["name"] == "new_img" else
view(s["name"], s["numel"] * _dtype(s["dtype"]).itemsize, s["dtype"]) for s in model["inputs"]}
def warp_inputs(part: dict, tfm: str) -> dict:
return {s["slot"]: view("frames" if s["name"] == "input_frame" else tfm, s["numel"] * _dtype(s["dtype"]).itemsize, s["dtype"]) for s in part["inputs"]}
warp_inputs = {s["slot"]: view("frames" if s["name"] == "input_frame" else "tfm", s["numel"] * _dtype(s["dtype"]).itemsize, s["dtype"])
for s in warp["inputs"]}
def remap(calls: list, inputs: dict, buffers: int, lib: int) -> list:
return [call | ({"lib": [call["lib"][0] + lib, call["lib"][1] + lib]} if "lib" in call else {}) |
@@ -75,11 +62,8 @@ def frame_part(warp_path: str, parts: dict, extras: dict, light_warp_path: str |
header = {"input_names": names, "returns": {"kind": "NoneType", "tensors": []}, "check": None,
"inputs": [{"slot": names.index("host"), "name": "host", "dtype": u8, "numel": packed, "device": "NPY"}] +
[s | {"slot": slots[s["slot"]]} for s in model["inputs"] if s["slot"] in slots],
"buffers": model["buffers"] + [b | ({"init": [b["init"][0] + d, b["init"][1] + d]} if "init" in b else {})
for part, d in ((warp, shift), (light_warp, light_shift)) for b in part["buffers"]] +
"buffers": model["buffers"] + [b | ({"init": [b["init"][0] + shift, b["init"][1] + shift]} if "init" in b else {}) for b in warp["buffers"]] +
[{"nbytes": packed, "device": model["inputs"][0]["device"], "dtype": u8}],
"calls": [{"kind": "copy", "args": [{"buffer": upload, "offset": 0, "nbytes": packed, "dtype": u8}, {"input": names.index("host")}]}] +
remap(warp["calls"], warp_inputs(warp, "tfm"), len(model["buffers"]), shift) +
remap(light_warp["calls"], warp_inputs(light_warp, "light_tfm"), len(model["buffers"]) + len(warp["buffers"]), light_shift) +
remap(model["calls"], model_inputs, 0, 0)}
return {"run": (header, _Blobs(model_blob, warp_blob, light_blob) if light else _Blobs(model_blob, warp_blob))}
remap(warp["calls"], warp_inputs, len(model["buffers"]), shift) + remap(model["calls"], model_inputs, 0, 0)}
return {"run": (header, _Blobs(model_blob, warp_blob))} # replaces the model's run: one set of buffers, no separate warp
+2 -2
View File
@@ -264,8 +264,8 @@ def _program_uop(call: dict, lib: bytes):
parameter = UOp.param(call["globals"][compact_slot], _dtype(dtype), math.prod(shape))
image = (shape[0], shape[1]) if len(shape) == 3 and shape[2] == 4 else None
parameters.append(parameter.replace(arg=_fields(parameter.arg, name=name, image=image)))
info = _fields(ProgramInfo, global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), params=tuple(parameters),
globals=tuple(call["globals"]), outs=tuple(call["outs"]), ins=tuple(call["ins"]), target=_fields(Target, **call["target"]))
info = _fields(ProgramInfo, global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), globals=tuple(call["globals"]),
outs=tuple(call["outs"]), ins=tuple(call["ins"]), target=_fields(Target, **call["target"]))
return UOp(Ops.PROGRAM, src=(UOp(Ops.SINK, arg=KernelInfo(name=call["name"])), UOp(Ops.LINEAR, src=tuple(parameters)),
UOp(Ops.SOURCE, arg=""), UOp(Ops.BINARY, arg=lib)), arg=info)
@@ -1,22 +0,0 @@
"""
Copyright (c) 2021-, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
"""
import numpy as np
from openpilot.common.transformations.camera import get_view_frame_from_calib_frame, view_frame_from_device_frame
from openpilot.common.transformations.orientation import rot_from_euler
LIGHT_INPUT_SIZE = (512, 256)
light_intrinsics = np.array([
[1000.0, 0.0, 256.0],
[0.0, 1000.0, 200.0],
[0.0, 0.0, 1.0]])
calib_from_light = np.linalg.inv((light_intrinsics @ get_view_frame_from_calib_frame(0, 0, 0, 0))[:, :3])
def get_light_warp_matrix(device_from_calib_euler: np.ndarray, intrinsics: np.ndarray) -> np.ndarray:
camera_from_calib = intrinsics @ view_frame_from_device_frame @ rot_from_euler(device_from_calib_euler)
return (camera_from_calib @ calib_from_light).astype(np.float32)
@@ -1,153 +0,0 @@
import json
import os
import shutil
import struct
import sys
import tempfile
import time
os.environ.setdefault("DEV", "MOCKUSB+AMD")
QCOM = "QCOM" in os.environ["DEV"]
ON_MOCK = os.environ["DEV"].startswith("MOCK")
ARCH, HARDWARE = ("a630", "comma 3x/4 (a630)") if QCOM else ("gfx1200", "chestnut")
os.environ.setdefault("CHECK_OOB", "0")
os.environ.setdefault("GMMU", "0")
if sys.platform == "darwin" and os.environ["DEV"].startswith("MOCKUSB"): # the mock's VRAM is a memfd, which macOS lacks: an unlinked file is the same
from tinygrad.runtime.autogen import libc
def _memfd_create(name, flags):
fd, path = tempfile.mkstemp()
os.unlink(path)
return fd
libc.memfd_create = _memfd_create
import numpy as np
from openpilot.sunnypilot.modeld_v2.helpers import ALIGNMENT, MAGIC, PlainJit, _device, read_parts
class ParityError(RuntimeError):
pass
def _emulator():
try:
if QCOM:
from test.mockgpu.qcom import emu
else:
from test.mockgpu.amd import emu
except ImportError as e:
raise ParityError(f"tinygrad's {'qcom' if QCOM else 'amd'} mock is not importable: put tinygrad_repo on PYTHONPATH") from e
if not hasattr(emu, "hw_unverified"):
raise ParityError(f"this tinygrad's emulator has no {ARCH} rounding model, its check frames would not match a {HARDWARE}")
return emu
def unverified(header: dict, blob: memoryview) -> dict[str, int]:
emu, out = _emulator(), {}
for call in header["calls"]:
if call["kind"] != "program":
continue
if (arch := call["target"].get("arch", "")).split(",")[0] != ARCH: # IMAGE builds append ",IMAGE_PITCH_ALIGNMENT=64"
out[f"arch={arch}"] = out.get(f"arch={arch}", 0) + 1
continue
for sig, n in emu.hw_unverified(bytes(blob[slice(*call["lib"])])).items():
out[sig] = out.get(sig, 0) + n
return out
def _array(blob: memoryview, stored: dict) -> np.ndarray:
return np.frombuffer(blob[slice(*stored["data"])], dtype=np.dtype(stored["dtype"])).reshape(stored["shape"]).copy()
def replay(header: dict, blob: memoryview, shared: dict) -> list[np.ndarray]:
from tinygrad import Tensor
check = header["check"]
jit = PlainJit(header, blob, None, verify=False, shared=shared)
kwargs: dict = {}
for path, stored, device in check["inputs"]:
tensor = Tensor(_array(blob, stored), device=_device(device, None)).realize()
kwargs[path[0]] = tensor if len(path) == 1 else {**kwargs.get(path[0], {}), path[1]: tensor}
result = jit(**kwargs)
outputs = kwargs[check["output_container"]].values() if check["output_container"] else result
return [t.numpy() for t in (outputs if isinstance(outputs, (list, tuple)) or check["output_container"] else [outputs])]
def _parts_with_offsets(path: str) -> tuple[dict, dict]:
parts, _ = read_parts(path)
with open(path, "rb") as f:
head = f.read(len(MAGIC) + 8)
start = len(MAGIC) + 8 + struct.unpack_from("<Q", head, len(MAGIC))[0]
f.seek(len(MAGIC) + 8)
entries = json.loads(f.read(start - len(MAGIC) - 8))["parts"]
base = start + -start % ALIGNMENT
offsets = {tuple(e["key"]) if isinstance(e["key"], list) else e["key"]: base + e["blob"][0] for e in entries}
return parts, offsets
def _gate(parts: dict) -> None:
missing: dict[str, int] = {}
for header, blob in parts.values():
if "check" not in header:
raise ParityError("artifact has no check frame")
for sig, n in unverified(header, blob).items():
missing[sig] = missing.get(sig, 0) + n
if missing:
raise ParityError(f"kernels use instructions the emulator is not verified against a {HARDWARE} for: " +
", ".join(f"{s} ({n}x)" for s, n in sorted(missing.items(), key=lambda kv: -kv[1])))
def check(path: str) -> None:
parts, _ = read_parts(path)
if ON_MOCK:
_gate(parts)
shared: dict = {}
for key, (header, blob) in parts.items():
got = replay(header, blob, shared)
for i, (g, stored) in enumerate(zip(got, header["check"]["outputs"], strict=True)):
want = _array(blob, stored)
if g.shape != want.shape or not np.array_equal(g.view(np.uint8), want.view(np.uint8)):
n = int(np.count_nonzero(g != want)) if g.shape == want.shape else -1
where = "mock" if ON_MOCK else "device"
raise ParityError(f"part {key} output {i}: the {where}'s check frame differs from the stored one in {n} of {want.size} values")
def record(path: str, out: str) -> None:
parts, offsets = _parts_with_offsets(path)
_gate(parts)
shared, writes = {}, []
for key, (header, blob) in parts.items():
for g, stored in zip(replay(header, blob, shared), header["check"]["outputs"], strict=True):
want = np.dtype(stored["dtype"])
if tuple(g.shape) != tuple(stored["shape"]) or g.dtype != want:
raise ParityError(f"part {key}: replayed output {g.dtype}{g.shape} does not fit the stored {want}{tuple(stored['shape'])}")
writes.append((offsets[key] + stored["data"][0], np.ascontiguousarray(g).tobytes()))
shutil.copyfile(path, out)
with open(out, "r+b") as f:
for offset, data in writes:
f.seek(offset)
f.write(data)
def main(argv: list[str]) -> int:
if len(argv) < 2 or argv[0] not in ("check", "record") or (argv[0] == "record" and len(argv) != 3):
print("usage: mock_parity check ARTIFACT... | record ARTIFACT OUT", file=sys.stderr)
return 2
try:
if argv[0] == "record":
st = time.perf_counter()
record(argv[1], argv[2])
print(f"{argv[2]}: check frame recorded on the mock in {time.perf_counter() - st:.0f}s")
return 0
for path in argv[1:]:
st = time.perf_counter()
check(path)
print(f"{os.path.basename(path)}: bit-identical to its {HARDWARE} check frame on the {'mock' if ON_MOCK else 'device'} ({time.perf_counter() - st:.0f}s)")
return 0
except ParityError as e:
print(f"PARITY FAIL: {e}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
@@ -11,7 +11,7 @@ import pickle
import numpy as np
from openpilot.common.basedir import BASEDIR
from openpilot.sunnypilot.modeld_v2.frame_fusion import host_shapes, pack_layout
from openpilot.sunnypilot.modeld_v2.frame_fusion import pack_layout
from openpilot.sunnypilot.modeld_v2.helpers import load_model
from openpilot.sunnypilot.modeld_v2.compile_modeld import (POLICY_INPUTS, derive_frame_skip,
make_split_input_queues, make_supercombo_input_queues)
@@ -172,7 +172,7 @@ class NativeTinygradAdapter(BaseModelAdapter):
self.input_queues = {name: Tensor(np.zeros(shape, dtype=dtype), device=self.model_device).realize()
for name, (shape, dtype) in self.input_shapes.items() if name in self.state_pairs}
shapes = host_shapes({name: shape for name, (shape, _) in self.input_shapes.items()}, self.state_pairs)
shapes = {name: shape for name, (shape, _) in self.input_shapes.items() if name not in self.state_pairs and name != 'new_img'}
offsets, packed = pack_layout(shapes, self.warp_frame_size)
self.packed_input = np.zeros(packed, dtype=np.uint8)
self.host = Tensor(self.packed_input, device='NPY')
+2 -15
View File
@@ -44,7 +44,6 @@ from openpilot.sunnypilot.modeld_v2.blendful import Blendful
from openpilot.sunnypilot.modeld_v2.constants import ModelConstants, Plan
from openpilot.sunnypilot.modeld_v2.meta_helper import load_meta_constants
from openpilot.sunnypilot.modeld_v2.camera_offset_helper import CameraOffsetHelper
from openpilot.sunnypilot.modeld_v2.light_frame import get_light_warp_matrix
from openpilot.sunnypilot.livedelay.helpers import get_lat_delay
from openpilot.sunnypilot.modeld_v2.modeld_base import ModelStateBase
from openpilot.sunnypilot.modeld_v2.helpers import load_model
@@ -172,7 +171,7 @@ class ModelState(ModelStateBase):
np.where(inputs[desire_key] - self.prev_desire > .99, inputs[desire_key], 0)
self.prev_desire[:] = inputs[desire_key]
for key in ('traffic_convention', 'lateral_control_params', 'action_t', 'speed_limit'):
for key in ('traffic_convention', 'lateral_control_params', 'action_t'):
if key in self.numpy_inputs and key in inputs:
self.numpy_inputs[key][:] = inputs[key]
@@ -180,8 +179,6 @@ class ModelState(ModelStateBase):
for i, key in enumerate(self._vision_input_names):
if key in transforms:
self.numpy_inputs['tfm'][i] = transforms[key].reshape(3, 3)
if 'light_tfm' in self.numpy_inputs and 'light' in transforms:
self.numpy_inputs['light_tfm'][0] = transforms['light'].reshape(3, 3)
else:
self.numpy_inputs['tfm'][:, :] = transforms[self._road_key].reshape(3, 3)
self.numpy_inputs['big_tfm'][:, :] = transforms[self._wide_key].reshape(3, 3)
@@ -341,7 +338,7 @@ def main(demo=False):
pub_socks = ["modelV2", "drivingModelData", "cameraOdometry", "modelDataV2SP"] + (["chestnutGpuState"] if CHESTNUT else [])
pm = PubMaster(pub_socks)
sm = SubMaster(["deviceState", "carState", "narrowRoadCameraState", "extrinsicsCalibration", "driverMonitoringState",
"carControl", "controlsState", "lateralDelay", "longitudinalPlanSP"])
"carControl", "controlsState", "lateralDelay"])
publish_state = PublishState()
chestnut_state = ChestnutGpuState(pm, model.chestnut) if CHESTNUT else None
@@ -354,7 +351,6 @@ def main(demo=False):
model_transform_main = np.zeros((3, 3), dtype=np.float32)
model_transform_extra = np.zeros((3, 3), dtype=np.float32)
model_transform_light = np.zeros((3, 3), dtype=np.float32)
live_calib_seen = False
buf_main, buf_extra = None, None
meta_main = FrameMeta()
@@ -373,9 +369,6 @@ def main(demo=False):
prev_action = log.ModelDataV2.Action()
DH = DesireHelper()
desire_frames = [q.shape[0] for m in (model, small_model) if m is not None for name, q in m.adapter.input_queues.items() if 'desire' in name]
if desire_frames:
DH.turn_repulse = min(desire_frames) - 2
meta_constants = load_meta_constants()
RELC = RoadEdgeLaneChangeController()
blendful = Blendful()
@@ -432,7 +425,6 @@ def main(demo=False):
model_transform_main = get_warp_matrix(device_from_calib_euler, main_intrinsics, False).astype(np.float32)
model_transform_extra = get_warp_matrix(device_from_calib_euler, dc.wide_road.intrinsics, True).astype(np.float32)
model_transform_main, model_transform_extra = camera_offset_helper.update(model_transform_main, model_transform_extra, sm, main_wide_camera)
model_transform_light = get_light_warp_matrix(device_from_calib_euler, main_intrinsics)
live_calib_seen = True
traffic_convention = np.zeros(2)
@@ -454,7 +446,6 @@ def main(demo=False):
bufs = {name: buf_extra if 'big' in name else buf_main for name in model.vision_input_names}
transforms = {name: model_transform_extra if 'big' in name else model_transform_main for name in model.vision_input_names}
transforms['light'] = model_transform_light
frame_delay = DT_MDL # compensate for time passed since the frame was captured: current_time - timestamp_eof is 50ms on average
action_delay = DT_MDL / 2 # middle of the interval between model output (current state) and next frame (expected state)
@@ -472,10 +463,6 @@ def main(demo=False):
if 'action_t' in model.numpy_inputs:
inputs['action_t'] = np.array([lat_action_t, long_action_t], dtype=np.float32)
if 'speed_limit' in model.numpy_inputs:
resolver = sm['longitudinalPlanSP'].speedLimit.resolver
inputs['speed_limit'] = np.array([resolver.speedLimitFinal if resolver.speedLimitValid else 0.0], dtype=np.float32)
mt1 = time.perf_counter()
try:
send_chestnut = (chestnut_state is not None and
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c08cffa27671d6773e316fdf4dcab80ffce2c4cc6fa28775615bc6726a63d7ba
size 3473263
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:6cddff2e3320057735fd4812d1ef9cc7d6d9fe405e3b6e7c4df85ee654a0e886
size 7699823
@@ -80,8 +80,7 @@ def serialize_jit(jit, kernel_linear, check: dict | None = None) -> tuple[dict,
if ast.op is Ops.PROGRAM:
elf, info = ast.to_elf(), ast.arg
global_size, local_size = info.launch_dims({})
device_target = dataclasses.asdict(dataclasses.replace(elf.target, interface=elf.target.interface.removeprefix("MOCK")))
calls.append({"kind": "program", "name": elf.name, "lib": blob.add(bytes(elf.lib)), "target": device_target,
calls.append({"kind": "program", "name": elf.name, "lib": blob.add(bytes(elf.lib)), "target": dataclasses.asdict(elf.target),
"signature": kernel_signature(ast, elf),
"global_size": [int(x) for x in global_size], "local_size": [int(x) for x in local_size],
"globals": list(info.globals), "outs": list(info.outs), "ins": list(info.ins), "args": arguments})
@@ -6,7 +6,6 @@ See the LICENSE.md file in the root directory for more details.
"""
import os
import re
import struct
import sys
import tempfile
import unittest
@@ -26,8 +25,8 @@ if sys.platform == "darwin" and os.environ.get("DEV", "").startswith("MOCKUSB"):
from openpilot.common.test import OpenpilotTestCase
from openpilot.sunnypilot.modeld_v2 import convert_upstream_pkl
from openpilot.sunnypilot.modeld_v2.convert_upstream_pkl import _elf, convert
from openpilot.sunnypilot.modeld_v2.helpers import ALIGNMENT, PlainJit, load_model
from openpilot.sunnypilot.modeld_v2.convert_upstream_pkl import convert
from openpilot.sunnypilot.modeld_v2.helpers import PlainJit, load_model
from openpilot.sunnypilot.modeld_v2.plain_artifact_compiler import capture_frame
IMG, DESIRE, STATE, OUT = (6, 8, 8), (8,), (4, 16), (1, 32)
@@ -120,18 +119,3 @@ class TestConvertUpstreamPkl(OpenpilotTestCase):
bundle = load_model(out, cache=False, verify=True)
assert bundle["metadata"] == {"name": "tiny"} and bundle["input_specs"] == self.input_specs and bundle["output_specs"] == self.output_specs
self.assert_matches_eager(bundle["run"], 6)
class TestKernelElf(OpenpilotTestCase):
def test_sections_follow_the_code(self):
from tinygrad.runtime.support.elf import elf_loader
code = bytes(range(1, 129))
for rodata, entry in ((len(code), 0), (0, 256)):
image = bytearray(code + bytes(64) if entry < rodata else bytes(256) + code)
struct.pack_into("<q", image, rodata + 16, entry - rodata)
loaded, sections, _ = elf_loader(_elf(bytes(image), rodata, entry))
headers = {section.name: section.header for section in sections}
assert bytes(loaded[ALIGNMENT:]) == bytes(image), "the uploaded program differs from the pickle's"
assert headers[".rodata"].sh_addr == ALIGNMENT + rodata, "the loader reads the kernel descriptor at .rodata"
text = headers[".text"]
assert text.sh_addr <= ALIGNMENT + entry < text.sh_addr + text.sh_size, "the amd mock sizes the code from the entry to the end of .text"
@@ -17,4 +17,4 @@ class TestRegistryModels(unittest.TestCase):
self.skipTest("MODEL_BASE_NAME env var not set, skipping integration test.")
bundle = load_model(os.path.join(os.environ.get("MODEL_DIR", "/tmp/models"), base_name), {"QCOM": "CPU", "AMD": "CPU"}, verify=False, cache=False)
jits = [value for value in bundle.values() if isinstance(value, PlainJit)]
assert "metadata" in bundle and jits and all(len(jit.captured._linear.src) > 0 for jit in jits)
assert "metadata" in bundle and jits and all(len(jit.captured.linear.src) > 0 for jit in jits)
Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.5 KiB

@@ -103,7 +103,9 @@ def setup_interfaces(CI: CarInterfaceBase, params: Params | None = None) -> None
def initialize_params(params) -> list[dict[str, Any]]:
keys: list = []
keys: list = [
"SmoothStopSP",
]
# hyundai
keys.extend([
@@ -126,6 +128,7 @@ def initialize_params(params) -> list[dict[str, Any]]:
keys.extend([
"ToyotaEnforceStockLongitudinal",
"ToyotaStopAndGoHack",
"ToyotaEnhancedBsm",
"ToyotaTSS2Smooth",
])
@@ -12,11 +12,13 @@ from openpilot.cereal import log, custom
from opendbc.car import structs
from openpilot.common.params import Params
from openpilot.common.swaglog import cloudlog
from openpilot.selfdrive.controls.lib.longcontrol import LongControl
from openpilot.sunnypilot import PARAMS_UPDATE_PERIOD
from openpilot.sunnypilot.livedelay.helpers import get_lat_delay
from openpilot.sunnypilot.modeld_v2.modeld_base import ModelStateBase
from openpilot.sunnypilot.selfdrive.controls.lib.blinker_pause_lateral import BlinkerPauseLateral
from openpilot.sunnypilot.selfdrive.controls.lib.latcontrol_torque_v0 import LatControlTorque as LatControlTorqueV0
from openpilot.sunnypilot.selfdrive.controls.lib.longcontrol_sp import LongControlSP
class ControlsExt(ModelStateBase):
@@ -34,6 +36,11 @@ class ControlsExt(ModelStateBase):
self.sm_services_ext = ['radarState', 'selfdriveStateSP']
self.pm_services_ext = ['carControlSP']
def initialize_longitudinal_control(self, lo_c: LongControl) -> LongControl:
if self.CP_SP.smoothStopDecelRate > 0.0:
return LongControlSP(self.CP, self.CP_SP)
return lo_c
def initialize_lateral_control(self, lac, CI, dt):
enforce_torque_control = self.params.get_bool("EnforceTorqueControl")
torque_versions = self.params.get("TorqueControlTune")
@@ -0,0 +1,57 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
"""
import numpy as np
from openpilot.common.realtime import DT_CTRL
from openpilot.selfdrive.controls.lib.longcontrol import LongControl, LongCtrlState
TAPER_JERK = 0.25
TAPER_MAX_DECEL = 1.0
STALL_TIME = 1.0
MAX_JERK = 2.0
class LongControlSP(LongControl):
def __init__(self, CP, CP_SP):
super().__init__(CP, CP_SP)
self.was_tapering = False
self.stopping_time = 0.0
def update(self, active, CS, a_target, should_stop, accel_limits):
prev_output_accel = self.last_output_accel
smooth_stop = self.CP_SP.smoothStopDecelRate > 0.0
tapering = smooth_stop and not should_stop and -TAPER_MAX_DECEL < a_target < 0.0
if tapering:
accel_limits = (max(accel_limits[0], -float(np.sqrt(2.0 * TAPER_JERK * max(CS.vEgo, 0.0)))), accel_limits[1])
output_accel = super().update(active, CS, a_target, should_stop, accel_limits)
if smooth_stop:
if self.long_control_state == LongCtrlState.stopping:
output_accel = self.stopping_accel(prev_output_accel, CS, accel_limits)
elif self.long_control_state == LongCtrlState.pid and (tapering or self.was_tapering):
max_change = MAX_JERK * DT_CTRL
output_accel = float(np.clip(output_accel, prev_output_accel - max_change, prev_output_accel + max_change))
self.last_output_accel = output_accel
if self.long_control_state != LongCtrlState.stopping:
self.stopping_time = 0.0
self.was_tapering = tapering
return float(output_accel)
def stopping_accel(self, accel, CS, accel_limits):
self.stopping_time += DT_CTRL
if CS.standstill or self.stopping_time > STALL_TIME:
if accel > self.CP.stopAccel:
accel = min(accel, 0.0) - self.CP_SP.smoothStopDecelRate * DT_CTRL
else:
target = -float(np.sqrt(2.0 * TAPER_JERK * max(CS.vEgo, 0.0)))
max_change = MAX_JERK * DT_CTRL
accel = min(accel, 0.0)
accel += float(np.clip(target - accel, -max_change, max_change))
return float(np.clip(accel, *accel_limits))
@@ -0,0 +1,165 @@
"""
Copyright (c) 2021-, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
"""
from types import SimpleNamespace
import pytest
from opendbc.car.structs import car
from openpilot.common.realtime import DT_CTRL
from openpilot.cereal import custom
from openpilot.selfdrive.controls.lib.longcontrol import LongCtrlState
from openpilot.sunnypilot.selfdrive.controls.lib.longcontrol_sp import LongControlSP, MAX_JERK, STALL_TIME, TAPER_JERK, TAPER_MAX_DECEL
SMOOTH_STOP_DECEL_RATE = 0.3
LIMITS = (-3.0, 2.0)
def make_control(stop_accel=-2.0, rate=SMOOTH_STOP_DECEL_RATE):
CP = car.CarParams.new_message()
CP.stopAccel = stop_accel
CP.longitudinalTuning.kiBP = [0.0]
CP.longitudinalTuning.kiV = [0.0]
CP_SP = custom.CarParamsSP.new_message()
CP_SP.smoothStopDecelRate = rate
return LongControlSP(CP, CP_SP)
def make_cs(v_ego, a_ego=0.0, standstill=False, cruise_standstill=False):
return SimpleNamespace(vEgo=v_ego, aEgo=a_ego, brakePressed=False, standstill=standstill,
cruiseState=SimpleNamespace(standstill=cruise_standstill))
def start_stopping(loc, a_target=-0.5):
for _ in range(100):
loc.update(True, make_cs(0.5, a_target), a_target, False, LIMITS)
return loc.update(True, make_cs(0.3, a_target), a_target, True, LIMITS)
def test_decel_ramps_at_stopping_rate():
loc = make_control()
out = start_stopping(loc)
assert loc.long_control_state == LongCtrlState.stopping
for _ in range(100):
prev, out = out, loc.update(True, make_cs(0.0, standstill=True), -0.5, True, LIMITS)
assert out == pytest.approx(prev - SMOOTH_STOP_DECEL_RATE * DT_CTRL)
def test_no_step_at_handover():
loc = make_control()
for _ in range(100):
out = loc.update(True, make_cs(0.5, -0.5), -0.5, False, LIMITS)
assert abs(loc.update(True, make_cs(0.3, -0.5), -0.5, True, LIMITS) - out) <= MAX_JERK * DT_CTRL + 1e-6
def test_decel_only_grows_after_standstill():
loc = make_control()
start_stopping(loc)
outs = [loc.update(True, make_cs(0.0, standstill=True), -0.5, True, LIMITS) for _ in range(300)]
assert all(b <= a for a, b in zip(outs, outs[1:], strict=False))
def test_stops_at_stop_accel():
loc = make_control(-1.5)
start_stopping(loc)
for _ in range(int(10.0 / DT_CTRL)):
out = loc.update(True, make_cs(0.0, standstill=True), -0.5, True, LIMITS)
assert out == pytest.approx(-1.5, abs=0.01)
def test_positive_output_starts_from_zero():
loc = make_control()
loc.update(True, make_cs(0.5), 0.3, False, LIMITS)
assert start_stopping(loc, a_target=0.3) <= 0.0
def test_repeatable():
runs = []
for _ in range(2):
loc = make_control()
runs.append([start_stopping(loc)] + [loc.update(True, make_cs(0.1), -0.5, True, LIMITS) for _ in range(300)])
assert runs[0] == runs[1]
def test_disabled_uses_stock_stopping_behavior():
loc = make_control(rate=0.0)
start_stopping(loc)
prev = loc.last_output_accel
assert loc.update(True, make_cs(0.1), -0.5, True, LIMITS) == pytest.approx(prev - 1.0 * DT_CTRL)
@pytest.mark.parametrize("rate", [0.3, 0.8, 2.0])
def test_rate_comes_from_car_params(rate):
loc = make_control(rate=rate)
out = start_stopping(loc)
assert loc.update(True, make_cs(0.0, standstill=True), -0.5, True, LIMITS) == pytest.approx(out - rate * DT_CTRL)
def settle(loc, v, a_target, frames=600):
for _ in range(frames):
out = loc.update(True, make_cs(v, a_target), a_target, False, LIMITS)
return out
def test_gentle_braking_fades_with_speed():
loc = make_control()
outs = [settle(loc, v, -0.6) for v in (1.0, 0.4, 0.15, 0.05)]
assert outs[0] < outs[1] < outs[2] < outs[3]
assert abs(outs[3]) < 0.2
def test_decel_never_exceeds_profile_while_tapering():
loc = make_control()
for v in (0.8, 0.5, 0.2, 0.1):
assert settle(loc, v, -0.6) >= -((2.0 * TAPER_JERK * v) ** 0.5) - 1e-6
def test_hard_braking_is_not_tapered():
loc = make_control()
target = -(TAPER_MAX_DECEL + 0.5)
assert settle(loc, 0.1, target) <= target
def test_taper_off_without_smooth_stop():
loc = make_control(rate=0.0)
assert settle(loc, 0.1, -0.6) <= -0.6
def test_taper_never_steps_the_output():
loc = make_control()
prev, v = None, 1.0
for target in [-1.2] * 30 + [-0.9] * 300:
v = max(v - 0.4 * DT_CTRL, 0.1)
out = loc.update(True, make_cs(v, -0.4), target, False, LIMITS)
if prev is not None:
assert abs(out - prev) <= MAX_JERK * DT_CTRL + 1e-6
prev = out
def test_stopping_follows_taper_until_standstill():
loc = make_control()
start_stopping(loc)
for _ in range(50):
out = loc.update(True, make_cs(0.1, -0.3), -0.5, True, LIMITS)
assert out == pytest.approx(-((2.0 * TAPER_JERK * 0.1) ** 0.5), abs=0.01)
def test_stopping_ramps_only_after_standstill():
loc = make_control()
start_stopping(loc)
for _ in range(50):
out = loc.update(True, make_cs(0.1, -0.3), -0.5, True, LIMITS)
after = loc.update(True, make_cs(0.0, standstill=True), -0.5, True, LIMITS)
assert after == pytest.approx(out - SMOOTH_STOP_DECEL_RATE * DT_CTRL)
def test_creeping_car_is_eventually_braked():
loc = make_control()
start_stopping(loc)
for _ in range(int((STALL_TIME + 5.0) / DT_CTRL)):
out = loc.update(True, make_cs(0.1, -0.1), -0.5, True, LIMITS)
assert out < -0.5
+14 -3
View File
@@ -13,6 +13,7 @@ from opendbc.car.subaru.values import CAR as SUBARU_CAR, SubaruFlags
from opendbc.car.toyota.values import CAR as TOYOTA_CAR, ToyotaFlags
from opendbc.sunnypilot.car.interfaces import has_bsm
from opendbc.sunnypilot.car.tesla.values import TeslaFlagsSP
from opendbc.sunnypilot.car.toyota.values import ToyotaFlagsSP
from openpilot.common.params import Params
from openpilot.common.swaglog import cloudlog
from openpilot.common.hardware import HARDWARE
@@ -44,6 +45,7 @@ CAPABILITY_FIELDS = (
"device_type",
"subaru_has_sng",
"hyundai_alpha_long_available",
"toyota_enhanced_bsm_available",
"toyota_tss2",
"custom_model_active",
)
@@ -68,6 +70,7 @@ CAPABILITY_LABELS: dict[str, str] = {
"device_type": "Device type",
"subaru_has_sng": "Subaru Stop-and-Go available",
"hyundai_alpha_long_available": "Hyundai Alpha Longitudinal available",
"toyota_enhanced_bsm_available": "Toyota Enhanced BSM available",
"toyota_tss2": "Toyota TSS2 platform",
"custom_model_active": "Non-Default model selected",
}
@@ -89,6 +92,7 @@ CAR_CAPABILITY_FIELDS = (
"subaru_has_sng",
"hyundai_alpha_long_available",
"toyota_tss2",
"toyota_enhanced_bsm_available",
)
# Explicit defaults for non-boolean capability fields
@@ -104,7 +108,7 @@ def _bundle_field(bundle: dict | None, key: str) -> str:
return bundle.get(key, "") if isinstance(bundle, dict) else ""
def _resolve_brand_capabilities(caps: dict, bundle_platform: str, CP) -> None:
def _resolve_brand_capabilities(caps: dict, bundle_platform: str, CP, CP_SP=None) -> None:
"""Set brand-specific capabilities from bundle platform or CarParams fallback.
Bundle (manual car selection) is a pre-fingerprint approximation.
@@ -138,11 +142,18 @@ def _resolve_brand_capabilities(caps: dict, bundle_platform: str, CP) -> None:
elif brand == "toyota":
if bundle_platform:
try:
caps["toyota_tss2"] = bool(TOYOTA_CAR[bundle_platform].config.flags & ToyotaFlags.TSS2)
flags = TOYOTA_CAR[bundle_platform].config.flags
caps["toyota_tss2"] = bool(flags & ToyotaFlags.TSS2)
caps["toyota_enhanced_bsm_available"] = bool(flags & ToyotaFlags.TSS2) and not bool(flags & ToyotaFlags.SECOC)
except KeyError:
cloudlog.exception(f"capabilities: unknown toyota platform {bundle_platform!r}")
elif CP is not None:
caps["toyota_tss2"] = bool(CP.flags & ToyotaFlags.TSS2)
# same gate as _initialize_toyota: TSS2 without SecOC, or TSS-P with a smartDSU and openpilot
# longitudinal control (the bundle path can't see 0x2FF, so it stays TSS2-only)
sdsu_bsm = CP_SP is not None and bool(CP_SP.flags & ToyotaFlagsSP.SMART_DSU) and bool(CP.openpilotLongitudinalControl)
caps["toyota_enhanced_bsm_available"] = (not bool(CP.flags & ToyotaFlags.SECOC) and
(bool(CP.flags & ToyotaFlags.TSS2) or sdsu_bsm))
def generate_capabilities(params: Params | None = None) -> dict:
@@ -217,7 +228,7 @@ def generate_capabilities(params: Params | None = None) -> dict:
if CP is not None:
caps["has_bsm"] = has_bsm(CP, CP_SP)
_resolve_brand_capabilities(caps, bundle_platform, CP)
_resolve_brand_capabilities(caps, bundle_platform, CP, CP_SP)
return caps
@@ -156,7 +156,7 @@ The tables below describe the **compiled** `settings_ui.json` schema — what th
**Enablement rules**: Grayed out (disabled) when rules fail. Frontend shows a contextual badge explaining why.
**Capability fields** (referenced in rules): `has_longitudinal_control`, `has_icbm`, `icbm_available`, `torque_allowed`, `brand`, `pcm_cruise`, `alpha_long_available`, `steer_control_type`, `has_bsm`, `is_release`, `is_sp_release`, `is_development`, `tesla_has_vehicle_bus`, `has_stop_and_go`, `stock_longitudinal`
**Capability fields** (referenced in rules): `has_longitudinal_control`, `has_icbm`, `icbm_available`, `torque_allowed`, `brand`, `pcm_cruise`, `alpha_long_available`, `steer_control_type`, `has_bsm`, `is_release`, `is_sp_release`, `is_development`, `tesla_has_vehicle_bus`, `has_stop_and_go`, `stock_longitudinal`, `toyota_enhanced_bsm_available`
---
@@ -633,6 +633,23 @@
}
]
},
{
"key": "SmoothStopSP",
"widget": "toggle",
"needs_onroad_cycle": true,
"title": "Smooth Stop",
"description": "Brakes gently as sunnypilot comes to a final stop to reduce the lurch at the end. Applies to all vehicles with sunnypilot longitudinal control.",
"enablement": [
{
"type": "not_engaged"
},
{
"type": "capability",
"field": "has_longitudinal_control",
"equals": true
}
]
},
{
"key": "IntelligentCruiseButtonManagement",
"widget": "toggle",
@@ -1412,6 +1429,12 @@
}
]
},
{
"key": "TripSummary",
"widget": "toggle",
"title": "Trip Summary",
"description": "Show a summary of distance and engagement when you shift into park after a drive."
},
{
"key": "RocketFuel",
"widget": "toggle",
@@ -2361,6 +2384,28 @@
}
]
},
{
"key": "ToyotaEnhancedBsm",
"widget": "toggle",
"needs_onroad_cycle": true,
"title": "Enhanced Blind Spot Monitoring (Alpha)",
"description": "sunnypilot will poll the blind spot monitor diagnostics directly from the radar. Use this on vehicles whose blind spot monitors do not report blind spot status over the standard CAN signals. This is an alpha feature. Use at your own risk.",
"enablement": [
{
"type": "not_engaged"
},
{
"type": "capability",
"field": "toyota_enhanced_bsm_available",
"equals": true
},
{
"type": "capability",
"field": "stock_longitudinal",
"equals": false
}
]
},
{
"key": "ToyotaTSS2Smooth",
"widget": "toggle",
@@ -36,6 +36,16 @@ sections:
description: "Adds a small push on top of sunnypilot's own acceleration, stronger in Aggressive and gentler in Relaxed. Eases off in curves and when sunnypilot slows or stops. Never goes past your set speed, and lead-car following still comes first."
enablement:
- $ref: '#/macros/longitudinal'
- key: SmoothStopSP
widget: toggle
needs_onroad_cycle: true
title: Smooth Stop
description: Brakes gently as sunnypilot comes to a final stop to reduce the lurch at the end. Applies to all vehicles with sunnypilot longitudinal control.
enablement:
- type: not_engaged
- type: capability
field: has_longitudinal_control
equals: true
- key: IntelligentCruiseButtonManagement
widget: toggle
title: Intelligent Cruise Button Management (ICBM) (Alpha)
@@ -101,6 +101,21 @@ sections:
- type: param
key: ToyotaEnforceStockLongitudinal
equals: false
- key: ToyotaEnhancedBsm
widget: toggle
needs_onroad_cycle: true
title: Enhanced Blind Spot Monitoring (Alpha)
description: sunnypilot will poll the blind spot monitor diagnostics directly from the radar. Use this on vehicles whose
blind spot monitors do not report blind spot status over the standard CAN signals. This is an alpha feature. Use at
your own risk.
enablement:
- $ref: '#/macros/not_engaged'
- type: capability
field: toyota_enhanced_bsm_available
equals: true
- type: capability
field: stock_longitudinal
equals: false
- key: ToyotaTSS2Smooth
widget: toggle
needs_onroad_cycle: true
@@ -39,6 +39,10 @@ sections:
description: Show a timer on the HUD when the car is at a standstill.
visibility:
- $ref: '#/macros/hide_on_mici'
- key: TripSummary
widget: toggle
title: Trip Summary
description: Show a summary of distance and engagement when you shift into park after a drive.
- key: RocketFuel
widget: toggle
title: Real-time Acceleration Bar
@@ -70,12 +70,18 @@ class TestOpaquePerBrandFlags(OpenpilotTestCase):
def test_hyundai_alpha_long_available_field_present(self):
assert "hyundai_alpha_long_available" in CAPABILITY_FIELDS
def test_toyota_enhanced_bsm_available_field_present(self):
assert "toyota_enhanced_bsm_available" in CAPABILITY_FIELDS
def test_subaru_has_sng_default_false(self, caps):
assert caps["subaru_has_sng"] is False
def test_hyundai_alpha_long_available_default_false(self, caps):
assert caps["hyundai_alpha_long_available"] is False
def test_toyota_enhanced_bsm_available_default_false(self, caps):
assert caps["toyota_enhanced_bsm_available"] is False
def test_toyota_tss2_field_present(self):
assert "toyota_tss2" in CAPABILITY_FIELDS
@@ -207,6 +207,7 @@ class TestNotEngagedReplacement(OpenpilotTestCase):
"AlphaLongitudinalEnabled",
"ToyotaEnforceStockLongitudinal",
"ToyotaStopAndGoHack",
"ToyotaEnhancedBsm",
"ToyotaTSS2Smooth",
], names=["key"])
def test_offroad_only_replaced_with_not_engaged(self, schema, key):
@@ -286,6 +286,7 @@ class TestKnownVehicleSettings(OpenpilotTestCase):
keys = {i["key"] for i in _brand_items(schema["vehicle_settings"].get("toyota"))}
assert "ToyotaEnforceStockLongitudinal" in keys
assert "ToyotaStopAndGoHack" in keys
assert "ToyotaEnhancedBsm" in keys
assert "ToyotaTSS2Smooth" in keys
def test_tesla_has_coop_steering(self, schema):
@@ -186,9 +186,6 @@ procs += [
# locationd
NativeProcess("locationd_llk", "openpilot/sunnypilot/selfdrive/locationd", ["./locationd"], only_onroad),
# achievements: read-only stats and unlocks, saved on the device
PythonProcess("achievementsd", "openpilot.sunnypilot.achievements.achievementsd", only_onroad),
]
if os.path.exists("../../sunnypilot/sunnylink/uploader.py"):
+12 -27
View File
@@ -13,41 +13,26 @@ from pathlib import Path
from datetime import datetime, UTC
_VERSION_REGEX = re.compile(r'^(?:\d+[a-zA-Z]+|\d+[vVbB]\d+|[vVbB]\d+|\d{4})$')
_DECIMAL_VERSION_REGEX = re.compile(r'^([vVbB]?)(\d+(?:\.\d+)+)$')
def _version(word: str) -> str | None:
match = _DECIMAL_VERSION_REGEX.match(word.strip('()[]{}.,;:'))
if match:
return match.group(1).upper() + match.group(2)
stripped = re.sub(r'[^a-zA-Z0-9]', '', word)
return stripped.upper() if _VERSION_REGEX.match(stripped) else None
def _truncate(short_name: str) -> str:
return short_name[:8].rstrip('.')
def create_short_name(full_name: str) -> str:
# Remove parentheses and extract alphanumeric words, keeping decimals for versioning
# Remove parentheses and extract alphanumeric words
clean_name = re.sub(r'\([^)]*\)', '', full_name)
words = [re.sub(r'[^a-zA-Z0-9.]', '', word) for word in clean_name.split()]
words = [word for word in words if word]
words = [re.sub(r'[^a-zA-Z0-9]', '', word) for word in clean_name.split() if re.sub(r'[^a-zA-Z0-9]', '', word)]
if len(words) == 1:
return _truncate((_version(words[0]) or words[0]).upper())
return words[0][:8].upper()
# Handle special case: Name + Version (e.g., "Word A1" -> "WordA1", "Model v1.1" -> "ModelV1.1")
if len(words) == 2 and re.match(r'^[A-Za-z]?\d+$', words[1]):
return _truncate((words[0] + (_version(words[1]) or words[1])).upper())
# Handle special case: Name + Version (e.g., "Word A1" -> "WordA1")
if len(words) == 2 and re.match(r'^[A-Za-z]\d+$', words[1]):
return (words[0] + words[1])[:8].upper()
result = ""
for word in words:
# Version or number patterns
version = _version(word)
if version:
result += version
if (re.match(r'^\d+[a-zA-Z]+$', word) or
re.match(r'^\d+[vVbB]\d+$', word) or
re.match(r'^[vVbB]\d+$', word) or
re.match(r'^\d{4}$', word)):
result += word.upper()
# All uppercase abbreviations (2-3 letters)
elif re.match(r'^[A-Z]{2,3}$', word):
result += word
@@ -60,7 +45,7 @@ def create_short_name(full_name: str) -> str:
result += word
else:
result += word[0].upper()
return _truncate(result)
return result[:8]
def create_pkl_name(full_name: str) -> str: