Compare commits

..

2 Commits

Author SHA1 Message Date
Prabhaav Pillai 88a3b5e2a5 Add car control sections and update projections in settings for AA support 2026-10-27 17:10:33 -04:00
Prabhaav Pillai 8ac6b224d7 Add support for host runtime and Bluetooth integration in StarPilot 2026-10-25 17:51:38 -04:00
2529 changed files with 37277 additions and 368705 deletions
+1 -15
View File
@@ -14,8 +14,6 @@ openpilot/selfdrive/modeld/models/big_*_tinygrad.pkl -text
openpilot/starpilot/galaxy/web/vendor/vue/vue.esm-browser.js -text
*.apk -text
openpilot/starpilot/galaxy/bin/hev-socks5-tunnel_linux_arm64 -text
openpilot/starpilot/galaxy/bin/hostapd_linux_arm64 -text
openpilot/starpilot/galaxy/bin/hostapd_cli_linux_arm64 -text
openpilot/starpilot/soloist/pulseaudio/bin/pulseaudio -text
openpilot/starpilot/soloist/pulseaudio/bin/pactl -text
openpilot/starpilot/soloist/bluealsa/bin/bluealsa-aplay -text
@@ -41,6 +39,7 @@ tinygrad_model_runtime/** !filter !diff !merge -text
*.a -text
*.so -text
*.so.* -text
openpilot/cereal/messaging/bridge -text
openpilot/selfdrive/pandad/pandad -text
openpilot/tools/replay/replay -text
openpilot/system/camerad/camerad -text
@@ -49,17 +48,4 @@ openpilot/system/loggerd/encoderd -text
openpilot/system/loggerd/loggerd -text
openpilot/starpilot/galaxy/bin/frpc_linux_arm64 -text
openpilot/starpilot/maps/provider/mapd -text
openpilot/tools/arabesque_firefox/offline/*.deb -text
openpilot/tools/arabesque_firefox/offline/*.tar.xz -text
openpilot/selfdrive/test/process_replay/refs/starpilot/*.zst -text
# Published Panda object files are AArch64 ELF, not text.
openpilot/selfdrive/pandad/main.o -text
openpilot/selfdrive/pandad/panda.o -text
openpilot/selfdrive/pandad/panda_safety.o -text
openpilot/selfdrive/pandad/panda_usb.o -text
openpilot/selfdrive/pandad/pandad.o -text
openpilot/selfdrive/pandad/spi.o -text
# Compressed recorded regression fixture; retain exact binary bytes.
openpilot/starpilot/tests/testdata/ford_mache_lead_departure.json.gz -text
@@ -0,0 +1,68 @@
name: Android companion release
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: android-companion-release
cancel-in-progress: false
jobs:
build:
name: Test, build and sign the companion APK
runs-on: ubuntu-latest
timeout-minutes: 30
environment: android-release
defaults:
run:
working-directory: openpilot/starpilot/companion/android
steps:
- uses: actions/checkout@v7
with:
submodules: false
lfs: false
persist-credentials: false
- uses: actions/setup-java@v4
with:
distribution: temurin
java-version: '17'
- name: Install Android SDK platform and build tools
run: yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "platforms;android-37" "build-tools;37.0.0"
- name: Unit tests
run: ./gradlew testDebugUnitTest
- name: Restore signing key
env:
KEYSTORE_BASE64: ${{ secrets.COMPANION_KEYSTORE_BASE64 }}
run: |
umask 077
printf '%s' "$KEYSTORE_BASE64" | base64 -d > "$RUNNER_TEMP/companion.jks"
- name: Build and sign the APK
env:
COMPANION_KEYSTORE: ${{ runner.temp }}/companion.jks
COMPANION_KEY_PASSWORD: ${{ secrets.COMPANION_KEY_PASSWORD }}
run: ./build_apk.sh
- name: Remove signing key
if: always()
run: rm -f "$RUNNER_TEMP/companion.jks"
- name: Check the APK
working-directory: openpilot/starpilot/galaxy/bin
run: |
if unzip -l starpilot-companion.apk | grep -q 'assets/galaxy/'; then
echo 'APK still bundles Galaxy' >&2
exit 1
fi
BT="$ANDROID_HOME/build-tools/37.0.0"
"$BT/aapt2" dump badging starpilot-companion.apk | head -1
"$BT/apksigner" verify --print-certs starpilot-companion.apk | grep -E 'certificate (DN|SHA-256)'
cat starpilot-companion.apk.version
- uses: actions/upload-artifact@v4
with:
name: starpilot-companion-apk
path: |
openpilot/starpilot/galaxy/bin/starpilot-companion.apk
openpilot/starpilot/galaxy/bin/starpilot-companion.apk.version
if-no-files-found: error
retention-days: 30
+1 -6
View File
@@ -5,7 +5,7 @@ on:
branches:
- master
- Domathon
- Dom
- SecretGoodStarPilot
pull_request:
workflow_call:
inputs:
@@ -29,11 +29,6 @@ jobs:
steps:
- uses: commaai/timeout@v1
- uses: actions/checkout@v7
with:
sparse-checkout: |
/docs/
/openpilot/selfdrive/assets/icons_mici/settings/comma_icon.png
sparse-checkout-cone-mode: false
# Build
- name: Build docs
-77
View File
@@ -1,77 +0,0 @@
name: FLM offroad contracts
on:
push:
branches: [Dom]
paths:
- 'openpilot/starpilot/flm/**'
- 'openpilot/starpilot/lateral/**'
- 'openpilot/starpilot/galaxy/web/js/flm*.js'
- 'openpilot/starpilot/galaxy/tests/test_flm*.mjs'
- 'openpilot/selfdrive/controls/lib/latcontrol*'
- 'openpilot/selfdrive/controls/controlsd.py'
- 'openpilot/system/manager/process_config.py'
- '.github/workflows/flm-offroad.yaml'
- 'pyproject.toml'
- 'uv.lock'
workflow_dispatch:
inputs:
source_sha:
description: Full candidate commit SHA to qualify without creating a branch
type: string
required: false
permissions:
contents: read
concurrency:
group: flm-offroad-${{ github.ref }}
cancel-in-progress: true
env:
CI: 1
PYTHONPATH: ${{ github.workspace }}
RAYLIB_BACKEND: headless
jobs:
contracts:
name: offroad backend contracts
runs-on: ubuntu-24.04
timeout-minutes: 20
steps:
- uses: actions/checkout@v7
with:
ref: ${{ inputs.source_sha || github.sha }}
- name: Verify exact source under test
env:
SOURCE_SHA: ${{ inputs.source_sha }}
run: |
git rev-parse HEAD
test -z "$SOURCE_SHA" || test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
- run: ./tools/op.sh setup
- name: Check all FLM and integrated controller source
timeout-minutes: 1
run: |
ruff check openpilot/starpilot/flm openpilot/selfdrive/controls/controlsd.py openpilot/selfdrive/controls/lib/latcontrol_pid.py openpilot/selfdrive/controls/lib/latcontrol_torque.py openpilot/system/manager/process_config.py
ty check openpilot/starpilot/flm openpilot/selfdrive/controls/controlsd.py openpilot/selfdrive/controls/lib/latcontrol_pid.py openpilot/selfdrive/controls/lib/latcontrol_torque.py openpilot/system/manager/process_config.py
- name: Build native dependencies
timeout-minutes: 10
run: scons
- name: Test offroad recommendations, storage, startup and rollback contracts
timeout-minutes: 3
run: >-
tools/op.sh test
openpilot/starpilot/flm/tests/test_offroad_backend.py
--json-output ci-results/flm-offroad.json
- name: Test completed-drive and steering-result presentation
timeout-minutes: 1
run: |
node openpilot/starpilot/galaxy/tests/test_flm_training_results.mjs
node openpilot/starpilot/galaxy/tests/test_flm_tracking_results.mjs
- name: Retain offroad contract results
if: always()
uses: actions/upload-artifact@v4
with:
name: flm-offroad-contracts
path: ci-results/flm-offroad.json
if-no-files-found: warn
+1 -1
View File
@@ -2,7 +2,7 @@ name: iOS companion
on:
push:
branches: [Dom, secretgood-ios-integration]
branches: [SecretGoodStarPilot, secretgood-ios-integration]
paths:
- 'openpilot/starpilot/companion/ios/**'
- 'openpilot/starpilot/galaxy_bluetooth/**'
+1 -1
View File
@@ -2,7 +2,7 @@ name: safety qualification
on:
push:
branches: [master, Domathon, Dom]
branches: [master, Domathon, SecretGoodStarPilot]
pull_request:
workflow_dispatch:
+11 -33
View File
@@ -5,7 +5,7 @@ on:
branches:
- master
- Domathon
- Dom
- SecretGoodStarPilot
pull_request:
workflow_dispatch:
workflow_call:
@@ -35,19 +35,17 @@ jobs:
steps:
- uses: actions/checkout@v7
- name: Validate published device build receipt
if: github.event_name == 'push' && github.ref == 'refs/heads/Dom'
if: github.event_name == 'push' && github.ref == 'refs/heads/SecretGoodStarPilot'
run: python3 tools/ci/check_prebuilt_receipt.py --revision HEAD
- name: Check shipped device runtime dependencies
run: |
python3 -m unittest tools.laptop_device_build.tests.test_runtime_dependencies -v
python3 -c 'from pathlib import Path; assert b"starting pandad (comma hardware)" in Path("openpilot/selfdrive/pandad/pandad").read_bytes(), "Shipped pandad must use the comma hardware build profile"'
run: python3 -m unittest tools.laptop_device_build.tests.test_runtime_dependencies -v
- name: Validate tracked dependencies
run: python3 tools/vendor/check.py --revision HEAD
- run: ./tools/op.sh setup
id: setup
- name: Build native dependencies for packaging tests
id: build
run: scons -j2 openpilot/common/libparams_c.so msgq_repo/msgq/_ipc.so
run: scons -j2 openpilot/common/libparams_c.so msgq_repo/msgq/ipc_pyx.so
- name: Test dependency tools
if: ${{ !cancelled() && steps.build.outcome == 'success' }}
run: python3 -m unittest discover -s tools/vendor/tests -v
@@ -198,11 +196,6 @@ jobs:
run: |
uv pip install --python .venv/bin/python --only-binary av av==19.0.1
.venv/bin/python -c "import av; assert 'libx264' in av.codecs_available"
- name: Provide pinned OpenCV DNN for vision contract tests
if: ${{ !cancelled() && steps.build.outcome == 'success' }}
run: |
uv pip install --python .venv/bin/python --no-deps --only-binary :all: opencv-python-headless==4.11.0.86
.venv/bin/python -c "import cv2; assert cv2.__version__ == '4.11.0'; assert callable(cv2.dnn.readNetFromONNX); assert callable(cv2.dnn.NMSBoxes)"
- name: Run unit tests
if: ${{ !cancelled() && steps.build.outcome == 'success' }}
timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
@@ -214,15 +207,6 @@ jobs:
if: ${{ !cancelled() && steps.build.outcome == 'success' }}
timeout-minutes: 5
run: tools/op.sh test opendbc_repo/opendbc/car/gm/tests --json-output gm-test-results.json
- name: Test shared GM discovery, routing, and manual overrides
if: ${{ !cancelled() && steps.build.outcome == 'success' }}
timeout-minutes: 2
run: >-
.venv/bin/python -m pytest -q --import-mode=importlib
opendbc_repo/opendbc/car/gm/tests/test_shared_discovery.py
opendbc_repo/opendbc/car/gm/tests/test_automatic_discovery.py
opendbc_repo/opendbc/car/gm/tests/test_bolt_vin_discovery.py
--junitxml=gm-discovery-results.xml
- name: Test native independent-axis protocol
if: ${{ !cancelled() && steps.build.outcome == 'success' }}
run: |
@@ -230,7 +214,6 @@ jobs:
openpilot/selfdrive/pandad/tests/test_pandad_canprotocol
openpilot/selfdrive/pandad/tests/test_aol_protocol
openpilot/selfdrive/pandad/tests/test_aol_wire
openpilot/selfdrive/pandad/tests/test_panda_faults
openpilot/selfdrive/pandad/tests/test_producer_timing
openpilot/selfdrive/pandad/tests/test_producer_sink
- name: Test map provider to host messaging
@@ -244,14 +227,13 @@ jobs:
STARPILOT_PYTHON: ${{ github.workspace }}/.venv/bin/python
run: go test -mod=readonly -race -run '^Test(MsgqNativeLayout|Shadow(GoPythonIPC|ExecutableHostGpsIPC))$' -count=1 -v
- name: Save complete unit test results
if: ${{ always() && (hashFiles('unit-test-results.json') != '' || hashFiles('gm-test-results.json') != '' || hashFiles('gm-discovery-results.xml') != '' || hashFiles('ci-results/host-test-diagnostics/**') != '') }}
if: ${{ always() && (hashFiles('unit-test-results.json') != '' || hashFiles('gm-test-results.json') != '' || hashFiles('ci-results/host-test-diagnostics/**') != '') }}
uses: actions/upload-artifact@v4
with:
name: unit-test-results
path: |
unit-test-results.json
gm-test-results.json
gm-discovery-results.xml
ci-results/host-test-diagnostics/
if-no-files-found: error
@@ -264,7 +246,7 @@ jobs:
strategy:
fail-fast: false
matrix:
shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
steps:
- uses: actions/checkout@v7
- name: Provide JavaScript runtime for Galaxy contract tests
@@ -297,7 +279,7 @@ jobs:
timeout-minutes: 15
env:
RAYLIB_BACKEND: headless
run: .venv/bin/python -m tools.ci.custom_pytest_shards run --index "${{ matrix.shard }}" --count 32 --output "ci-results/custom-pytest-${{ matrix.shard }}"
run: .venv/bin/python -m tools.ci.custom_pytest_shards run --index "${{ matrix.shard }}" --count 24 --output "ci-results/custom-pytest-${{ matrix.shard }}"
- name: Save collection, every outcome, JUnit and durable worker diagnostics
if: always()
uses: actions/upload-artifact@v4
@@ -320,7 +302,7 @@ jobs:
pattern: custom-pytest-*
path: ci-results/custom-pytest-shards
- name: Require every case passed or audited capability-unavailable
run: python3 -m tools.ci.custom_pytest_shards aggregate --directory ci-results/custom-pytest-shards --count 32 --output ci-results/custom-pytest-complete.json
run: python3 -m tools.ci.custom_pytest_shards aggregate --directory ci-results/custom-pytest-shards --count 24 --output ci-results/custom-pytest-complete.json
- name: Save complete custom catalog verdict
if: always()
uses: actions/upload-artifact@v4
@@ -348,8 +330,6 @@ jobs:
run: scons
- name: Gate every assigned lifecycle module serially
timeout-minutes: 10
env:
RAYLIB_BACKEND: headless
run: python3 tools/ci/run_vehicle_tests.py --suite aol-lifecycle --shard-index "${{ matrix.shard }}" --shard-count 16 --output "ci-results/aol-lifecycle-${{ matrix.shard }}"
- name: Save every lifecycle outcome and native build
if: always()
@@ -450,7 +430,7 @@ jobs:
strategy:
fail-fast: false
matrix:
shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
shard: [0, 1, 2, 3]
env:
FUZZ_SEED: '0'
steps:
@@ -460,9 +440,7 @@ jobs:
run: scons
- name: Run every assigned RELEASE target serially
timeout-minutes: 20
env:
RAYLIB_BACKEND: headless
run: python3 tools/ci/run_vehicle_tests.py --suite safety-release --shard-index "${{ matrix.shard }}" --shard-count 16 --output "ci-results/safety-release-${{ matrix.shard }}"
run: python3 tools/ci/run_vehicle_tests.py --suite safety-release --shard-index "${{ matrix.shard }}" --shard-count 4 --output "ci-results/safety-release-${{ matrix.shard }}"
- name: Save every RELEASE outcome and native build
if: always()
uses: actions/upload-artifact@v4
@@ -485,7 +463,7 @@ jobs:
pattern: safety-release-*
path: ci-results/safety-release-shards
- name: Require every RELEASE case exactly once and passed
run: python3 tools/ci/lifecycle_shards.py ci-results/safety-release-shards --suite safety-release --count 16 --output ci-results/safety-release-complete.json
run: python3 tools/ci/lifecycle_shards.py ci-results/safety-release-shards --suite safety-release --count 4 --output ci-results/safety-release-complete.json
- name: Save complete RELEASE verdict
if: always()
uses: actions/upload-artifact@v4
+1 -2
View File
@@ -49,8 +49,6 @@ bin/
!openpilot/starpilot/soloist/pulseaudio/modules/*.so
*.a
st[0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z]
!/openpilot/starpilot/system/starport/
!/openpilot/tools/starport/
*.unchunked
*.clb
*.class
@@ -74,6 +72,7 @@ openpilot/selfdrive/pandad/tests/test_aol_wire
openpilot/selfdrive/pandad/tests/test_panda_usb
openpilot/cereal/services.h
openpilot/cereal/gen
openpilot/cereal/messaging/bridge
openpilot/selfdrive/ui/translations/tmp
openpilot/selfdrive/car/tests/cars_dump
openpilot/system/camerad/camerad
+8 -18
View File
@@ -2,7 +2,7 @@
## Jetlink external inference
The comma-side Jetlink client and USB transport in `jetlink_repo` are by Zeph Leggett and the [Jetlink contributors](https://github.com/zoompilot/jetlink). They are vendored from revision `9cfde98eec1d7cc714d800504944cf1a3944513d` under the MIT license, retained in [jetlink_repo/LICENSE](jetlink_repo/LICENSE). StarPilot's adapter (`openpilot/starpilot/models/jetlink_adapter.py`) builds on Jetlink's openpilot layer and owns its param names, Always On Lateral rule, model runner reset, settings and alerts. Companion applications and inference servers remain distributed by Jetlink.
The comma-side Jetlink client and USB transport in `jetlink_repo` are by Zeph Leggett and the [Jetlink contributors](https://github.com/zoompilot/jetlink). They are vendored from revision `d24328d849b0e5154b55f149dc18c5dfe8bd79cc` under the MIT license, retained in [jetlink_repo/LICENSE](jetlink_repo/LICENSE). StarPilot's adapter (`openpilot/starpilot/models/jetlink_adapter.py`) builds on Jetlink's openpilot layer and owns its param names, Always On Lateral rule, model runner reset, settings and alerts. Companion applications and inference servers remain distributed by Jetlink.
## MRR35 Hyundai angle and radar
@@ -12,11 +12,15 @@ The copied MRR35 generator source and DBC definition are at `opendbc_repo/opendb
## Ford support adapted from BluePilot
Ford lateral control vendors [BluePilot lateral](https://github.com/BluePilotDev/bluepilot-lateral) v0.2.1 (`2b9dc6cc784faac47a1885d0bda0561f7b76f74d`), developed by Alan Polk (`alan-polk`) and the BluePilot contributors. Alan provided the [StarPilot integration in PR #269](https://github.com/StarPilotHQ/StarPilot/pull/269); his commits and authorship are retained. BluePilot maintains the library and its integration; StarPilot keeps its settings, AOL policy, packaging and integration tests separate. Complete contributor and license notices remain in [the library credits](opendbc_repo/opendbc/bluepilot_lateral/CREDITS.md) and [license](opendbc_repo/opendbc/bluepilot_lateral/LICENSE).
The Ford lateral strategies, extended CAN constructors, camera-radar filtering and native checks are adapted from StarPilot `2efe6017bfe14cee4807833e05f9a7f6ba255cdb`. Their lineage includes BluePilot's `bp-7.0` Ford work, principally by Alan Polk, with contributions from John Christman, Jacob Neulight, tonesto7, Haibin Wen and other sunnypilot and BluePilot contributors. The reference snapshot is [`e1d051d7ba270261b4455068bd68f1a58db15a4a`](https://github.com/BluePilotDev/bluepilot/commit/e1d051d7ba270261b4455068bd68f1a58db15a4a); the exact checkout used for the earlier StarPilot import was not recorded.
The Ford CAN FD camera-lead radar filtering is adapted from StarPilot `2efe6017bfe14cee4807833e05f9a7f6ba255cdb`. Its lineage includes BluePilot's `bp-7.0` Ford work, principally by Alan Polk, with contributions from John Christman, Jacob Neulight, tonesto7, Haibin Wen and other sunnypilot and BluePilot contributors. The reference snapshot is [`e1d051d7ba270261b4455068bd68f1a58db15a4a`](https://github.com/BluePilotDev/bluepilot/commit/e1d051d7ba270261b4455068bd68f1a58db15a4a); the exact checkout used for the earlier StarPilot import was not recorded.
Relevant lineage includes the controller and integration work in [`db2bdff05`](https://github.com/BluePilotDev/bluepilot/commit/db2bdff05df103d71df62f45c2a3cb5211aba6e6) and [`d0aac605f`](https://github.com/BluePilotDev/bluepilot/commit/d0aac605f99d37e9da205e419f7989c1e9eaa386), extended CAN and safety work in [`8f8d6d15f`](https://github.com/BluePilotDev/bluepilot/commit/8f8d6d15f0a590f42b78de964ffb0d0af7f5d63d), and manual-turn detection in [`97867c1eb`](https://github.com/BluePilotDev/bluepilot/commit/97867c1eb57b7472f6fc3de62f0fef576e5a5497).
The radar interface preserves that attribution while adapting to the current interfaces. Upstream contributors do not maintain or endorse this adaptation. Applicable notices are retained in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md).
The vehicle strategy and CAN/native owners preserve that attribution while adapting to the current controller interfaces and shared safety limits. Upstream contributors do not maintain or endorse this adaptation. Applicable notices are retained in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md).
## Optional Ford angle controller
The vendored `opendbc_repo/opendbc/bluepilot_lateral` angle core, codecs and Ford-only safety hooks are adapted directly from [BluePilot bp-7.0 e1d051d7ba270261b4455068bd68f1a58db15a4a](https://github.com/BluePilotDev/bluepilot/commit/e1d051d7ba270261b4455068bd68f1a58db15a4a), principally by Alan Polk, with John Christman, Jacob Neulight, Nathan Ingraham, Praeuner and other BluePilot, sunnypilot and comma contributors. Exact source paths and intentionally stricter safety checks are recorded in that folder. Both upstream `LICENSE` and `LICENSE.md` notices are retained. Its layout follows Alan Polk's September 2026 library proposal; this is a StarPilot adapter, not a published BluePilot library release.
## RAV4 Continental ARS510 receiver
@@ -29,17 +33,3 @@ Spotify's Bluetooth output binds the comma's BlueALSA A2DP source to the selecte
## Tesla Pre-AP pedal control
The Pre-AP configuration, pedal codec/feedback, VirtualDAS feedforward table, grade compensation, torque-zero estimator and vehicle-specific gains preserve firestar5683's intentional Dom work, including `7837a3c1aab`, `aed0b7fdb9e9` and pinned `b220a37f4bd119e402225a7aa16a7cd84878d0af`. Original author attribution: firestar5683 <168790843+firestar5683@users.noreply.github.com>. Startup/calibration ownership, current upstream button/PID interfaces and native command bounds are adapted to the current StarPilot architecture; the old permissive native gateway is not transplanted. Existing source/license notices remain intact.
## Subaru angle steering from Jacob Waller's fork
Subaru angle support includes implementation maintained and integrated by **Jacob Waller (jacobw)** in [jacobwaller/openpilot, jul-angle-based](https://github.com/jacobwaller/openpilot/tree/542816196a18dba5f6f0af093c1ece4f415b5a22), with its [opendbc implementation at 8acbeccdab45a47aa7af170d2d17b61f0eb5e4f0](https://github.com/jacobwaller/opendbc/tree/8acbeccdab45a47aa7af170d2d17b61f0eb5e4f0). Jacob authored the VehicleModel angle-limit type correction and integrated the Crosstrek support. The branch also retains the original angle implementation and follow-up work by **lukasloetkolben and elkoled**, alongside comma contributors; their authorship remains theirs.
The inherited source covers Subaru angle platform/firmware definitions, `ES_Status` cruise state, angle controller/CAN integration, vehicle-model limits and native angle enforcement. StarPilot has additional platform-specific limits, driver handoff, discovery and AOL safety guards; this attribution does not claim identical tuning or complete parity with the reference fork. Original source commits, author names, emails and titles are recorded in [the Subaru origin history](opendbc_repo/opendbc/car/subaru/ORIGIN_HISTORY.tsv).
The explicit StarPilot [jacobw Subaru adapter](opendbc_repo/opendbc/car/subaru/JACOBW.md) additionally integrates that branch's low-speed filter, target-first limiting and Crosstrek EPS envelope. Its preserved stricter guards and platform scope are documented alongside the code.
## StarPilot Ford demand tuning
The previous optional StarPilot Controller, formerly in `openpilot/starpilot/car/ford/starpilot_tuning.py`, restored demand tuning from StarPilot `8da0c9d916^`, descended from BluePilot bp-7.0 `e1d051d7ba270261b4455068bd68f1a58db15a4a` through StarPilot `2efe6017bfe14cee4807833e05f9a7f6ba255cdb`. Contributors to that lineage include Alan Polk, John Christman, Jacob Neulight, tonesto7 and Haibin Wen. The current protocol adapter is maintained by StarPilot separately from the pinned BluePilot library; this attribution does not imply BluePilot maintains or endorses the adapted tune.
The current StarPilot Controller replaces that historical curvature-primary tune with a separately maintained adaptation of BluePilot v0.2.1's angle strategy (`2b9dc6cc784faac47a1885d0bda0561f7b76f74d`, `core/lateral_angle_ext.py`). Alan Polk is the principal controller author; John Christman contributed angle-mode anti-stall work, Jacob Neulight the pinion/shadow measurement, Nathan Ingraham and Praeuner high-speed damping, and other contributors are retained in the pinned library's `CREDITS.md`. StarPilot adds bounded small-signal conditioning ahead of the unchanged angle limit chain. This adaptation is not maintained or endorsed by BluePilot; the pinned library and its imported hooks are unchanged.
-202
View File
@@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+11 -87
View File
@@ -1,90 +1,16 @@
# StarPilot
StarPilot is an independent fork of [openpilot](https://github.com/commaai/openpilot), with custom driving controls, vehicle integrations, raylib interfaces for comma 3 and comma four, and the Galaxy companion interface.
[![Ask DeepWiki](https://img.shields.io/badge/Ask-DeepWiki-blue?logo=readthedocs&logoColor=white)](https://deepwiki.com/StarPilotHQ/StarPilot)
[![Discord](https://img.shields.io/discord/1387432184121393333?label=Discord)](https://firestar.link/discord)
[![Last Updated](https://img.shields.io/github/last-commit/StarPilotHQ/StarPilot/StarPilot)](https://github.com/StarPilotHQ/StarPilot)
[![Wiki](https://img.shields.io/badge/Wiki-StarPilot-blue?logo=wiki)](https://wiki.firestar.link)
[![iOS App Store](https://img.shields.io/badge/iOS-App_Store-black?logo=apple&logoColor=white)](https://apps.apple.com/ca/app/starpilot-bt/id6821039119)
[![Android Beta](https://img.shields.io/badge/Android-Join_Beta-green?logo=android&logoColor=white)](https://groups.google.com/g/galaxy-starpilot-android-beta)
**Domathon is the development branch for StarPilot 7.0.** It rebuilds StarPilot on a current openpilot foundation. Feature migration and vehicle qualification are still in progress; this branch is not a completed replacement for the existing Dom release.
**StarPilot** is a custom fork of [comma.ai's openpilot](https://comma.ai/openpilot),
an open source driver assistance system.
## Explore the project
**This is the StarPilot 7.0 development branch (`SecretGoodStarPilot`). Feature migration,
vehicle qualification, and Auto Learn validation are still in progress.**
Openpilot provides
* Automated Lane Centering
* Adaptive Cruise Control
* Lane Change Assist
* Driver Monitoring *without wheel nags*
Ford-specific lateral-control and vehicle-support work includes substantial adaptations from
[BluePilot](https://github.com/BluePilotDev/bluepilot/tree/bp-7.0), principally developed by
[Alan Polk](https://github.com/alan-polk) with additional BluePilot contributors. StarPilot's
implementation has since diverged, but that does not erase its lineage. See [CREDITS.md](CREDITS.md)
for the code-level provenance and [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for applicable
upstream notices and terms. BluePilot and its contributors do not maintain or endorse StarPilot;
please direct support requests for this adaptation to the StarPilot project.
Hyundai, Kia, and Genesis angle steering and related vehicle support include substantial adaptations
from [sunnypilot](https://github.com/sunnypilot/sunnypilot/tree/hkg-angle-steering-2025) and its
[opendbc angle-steering branch](https://github.com/sunnypilot/opendbc/tree/hkg-angle-steering-2025).
StarPilot's implementation has diverged significantly; the upstream contributors do not maintain
this adaptation. Detailed code lineage is recorded in [CREDITS.md](CREDITS.md).
StarPilot has a vibrant, welcoming community [discord](https://firestar.link/discord).
Stop by to chat or ask questions!
If your car isn't driving as well as you'd like, [open a ticket so we can help](https://firestar.link/logs)!
## Documentation
Please see [https://wiki.firestar.link](https://wiki.firestar.link) for hardware lists,
installation guides, and software configuration.
## Features
* **Full support for Comma C3, C3X, and C4**
* Full support for chestnut/eGPU on C3, C3X, and C4. C3/C3X may require special gps handling.
* Model switcher with all of comma's tinygrad driving models
* StarPilot custom torque controller
* StarPilot custom longitudinal planner for both radar and VoACC (visual only, radar-less) vehicles
* **Custom tuning for an expanding list of vehicles**
* Firestar Lateral Method -
Auto tuning system to adapt openpilot to *your* car's unique lateral characteristics.
* Galaxy: StarPilot's portal to configure your comma device using your phone from anywhere.
Download models, change settings, update software, diagnostics, manage dashcam video, and more.
* Navigation - Display routes and give the driving model hints on taking exits and turns.
* Always On Lateral (full time steering assist)
* Lane Centering - Enable to center between detected lane lines,
with graceful fallback to model E2E lateral outputs and safety breakout
* Force Stop 2.0 - Better stops at stop signs and traffic lights.
* Speed Limit Controller - Use dashboard, map data, and/or vision detection for speed limit signs
* Curve Speed Controller - Learns from how you take curves
* Conditional Experimental Mode (CEM)
* Conditional Chill Mode (CCM)
* Driving Profiles and Traffic Mode
* Custom themes
* Alert volume customization
* Comma Pedal Interceptor support
* Bluetooth support - Connect keyboards, controllers, steering wheel remotes, and more.
* Sentry mode - Record when motion is detected while parked.
* V-ASM - Detect vehicles in the adjacent/blind spots using the driver-facing camera.
* Additional car support - Toyota SDSU, ZSS, CAN filter; pre-AP tesla; ford; Polestar 2; and more.
* And more!
## GM-only Features
* Increased LKAS fault resiliency
* ASCM_INT and SASCM support
* Custom lateral torque controller, with special tuning for Bolts
* 50% extra torque on 2017 Chevy Bolt
* Improved lateral and longitudinal tuning
* Dashboard cruise control display speed spoofing for vehicles with pedal interceptor
* Extra steering wheel button functionality for vehicles with pedal interceptor
* Optional toggle to boot comma when remote starting your vehicle
- [Developer commands](tools/STARPILOT_DEVELOPMENT.md): host tools, replay, and device builds.
- [Driving interfaces](openpilot/starpilot/ui/README.md) and [Galaxy](openpilot/starpilot/galaxy/README.md): native and companion controls.
- [Driving models](openpilot/starpilot/models/README.md): model management, artifact compatibility, and Model Laboratory status.
- [Builds and developer tools](docs/how-to/laptop-device-build.md): local development and device builds.
- [Dependency maintenance](tools/vendor/README.md): tracked source folders and upstream sync records.
## Development
@@ -110,10 +36,8 @@ StarPilot retains upstream vehicle and Panda safety interfaces and adds regressi
Support depends on the vehicle configuration and enabled feature. An existing vehicle identifier or settings page does not by itself establish driving support. Model Laboratory currently exposes management controls, but paired driving inference remains unavailable pending integration and Chestnut validation.
## Third-Party Notices
## Upstream and attribution
* Portions of this software include modified versions of the Material Design Icons provided by Google under the Apache License 2.0. A copy of the license is included in the `LICENSE-MDI` file.
* Ford support includes software adapted from BluePilot's `bp-7.0` branch. The source repository contains both a standard MIT notice and a separate custom SUNNYPILOT LLC notice; StarPilot preserves both in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md).
* Hyundai, Kia, and Genesis support includes software adapted directly from sunnypilot's HKG angle-steering branch and sunnypilot/opendbc. StarPilot preserves the applicable notices in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md).
The current openpilot baseline is [`78dccf0b482ea1298fd1f6fa59b0554187b02ce6`](https://github.com/commaai/openpilot/commit/78dccf0b482ea1298fd1f6fa59b0554187b02ce6). Its history remains intact, followed by the StarPilot foundation and feature commits. Upstream development continues independently; the baseline changes only after an update has been integrated and checked.
> This project uses software from Haibin Wen and SUNNYPILOT LLC and is licensed under a custom license requiring permission for use.
See [LICENSE](LICENSE), [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md), and [CREDITS.md](CREDITS.md) for licensing and source attribution. Component licenses remain with their source folders. StarPilot is maintained independently of comma.
+16 -17
View File
@@ -57,7 +57,7 @@ assert arch in [
"Darwin", # macOS arm64 (x86 not supported)
]
pkg_names = ['acados', 'capnproto', 'ffmpeg', 'json11', 'ncurses', 'zstd']
pkg_names = ['acados', 'capnproto', 'ffmpeg', 'json11', 'ncurses', 'zeromq', 'zstd']
if arch != 'Darwin':
pkg_names.append('libusb')
pkgs = [importlib.import_module(name) for name in pkg_names]
@@ -172,7 +172,7 @@ env = Environment(
COMPILATIONDB_USE_ABSPATH=True,
REDNOSE_ROOT="#rednose_repo",
tools=["default", "cython", "compilation_db", "rednose_filter"],
toolpath=["#rednose_repo/site_scons/site_tools"],
toolpath=["#msgq_repo/site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
)
# SCons' Darwin linker tool doesn't define the variables used to expand RPATH.
if arch == "Darwin":
@@ -292,7 +292,7 @@ SConscript(['opendbc_repo/opendbc/can/SConscript'])
# Build common module
SConscript(['openpilot/common/SConscript'])
Import('_common')
common = [_common, 'json11']
common = [_common, 'json11', 'zmq']
Export('common')
# Build messaging (cereal + msgq + socketmaster + their dependencies)
@@ -318,7 +318,7 @@ SConscript([
if arch == "comma_arm64":
SConscript(['openpilot/system/camerad/SConscript'])
SConscript(['openpilot/starpilot/system/starport/SConscript'])
SConscript(['openpilot/starpilot/system/android_auto/SConscript'])
# Build selfdrive
SConscript([
@@ -329,7 +329,6 @@ SConscript([
'openpilot/selfdrive/ui/SConscript',
'openpilot/starpilot/audio/native/SConscript',
'openpilot/starpilot/models/runtime_bridge/SConscript',
'openpilot/starpilot/ui/native/SConscript',
'openpilot/starpilot/flm/SConscript',
])
@@ -348,7 +347,7 @@ if GetOption('extras') and arch != "comma_arm64":
env.CompilationDatabase('compile_commands.json')
# progress output
def get_scons_nodes(nodes):
def count_scons_nodes(nodes):
seen = set()
stack = list(nodes)
@@ -357,28 +356,28 @@ def get_scons_nodes(nodes):
if node in seen:
continue
seen.add(node)
# Only scan directories; generated sources are not ready yet.
stack += (node.prerequisites or []) + node.children(scan=isinstance(node, SCons.Node.FS.Dir))
executor = node.get_executor()
if executor is not None:
stack += executor.get_all_prerequisites() + executor.get_all_children()
return seen
return len(seen)
progress_interval = 5
progress_nodes = get_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry))
progress_total = max(1, len(progress_nodes))
progress_count = 0
progress_total = max(1, count_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)))
def progress_function(node):
if node not in progress_nodes:
global progress_count
if progress_count >= progress_total:
return
progress_nodes.remove(node)
if len(progress_nodes) % progress_interval:
return
progress = round(100. * (1 - len(progress_nodes) / progress_total), 1)
progress_count = min(progress_count + progress_interval, progress_total)
progress = round(100. * progress_count / progress_total, 1)
sys.stderr.write("\rBuilding: %5.1f%%" % progress if sys.stderr.isatty() else "progress: %.1f\n" % progress)
if progress == 100. and sys.stderr.isatty():
sys.stderr.write("\n")
sys.stderr.flush()
Progress(progress_function)
Progress(progress_function, interval=progress_interval)
AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)
# Release CI rebuilds tracked device outputs for the host architecture.
+12 -23
View File
@@ -1,16 +1,5 @@
# Third-party notices
## Arabesque Firefox
The offline payload contains original Mozilla and Ubuntu archives. The pinned
versions, upstream download URLs and hashes are listed in
[`runtime.json`](openpilot/tools/arabesque_firefox/runtime.json). Mozilla publishes
the [Firefox 153.6.0esr source](https://archive.mozilla.org/pub/firefox/releases/153.6.0esr/source/)
and [Firefox notices](https://www.mozilla.org/en-US/about/legal/firefox/).
The Ubuntu archives retain their package copyright files. The
[Ubuntu package archive](https://ubuntu.com/project/docs/how-ubuntu-is-made/concepts/package-archive/)
provides source packages alongside the binaries.
## Jetlink
Jetlink's comma client is copyright (c) 2026 Zeph Leggett and distributed under the MIT License. Its complete license and permission notice are retained in [jetlink_repo/LICENSE](jetlink_repo/LICENSE); source and revision are recorded in [CREDITS.md](CREDITS.md#jetlink-external-inference) and `upstream-sync.json`.
@@ -84,7 +73,17 @@ the exclusion of the other.
## BluePilot Ford support
The Ford CAN FD camera-radar filtering includes work adapted from BluePilot's `bp-7.0` source, through StarPilot `2efe6017bfe14cee4807833e05f9a7f6ba255cdb`. See [CREDITS.md](CREDITS.md#ford-support-adapted-from-bluepilot) for contributors and source revisions. The upstream extension-file and `LICENSE.md` notices reproduced above also apply to that source; they are retained without resolving the difference between their published license descriptions.
The Ford lateral strategies, CAN constructors, camera-radar filtering and native checks include work adapted from BluePilot's `bp-7.0` source, through StarPilot `2efe6017bfe14cee4807833e05f9a7f6ba255cdb`. See [CREDITS.md](CREDITS.md#ford-support-adapted-from-bluepilot) for contributors and source revisions. The upstream extension-file and `LICENSE.md` notices reproduced above also apply to the Ford extension sources; they are retained without resolving the difference between their published license descriptions.
## Optional BluePilot angle library adapter
The angle-only adapter in `opendbc_repo/opendbc/bluepilot_lateral` is sourced directly from BluePilot bp-7.0 commit e1d051d7ba270261b4455068bd68f1a58db15a4a. The complete upstream MIT notice is retained in that folder's `LICENSE`, and the simultaneously published Custom MIT notice is retained verbatim in `LICENSE.md`. The applicable SUNNYPILOT acknowledgments reproduced above are also retained in its `CREDITS.md`. This preserves the published terms without resolving their scope or representing the adapted extension as unambiguously plain MIT.
The optional, default-disabled angle lane-centering trim additionally derives from
BluePilot bp-dev e22afa6be9b881fa784c92ebb316db47728a3d81. Its exact source and
contributors are recorded in the library CREDITS.md; the same root/opendbc MIT
and Custom MIT notices coexist unchanged at that revision. This adds attribution,
not a license-scope decision or permission grant.
## Continental ARS510 radar
@@ -130,7 +129,7 @@ UI framework is bundled.
## Galaxy Bluetooth companion source snapshot
The Galaxy Bluetooth phone bridge and iOS app from https://github.com/StarPilotHQ/StarPilot
The Galaxy Bluetooth phone bridge and iOS app from https://github.com/firestar5683/StarPilot
(natehuby76, PRs #224/#225) are included under `openpilot/starpilot/galaxy_bluetooth/`
and `openpilot/starpilot/companion/ios/`, with their license files (`UPSTREAM-LICENSE`, `openpilot/starpilot/companion/ios/UPSTREAM-GALAXY-LICENSE`).
The bridge runs on openpilot's Python with PyCryptodome 3.23.0 (BSD/public-domain
@@ -141,13 +140,3 @@ dbus-next 0.2.3 (MIT), ifaddr 0.2.0 (MIT) and Zeroconf 0.151.3
`vendor/VENDOR.json` records each PyPI source file and its SHA-256. Zeroconf is
kept as separate, unmodified source files that can be replaced with another
compatible version, as the LGPL requires.
## Subaru angle support from jacobw
The Subaru angle implementation includes work from Jacob Waller's `jul-angle-based` openpilot/opendbc branches and their upstream contributors. See [CREDITS.md](CREDITS.md#subaru-angle-steering-from-jacob-wallers-fork) and `opendbc_repo/opendbc/car/subaru/ORIGIN_HISTORY.tsv` for pinned sources, original authors and titles. The source repositories publish the comma MIT license; the complete applicable notice remains in the project `LICENSE` and `opendbc_repo/LICENSE`. StarPilot's adaptations preserve additional native and caller safety guards.
## StarPilot Ford demand tuning
The historical tuning formerly in `openpilot/starpilot/car/ford/starpilot_tuning.py` included the BluePilot bp-7.0-derived work identified in [CREDITS.md](CREDITS.md#starpilot-ford-demand-tuning). The published upstream license notices retained above and in the pinned BluePilot library remain applicable to the adapted source. StarPilot's separate protocol adapter does not modify those notices or resolve their scope.
The angle-primary adaptation in `openpilot/starpilot/car/ford/starpilot_controller.py` is derived from BluePilot v0.2.1. It retains the complete MIT copyright and permission notice in `opendbc_repo/opendbc/bluepilot_lateral/LICENSE`, including Comma.ai, BluePilot and StarPilot contributors. See the contributor attribution in `CREDITS.md`.
+1 -1
View File
@@ -77,7 +77,7 @@ while [[ $# -gt 0 ]]; do
;;
--cereal|-cereal)
shortcut_targets+=(openpilot/cereal/libcereal.a openpilot/cereal/libsocketmaster.a
openpilot/cereal/services.h)
openpilot/cereal/messaging/bridge openpilot/cereal/services.h)
;;
-j|--jobs)
passthrough_args+=("$1")
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
profile="${1:-c4}"
case "${profile}" in
c3) big=1 ;;
c4) big=0 ;;
*) echo "Usage: ./device [c3|c4]" >&2; exit 2 ;;
esac
# Run the real manager with host hardware/cloud disabled. The manager owns the
# UI and Galaxy processes, just as it does on a comma device.
export STARPILOT_CLOUD_DISABLED=1
export BIG="${big}"
export BLOCK="${BLOCK:+${BLOCK},}replay_owner,map_snapshot_operations"
exec "${ROOT_DIR}/dev" device
+1 -26
View File
@@ -5,8 +5,6 @@ openpilot ALC and openpilot LDW do not automatically drive the vehicle or reduce
While changing lanes, openpilot is not capable of looking next to you or checking your blind spot. Only nudge the wheel to initiate a lane change after you have confirmed it's safe to do so.
Unexpected steering may occur. openpilot ALC may not steer to avoid stopped or slow-moving vehicles, cross traffic, construction barriers or cones, motorcycles, children, pedestrians, animals, or other objects on the road.
Many factors can impact the performance of openpilot ALC and openpilot LDW, causing them to be unable to function as intended. These include, but are not limited to:
* Poor visibility (heavy rain, snow, fog, etc.) or weather conditions that may interfere with sensor operation.
@@ -15,14 +13,6 @@ Many factors can impact the performance of openpilot ALC and openpilot LDW, caus
* The device is mounted incorrectly.
* When in sharp curves, like on-off ramps, intersections etc...; openpilot is designed to be limited in the amount of steering torque it can produce.
* In the presence of restricted lanes or construction zones.
* When the lane markings are unclear, worn, missing, or they cross each other.
* Tar marks, shadows, road cracks, temporary or construction lane markings, or other road imperfections.
* When lanes merge or split.
* When entering a highway on-ramp or exiting an off-ramp.
* On roads with intersections or roundabouts.
* On slippery roads.
* When towing a trailer.
* When entering or exiting a tunnel.
* When driving on highly banked roads or in presence of strong cross-wind.
* Extremely hot or cold temperatures.
* Bright light (due to oncoming headlights, direct sunlight, etc.).
@@ -34,8 +24,6 @@ The list above does not represent an exhaustive list of situations that may inte
openpilot ACC and openpilot FCW are not systems that allow careless or inattentive driving. It is still necessary for the driver to pay close attention to the vehicle’s surroundings and to be ready to re-take control of the gas and the brake at all times.
openpilot ACC may not stop your vehicle in time to avoid a collision with a vehicle that quickly stops in front of you. openpilot ACC may occasionally cause your vehicle to brake when not required or when you are not expecting it.
Many factors can impact the performance of openpilot ACC and openpilot FCW, causing them to be unable to function as intended. These include, but are not limited to:
* Poor visibility (heavy rain, snow, fog, etc.) or weather conditions that may interfere with sensor operation.
@@ -47,20 +35,9 @@ Many factors can impact the performance of openpilot ACC and openpilot FCW, caus
* In presence of traffic signs or stop lights, which are not detected by openpilot at this time.
* When the posted speed limit is below the user selected set speed. openpilot does not detect speed limits at this time.
* In presence of vehicles in the same lane that are not moving.
* When a vehicle you are following moves out of your driving path and a stationary or slow-moving vehicle or object is in front of you.
* Narrow vehicles, such as bicycles or motorcycles.
* Vehicles or objects only partially in the driving lane.
* On curves, openpilot ACC may not detect a vehicle in the same lane, and may accelerate to the set speed.
* Cross traffic.
* Oncoming vehicles.
* Animals or objects in the road.
* Rare objects such as trailers, ramps, cargo, open doors, etc. protruding from vehicles.
* When abrupt braking maneuvers are required. openpilot is designed to be limited in the amount of deceleration and acceleration that it can produce.
* When surrounding vehicles perform close cut-ins from neighbor lanes.
* Driving on hills, narrow, or winding roads.
* On slippery roads.
* When towing a trailer.
* When entering or exiting a tunnel.
* Extremely hot or cold temperatures.
* Bright light (due to oncoming headlights, direct sunlight, etc.).
* Interference from other equipment that generates radar waves.
@@ -69,9 +46,7 @@ The list above does not represent an exhaustive list of situations that may inte
## Limitations of openpilot DM
openpilot DM displays a series of escalating warnings. Disregard of these warnings may result in openpilot being unavailable for a period of time.
openpilot DM should not be considered an exact measurement of the alertness of the driver. openpilot DM cannot always detect if the driver is tired or inattentive.
openpilot DM should not be considered an exact measurement of the alertness of the driver.
Many factors can impact the performance of openpilot DM, causing it to be unable to function as intended. These include, but are not limited to:
-6
View File
@@ -4,12 +4,6 @@ openpilot is an Adaptive Cruise Control (ACC) and Automated Lane Centering (ALC)
Like other ACC and ALC systems, openpilot is a failsafe passive system and it requires the
driver to be alert and to pay attention at all times.
openpilot is not an autonomous driving system. It is not a collision avoidance system or a
substitute for safe and attentive driving. The driver is always responsible for the safe
operation of the vehicle and must be ready to take control at any time. The driver can override
openpilot at any time by braking, accelerating, or steering the vehicle. openpilot may
disengage without warning. Read the [limitations](https://github.com/commaai/openpilot/blob/master/docs/LIMITATIONS.md) carefully before using openpilot.
To assist the driver in maintaining alertness, openpilot includes a driver monitoring feature
that alerts when it detects driver distraction.
-8
View File
@@ -1,8 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PYTHON_BIN="${HEATDEATH_PYTHON:-python3}"
if [[ -z "${HEATDEATH_PYTHON:-}" && -x /usr/local/venv/bin/python3 ]]; then
PYTHON_BIN=/usr/local/venv/bin/python3
fi
exec "$PYTHON_BIN" -B "$ROOT_DIR/tools/heatdeath/heatdeath.py" "$@"
+1 -1
View File
@@ -10,4 +10,4 @@ jetlink: run openpilot's large driving models on an attached Jetson.
# so does the release check (macos/scripts/check-version.sh). The comma runs
# jetlink from a checkout, not an install, so this is also all it has. The Swift
# reads it from Pinned.swift: run JetlinkKit/Scripts/make_pins.py after a bump.
__version__ = '0.9.0'
__version__ = '0.8.7'
+6 -21
View File
@@ -224,17 +224,12 @@ class JetlinkClient:
# -- handshake ------------------------------------------------------------
def hello(self, timeout: float = 5.0, pick: dict | None = None) -> dict:
def hello(self, timeout: float = 5.0) -> dict:
"""Introduce this client. The server starts its session over on a hello,
so this is also how a new owner of the gadget takes over one the server
never saw end; see Session.greet in JetlinkServer. `pick` is the comma's
own big-model pick with its age and its date by the comma's clock, {ref,
sha256, nbytes, name, age_s, at}; the answer's `pick` is the newer of it
and the server's (PickStore)."""
never saw end; see Session.greet in JetlinkServer."""
seq = self._next_seq()
client = {'nonce': self.nonce, 'name': self.name}
if pick is not None:
client['pick'] = pick
try:
link = self.t.link_info()
except Exception:
@@ -306,8 +301,7 @@ class JetlinkClient:
frame_skip: int = DEFAULT_FRAME_SKIP,
progress: ProgressFn | None = None,
build_timeout: float = 900.0,
should_stop: StopFn | None = None,
fetch: bool = False, ref: str | None = None) -> ModelSpec:
should_stop: StopFn | None = None) -> ModelSpec:
"""Make the server ready to run this model, uploading and building if needed.
Blocks until ready or the build fails, and returns the spec the server
@@ -315,12 +309,8 @@ class JetlinkClient:
parser rejects an export (tinygrad and the org.tinygrad domain) still runs.
`onnx_path` None means the caller cannot upload (modeld) and a missing
engine is EngineMissing. `fetch` asks a server that lacks the model to
download it itself (state downloading, stage download) rather than answer
need_upload; one that cannot, or is too old to, answers need_upload as
before. `ref` is the catalog ref the model is known by.
`progress(stage, frac, msg)` has stage in
download/upload/patch/parse/build/load. `should_stop` is polled during long waits.
engine is EngineMissing. `progress(stage, frac, msg)` has stage in
upload/patch/parse/build/load. `should_stop` is polled during long waits.
"""
self.progress_cb = progress
self._should_stop = should_stop or (lambda: False)
@@ -328,12 +318,7 @@ class JetlinkClient:
self._engine_state = None
seq = self._next_seq()
request = {'sha256': sha256, 'nbytes': nbytes, 'frame_skip': frame_skip}
if fetch:
request['fetch'] = True
if ref:
request['ref'] = ref
self.t.send_json(P.Msg.ENGINE_REQ, seq, request)
self.t.send_json(P.Msg.ENGINE_REQ, seq, {'sha256': sha256, 'nbytes': nbytes, 'frame_skip': frame_skip})
resp = json.loads(bytes(self._expect(P.Msg.ENGINE_RESP, seq, 60.0).payload))
self._engine_state = resp
log.info("server engine state: %s (%s)", resp['state'], resp.get('detail', ''))
+1 -1
View File
@@ -55,7 +55,7 @@ POLL = 0.5
# what a borrower passes on of the server's hello (Loan.note_server): the owner
# never speaks the protocol, and sleep_after is how it knows whether letting
# go of the gadget lets the Jetson sleep. The rest is for its status record
SERVER_FIELDS = ('protocol', 'device', 'backend', 'runtime_version', 'trt_version', 'sleep_after', 'pick', 'fetches')
SERVER_FIELDS = ('protocol', 'device', 'backend', 'runtime_version', 'trt_version', 'sleep_after')
def _send(conn: socket.socket, msg: dict) -> None:
+8 -29
View File
@@ -40,7 +40,7 @@ from jetlink.openpilot.interface import MODES, STATES, Keys, ModelFace, Openpilo
from jetlink.openpilot.parts import Parts, for_this_process
from jetlink.openpilot.status import Status
API = 4
API = 3
__all__ = ['API', 'MODES', 'STATES', 'Jetlink', 'Keys', 'ModelFace', 'Openpilot', 'OwnerConfig', 'Status', 'bind',
'conformance', 'SHUTDOWN_TIMEOUT', 'Flags', 'SwapEvents']
@@ -302,34 +302,13 @@ class Jetlink:
self._shutdown_asked = time.monotonic()
return True
def lists_big_models(self) -> bool:
"""Is the model manager's big-model list jetlink's (big_catalog) rather
than sunnypilot's? With no chestnut fitted. Hardware, not the link
setting: the model manager drops a pick its catalog does not list."""
def should_extend_catalog(self) -> bool:
"""Should the model manager's big-model catalog carry the newer catalogs'
models? With no chestnut fitted. Hardware, not the link setting: the model
manager drops a pick its catalog does not list."""
return not self._parts.chestnut_fitted()
def big_catalog(self, selector: int) -> dict | None:
"""The model index as the model manager's big-model catalog, every bundle
at `selector` (its REQUIRED_JSON_VERSION), fetched again
(Models.big_catalog). None when it cannot be had, offline say, and the
model manager keeps the list it has."""
from jetlink.registry.index import RegistryError
try:
return self._parts.models.big_catalog(selector)
except RegistryError as e:
self._log.warning("jetlink: could not fetch the model index: %s", e)
return None
def listed_ref(self, ref: str) -> str | None:
"""The ref big_catalog() lists a pick's model under, for a pick a new list
no longer names: a recompile is listed once, under its newest ref. None
while the list is not jetlink's (lists_big_models)."""
return self._parts.models.listed_ref(ref) if self.lists_big_models() else None
# API 3's, until the next minor release: the catalog is left as fetched.
# Its picks still drive, since the index knows every ref it ever saw
def should_extend_catalog(self) -> bool:
return False
def extend_catalog(self, catalog: dict) -> dict:
return catalog
"""The big-model catalog the model manager fetched, with the models newer
catalogs list folded in. Never raises."""
return self._parts.models.big_catalog(catalog)
+5 -17
View File
@@ -11,10 +11,9 @@ manager, hardwared, the UI and the model manager call these on every device,
link on or off, and modeld on every drive: whatever jetlink does wrong turns
the link off and is logged, and never takes one of them down. Each hook has
the answer it gets with the link off, and gives it on any failure. They are
not all False: attach() keeps modeld's model, big_catalog() is None so the
model manager keeps its own list, and shutdown_step() says yes to a power-off
that is wanted, since a broken jetlink must never keep a comma on to drain the
car's battery. selfdrived's
not all False: attach() keeps modeld's model, extend_catalog() the catalog,
and shutdown_step() says yes to a power-off that is wanted, since a broken
jetlink must never keep a comma on to drain the car's battery. selfdrived's
swap events (swap_events()) answer no event at all.
The fork's guard, which builds Hooks or else Off inside a function, and its
@@ -40,7 +39,7 @@ from jetlink.openpilot import API, Flags, Jetlink, SwapEvents, bind
# the APIs a fork's adapter may be written to: today's, and after a bump the
# previous one too, for one minor release (jetlink/openpilot/__init__.py)
COMPATIBLE = frozenset({API, 3})
COMPATIBLE = frozenset({API})
# the hooks that ask jetlink with link_on() saying off. status(): the UI shows
# the big model's name from the model library. shutdown_step(): hardwared
@@ -48,7 +47,7 @@ COMPATIBLE = frozenset({API, 3})
# and its wait, for a link turned off while one is pending. The model
# manager's: the catalog is hardware and the library, not the setting
BINDS_WITH_LINK_OFF = frozenset({'status', 'shutdown_step', 'request_shutdown', 'shutdown_pending', 'model_state',
'lists_big_models', 'big_catalog', 'listed_ref', 'should_extend_catalog', 'extend_catalog'})
'should_extend_catalog', 'extend_catalog'})
# what selfdrived is told when the swap events cannot run: none at all
NO_EVENTS = Flags(False, False, False, False, False)
@@ -157,17 +156,6 @@ class _Hooks:
def model_state(self, ref: str) -> str | None:
return self._call('model_state', None, ref)
def lists_big_models(self) -> bool:
return self._call('lists_big_models', False)
def big_catalog(self, selector: int) -> dict | None:
"""None with the link off or on any failure: the model manager keeps its list."""
return self._call('big_catalog', None, selector)
def listed_ref(self, ref: str) -> str | None:
return self._call('listed_ref', None, ref)
# API 3's, until the next minor release
def should_extend_catalog(self) -> bool:
return self._call('should_extend_catalog', False)
+4 -37
View File
@@ -45,24 +45,15 @@ class Keys:
offroad: str # BOOL: is the car parked (manager's IsOffroad)
progress: str # JSON {stage, frac, msg, drops}: provisioning and join progress
spec: str # JSON: the built model's spec and whether its engine is built
pointers: str # JSON: catalog ref -> {oid, size}
# sunnypilot's model manager: the big-model pick, JSON {ref, displayName},
# and the big-model catalog, JSON {bundles}. Without one both are None, and
# jetlink runs its own default big model, or the one picked on the server.
# A fork with a picker of its own and no catalog (StarPilot) declares only
# big_model, and may name its pick by the ONNX's hash, {sha256,
# displayName}; a newer pick made on the server is written into it
# jetlink runs its own default big model
big_model: str | None = None
catalog: str | None = None
# BOOL: an iPhone on a direct cable is asked to charge from the comma
# (comma.port). Off when unset, and without the key
charge_phone: str | None = None
# STRING: where the model manager takes a pick by its ref and stores it in
# big_model (sunnypilot's ModelManager_DownloadRef). A newer pick made on
# the server is written there, so the comma's own list shows it; without
# the key it drives all the same
select: str | None = None
# API 3's kept pointers, unread since the model index: until the next minor release
pointers: str | None = None
@dataclass(frozen=True)
@@ -98,6 +89,7 @@ class StatusSide(Protocol):
"""Every reader: manager, the UI, hardwared, the model manager."""
keys: Keys
log: Any # logging.Logger-like (cloudlog): debug, info, warning, error, exception
catalog_selector: int # the model manager's REQUIRED_JSON_VERSION; 0 without a model manager
def params_dir(self) -> Path:
"""The params store's directory, by params.cc's rule, worked out on every
@@ -160,16 +152,6 @@ class ModelSide(WorkerSide, Protocol):
frame thread at a hand-back, so it zeroes the history in place and
allocates and compiles nothing; warp.reset_queues makes one."""
def model_warp(self, small: Any, cam_w: int, cam_h: int, model_w: int, model_h: int) -> FrameWarp | None:
"""Optional (OPTIONAL): the warp the link sends frames through, warmed,
or None for openpilot's driving warp pickle at warp_path (warp.Warp). For
a fork whose runtime cannot load that pickle, typically with its small
model's own warp. Asked once on modeld's main thread before the frame
loop, after the small model's warmup; raising keeps the drive on the
small model, as a pickle that does not load does. The offroad checks
still look for the pickle (Warps.built): only modeld has a small model
to ask, so warp_path must name a file either way."""
class Openpilot(ModelSide, Protocol):
"""The whole adapter: one object implements every side."""
@@ -216,28 +198,13 @@ class JoinedModel(Protocol):
"""Let go of the link at modeld's exit."""
class FrameWarp(Protocol):
"""What an adapter's model_warp returns, warp.Warp's shape, run on modeld's
frame thread. Only one of the small and the large model runs a frame, so a
warp that shares the small model's JIT and buffers is never in use by both.
On a comma, a warp that reads its GPU output back moves it first with
warp.make_coherent, before the warp's JIT first runs."""
output: Any # the warped frame, (2, 6, model_h/2, model_w/2) uint8 as bytes: read after wait(), until the next start()
def start(self, frame: int, big_frame: int, tfm: Any, big_tfm: Any) -> None:
"""Warp the camera buffers at these addresses under their 3x3 transforms."""
def wait(self) -> None:
"""Return once `output` holds what start() warped."""
# The members of each protocol an implementation may lack: jetlink reads them
# with getattr and a fallback. conformance() passes an object without one and
# holds one with it to its signature, and adding one keeps API. Kept out of
# the protocols, where a name would be a member itself; a type checker still
# asks for these, as typing has no optional member
OPTIONAL: dict[type, tuple[str, ...]] = {
ModelSide: ('small_reset', 'model_warp'),
ModelSide: ('small_reset',),
SmallModel: ('desire_key', 'numpy_inputs', 'lat_delay', 'PLANPLUS_CONTROL'),
JoinedModel: ('get_action_from_model', 'constants', 'LAT_SMOOTH_SECONDS', 'LONG_SMOOTH_SECONDS', 'lat_delay'),
}
+5 -12
View File
@@ -116,7 +116,7 @@ LAGS = (BEHIND, BAD_OUTPUT)
RECONNECTING = {LOST: ('reconnecting', 'lost'), BEHIND: ('fell behind, reconnecting', 'behind'),
NOT_UP: ('load failed, reconnecting', 'load_failed'),
BAD_OUTPUT: ('bad output, reconnecting', 'bad_output')}
# how long a model name the comma's index did not have stays the sha's
# how long a model name the catalog had not resolved stays the sha's
NAME_RETRY = 30.0
@@ -334,7 +334,7 @@ class JoiningModelState:
self._demote(BEHIND)
elif self._active is not self._small and self._frame_drop_ratio > DROP_LIMIT:
# modeld is behind the large model; this frame is the small model's
self._log.warning("jetlink: modeld dropped %.2f %% of camera frames behind the large model, " +
self._log.warning("jetlink: modeld dropped %.2f %% of camera frames behind the large model, "
"the small model drives from this one", self._frame_drop_ratio * 100)
self._demote(BEHIND)
self._maybe_swap()
@@ -694,9 +694,8 @@ class JoiningModelState:
'lags': self._lags}
def _name_of(self, sha: str, names: dict) -> str:
"""The index's name, kept once it is one. The sha's stand-in (a model the
comma's index does not list yet, or a lookup that failed) is asked again
after NAME_RETRY."""
"""The catalog's name, kept once it is one. The sha's stand-in (a row not
resolved yet, or a lookup that failed) is asked again after NAME_RETRY."""
name, retry_at = names.get(sha, (None, 0.0))
if time.monotonic() >= retry_at:
try:
@@ -746,7 +745,6 @@ def join(parts, cam_w: int, cam_h: int, small) -> JoiningModelState:
link = links.Link(parts.log)
def prepare():
from jetlink.openpilot.interface import FrameWarp, conformance
from jetlink.openpilot.warp import Warp, prepare_reset
# the gadget first, so the Jetson enumerates while the warp loads. Left to
# the join thread the bind lands ~3 s later, behind the small model's
@@ -761,12 +759,7 @@ def join(parts, cam_w: int, cam_h: int, small) -> JoiningModelState:
try:
# the fork's own reset of its small model, else jetlink's from its shape
ready['reset_small'] = getattr(op, 'small_reset', lambda small: None)(small) or prepare_reset(small)
# the fork's own warp, else openpilot's driving warp pickle
warp = getattr(op, 'model_warp', lambda *args: None)(small, cam_w, cam_h, *geometry)
if warp is None:
warp = Warp(parts.warps.load(cam_w, cam_h, *geometry), face.frame_size(cam_w, cam_h))
elif problems := conformance(warp, FrameWarp):
raise RuntimeError(f"the adapter's model_warp is not a warp: {', '.join(problems)}")
warp = Warp(parts.warps.load(cam_w, cam_h, *geometry), face.frame_size(cam_w, cam_h))
except Exception:
link.close()
raise
+16 -28
View File
@@ -43,8 +43,6 @@ PRESENT_TIMEOUT = 5.0
# hardware; the ceiling is only there so a server that has stopped answering
# does not hold the caller for the rest of the day.
BUILD_TIMEOUT = 1800.0
# and on top of that, a server downloading the model itself: 1.75 GB at 1 MB/s
FETCH_TIMEOUT = 1800.0
class ModelMissing(RuntimeError):
@@ -225,16 +223,15 @@ def connect_patiently(link: Link):
# -- making the Jetson ready ------------------------------------------------------
def identity(parts, entry: dict) -> tuple[str, int]:
"""The picked model's sha256 and byte count, from the model index, so the
comma can name the model without holding or hashing the ONNX. A row has
them already where the comma's copy of the index knows the model; one it
does not fetches the index again, the one part of provisioning that may
need the internet."""
"""The picked model's sha256 and byte count, from the catalog model's LFS
pointer, so the comma can name the model without holding or hashing the
ONNX. Looked up once per model ever and kept in a param: the one part of
provisioning that needs the internet."""
sha256, nbytes = entry.get('oid'), entry.get('size')
if sha256 and nbytes:
return sha256, int(nbytes)
parts.progress.report('connect', 0.0, 'finding model', code='finding_model')
return parts.models.resolve(entry.get('ref'), sha256)
return parts.models.resolve_pointer(entry['ref'])
# What has already been hashed in this process, keyed on the file as it was
@@ -281,27 +278,21 @@ def verified_upload(log, model_path: Path | None, sha256: str, nbytes: int) -> P
def ensure(parts, client, sha256: str, nbytes: int, model_path: Path | None, *,
progress=None, should_stop=None, build_timeout: float = BUILD_TIMEOUT, ref: str | None = None,
fetch: bool = True):
progress=None, should_stop=None, build_timeout: float = BUILD_TIMEOUT):
"""Make the server ready for this model and remember what it answered.
Asks without the file first: the server answers from the sha alone when it
has the model, which is every parked poll and every join. Without the file
here either, the server is asked to download the model itself, which only
the Jetson needs. EngineMissing means the Jetson has neither the plan nor
the bytes and could not get them, and neither has the caller. `fetch` False
never asks for the download: a stand-in is a model the server has. `ref`,
the catalog's name for the model, goes with a download."""
has the model, which is every parked poll and every join. EngineMissing means
the Jetson has neither the plan nor the bytes, and neither has the caller."""
from jetlink.client import EngineMissing
fetch = fetch and model_path is None
ask = functools.partial(client.ensure_engine, sha256, nbytes, progress=progress, should_stop=should_stop)
ask = functools.partial(client.ensure_engine, sha256, nbytes, progress=progress,
build_timeout=build_timeout, should_stop=should_stop)
try:
spec = ask(onnx_path=None, fetch=fetch, ref=ref if fetch else None,
build_timeout=build_timeout + (FETCH_TIMEOUT if fetch else 0.0))
spec = ask(onnx_path=None)
except EngineMissing:
upload = verified_upload(parts.log, model_path, sha256, nbytes)
if upload is None:
raise
spec = ask(onnx_path=upload, build_timeout=build_timeout)
spec = ask(onnx_path=upload)
parts.spec.store(spec)
return spec
@@ -324,7 +315,8 @@ def stand_in(parts, hello: dict, pick: str, built) -> tuple[str, int] | None:
if pick in have:
return None
if loaded:
# its size names it to the server: the record's, else the index's
# its size names it to the server, and only a catalog model resolved
# here, or the record, has one
nbytes = built.nbytes if built is not None and built.sha256 == loaded else parts.models.size_for(loaded)
if nbytes:
return loaded, nbytes
@@ -354,15 +346,11 @@ def open_link(parts, link: Link, should_stop=None):
# booting; both resolve on their own
client = connect_patiently(link)
try:
sent = parts.models.hello_pick()
hello = parts.models.heard(client.hello(timeout=10.0, pick=sent), sent)
hello = client.hello(timeout=10.0)
parts.log.warning("jetlink: %s trt %s, engine %s, loaded %s",
hello.get('device'), hello.get('trt_version'),
hello.get('engine_state'), str(hello.get('loaded'))[:16])
link.note_server(hello)
# a newer pick on the server drives, and is ours from now on
parts.models.adopt()
selected = parts.models.selected_model() or selected
sha256, nbytes = identity(parts, selected)
path = parts.models.shipped_model_path()
built = parts.spec.ready_spec()
@@ -382,7 +370,7 @@ def open_link(parts, link: Link, should_stop=None):
# server that restarted reloads from the plan cache, 13 to 25 s; one
# that has never seen this model builds it, 102 to 294 s
spec = ensure(parts, client, sha256, nbytes, path, progress=parts.progress.report_with_eta,
should_stop=should_stop, ref=selected.get('ref'), fetch=standin is None)
should_stop=should_stop)
client.deadline = INFERENCE_TIMEOUT
return client, spec
except EngineMissing as e:
@@ -157,9 +157,9 @@ class JetlinkModelState:
# model manager key off this flag
self.chestnut = True
# a warp.Warp or the adapter's model_warp, loaded and warmed ahead: its
# first call links the JIT, and this runs on modeld's frame thread. Frames
# go out of its output as they are, over either transport
# a warp.Warp, loaded and warmed ahead: its first call links the JIT, and
# this runs on modeld's frame thread. Frames go out of its output as they
# are, over either transport
self.warp = warp
self.input_shapes = spec.input_shapes
@@ -305,9 +305,9 @@ class JetlinkModelState:
# over the cable: how frames go, how many never came back, and what the
# kernel dropped on the way in (softnet), since the link came up
drops = getattr(self.client.t, 'net_drops', lambda: None)()
cable = (f"; frames {'as datagrams' if self.client.t.datagrams else 'on the stream'}, " +
cable = (f"; frames {'as datagrams' if self.client.t.datagrams else 'on the stream'}, "
f"{self.client.frames_lost} lost, net drops {drops}" if drops is not None else '')
self._log.warning("jetlink: frame %d warp %.1f data %.1f send %.1f wait %.1f ms%s; " +
self._log.warning("jetlink: frame %d warp %.1f data %.1f send %.1f wait %.1f ms%s; "
"server gpu %.1f queue %.1f total %.1f ms; ffs maxima prepare %.1f read_wait %.1f handoff %.1f ms%s",
self._frame_id, (frame.t1 - frame.t0) * 1e3, (frame.t2 - frame.t1) * 1e3,
(frame.sent - frame.t2) * 1e3, (t4 - waiting_from) * 1e3,
+159 -380
View File
@@ -7,246 +7,132 @@ See the LICENSE file in the root directory for more details.
Which large model runs, and where its bytes are.
The large model is the model manager's big-model pick, the same slot a
chestnut runs from (Keys.big_model). It names a comma commit, and what the
Jetson runs is the ONNX that commit was built from: the model index
(jetlink.registry.index) says which, for every big-model commit, and lists
what can be picked. The comma keeps the last index that parsed under
model_dir(), so it names, sizes and lists its models with no network; a pick
the copy does not know fetches it again (resolve).
Without a chestnut fitted the model manager's big-model list is the index's
(big_catalog), so what the comma lists is what the Jetson runs. With one, the
list is sunnypilot's own, and jetlink stays out of it.
chestnut runs from (Keys.big_model). Its bundles are tinygrad pkls for a GPU
the Jetson does not have, but each one names the comma commit it was compiled
from, and that commit's ONNX in comma's LFS is what the Jetson runs. The
catalog says what exists, the slot says which one, the pointer at the commit
says which bytes. A commit that ships a precompiled pkl instead names its
export, and jetlink's registry follows that to comma's model repo; see
jetlink.registry.lfs.
Without a model manager (Keys.big_model and Keys.catalog None) there is no
pick and no catalog, and the large model is the one picked on the server, else
jetlink's own default, named by its ref.
A fork with a picker of its own but no catalog jetlink reads (Keys.big_model
without Keys.catalog, StarPilot's) may name its pick by the ONNX's sha256
instead of a ref, which is all its catalog knows of a model; the index sizes
it. With no catalog no model manager owns the slot either, so jetlink writes a
newer pick from the server into it itself, and the fork's picker shows what
drives.
pick and no catalog, and the large model is jetlink's own default, named by its
ref: its pointer resolves from the ref as any pick's does.
"""
from __future__ import annotations
import json
import os
import re
import time
from pathlib import Path
from jetlink.registry.index import (DEFAULT_BIG_MODEL_NAME, DEFAULT_BIG_MODEL_REF, ModelIndex, NetworkError, Pointer, RegistryError,
is_ref, is_sha256, parse_index)
from jetlink.registry.catalog import DEFAULT_BIG_MODEL_NAME, DEFAULT_BIG_MODEL_REF
POINTER_TIMEOUT = 10.0
_REF = re.compile(r'[0-9a-f]{40}')
# the build date a catalog name ends in, " (September 17, 2026)"; any other parenthesis stays
_TRAILING_DATE = re.compile(r' \([A-Za-z]+ \d{1,2}, \d{4}\)$')
# the index and the slot are a file and a JSON param, and the UI names the
# active model every frame; the status line can lag a new pick by this long
# the index and the slot are JSON params, and the UI names the active model
# every frame; the status line can lag a new pick by this long
INDEX_TTL = 2.0
# the comma's copy of the model index, under model_dir()
INDEX_FILE = 'index.json'
# a pick the comma's copy does not know fetches the index again, at most this often
INDEX_MISS_RETRY = 60.0
# A big model's download over a car's connection: the waits between the
# attempts of one provisioning run, one more attempt than waits. Each carries
# on from the .part the last left, so a stall costs the wait, not the bytes
# (lfs.lfs_download)
DOWNLOAD_RETRY_DELAYS = (5.0, 15.0, 30.0, 60.0, 120.0)
# server_pick() before this process has heard a hello
_UNHEARD = object()
# heard() told nothing of the hello's own pick: an older caller
_UNTOLD = object()
# the slot as jetlink saw it last, under model_dir(): to tell a slot the user
# emptied (the default, chosen) from one never filled
SLOT_STATE = 'slot.json'
def _dated(pick) -> dict | None:
"""A server's pick as its hello carries it, dated by this clock:
{ref, sha256, nbytes, name, at}. None for none, or one naming no model."""
if not isinstance(pick, dict) or not isinstance(pick.get('age_s'), (int, float)):
return None
ref = pick['ref'] if is_ref(pick.get('ref')) else None
sha256 = pick['sha256'] if is_sha256(pick.get('sha256')) else None
if ref is None and sha256 is None:
return None
return {'ref': ref, 'sha256': sha256, 'nbytes': pick.get('nbytes'), 'name': pick.get('name'),
'at': time.time() - float(pick['age_s'])}
class Models:
def __init__(self, op):
self.op = op
# (looked at, the file's (mtime, size), the index it held and its rows)
self._index_cache: tuple[float, tuple | None, ModelIndex | None, list[dict]] | None = None
self._refresh_after = 0.0
self._slot_cache: tuple[float, dict | None, float | None] | None = None
# SLOT_STATE as this process last read or wrote it
self._slot_state: dict | None = None
# the pick of the last hello this process heard, and the owner's record
# of the last hello any process heard
self._heard: dict | None | object = _UNHEARD
self._record_cache: tuple[float, dict | None] | None = None
self._index_cache: tuple[float, list[dict], dict[str, dict]] | None = None
self._slot_cache: tuple[float, dict | None] | None = None
# -- the model index --------------------------------------------------------
def catalog(self) -> list[dict]:
"""sunnypilot's big-model bundles as {name, ref}, newest first, as the model
manager's own picker lists them. From the cached JSON rather than the parsed
bundles: parsing builds capnp objects and writes chunk manifests, for two
fields. Never raises."""
try:
key = self.op.keys.catalog
bundles = ((self.op.get(key) if key else None) or {}).get('bundles', [])
found = [b for b in bundles if _REF.fullmatch(str(b.get('ref')))
and int(b.get('minimum_selector_version', 0)) == self.op.catalog_selector]
except Exception:
self.op.log.exception("jetlink: could not read the big-model catalog")
return []
found.sort(key=lambda b: int(b.get('index', 0)), reverse=True)
return [{'name': str(b.get('display_name') or b['ref'][:10]), 'ref': b['ref']} for b in found]
def index_path(self) -> Path:
return self.model_dir() / INDEX_FILE
def pointers(self) -> dict[str, dict]:
value = self.op.get(self.op.keys.pointers)
return value if isinstance(value, dict) else {}
def index(self) -> ModelIndex | None:
"""The comma's copy of the model index, None before the first fetch.
Looked at once per INDEX_TTL and parsed again only when the file
changed. Never raises."""
return self._read_index()[0]
def fetch_pointer(self, ref: str) -> tuple[str, int]:
"""The oid and size of the ONNX a comma commit names, in its tree or, for a
precompiled-pkl commit, in comma's model repo. The Jetson's registry does
the same lookup, so the two ends agree on every model's identity."""
from jetlink.registry.lfs import fetch_pointer as registry_fetch_pointer
pointer = registry_fetch_pointer(ref, timeout=POINTER_TIMEOUT)
return pointer.oid, pointer.size
def resolve_pointer(self, ref: str) -> tuple[str, int]:
"""The oid and size behind a catalog model, fetched the first time and kept for good."""
known = self.pointers()
if ref in known:
return known[ref]['oid'], int(known[ref]['size'])
oid, size = self.fetch_pointer(ref)
known[ref] = {'oid': oid, 'size': size}
self._keep_pointers(known)
self.op.log.warning("jetlink: %s is %s, %d MB", ref[:10], oid[:16], size >> 20)
return oid, size
def _keep_pointers(self, known: dict[str, dict]) -> None:
# blocking: the next lookup reads this back, and a put still in flight would be lost under it
self.op.put(self.op.keys.pointers, known, block=True)
self._index_cache = None
@staticmethod
def _row(name: str, ref: str, known: dict[str, dict]) -> dict:
p = known.get(ref) or {}
return {'name': name, 'ref': ref, 'oid': p.get('oid'), 'size': int(p['size']) if p.get('size') else None}
def _index(self) -> tuple[list[dict], dict[str, dict]]:
"""The catalog's rows and the pointers behind them, read together once per INDEX_TTL."""
now = time.monotonic()
if self._index_cache is None or now - self._index_cache[0] >= INDEX_TTL:
known = self.pointers()
self._index_cache = (now, [self._row(b['name'], b['ref'], known) for b in self.catalog()], known)
return self._index_cache[1], self._index_cache[2]
def model_index(self) -> list[dict]:
"""Every model the index lists, {name, ref, oid, size}, in its order. No network."""
return self._read_index()[1]
def _read_index(self) -> tuple[ModelIndex | None, list[dict]]:
now = time.monotonic()
cache = self._index_cache
if cache is not None and now - cache[0] < INDEX_TTL:
return cache[2], cache[3]
path = self.index_path()
try:
st = path.stat()
seen = (st.st_mtime_ns, st.st_size)
except OSError:
seen = None
if cache is not None and cache[1] == seen:
# the same file, parsed or not: a copy that does not parse is told of once
index = cache[2]
else:
index = None
if seen is not None:
try:
index = parse_index(json.loads(path.read_text()))
except (OSError, ValueError, RegistryError):
self.op.log.exception("jetlink: the comma's model index does not parse")
self._keep_index(now, seen, index)
return self._index_cache[2], self._index_cache[3]
def _keep_index(self, now: float, seen: tuple | None, index: ModelIndex | None) -> None:
rows = self._index_cache[3] if self._index_cache is not None and self._index_cache[2] is index else \
[{'name': m.name, 'ref': m.ref, 'oid': m.oid, 'size': m.size} for m in index.models] if index else []
self._index_cache = (now, seen, index, rows)
def refresh_index(self) -> ModelIndex:
"""The model index, fetched and kept as the comma's copy, which is left
alone when nothing changed so no reader parses it again. Raises a
RegistryError for one that cannot be had or used, and keeps the copy."""
from jetlink.comma import gadget
from jetlink.registry.index import fetch_index
data, index = fetch_index()
path = self.index_path()
if gadget.read_record(path) != data:
gadget.write_record(path, data)
st = path.stat()
self._keep_index(time.monotonic(), (st.st_mtime_ns, st.st_size), index)
return index
def _row(self, name: str, ref: str | None, sha256: str | None = None) -> dict:
"""A model as a row: its ONNX by its ref, from the index, else by its hash
and sized from the index. oid or size is None where the index does not know."""
index = self.index()
pointer = index.pointer(ref) if index is not None and ref else None
if pointer is not None:
return {'name': name, 'ref': ref, 'oid': pointer.oid, 'size': pointer.size}
return {'name': name, 'ref': ref, 'oid': sha256, 'size': index.size(sha256) if index is not None and sha256 else None}
def resolve(self, ref: str | None, sha256: str | None) -> tuple[str, int]:
"""A pick's oid and size, from the index; one the comma's copy does not
know fetches it again, at most once per INDEX_MISS_RETRY. Raises a
RegistryError for a model the index does not know, or no index at all."""
if (found := self._identity(ref, sha256)) is None and time.monotonic() >= self._refresh_after:
self._refresh_after = time.monotonic() + INDEX_MISS_RETRY
self.refresh_index()
found = self._identity(ref, sha256)
if found is None:
raise RegistryError(f"the model index does not know {(ref or sha256 or 'the model')[:16]}")
return found
def _identity(self, ref: str | None, sha256: str | None) -> tuple[str, int] | None:
row = self._row('', ref, sha256)
return (row['oid'], int(row['size'])) if row['oid'] and row['size'] else None
"""Every catalog model, {name, ref, oid, size}. oid and size are None until
the model has been selected and resolved. No network."""
return self._index()[0]
def selected_slot(self) -> dict | None:
"""The big-model slot's pick as {name, ref, sha256}, from the raw param
rather than a parsed bundle, and memoised with when it was made
(picked_at): the UI asks every frame. It names its model by the catalog's
ref, by the ONNX's sha256 (a fork whose catalog knows only that), or by
both; the one it lacks is None."""
"""The big-model slot's pick as {name, ref}, from the raw param rather than a
parsed bundle, and memoised: the UI asks every frame."""
now = time.monotonic()
if self._slot_cache is not None and now - self._slot_cache[0] < INDEX_TTL:
return self._slot_cache[1]
key = self.op.keys.big_model
slot = self.op.get(key) if key else None
slot = slot if isinstance(slot, dict) else {}
ref = slot['ref'] if isinstance(slot.get('ref'), str) and slot['ref'] else None
sha256 = slot['sha256'] if is_sha256(slot.get('sha256')) else None
pick, at = None, None
if ref or sha256:
pick = {'name': str(slot.get('displayName') or (ref or sha256)[:10]), 'ref': ref, 'sha256': sha256}
try:
at = os.stat(Path(self.op.params_dir()) / key).st_mtime
except OSError:
pass
self._note_slot(pick)
self._slot_cache = (now, pick, at)
ref = slot.get('ref') if isinstance(slot, dict) else None
pick = None
if isinstance(ref, str) and ref:
pick = {'name': str(slot.get('displayName') or ref[:10]), 'ref': ref}
self._slot_cache = (now, pick)
return pick
def _note_slot(self, pick: dict | None) -> None:
"""Keep SLOT_STATE in step with the slot: what is in it, and when it was
emptied. A pick taken out is the default chosen (the model manager's
Default, here or on sunnylink), a pick like any other, which the hello
names and a server's older pick must not undo. Read again only when the
slot no longer matches what this process knows, so per process it costs
a file read at a change. Never raises."""
last = {'ref': pick['ref'], 'sha256': pick['sha256']} if pick else None
state = self._slot_state
if state is not None and state.get('last') == last:
return
path = self.model_dir() / SLOT_STATE
try:
state = json.loads(path.read_text())
state = state if isinstance(state, dict) else {}
except (OSError, ValueError):
state = {}
if state and state.get('last') == last:
self._slot_state = state
return
if last is None and not state:
# empty with nothing known before it: never picked, as far as we know
self._slot_state = {'last': None, 'cleared_at': None}
return
state = {'last': last, 'cleared_at': time.time() if last is None else None}
self._slot_state = state
from jetlink.comma import gadget
try:
gadget.write_record(path, state)
except OSError:
self.op.log.exception("jetlink: could not keep the big-model slot's state")
def default_chosen_at(self) -> float | None:
"""When the slot was emptied by this clock, the default chosen; None while
it holds a pick, or was never filled."""
if self.selected_slot() is not None or self._slot_state is None:
return None
at = self._slot_state.get('cleared_at')
return float(at) if isinstance(at, (int, float)) else None
def default_model(self) -> dict | None:
"""What runs with no pick: the index's default, else the first it lists.
Not the chestnut's default, which is the model in the tree. Before the
comma has an index, jetlink's own by its ref, which a provisioning run
resolves by fetching one."""
if (index := self.index()) is None:
return self._row(DEFAULT_BIG_MODEL_NAME, DEFAULT_BIG_MODEL_REF)
"""What runs with no pick: jetlink's default big model, else the newest the
catalog lists. Not the chestnut's default, which is the model in the tree.
A fork with no catalog at all (Keys.catalog None) gets jetlink's default by
its ref; one whose catalog has not been fetched yet waits for it."""
if self.op.keys.catalog is None:
return self._row(DEFAULT_BIG_MODEL_NAME, DEFAULT_BIG_MODEL_REF, self._index()[1])
models = self.model_index()
return next((m for m in models if m['ref'] == index.default), models[0] if models else None)
return next((m for m in models if m['ref'] == DEFAULT_BIG_MODEL_REF), models[0] if models else None)
def default_model_name(self) -> str | None:
"""default_model()'s name without the catalog's trailing build date, the
@@ -256,156 +142,12 @@ class Models:
return _TRAILING_DATE.sub('', model['name']) if model else None
def selected_model(self) -> dict | None:
"""The model picked last, here or on the server: the slot's pick, listed
or not (the index knows every ref it ever saw), unless the server's last
hello named a newer one. With neither, default_model()."""
if (theirs := self._newer_server_pick()) is not None:
return self._server_row(theirs)
if (own := self.selected_slot()) is not None:
return self._own_row(own)
"""The slot's pick, listed in the catalog or not: a ref is enough to find its
pointer. With no pick, default_model()."""
if (pick := self.selected_slot()) is not None:
return self._row(pick['name'], pick['ref'], self._index()[1])
return self.default_model()
def _own(self) -> tuple[dict | None, float | None]:
"""Our pick as a row, and when it was made by this clock: the slot's, or
the default when it was chosen by emptying the slot. (None, None) with
neither."""
if (own := self.selected_slot()) is not None:
return self._own_row(own), self.picked_at()
if (at := self.default_chosen_at()) is not None and (default := self.default_model()) is not None:
return default, at
return None, None
def _own_row(self, own: dict) -> dict:
return self._row(own['name'], own['ref'], own['sha256'])
# -- picks on both ends -----------------------------------------------------
# A pick is made on either end: in the comma's list (the default included,
# chosen by emptying the slot), or on the server (its web page, an app's Use
# Model). Each hello carries the comma's own pick, when it was made by this
# clock and how long ago, and the server answers with the newer of that and
# its own (PickStore in JetlinkServer), which it settles by what the comma
# named before rather than by its clock where it can: a Jetson's clock
# starts again behind at every power-on. The answer is kept with the rest of
# the hello (gadget.SERVER, through the owner) along with the pick the hello
# carried, and selected_model() takes the server's when that is still our
# pick: the server chose knowing it. So a model picked on the server drives
# without being picked here too, and one picked here goes to the server at
# the next hello.
def picked_at(self) -> float | None:
"""When the slot was last written, by this clock: when the pick was made.
None with no pick of our own, or no date for it."""
return self._slot_cache[2] if self.selected_slot() is not None and self._slot_cache else None
def hello_pick(self) -> dict | None:
"""Our own pick, for the hello: {ref, sha256, nbytes, name, age_s, at},
its oid and size where the index knows them (no network here), `at` when it
was made by this clock. None with nothing picked here, or no date for it:
the server's pick, if any, stands."""
row, at = self._own()
if row is None or at is None or not (row['ref'] or row['oid']):
return None
return {'ref': row['ref'], 'sha256': row['oid'], 'nbytes': row['size'], 'name': row['name'],
'age_s': round(max(0.0, time.time() - at), 1), 'at': round(at, 3)}
def heard(self, hello: dict, asked=_UNTOLD) -> dict:
"""The server's hello with its `pick` dated by this clock (_dated), and
`asked`, the hello_pick() the hello carried, kept with it: which is how
the owner keeps it (lending.SERVER_FIELDS) and server_pick() reads it.
This process goes by it at once, whatever the owner does with it."""
self._heard = _dated(hello.get('pick'))
if self._heard is not None and asked is not _UNTOLD:
self._heard['asked'] = {k: asked.get(k) for k in ('ref', 'sha256', 'at')} if isinstance(asked, dict) else None
return {**hello, 'pick': self._heard} if self._heard is not None else hello
def server_pick(self) -> dict | None:
"""The pick the server last answered a hello with, {ref, sha256, nbytes,
name, at}, or None: a server too old to say, or one with no pick. What
this process heard last, else the owner's record of the last hello: the
UI says no hello, and reads it every frame."""
if self._heard is not _UNHEARD:
return self._heard
now = time.monotonic()
if self._record_cache is None or now - self._record_cache[0] >= INDEX_TTL:
from jetlink.comma import gadget
pick = (gadget.read_record(gadget.SERVER) or {}).get('pick')
self._record_cache = (now, pick if isinstance(pick, dict) and isinstance(pick.get('at'), (int, float)) else None)
return self._record_cache[1]
def _newer_server_pick(self) -> dict | None:
"""The server's last pick, when it is another model than ours and was
picked after it, or we have none. The server settled that when the hello
carried this pick of ours; a pick made here since that hello is newer.
An answer kept without the pick it was asked with (an older jetlink) goes
by its date."""
theirs = self.server_pick()
if theirs is None:
return None
own, at = self._own()
if own is None:
return theirs
if self._same(theirs, own):
return None
if 'asked' in theirs:
asked = theirs['asked']
fresh = isinstance(asked, dict) and self._same(asked, own) and isinstance(asked.get('at'), (int, float)) \
and at is not None and abs(asked['at'] - at) < 1.0
return theirs if fresh else None
return None if theirs['at'] <= (at or 0.0) else theirs
@staticmethod
def _same(pick: dict, row: dict) -> bool:
"""One model: by the catalog ref where both have one, since two refs can
share one export, else by the hash. The server's PickStore.sameModel."""
if pick.get('ref') and row['ref']:
return pick['ref'] == row['ref']
return bool(pick.get('sha256')) and pick['sha256'] == row['oid']
def _server_row(self, pick: dict) -> dict:
"""The server's pick as a row: named as we list it, with the server's oid
and size, which name the bytes it has."""
listed = next((m for m in self.model_index() if pick['ref'] and m['ref'] == pick['ref']), None)
row = self._row(listed['name'] if listed else pick.get('name') or (pick['ref'] or pick['sha256'])[:16],
pick['ref'], pick['sha256'])
if pick['sha256'] and pick.get('nbytes'):
row['oid'], row['size'] = pick['sha256'], int(pick['nbytes'])
return row
def adopt(self) -> None:
"""Make a newer pick from the server ours. With a catalog, ask the model
manager for it (Keys.select), so the comma's own list shows what drives:
only a model the catalog lists, which is all the model manager takes; any
other drives all the same (selected_model). Without one, no model manager
owns the slot, so the pick is written into it, whatever the model. Never
raises."""
try:
if (theirs := self.server_pick()) is None or self._newer_server_pick() is None:
return
model, keys = self._server_row(theirs), self.op.keys
if keys.catalog is None:
if keys.big_model:
self.op.log.warning("jetlink: %s was picked on the server, picking it here too", model['name'])
slot = {k: v for k, v in (('displayName', model['name']), ('ref', model['ref']), ('sha256', model['oid'])) if v}
self.op.put(keys.big_model, slot, block=True)
self._slot_cache = None
return
# a download the user asked for that the model manager has not taken
# yet is left alone: the next hello asks again
key = keys.select
if key and model['ref'] and self.op.get(key) is None and model['ref'] in self._catalog_refs():
self.op.log.warning("jetlink: %s was picked on the server, picking it here too", model['name'])
self.op.put(key, model['ref'])
self._slot_cache = None
except Exception:
self.op.log.exception("jetlink: could not take the server's pick")
def _catalog_refs(self) -> set[str]:
"""The refs the model manager's catalog lists: the index's without a
chestnut fitted (big_catalog), else sunnypilot's."""
key = self.op.keys.catalog
bundles = ((self.op.get(key) if key else None) or {}).get('bundles')
return {b['ref'] for b in bundles if isinstance(b, dict) and is_ref(b.get('ref'))} if isinstance(bundles, list) else set()
def selected_model_name(self) -> str | None:
selected = self.selected_model()
return selected['name'] if selected else None
@@ -459,6 +201,8 @@ class Models:
the run for the next one. Each attempt asks the LFS server again:
the address it hands out expires. `retrying(error, delay)` hears of each
wait; a run that is stopped stops waiting."""
from jetlink.registry.catalog import NetworkError
from jetlink.registry.lfs import Pointer
model = self.selected_model()
if model is None or not model['oid']:
return None
@@ -482,23 +226,30 @@ class Models:
time.sleep(min(1.0, end - time.monotonic()))
def _fetch_once(self, pointer, dest: Path, progress, should_stop) -> Path:
from jetlink.registry.lfs import lfs_download, lfs_hrefs
for endpoint, href in lfs_hrefs(self.lfs_endpoints(), pointer):
from jetlink.registry.catalog import NetworkError
from jetlink.registry.lfs import lfs_download, lfs_resolve
for endpoint in self.lfs_endpoints():
href = lfs_resolve(endpoint, pointer)
if href is None:
continue
self.op.log.warning("jetlink: fetching the large model (%d MB) from %s", pointer.size >> 20, endpoint)
return lfs_download(href, pointer, dest, progress=progress, should_stop=should_stop)
raise NetworkError(f"no LFS server has {pointer.oid[:16]}")
def _row_for(self, oid: str) -> dict | None:
"""The catalog model with this oid, among those we have resolved."""
return next((m for m in self.model_index() if m['oid'] == oid), None)
def name_for(self, oid: str) -> str:
"""The index's name for the model with this oid, or the oid's first 16
characters when it lists none."""
index = self.index()
listed = index.by_oid.get(oid) if index is not None else None
return listed.name if listed is not None else oid[:16]
"""The catalog's name for the model with this oid, or the oid's first 16
characters when it lists none we have resolved."""
row = self._row_for(oid)
return row['name'] if row is not None else oid[:16]
def size_for(self, oid: str) -> int | None:
"""The byte count of the ONNX with this oid, as the index knows it."""
index = self.index()
return index.size(oid) if index is not None else None
"""The byte count of the catalog model with this oid, once resolved here."""
row = self._row_for(oid)
return row['size'] if row is not None else None
def has_file(self, model: dict) -> bool:
"""Is this model's ONNX on the comma, whole? Its size is the cheap check."""
@@ -511,26 +262,54 @@ class Models:
# -- the model manager's catalog --------------------------------------------
def big_catalog(self, selector: int) -> dict:
"""The index's models as a catalog in sunnypilot's shape, for the model
manager's big-model list when no chestnut is fitted: what the comma lists
is then what the Jetson runs, each with the index's fields, in its folder
and order. Only `selector` is the fork's (its model manager's
REQUIRED_JSON_VERSION, which it parses), and no artifacts: a chestnut
has nothing to download, and an accelerator runs the ONNX. The index is
fetched again; raises a RegistryError when it cannot be, so the model
manager keeps the list it has."""
models = self.refresh_index().models
return {'bundles': [{'ref': m.ref, 'display_name': m.name, 'short_name': m.short_name or m.ref[:7], 'index': len(models) - i,
'is_big': True, 'is_20hz': m.is_20hz, 'generation': m.generation, 'environment': m.environment,
'runner': m.runner, 'minimum_selector_version': str(selector), 'build_time': m.build_time,
'overrides': {'folder': m.folder}, 'models': []} for i, m in enumerate(models)]}
def big_catalog(self, catalog: dict) -> dict:
"""The big-model catalog with every newer one sunnypilot has published folded in.
A Jetson runs the commit's ONNX, so a model sunnypilot only builds for its next
runtime is still one it can run; see jetlink.registry.catalog.fetch_catalogs.
Never raises: a probe that fails keeps what the last one found."""
try:
from jetlink.registry.catalog import fetch_catalogs, merge_catalogs
fetched = fetch_catalogs()
self.keep_pins(fetched.get('pointers', {}))
merged = merge_catalogs([catalog, fetched], selector=self.op.catalog_selector)
merged.pop('pointers', None) # the pins are kept above, never in the model manager's catalog
added = len(merged.get('bundles', [])) - len(catalog.get('bundles', []))
if added <= 0:
return catalog
self.op.log.warning("jetlink: %d model(s) only newer catalogs list", added)
return merged
except Exception:
self.op.log.exception("jetlink: could not check for newer catalogs")
return self._found_before(catalog)
def listed_ref(self, ref: str) -> str | None:
"""The ref the index lists a model under: its own, or the newest one with
the same ONNX (a recompile is listed once). None for a model it does not
list. No network."""
index = self.index()
listed = index.listed(ref) if index is not None and is_ref(ref) else None
return listed.ref if listed is not None else None
def keep_pins(self, pins: dict[str, dict]) -> None:
"""Writes the list's pins over the kept pointers (see catalog.pinned_pointers)."""
known = self.pointers()
changed = {ref: pin for ref, pin in pins.items() if known.get(ref) != pin}
if not changed:
return
known.update(changed)
self._keep_pointers(known)
for ref, pin in changed.items():
self.op.log.warning("jetlink: %s is pinned to %s", ref[:10], pin['oid'][:16])
def _found_before(self, catalog: dict) -> dict:
"""The catalog with the models the last probe found folded in again, from the
model manager's cached copy: every entry there the fetched catalog does not
list. Not only merge_catalogs' entries with no artifacts: a newer catalog at
the same selector version comes through with its builds (v26 lists Cinque
Terre V3 that way). Dropped with a probe that failed, a pick only a newer
catalog lists would be reset at the manager's next start and the owner would
provision the default in its place; a refresh on a flaky network is enough.
A model sunnypilot withdrew stays listed until a probe works."""
try:
listed = {b.get('ref') for b in catalog.get('bundles', [])}
key = self.op.keys.catalog
cached = ((self.op.get(key) if key else None) or {}).get('bundles', [])
kept = [b for b in cached if isinstance(b, dict) and b.get('ref') not in listed]
except Exception:
return catalog
if not kept:
return catalog
self.op.log.warning("jetlink: keeping %d model(s) the last probe found", len(kept))
return {**catalog, 'bundles': [*catalog.get('bundles', []), *kept]}
+32 -54
View File
@@ -126,8 +126,10 @@ class ProvisioningRun:
return path
def needs_download(self) -> bool:
"""Could the Jetson need the picked model from the comma? Only while its
engine is not known to be built and the file is not here."""
"""Does the comma have to fetch the picked model before a Jetson can build
it? Only while its engine is not known to be built and the file is not here.
Asked without the Jetson: a download needs only the internet, and a car
whose Jetson goes off with the ignition has no Jetson while parked."""
parts = self.parts
entry = parts.models.selected_model()
if entry is None:
@@ -138,17 +140,12 @@ class ProvisioningRun:
return parts.models.shipped_model_path() is None
def provision(self) -> bool:
"""Make the Jetson ready for the picked model; a host is attached. The
hello settles which pick that is, the comma's or a newer one made on the
server. A server without the model downloads it itself; only one that
cannot (no internet, or too old) gets the bytes from the comma, fetched
here then."""
"""Make the Jetson ready for the selected model; a host is attached. run()
has fetched the file already unless the spec record said the model was
built; a server that lost it since asks for the bytes, and they are
fetched then."""
from jetlink.client import EngineMissing
parts = self.parts
parts.progress.report('connect', 0.0, 'connecting', code='connecting')
hello = self.hello()
self.log.warning("jetlink: server %s trt %s", hello.get('device'), hello.get('trt_version'))
parts.models.adopt()
entry = parts.models.selected_model()
if entry is None:
# no catalog yet; not an error
@@ -157,24 +154,27 @@ class ProvisioningRun:
return False
sha256, nbytes = link.identity(parts, entry)
# only needed if the server turns out not to have this model and cannot
# get it; None is a legitimate state here, see EngineMissing below
# only needed if the server turns out not to have this model; None is a
# legitimate state here, see EngineMissing below
model_path = parts.models.shipped_model_path()
self.log.warning("jetlink: provisioning %s (%d MB, sha %s)",
entry.get('name', sha256[:16]), nbytes >> 20, sha256[:16])
parts.progress.report('connect', 0.0, 'connecting', code='connecting')
def ensure(path):
return link.ensure(parts, self.client, sha256, nbytes, path, progress=parts.progress.report_with_eta,
should_stop=lambda: self.stop, ref=entry.get('ref'))
should_stop=lambda: self.stop)
hello = self.hello()
self.log.warning("jetlink: server %s trt %s", hello.get('device'), hello.get('trt_version'))
try:
spec = ensure(model_path)
except EngineMissing:
# the server has nothing to build from and could not download it, and
# neither have we: fetch the model and hand it over in this run. The
# owner lends the link for the whole run; leaving after the download
# would put the build a WORKER_BACKOFF later
# the server has nothing to build from, and neither have we: fetch the
# model and hand it over in this run. The owner lends the link for the
# whole run; leaving after the download would put the build a
# WORKER_BACKOFF later
if model_path is not None or (model_path := self.fetch_model()) is None:
raise
# a download of minutes can outlast the session; a hello starts a new one
@@ -186,36 +186,16 @@ class ProvisioningRun:
return True
def hello(self) -> dict:
"""Say hello with our pick, and pass the answer on to the owner, which
needs its sleep_after to decide whether letting go of the gadget is worth
what it costs, never speaks the protocol to learn it, and keeps the
server's pick and whether it downloads for the comma (Models.heard)."""
models = self.parts.models
sent = models.hello_pick()
hello = models.heard(self.client.hello(timeout=10.0, pick=sent), sent)
"""Say hello, and pass the answer on to the owner, which needs its
sleep_after to decide whether letting go of the gadget is worth what it
costs and never speaks the protocol to learn it."""
hello = self.client.hello(timeout=10.0)
if self.loan is not None:
self.loan.note_server(hello)
return hello
# -- the parked car -------------------------------------------------------
def without_jetson(self) -> None:
"""No Jetson came to the bind: a car whose Jetson is switched with the
ignition has none while parked. Only the Jetson needs the model, and one
that can download it gets it itself when it is on; the comma fetches it
now only for a Jetson that cannot (no internet when it was last here, or
too old to say), so the build can start the moment it is."""
if not self.has_work():
return
if self.needs_download():
if server_fetches():
self.parts.progress.report('waiting', 0.0, 'the jetson downloads it when it is on', code='waiting_for_host')
return
if self.fetch_model() is None:
return
self.parts.progress.report('waiting', 0.0, 'downloaded, builds when the jetson is on',
code='downloaded_waiting_host')
def has_work(self) -> bool:
"""Is there a reason to wake the Jetson? Only things the link can fix count."""
spec = self.parts.spec.load()
@@ -279,21 +259,25 @@ class ProvisioningRun:
self.log.warning("jetlink: no warp built for this camera, nothing to provision for")
return True
# Nothing to build, unless a model was picked on the server since: a
# Jetson awake on the bus is asked, which costs a hello and no wake. Not
# a phone's cable, which is there only once the app dials in
if not self.has_work() and not (gadget.host_attached() and gadget.link_kind(self.parts.settings.mode()) == 'usb'):
if not self.has_work():
self.log.warning("jetlink: nothing to provision")
return True
finished = False
try:
# the bytes first while no host is on the cable: a parked car whose
# Jetson is switched with the ignition never has one to wait for. A host
# that is there is asked first, since one with the model built needs
# nothing from the comma; provision() fetches it if the host has not
if self.needs_download() and not gadget.host_attached() and self.fetch_model() is None:
return False
if not self.open_link():
return False
if not self.wait_for_jetson():
self.log.warning("jetlink: no jetson within %.0f s, leaving it for the next run", WAKE_TIMEOUT)
self.close_link()
self.without_jetson()
if self.parts.models.shipped_model_path() is not None:
self.parts.progress.report('waiting', 0.0, 'downloaded, builds when the jetson is on',
code='downloaded_waiting_host')
return False
finished = self.provision()
except Exception:
@@ -304,12 +288,6 @@ class ProvisioningRun:
return finished
def server_fetches() -> bool:
"""Did the server the comma last spoke to say it downloads a model itself
(its hello's `fetches`, as the owner keeps it)? No for one too old to say."""
return (gadget.read_record(gadget.SERVER) or {}).get('fetches') is True
def main(argv: list[str] | None = None) -> None:
"""One provisioning round over the fork's adapter, as the owner starts it.
Exits 0 when nothing is left to do, and 1 when the owner should try again."""
+8 -14
View File
@@ -31,17 +31,14 @@ PROGRESS_MIN_INTERVAL = 0.25
# what a progress record's `code` may be, for a fork that words the line
# itself; `msg` stays the English one. `args`, where a code has them:
# downloading, download and upload {got_mb, total_mb}, download_stalled
# {retry_s}, reconnecting {why: lost, behind, load_failed or bad_output},
# build {eta_s}. downloading is the comma fetching the model, download the
# server fetching it itself. From download on they are stage names, and the
# server's stages are its own:
# downloading and upload {got_mb, total_mb}, download_stalled {retry_s},
# reconnecting {why: lost, behind, load_failed or bad_output}, build {eta_s}.
# From upload on they are stage names, and the server's stages are its own:
# a server can add one, so a fork falls back to `msg` for a code it does not
# know. A server message with no stage has no code
PROGRESS_CODES = ('waiting_for_host', 'finding_model', 'model_missing', 'ready_waiting_window', 'reconnecting',
'connecting', 'downloading', 'download_stalled', 'download_failed', 'downloaded_waiting_host',
'engine_ready', 'failed', 'download', 'upload', 'patch', 'parse', 'convert', 'compile', 'build',
'load', 'warm')
'engine_ready', 'failed', 'upload', 'patch', 'parse', 'convert', 'compile', 'build', 'load', 'warm')
def estimated_build_seconds(size: int) -> int:
@@ -107,15 +104,12 @@ class Progress:
size. Only the build: the upload reports MB of MB and a connect has nothing
to predict."""
args = None
size = self._size() if self._size is not None and stage in ('build', 'upload', 'download') else None
size = self._size() if self._size is not None and stage in ('build', 'upload') else None
if size and stage == 'build':
left = estimated_build_seconds(size) * max(0.0, 1.0 - frac)
msg, args = _eta(left), {'eta_s': round(left)}
elif size:
args = {'got_mb': int(frac * size) >> 20, 'total_mb': size >> 20}
if stage == 'download' and args:
# the server downloading the model itself: not the comma's own 'downloading'
msg = f"jetson downloading {args['got_mb']} of {args['total_mb']} MB"
self.report(stage, frac, msg, code=stage or None, args=args)
@@ -433,11 +427,11 @@ def read(parts, mode: str) -> Status:
def model_state(parts, ref: str) -> str | None:
"""One listed model, for the picker's list: 'ready' when the Jetson has
"""One catalog model, for the picker's list: 'ready' when the Jetson has
built it (the spec record), 'downloaded' when the comma has its file, else
None. Records and one stat."""
None. Records and one stat; a model never resolved has neither."""
model = next((m for m in parts.models.model_index() if m['ref'] == ref), None)
if model is None:
if model is None or not model.get('oid'):
return None
if parts.spec.engine_ready_for(model['oid']):
return 'ready'
+9 -3
View File
@@ -16,8 +16,8 @@ capnp, swaglog loads zmq, and model_face() imports modeld itself, so it is for
modeld alone, where that module is already loaded.
Data are class attributes, behaviour small methods, each a fork may override;
one override point per value: a fork sets KEYS, and the interface's keys
reads it. It is jetlink's contract:
one override point per value: a fork sets KEYS and CATALOG_SELECTOR, and the
interface's keys and catalog_selector read them. Both are jetlink's contract:
renaming or removing one, or changing what one means, is a breaking change
and bumps API. scripts/upstream (the upstream-paths CI job) checks the paths
and literals here against commaai/openpilot master weekly.
@@ -57,7 +57,9 @@ class Stock:
# the canonical names, which a fork declares in params_keys.h. No model
# manager upstream: no big-model slot or catalog, so jetlink runs its default
KEYS = Keys(link='JetlinkLink', offroad='IsOffroad', progress='JetlinkProgress', spec='JetlinkSpec',
charge_phone='JetlinkChargePhone')
pointers='JetlinkModelPointers', charge_phone='JetlinkChargePhone')
# the model manager's catalog version; none upstream
CATALOG_SELECTOR = 0
# comma's chestnut, running and in its ROM (common.hardware.usb's
# CHESTNUT_USB_IDS + CHESTNUT_ROM_USB_IDS, which CI holds this to): written
# out, as the owner cannot import that package
@@ -86,6 +88,10 @@ class Stock:
def keys(self) -> Keys:
return self.KEYS
@property
def catalog_selector(self) -> int:
return self.CATALOG_SELECTOR
@cached_property
def log(self):
from openpilot.common.swaglog import cloudlog
+1 -2
View File
@@ -192,8 +192,7 @@ _MODEL = object()
HOOK_CALLS = {'should_run': [(False, None, None)], 'status': [()], 'reason': [()], 'prepare': [(False,), (True,)],
'attach': [(_MODEL, object(), 1928, 1208)], 'handed_over': [(_MODEL,)], 'state': [(_MODEL,)],
'shutdown_step': [(False,), (True, 'test')], 'request_shutdown': [('test',)], 'shutdown_pending': [()],
'model_state': [('f' * 40,)], 'lists_big_models': [()], 'big_catalog': [(21,)], 'listed_ref': [('f' * 40,)],
'should_extend_catalog': [()], 'extend_catalog': [({'bundles': []},)],
'model_state': [('f' * 40,)], 'should_extend_catalog': [()], 'extend_catalog': [({'bundles': []},)],
'runtime': [()], 'swap_events': [(0.01,)]}
+5 -10
View File
@@ -161,7 +161,7 @@ class Warp:
out = jit.captured.ret.uop.base.buffer
self._device = out.device
if self._device.startswith('QCOM'):
make_coherent(out)
_make_coherent(out)
# the two buffers every frame passes, written in place by start()
self._tensors = {name: Tensor(np.zeros(shape, dtype=dtype), device=device).realize()
for name, (shape, dtype, device) in specs.items()}
@@ -191,15 +191,10 @@ class Warp:
self._replay(self._inputs, {})
def make_coherent(out) -> None:
"""Move a QCOM buffer to KGSL memory flagged IOCOHERENT and write-back, so
the CPU reads it through its cache once the GPU is done (see Warp). Raises
if the kernel keeps either flag back (the allocation's record holds its
answer): write-back memory that is not coherent reads stale frames.
For an adapter's model_warp too. Move a buffer before the first call of
the JIT that writes it: that call links its address into the JIT, which
would keep writing the memory this frees."""
def _make_coherent(out) -> None:
"""Move a QCOM buffer to KGSL memory flagged IOCOHERENT and write-back.
Raises if the kernel keeps either flag back (the allocation's record holds
its answer): write-back memory that is not coherent reads stale frames."""
from tinygrad.device import Device
dev = Device[out.device]
mem = dev._gpu_alloc(out.nbytes, flags=COHERENT_WRITEBACK)
+2 -2
View File
@@ -4,8 +4,8 @@ Copyright (c) 2026-, Zeph Leggett.
This file is part of jetlink and is licensed under the MIT License.
See the LICENSE file in the root directory for more details.
Which large models exist and how to get their bytes: the model index, which
names every big model's ONNX (index), and an ONNX by its oid (lfs).
Which large models exist and how to get their bytes: sunnypilot's catalog
(catalog) and a model's ONNX by its git-lfs pointer (lfs).
The comma imports both, so this package stays stdlib only and imports nothing
itself. The server's side of it is JetlinkRegistry in Swift, held to these
+211
View File
@@ -0,0 +1,211 @@
"""
Copyright (c) 2026-, Zeph Leggett.
This file is part of jetlink and is licensed under the MIT License.
See the LICENSE file in the root directory for more details.
sunnypilot's big-model catalog: which large models exist and which commit each one is.
The catalog is the same JSON the comma's model manager caches for its chestnut
slot, so a model picked here is a model the comma can ask for. A bundle's
artifacts are tinygrad pkls for a GPU we do not have; the only field that
matters is `ref`, the comma openpilot commit the bundle was compiled from,
because that commit's ONNX is what a jetlink server runs.
Stdlib only, and every network call takes an `opener` so tests stay offline.
"""
from __future__ import annotations
import json
import re
import urllib.error
import urllib.request
from dataclasses import dataclass
CATALOG_URL_TEMPLATE = 'https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_chestnut_v{version}.json'
# v26 is v25 plus Cinque Terre V3, at the same selector version. v27 onwards
# are selector 20 and a newer tinygrad, for sunnypilot's next sync.
CATALOG_VERSION = 26
CATALOG_URL = CATALOG_URL_TEMPLATE.format(version=CATALOG_VERSION)
# sunnypilot publishes a new catalog version for new models or a new runtime,
# and keeps the old ones. The versions after the pinned one are probed up to
# the first that is not there, so a model published later is listed without a
# release of this package. See merge_catalogs.
PROBE_LIMIT = 10
# zoompilot's own big models: ones sunnypilot does not list (yet), such as
# previews from comma's open pull requests, in the catalog's shape. Merged
# after sunnypilot's catalogs, so their entry for the same commit wins. An
# edit on main reaches every comma within the hour, with no release.
EXTRA_CATALOG_URL = 'https://raw.githubusercontent.com/zoompilot/jetlink/refs/heads/main/catalog/extra_big_models.json'
# The selector version the fork requires (REQUIRED_JSON_VERSION on the comma).
# It is a string in the JSON; bundles at any other version describe fields we
# would misread.
REQUIRED_SELECTOR_VERSION = 19
# Cinque Terre V3 Model (September 17, 2026)
DEFAULT_BIG_MODEL_REF = 'bf3e3631b3f91d92a1020a5e0dd4298b93ff4244'
# its name as sunnypilot's catalog lists it, without the build date: what a
# comma with no catalog to name it from shows (jetlink.openpilot, on a fork
# without sunnypilot's model manager)
DEFAULT_BIG_MODEL_NAME = 'Cinque Terre V3 Model'
CATALOG_TIMEOUT = 10.0
_REF = re.compile(r'[0-9a-f]{40}')
_SHA256 = re.compile(r'[0-9a-f]{64}')
class RegistryError(Exception):
"""Anything the registry refuses to do."""
class NetworkError(RegistryError):
"""A server could not be reached or did not answer sensibly."""
class VerifyError(RegistryError):
"""Bytes arrived, but not the bytes that were asked for."""
class NotFound(NetworkError):
"""The server answered that there is nothing at the URL."""
@dataclass(frozen=True)
class CatalogModel:
name: str
short_name: str
ref: str
build_time: str
index: int
def is_ref(value: str) -> bool:
"""A comma openpilot commit: 40 lowercase hex characters."""
return isinstance(value, str) and _REF.fullmatch(value) is not None
def is_sha256(value: str) -> bool:
"""A model identity: 64 lowercase hex characters, which is also the LFS oid."""
return isinstance(value, str) and _SHA256.fullmatch(value) is not None
def parse_catalog(data: dict) -> list[CatalogModel]:
"""The big-model bundles, newest first. No IO, and no bundle can raise.
A malformed entry in a catalog served to every comma must cost that entry and
nothing else, so each bundle is parsed in its own try.
"""
found: dict[str, CatalogModel] = {}
for bundle in _bundles(data):
try:
ref = bundle.get('ref')
if not is_ref(ref) or ref in found:
continue
if _selector(bundle) != REQUIRED_SELECTOR_VERSION or not bundle.get('is_big'):
continue
found[ref] = CatalogModel(name=str(bundle.get('display_name') or ref[:10]), short_name=str(bundle.get('short_name') or ''),
ref=ref, build_time=str(bundle.get('build_time') or ''), index=int(bundle.get('index', 0)))
except (AttributeError, TypeError, ValueError):
continue
return sorted(found.values(), key=lambda m: m.index, reverse=True)
def http_get(url: str, timeout: float, opener=None, limit: int = -1) -> bytes:
"""The body at `url`, at most `limit` bytes. A 404 is NotFound, and every
other failure a NetworkError, so a caller only catches what it treats apart."""
opener = opener or urllib.request.urlopen
try:
with opener(url, timeout=timeout) as response:
return response.read(limit)
except urllib.error.HTTPError as e:
raise (NotFound if e.code == 404 else NetworkError)(f"could not fetch {url}: {e}") from e
except (OSError, ValueError) as e:
raise NetworkError(f"could not fetch {url}: {e}") from e
def http_json(url: str, timeout: float, opener=None, kind: type = dict):
"""http_get, parsed, and of the type asked for."""
body = http_get(url, timeout, opener)
try:
data = json.loads(body.decode())
except ValueError as e:
raise NetworkError(f"{url} did not serve JSON: {e}") from e
if not isinstance(data, kind):
raise NetworkError(f"{url} did not serve a JSON {kind.__name__}")
return data
def fetch_catalog(url: str = CATALOG_URL, timeout: float = CATALOG_TIMEOUT, opener=None) -> dict:
"""The catalog JSON. Every failure, transport or content, is a NetworkError."""
return http_json(url, timeout, opener)
def fetch_catalogs(opener=None) -> dict:
"""The pinned catalog merged with every one sunnypilot has published since,
whose commits the server runs too, and then zoompilot's extra models. The
probe stops at the first version that is not there, and a missing extra list
adds nothing. Any other failure raises: a short list would drop a model
found only past it from whatever the caller caches, so the caller keeps its
last complete one instead."""
found = [fetch_catalog(CATALOG_URL, opener=opener)]
for v in range(CATALOG_VERSION + 1, CATALOG_VERSION + 1 + PROBE_LIMIT):
try:
found.append(fetch_catalog(CATALOG_URL_TEMPLATE.format(version=v), opener=opener))
except NotFound:
break
try:
extra = fetch_catalog(EXTRA_CATALOG_URL, opener=opener)
except NotFound:
extra = {}
merged, pins = merge_catalogs([*found, extra]), pinned_pointers(extra)
return {**merged, 'pointers': pins} if pins else merged
def pinned_pointers(data: dict) -> dict[str, dict]:
"""The extra list's well-formed pins, {ref: {'oid', 'size'}}: the ONNX jetlink
runs for that commit whatever a lookup said. See catalog/README.md."""
pins = data.get('pointers')
if not isinstance(pins, dict):
return {}
return {ref: {'oid': p['oid'], 'size': p['size']} for ref, p in pins.items()
if is_ref(ref) and isinstance(p, dict) and is_sha256(p.get('oid'))
and type(p.get('size')) is int and p['size'] > 0}
def merge_catalogs(catalogs: list[dict], selector: int = REQUIRED_SELECTOR_VERSION) -> dict:
"""One catalog in the first one's shape, listing every big model of them all.
A model some catalog lists at `selector` comes through as published, so a
chestnut can still fetch its build. One listed only at another selector
version, which is where sunnypilot puts every model once it moves runtimes,
comes from the newest catalog that has it, retyped to `selector` and with no
artifacts: an accelerator runs the commit's ONNX and needs nothing else from
the entry, and a chestnut has nothing to download.
"""
if not catalogs:
return {}
kept: dict[str, dict] = {}
others: dict[str, dict] = {}
for data in catalogs:
for bundle in _bundles(data):
ref = bundle.get('ref')
if not is_ref(ref):
continue
if _selector(bundle) == selector:
kept.setdefault(ref, bundle)
elif bundle.get('is_big'):
others[ref] = bundle # the newest catalog's entry wins
extra = [{**b, 'minimum_selector_version': str(selector), 'models': []} for ref, b in others.items() if ref not in kept]
return {**catalogs[0], 'bundles': [*kept.values(), *extra]}
def _bundles(data: dict) -> list[dict]:
bundles = data.get('bundles') if isinstance(data, dict) else None
return [b for b in bundles if isinstance(b, dict)] if isinstance(bundles, list) else []
def _selector(bundle: dict) -> int | None:
try:
return int(bundle.get('minimum_selector_version', 0))
except (TypeError, ValueError):
return None
-193
View File
@@ -1,193 +0,0 @@
"""
Copyright (c) 2026-, Zeph Leggett.
This file is part of jetlink and is licensed under the MIT License.
See the LICENSE file in the root directory for more details.
Which big models a Jetson can run, and which ONNX each one is: the model
index, zoompilot/jetlink-model-index.
A big model is a comma openpilot commit (its ref), and what a Jetson runs is
the ONNX that commit was built from. Finding that ONNX takes reading comma's
trees, its precompiled pkls' weights and its model repo, so the index repo does
it once an hour for every commit any catalog, comma master or comma pull
request has named, and publishes the answers:
default the model a Jetson runs with nothing picked
refs every big-model commit ever seen -> {oid, size}: an old pick still resolves
onnx every ONNX comma has published -> {size}: a pick by hash alone is sized
models what to list, one entry per ONNX, in display order, with its folder
and the catalog fields a list in sunnypilot's shape needs
The comma and the server each keep the last index that parsed, so both work
offline with what they had, and neither looks anything up itself.
Stdlib only, and every network call takes an `opener` so tests stay offline.
"""
from __future__ import annotations
import json
import re
import urllib.request
from dataclasses import dataclass, field
INDEX_URL = 'https://raw.githubusercontent.com/zoompilot/jetlink-model-index/main/index/v1.json'
# the index's own version: another one describes fields we would misread
SCHEMA = 1
INDEX_TIMEOUT = 10.0
# a listed model with no folder of its own
OTHER_FOLDER = 'Other Models'
# the runtime fields of a listed model the index has none for (an index older
# than them): what sunnypilot's big models are built for
RUNTIME = {'generation': '12', 'environment': 'development', 'runner': 'tinygrad'}
# what runs with nothing picked before the comma or server has an index, which
# names its own (ModelIndex.default): Cinque Terre V3 Model (September 17, 2026)
DEFAULT_BIG_MODEL_REF = 'bf3e3631b3f91d92a1020a5e0dd4298b93ff4244'
# its name without the build date: what a comma with no index to name it from
# shows (jetlink.openpilot, on a fork without sunnypilot's model manager)
DEFAULT_BIG_MODEL_NAME = 'Cinque Terre V3 Model'
_REF = re.compile(r'[0-9a-f]{40}')
_SHA256 = re.compile(r'[0-9a-f]{64}')
class RegistryError(Exception):
"""Anything the registry refuses to do."""
class NetworkError(RegistryError):
"""A server could not be reached or did not answer sensibly."""
class VerifyError(RegistryError):
"""Bytes arrived, but not the bytes that were asked for."""
class IndexInvalid(RegistryError):
"""An index that cannot be used: another schema, or nothing listed."""
def is_ref(value) -> bool:
"""A comma openpilot commit: 40 lowercase hex characters."""
return isinstance(value, str) and _REF.fullmatch(value) is not None
def is_sha256(value) -> bool:
"""A model identity: 64 lowercase hex characters, which is also the LFS oid."""
return isinstance(value, str) and _SHA256.fullmatch(value) is not None
@dataclass(frozen=True)
class Pointer:
oid: str # the ONNX SHA-256
size: int
@dataclass(frozen=True)
class IndexModel:
ref: str
oid: str
size: int
name: str # as sunnypilot's catalog names it, build date and all
short_name: str # '' when the index has none
folder: str
build_time: str # ISO-8601, or ''
pr: int | None # the comma pull request a model listed only as one is
is_20hz: bool
generation: str # the runtime it was built for, as sunnypilot's catalog says
environment: str
runner: str
@dataclass(frozen=True)
class ModelIndex:
models: tuple[IndexModel, ...] # in display order
refs: dict[str, Pointer] # every ref the index resolves, the listed ones included
sizes: dict[str, int] # every ONNX the index knows, by oid
default: str | None = None # the listed ref that runs with nothing picked
# the listed models by their ONNX: one entry per ONNX, so one each
by_oid: dict[str, IndexModel] = field(init=False, repr=False, compare=False)
def __post_init__(self):
object.__setattr__(self, 'by_oid', {m.oid: m for m in reversed(self.models)})
def pointer(self, ref: str) -> Pointer | None:
return self.refs.get(ref)
def size(self, oid: str) -> int | None:
return self.sizes.get(oid)
def listed(self, ref: str) -> IndexModel | None:
"""The listed model for a ref's ONNX: its own entry, or for a recompile of a
listed model, the newest ref's (each ONNX is listed once)."""
pointer = self.refs.get(ref)
return self.by_oid.get(pointer.oid) if pointer is not None else None
def _pointer(value) -> Pointer | None:
if not isinstance(value, dict) or not is_sha256(value.get('oid')):
return None
size = _size(value.get('size'))
return Pointer(value['oid'], size) if size else None
def _size(value) -> int | None:
# bool is an int; a size is not
return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None
def _text(value) -> str:
return value if isinstance(value, str) else ''
def parse_index(data) -> ModelIndex:
"""The index, or IndexInvalid naming why it cannot be used. No IO. An entry
that is malformed costs that entry and nothing else; an index of another
schema, or one listing nothing, is refused whole, so a caller keeps the
last one that parsed rather than an empty list."""
if not isinstance(data, dict):
raise IndexInvalid('the index is not a JSON object')
if data.get('schema') != SCHEMA:
raise IndexInvalid(f"the index is schema {data.get('schema')!r}, not {SCHEMA}")
refs = {ref: p for ref, value in _items(data.get('refs')) if is_ref(ref) and (p := _pointer(value))}
sizes = {oid: s for oid, value in _items(data.get('onnx')) if is_sha256(oid) and isinstance(value, dict) and (s := _size(value.get('size')))}
models: dict[str, IndexModel] = {}
for entry in data.get('models') if isinstance(data.get('models'), list) else []:
if not isinstance(entry, dict) or not is_ref(entry.get('ref')) or entry['ref'] in models or (p := _pointer(entry)) is None:
continue
ref = entry['ref']
models[ref] = IndexModel(ref=ref, oid=p.oid, size=p.size, name=_text(entry.get('name')) or ref[:10],
short_name=_text(entry.get('short_name')), folder=_text(entry.get('folder')) or OTHER_FOLDER,
build_time=_text(entry.get('build_time')), pr=_size(entry.get('pr')),
is_20hz=entry['is_20hz'] if isinstance(entry.get('is_20hz'), bool) else True,
**{k: _text(entry.get(k)) or v for k, v in RUNTIME.items()})
if not models:
raise IndexInvalid('the index lists no model')
# a listed model is what is shown and run, so its own entry names its ONNX
for m in models.values():
refs[m.ref] = Pointer(m.oid, m.size)
for p in refs.values():
sizes.setdefault(p.oid, p.size)
default = data.get('default')
return ModelIndex(models=tuple(models.values()), refs=refs, sizes=sizes, default=default if default in models else None)
def _items(value) -> list[tuple]:
return list(value.items()) if isinstance(value, dict) else []
def fetch_index(url: str = INDEX_URL, timeout: float = INDEX_TIMEOUT, opener=None) -> tuple[dict, ModelIndex]:
"""The index as served, and parsed: the raw JSON is what a caller keeps.
A transport failure is a NetworkError, an index that cannot be used
IndexInvalid; both are RegistryErrors, and either way nothing is kept."""
opener = opener or urllib.request.urlopen
try:
with opener(url, timeout=timeout) as response:
body = response.read()
except (OSError, ValueError) as e: # HTTPError is an OSError
raise NetworkError(f"could not fetch {url}: {e}") from e
try:
data = json.loads(body.decode())
except ValueError as e:
raise NetworkError(f"{url} did not serve JSON: {e}") from e
return data, parse_index(data)
+222 -37
View File
@@ -4,14 +4,27 @@ Copyright (c) 2026-, Zeph Leggett.
This file is part of jetlink and is licensed under the MIT License.
See the LICENSE file in the root directory for more details.
Getting a large model's ONNX by its oid, the one the model index names
(jetlink.registry.index).
Getting a large model's ONNX by its git-lfs oid.
The bytes are on comma's LFS servers. Since openpilot #39206 comma's own LFS
client stores objects in a HuggingFace bucket, read by oid; the older LFS
servers, and comma's model repo for the exports behind a precompiled pkl,
speak the LFS batch protocol, and which one has an object varies with the
model's age, so each is asked in turn.
comma overwrites one file per model, so the commit is the only name a big
model's ONNX has. GitHub's raw host serves the LFS pointer for any commit it
holds, merged or not, and that pointer carries the oid and size the comma will
ask a jetlink server for. The bytes themselves are on comma's LFS servers,
which is GitLab and not GitHub: each is asked in turn because which one has an
object varies with the model's age.
Since openpilot #38930 a commit can ship a precompiled tinygrad
pkl instead, with no ONNX in the tree at all; Cinque Terre V3 is one. Its
subject names the export ("Use f78ed37d for the precompiled eGPU driving
model") and the ONNX is in comma's HuggingFace model repo, in the folder that
id starts. sunnypilot's model builds find it the same way. That repo speaks
the LFS batch protocol too, so it is one more endpoint to ask.
Not every such commit names its export. ResAction's pull request (#39037)
adds the ONNX in one commit and replaces it with the pkl in the
next, subject "compiled", and a squash merge's subject only names the pull
request. The pointer is still in a diff: the one the commit's own patch
deletes, or the last one its pull request's patch carries.
The zoompilot fork downloads through this module too, asking its own
.lfsconfig endpoint first. No openpilot imports.
@@ -21,29 +34,49 @@ from __future__ import annotations
import hashlib
import json
import logging
import re
import shutil
import urllib.parse
import urllib.request
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from jetlink.registry.index import NetworkError, Pointer, RegistryError, VerifyError
__all__ = ['BUCKET', 'LFS_ENDPOINTS', 'Pointer', 'lfs_download', 'lfs_hrefs', 'lfs_resolve']
from jetlink.registry.catalog import NetworkError, NotFound, RegistryError, VerifyError, http_get, http_json, is_sha256
log = logging.getLogger('jetlink.registry')
# where comma's LFS client keeps objects since openpilot #39206. A HuggingFace
# bucket is not an LFS server: an object is read by its oid under it (lfs_resolve)
BUCKET = 'https://huggingface.co/buckets/commaai/openpilot-lfs'
BIG_ONNX = 'big_driving_supercombo.onnx'
POINTER_URL = 'https://raw.githubusercontent.com/commaai/openpilot/{ref}/openpilot/selfdrive/modeld/models/' + BIG_ONNX
# a commit's subject without the API, whose anonymous limit a car behind CGNAT shares
COMMIT_PATCH_URL = 'https://github.com/commaai/openpilot/commit/{ref}.patch'
# every commit of a pull request, one patch after another, for a squash merge's subject
PULL_PATCH_URL = 'https://github.com/commaai/openpilot/pull/{number}.patch'
# how much of a patch is read for the pointers in its diffs; a model commit's is 3 KB
PATCH_MAX = 256 << 10
# the diff header of the big ONNX, wherever the tree keeps selfdrive
_ONNX_DIFF = re.compile(r'^diff --git a/\S*selfdrive/modeld/models/' + re.escape(BIG_ONNX) + r' ')
# a squash merge's subject ends with its pull request
_PULL = re.compile(r'\(#([0-9]+)\)$')
DRIVING_MODELS_REPO = 'commaai/openpilot_driving_models'
DRIVING_MODELS_TREE_URL = f'https://huggingface.co/api/models/{DRIVING_MODELS_REPO}/tree/main'
LFS_ENDPOINTS = (
BUCKET,
'https://gitlab.com/commaai/openpilot-lfs.git/info/lfs', # every object, older and PR-branch models included
'https://huggingface.co/commaai/openpilot-lfs.git/info/lfs', # the current ones
'https://huggingface.co/commaai/openpilot-lfs.git/info/lfs', # where comma is moving them; the current ones
f'https://huggingface.co/{DRIVING_MODELS_REPO}.git/info/lfs', # the exports behind a precompiled pkl
)
# an export folder is a uuid; the subject names its first eight
_EXPORT_ID = re.compile(r'\b[0-9a-f]{8}\b')
# how comma's model commits name an export: the whole uuid, or its first eight
# and the checkpoint ('8d736683/12864'). A bare eight could be any hex word
_NAMED_EXPORT = r'\b{}[-/]'
LFS_MEDIA_TYPE = 'application/vnd.git-lfs+json'
POINTER_TIMEOUT = 10.0
CONNECT_TIMEOUT = 30.0
# A pointer is 134 bytes. Anything larger is the ONNX itself, served by a host
# that resolved the LFS filter for us, and reading a gigabyte to find that out
# is not on.
POINTER_MAX = 4096
CHUNK = 4 << 20
FREE_SLACK = 64 << 20
@@ -51,30 +84,190 @@ ProgressFn = Callable[[float], None]
StopFn = Callable[[], bool]
def _bucket_href(bucket: str, pointer: Pointer, timeout: float, opener) -> str | None:
"""A bucket's address for the object, or None when it does not have it or
cannot be reached: its first byte is asked for, as cheap as a HEAD."""
url = f"{bucket}/resolve/sha256/{pointer.oid}"
try:
with opener(urllib.request.Request(url, headers={'Range': 'bytes=0-0'}), timeout=timeout) as response:
response.read(1)
except (OSError, ValueError) as e:
log.warning("%s has no %s (%s)", bucket, pointer.oid[:16], e)
@dataclass(frozen=True)
class Pointer:
oid: str # the ONNX SHA-256
size: int
def parse_pointer_text(text: str) -> Pointer | None:
"""The oid and size in a git-lfs pointer's text, or None if it is not one."""
if not isinstance(text, str) or len(text.encode('utf-8', 'replace')) > POINTER_MAX:
return None
return url
oid = None
size = None
for line in text.splitlines():
key, _, value = line.partition(' ')
if key == 'oid':
oid = value.removeprefix('sha256:').strip()
elif key == 'size':
try:
size = int(value)
except ValueError:
return None
if not is_sha256(oid) or size is None or size <= 0:
return None
return Pointer(oid, size)
def fetch_pointer(ref: str, timeout: float = POINTER_TIMEOUT, opener=None) -> Pointer:
"""The oid and size of the ONNX at a comma commit: in its tree, or for a
commit that ships a precompiled pkl instead, the ONNX it was built from."""
try:
text = http_get(POINTER_URL.format(ref=ref), timeout, opener, POINTER_MAX).decode('utf-8', 'replace')
except NotFound:
return fetch_export_pointer(ref, timeout=timeout, opener=opener)
pointer = parse_pointer_text(text)
if pointer is None:
raise RegistryError(f"{ref[:10]} did not serve an lfs pointer")
return pointer
def patch_subjects(text: str) -> list[str]:
"""Every commit's subject in a patch, in order: one for a commit's, one per
commit for a pull request's."""
lines = text.splitlines()
subjects = []
for i, line in enumerate(lines):
if line.startswith('Subject:'):
subject = [line.removeprefix('Subject:').strip()]
# a long subject is folded onto indented lines; the headers end at a blank one
for cont in lines[i + 1:]:
if not cont[:1].isspace() or not cont.strip():
break
subject.append(cont.strip())
subjects.append(re.sub(r'^\[PATCH[^\]]*\]\s*', '', ' '.join(subject)))
return subjects
def diff_pointer(text: str) -> Pointer | None:
"""The big ONNX's pointer in a patch's diffs, the last one wins: the pointer a
diff gives the file, or for a diff that deletes it, the pointer it had. A
commit that swaps the ONNX for a pkl deletes the ONNX the pkl was built from."""
found = None
section: list[str] | None = None
def close():
nonlocal found
if section is not None:
added = parse_pointer_text('\n'.join(x[1:] for x in section if x.startswith('+') and not x.startswith('+++')))
removed = parse_pointer_text('\n'.join(x[1:] for x in section if x.startswith('-') and not x.startswith('---')))
found = added or removed or found
for line in text.splitlines():
# each diff, and each commit of a pull request's patch, ends the one before
if line.startswith(('diff --git ', 'From ')):
close()
section = [] if _ONNX_DIFF.match(line) else None
elif section is not None:
section.append(line)
close()
return found
def commit_subject(ref: str, timeout: float = POINTER_TIMEOUT, opener=None) -> str:
"""A comma commit's subject line, from the head of its patch."""
url = COMMIT_PATCH_URL.format(ref=ref)
subjects = patch_subjects(http_get(url, timeout, opener, POINTER_MAX).decode('utf-8', 'replace'))
if not subjects:
raise RegistryError(f"{url} has no subject line")
return subjects[0]
def _tree(path: str, timeout: float, opener) -> list[dict]:
url = DRIVING_MODELS_TREE_URL + (f"/{urllib.parse.quote(path)}?recursive=true" if path else '')
return [e for e in http_json(url, timeout, opener, list) if isinstance(e, dict) and isinstance(e.get('path'), str)]
def fetch_export_pointer(ref: str, timeout: float = POINTER_TIMEOUT, opener=None) -> Pointer:
"""The big ONNX a precompiled-pkl commit was built from. In turn: the export
its subject names, in comma's model repo; the pointer its own diff deletes;
and for a squash merge, the last pointer its pull request carried, or the
export one of that pull request's commits names.
The newest export named wins: a pull request's commits are in order, and
Mountain Dew's (#39047) named three before the one it shipped. One that is
not in the model repo yet stops the search rather than falling back to an
older one, a different model."""
url = COMMIT_PATCH_URL.format(ref=ref)
patch = http_get(url, timeout, opener, PATCH_MAX).decode('utf-8', 'replace')
subjects = patch_subjects(patch)
if not subjects:
raise RegistryError(f"{url} has no subject line")
subject = subjects[0]
ids = _newest_first(subject)
pointer, missing = _export_pointer(ref, ids, subject, timeout, opener)
if pointer is not None:
return pointer
tried = [f"its subject {_not_uploaded(missing)}" if missing else f"no folder in {DRIVING_MODELS_REPO} for {', '.join(ids)}" if ids
else 'its subject names no export']
if (pointer := diff_pointer(patch)) is not None:
log.info("%s deletes %s %s", ref[:10], BIG_ONNX, pointer.oid[:16])
return pointer
tried.append(f"its diff has no {BIG_ONNX}")
if (pull := _PULL.search(subject)) is not None:
number = pull.group(1)
pull_patch = http_get(PULL_PATCH_URL.format(number=number), timeout, opener, PATCH_MAX).decode('utf-8', 'replace')
if (pointer := diff_pointer(pull_patch)) is not None:
log.info("%s merged #%s, whose last %s is %s", ref[:10], number, BIG_ONNX, pointer.oid[:16])
return pointer
if missing is None: # an export the subject names and the repo lacks stops the search here too
pull_subjects = ' '.join(patch_subjects(pull_patch))
pull_ids = [i for i in _newest_first(pull_subjects) if i not in ids]
pointer, pull_missing = _export_pointer(ref, pull_ids, pull_subjects, timeout, opener)
if pointer is not None:
return pointer
tried.append(f"pull request #{number} {_not_uploaded(pull_missing)}" if pull_missing else f"nor does pull request #{number}")
raise RegistryError(f"{ref[:10]} has no {BIG_ONNX}: {'; '.join(tried)} ({subject!r})")
def _newest_first(text: str) -> list[str]:
"""The eight-hex ids `text` names, the last mention first."""
return list(dict.fromkeys(reversed(_EXPORT_ID.findall(text))))
def _not_uploaded(export: str) -> str:
return f"names export {export}, which is not in {DRIVING_MODELS_REPO} yet"
def _export_pointer(ref: str, ids: list[str], subject: str, timeout: float, opener) -> tuple[Pointer | None, str | None]:
"""The ONNX in the first export folder one of `ids` starts, and None; or
None and the first id `subject` names as an export that has no folder, where
the search stops; or None and None when no id has a folder."""
if not ids:
return None, None
folders = [e['path'] for e in _tree('', timeout, opener) if e.get('type') == 'directory']
for export in ids:
matches = [f for f in folders if f.startswith(export)]
if not matches:
if re.search(_NAMED_EXPORT.format(export), subject):
return None, export
continue
if len(matches) > 1:
raise RegistryError(f"{ref[:10]}: {export} starts {len(matches)} folders in {DRIVING_MODELS_REPO}")
files = [e for e in _tree(matches[0], timeout, opener)
if e.get('type') == 'file' and e['path'].rsplit('/', 1)[-1] == BIG_ONNX]
if len(files) > 1:
# a subject that names the checkpoint, as '1a421175-.../12864' does, picks one
files = [f for f in files if f['path'].rsplit('/', 1)[0] in subject] or files
if len(files) != 1:
raise RegistryError(f"{ref[:10]}: {len(files)} copies of {BIG_ONNX} under {matches[0]}")
lfs = files[0].get('lfs') or {}
oid, size = lfs.get('oid'), lfs.get('size')
if not is_sha256(oid) or not isinstance(size, int) or size <= 0:
raise RegistryError(f"{files[0]['path']} is not an lfs object")
log.info("%s names export %s: %s", ref[:10], export, files[0]['path'])
return Pointer(oid, size), None
return None, None
def lfs_resolve(endpoint: str, pointer: Pointer, timeout: float = CONNECT_TIMEOUT, opener=None) -> str | None:
"""Ask one LFS server for a download href, or None if it does not have it.
A HuggingFace bucket, comma's (BUCKET) or one a checkout's .lfsconfig
names, is asked for the object itself.
A server that is down is not different from a server that lacks the object:
either way the caller moves to the next one, so nothing raises here.
"""
opener = opener or urllib.request.urlopen
if '/buckets/' in endpoint:
return _bucket_href(endpoint, pointer, timeout, opener)
body = json.dumps({
'operation': 'download',
'transfers': ['basic'],
@@ -101,14 +294,6 @@ def lfs_resolve(endpoint: str, pointer: Pointer, timeout: float = CONNECT_TIMEOU
return None
def lfs_hrefs(endpoints, pointer: Pointer, timeout: float = CONNECT_TIMEOUT, opener=None):
"""Each of `endpoints` that has the object, in order, with its download href."""
for endpoint in endpoints:
href = lfs_resolve(endpoint, pointer, timeout=timeout, opener=opener)
if href is not None:
yield endpoint, href
def lfs_download(href: str, pointer: Pointer, dest: Path, progress: ProgressFn | None = None,
should_stop: StopFn | None = None, opener=None) -> Path:
"""Stream to a .part file, hashing as we go, and only then take the name.
-4
View File
@@ -1,4 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
exec "${ROOT_DIR}/scripts/host_tool_runner.sh" jotpluggler "$@"
+4 -37
View File
@@ -57,37 +57,6 @@ EOF
fi
}
function update_is_ready {
[ -f "$1" ] && [ ! -L "$1" ]
}
# A validated legacy return records the OS it was staged against. Check it on
# every boot before swapping trees, including manual reboots.
# Keep this in sync with TRUSTED_LEGACY_RELEASES in updated.py.
TRUSTED_LEGACY_STARPILOT_COMMIT="4a553da9ac22a14e34513c011aed04c1e93b4619"
function staged_os_is_compatible {
local staged="$1" marker="$1/.git/starpilot-retain-agnos" commit
commit="$(git -C "$staged" rev-parse --verify 'HEAD^{commit}')" || return 1
if [ "$commit" = "$TRUSTED_LEGACY_STARPILOT_COMMIT" ]; then
[ -f "$marker" ] && [ -s "$marker" ] && [ ! -L "$marker" ] && [ -r /VERSION ] && [ "$(cat "$marker")" = "$(< /VERSION)" ]
else
[ ! -e "$marker" ] && [ ! -L "$marker" ]
fi
}
function reset_update_os_environment {
unset AGNOS_VERSION AGNOS_ACCEPTED_VERSIONS
}
function maybe_activate_legacy_agnos {
local repo="$1" verifier="$2" manifest="$3" commit
commit="$(git -C "$repo" rev-parse --verify 'HEAD^{commit}')" || return 1
if [ "$commit" != "$TRUSTED_LEGACY_STARPILOT_COMMIT" ] && [ ! -L "$manifest" ]; then
"$verifier" --verify "$manifest" && sudo reboot
fi
}
function launch {
# Remove orphaned git lock if it exists on boot
[ -f "$DIR/.git/index.lock" ] && rm -f "$DIR/.git/index.lock"
@@ -106,10 +75,8 @@ function launch {
if [ $? -eq 0 ]; then
echo "${DIR} has been modified, skipping overlay update installation"
else
if update_is_ready "${STAGING_ROOT}/finalized/.overlay_consistent"; then
if ! staged_os_is_compatible "${STAGING_ROOT}/finalized"; then
echo "Staged update requires a different or unverified AGNOS version; skipping installation."
elif [ ! -d /data/safe_staging/old_openpilot ]; then
if [ -f "${STAGING_ROOT}/finalized/.overlay_consistent" ]; then
if [ ! -d /data/safe_staging/old_openpilot ]; then
echo "Valid overlay update found, installing"
LAUNCHER_LOCATION="${BASH_SOURCE[0]}"
@@ -117,10 +84,10 @@ function launch {
mv "${STAGING_ROOT}/finalized" "$DIR"
cd "$DIR"
# releases before the openpilot/ move can't swap AGNOS on newer AGNOS, use ours
maybe_activate_legacy_agnos "$DIR" /data/safe_staging/old_openpilot/openpilot/common/hardware/comma/agnos.py system/hardware/tici/agnos.json
[ ! -L system/hardware/tici/agnos.json ] && /data/safe_staging/old_openpilot/openpilot/common/hardware/comma/agnos.py --verify system/hardware/tici/agnos.json && sudo reboot
echo "Restarting launch script ${LAUNCHER_LOCATION}"
reset_update_os_environment
unset AGNOS_VERSION
exec "${LAUNCHER_LOCATION}"
else
echo "openpilot backup found, not updating"
+1 -15
View File
@@ -1,10 +1,5 @@
#!/usr/bin/env bash
# Device Params must not depend on the platform used to compile native libraries.
if [ -f /AGNOS ] && [ -z "${PARAMS_ROOT:-}" ]; then
export PARAMS_ROOT="/data/params"
fi
export OMP_NUM_THREADS=1
export MKL_NUM_THREADS=1
export NUMEXPR_NUM_THREADS=1
@@ -26,18 +21,9 @@ fi
export QCOM_PRIORITY=12
if [ -z "$AGNOS_VERSION" ]; then
export AGNOS_VERSION="20.2.2"
export AGNOS_VERSION="19.8.1"
fi
export AGNOS_UPDATE_POLICY="auto"
# C3 selects its complete matching firmware chain in the AGNOS manifest loader.
# Unknown hardware retains its installed OS.
if [ -f /AGNOS ] || [ -e /sys/firmware/devicetree/base/model ]; then
case "$(tr -d '\0' 2>/dev/null < /sys/firmware/devicetree/base/model)" in
"comma tici"|"comma tizi"|"comma mici") ;;
*) export AGNOS_UPDATE_POLICY="retain" ;;
esac
fi
export STAGING_ROOT="/data/safe_staging"
-5
View File
@@ -291,11 +291,6 @@ struct MapdOut @0xa4f1eb3323f5f582 {
computedMonoTime @31 :UInt64; # Computation boot-time nanoseconds, not send time.
sourceGeneration @32 :UInt64;
producerSession @33 :UInt64; # Random nonzero process session.
# Data-only preview from the same qualified GPS fix; no controller smoothing.
previewSpeedLimit @34 :Float32;
previewDistance @35 :Float32;
previewBoundaryId @36 :Text;
previewWayId @37 :Int64;
enum SampleStatus {
unknown @0;
+309 -357
View File
@@ -2887,12 +2887,12 @@ type MapdOut capnp.Struct
const MapdOut_TypeID = 0xa4f1eb3323f5f582
func NewMapdOut(s *capnp.Segment) (MapdOut, error) {
st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 128, PointerCount: 7})
st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 112, PointerCount: 6})
return MapdOut(st), err
}
func NewRootMapdOut(s *capnp.Segment) (MapdOut, error) {
st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 128, PointerCount: 7})
st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 112, PointerCount: 6})
return MapdOut(st), err
}
@@ -3260,54 +3260,12 @@ func (s MapdOut) SetProducerSession(v uint64) {
capnp.Struct(s).SetUint64(104, v)
}
func (s MapdOut) PreviewSpeedLimit() float32 {
return math.Float32frombits(capnp.Struct(s).Uint32(76))
}
func (s MapdOut) SetPreviewSpeedLimit(v float32) {
capnp.Struct(s).SetUint32(76, math.Float32bits(v))
}
func (s MapdOut) PreviewDistance() float32 {
return math.Float32frombits(capnp.Struct(s).Uint32(112))
}
func (s MapdOut) SetPreviewDistance(v float32) {
capnp.Struct(s).SetUint32(112, math.Float32bits(v))
}
func (s MapdOut) PreviewBoundaryId() (string, error) {
p, err := capnp.Struct(s).Ptr(6)
return p.Text(), err
}
func (s MapdOut) HasPreviewBoundaryId() bool {
return capnp.Struct(s).HasPtr(6)
}
func (s MapdOut) PreviewBoundaryIdBytes() ([]byte, error) {
p, err := capnp.Struct(s).Ptr(6)
return p.TextBytes(), err
}
func (s MapdOut) SetPreviewBoundaryId(v string) error {
return capnp.Struct(s).SetText(6, v)
}
func (s MapdOut) PreviewWayId() int64 {
return int64(capnp.Struct(s).Uint64(120))
}
func (s MapdOut) SetPreviewWayId(v int64) {
capnp.Struct(s).SetUint64(120, uint64(v))
}
// MapdOut_List is a list of MapdOut.
type MapdOut_List = capnp.StructList[MapdOut]
// NewMapdOut creates a new list of MapdOut.
func NewMapdOut_List(s *capnp.Segment, sz int32) (MapdOut_List, error) {
l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 128, PointerCount: 7}, sz)
l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 112, PointerCount: 6}, sz)
return capnp.StructList[MapdOut](l), err
}
@@ -3437,318 +3395,312 @@ func NewMapdOut_GpsSource_List(s *capnp.Segment, sz int32) (MapdOut_GpsSource_Li
return capnp.NewEnumList[MapdOut_GpsSource](s, sz)
}
const schema_b526ba661d550a59 = "x\xda\x94zkp\x1c\xd5\x95\xff9\xdd3\x1a==" +
"\x1a\xdf1\xd8\x92\xc5\xd8\xc6\xc2\xc8A\xc4F\xf8\x1f\xe4" +
"\xe0\x08Y2\xb6U#\xacV\xdb\x16v\xe1\xaa\\M" +
"_\xcd\xb4\xdd\xea\x1ew\xf7H\x1a\x17.\x1b\xc7T\x01" +
"\x7f\xa8@6\x0f'\x156<LU\x92\x85\x84\xcd\xc2" +
"\x96\x97\"\xb5\x84\"U\x86\xc5U\x90\x0a\xa9<\xc8\xe2" +
"\xe4\x0bI\x96\x14\x9b,\xa9\xcdfC\xf5\xd6\xb9=\x8f" +
"\xd6#\x18}\xf1\xa8\x7f\xe7\xf4\xb9\xe7\x9e{\xee\xb9\xe7" +
"w\xdb[N\xb6\xdc\x16\xdb\xda\x96o\x05E{8\xde" +
"\x10\xa4\x8e\x1d\xfa\xb5\xed\x7f\xe74\xa4:18\xd4|" +
"\xa0k\xea\x85\xeb\x9e\x87X\x02\xa0oM\xc3ad\xbd" +
"\x0d\x09P\x83\xef~\x90\xed?\xfc\x9f/\xdf3_+" +
".\xd5\x9a\x1aN \xebjH\x00\xb05\x0d\xef\x02\x06" +
"w\x0e\xbd\xfc\xf9\xc9\xeb\xbeq/\xa4:\x95\xba2`" +
"\x1f&F\x90\xadJ\xd0K\xa9\x84\x13\x03\x0cz_-" +
"\xdd\xf9\xed5\x0f=\x02Z/6\x05/\xdb\xff\xdar" +
"\xf0\x95\xbb\xfe\x0b\xe2H:\x0f\xb5<\x80\xec\xa9\x16\xfa" +
"\xf3\xf1\x96w\x14\xc0\xe0\xef?\xf1~\xc9<q\xf8\xcb" +
"\x90\xbaV\x09\xce\xfc\xe9O\xd7\xf6\xfd\xc7\x1f\xce\x93\xe5" +
"\x9f\xad8\x83\xec\xf7+\xc8\x87\xdf\xae\xf8\x0a`\xb0\xf2" +
"9\xcc\xe7/^\xfa\xfa\x12\xd3\xeaON\"\x1bM\xd2" +
"\xb4z\x7f\x91Q\x87\x13\x85\xc7\x97\xd0\xeaI\x1eF\xb6" +
"CjYj\xdfm\x7f\x9elx\x12\xb4N\\4\xfb" +
"5\xa4\xd7\x9b\x0c_\x99@\xc0\xba_Z'\xb6G\xd4" +
"\xe5\xbc\x1fm\xef@\xf6L\xfb&\x80\xbeW\xdb'\x1a" +
"\x01\x83\xe1\xa1\x97^{\xe6\x9d'\x9fZ\x14\xadC\x1d" +
"g\x90Mw\xd0\x9c\xcc\x8eO\x01\x06\x07\xcek?\xbb" +
"a\xe6\xd2SKx;\xddq\x18\xd9=\x1d\xe4\xedw" +
"~\x91\xfb\xd2\xff\xff\xe6\xfb\xff\xb0\xc8\xde\x91\x8e\xedu" +
"{\xfb\x00\x83#\xcf\xbf\xbb\xf5\xf3\x1f\xbe\xf3\x9d%\xec" +
"\x9d${\x8fH{\x03\xef\xfc\xee\xe4\xe8\xba\x89g\x97" +
"\x1cu\xb2:\xeaE\xcc\xae7s\xf6\x85%\xb4\x8e\x90" +
"\xad\xe3R\xab\xf0\xe2\xdf}Q\xff\xf4\xf0\xf7\x97\xd0\x1a" +
"%[\\j\x9dUo\xbe,n\xdd\xf8\xf2\x12Z;" +
"HK\x93Z\xef\xff\xd3g\xde\xbce\xc7\xd1\x8b\x0bW" +
"E!\xbd\xde\x8e\x95\xc8vt\xc8\xe5\xee\xc8\xd0\xaal" +
"?4Q\xb4\xde\xfa\xc6\xbf-5t'\x0d\xddIF" +
"\xdf8\xf3X\xfe\x7f\x7f\xfe\xe5KK\x0dMZ\x9a\xd4" +
"\xeaz%\xf7\xf6\xdb\xfe\x997\x17\x85\xb8\xb7s'\xb2" +
"\x1d\x9dr\xd8N\x99\x0c\xfd\xfd/>\xfe\xfa\x03\xff\xfd" +
"\x13\xf2R]\x98bkG\x90\xedXK\x0b\xd2\xbf\x96" +
"6\xce\xb6\xec\xf0\xe7\xceM|\xf9\xe7K\x8c\xde\xd5u" +
"\x18\xd9\xd6.\x1a\xbd\xffs\xaf}\xed+\xf1\xaf\xfc;" +
"\x99T\x16\xa8\xb5u\xedD\xd6\xd5%\xf7b\xd7,`" +
"p\xfd\x9b?y\xff\x86]\xcf\xfdq\xd1\x8e9\xd9u" +
"\x14\xd9#R\xf3\xa1\xae<`}\xf3\xcd7\x1bn\xc5" +
"\x7f!\xedK]W\x03\xf4\xbd\xd5\x95H\x00\x06k\x1e" +
"}to\xd3{W}\xb0\x84\xaf\xbf\xec>\x8c\xec\x0f" +
"\xdd\xe4\xeb\x93\xb1\xe2\x87\x9f>\xfb\xd0\xff,\xa1u\x89" +
"\xb4~%\xb56\xff\xa6\xfb\x8ew\xef\xfa\xcc_\x16\xc5" +
"\xf3\xfb\xdd\x93\xc8\xde\xec&'/u\x9f\x02\x0c\x9e\xf0" +
"\xc6.\xbev\xe4\xc9\xbf,tR.z\xdbug\x90" +
"\xad\xbfN\x86\xeb:\x19\xfd\xd4?\xcf\xde\xf7\xfb\xc1\xc9" +
"\xbf.1\xfc\xa3\x9b&\x91\xfd\xe3&\x1a\xfe\xd4\xb9g" +
"\xdf\xd5\xcf\xdd\x17,\xcc\xa4\xb0\x08mz\x01\xd9S\x9b" +
"\xc8\x85\xc77}\x17z\x83\x9cp\x05\xb7>\x99\x8b\x95" +
"<\xdf\x99\xfedN\xfe\xdc\x98\xe3E\xbb\xb8}H>" +
"\x8c\x0bO\xb83B5n\x1eC\xfcH}\xdd\xe7\xee" +
"\x98i9\xfe\x10w\x87\x06\x1c\xdbw\x1dk\x0cQk" +
"Wc\x001\x04H\xf1\xc3\x00\xdagU\xd4,\x05S" +
"\x88i$\xd0\xfc\x02\x80f\xa9\xa8\xcd)\x98R\x944" +
"*\x00\xa9\x12i\xfa*j\xa7\x15L\xa9j\x1aU\x80" +
"\xd4\xc9\x11\x00\xedn\x15\xb5\xfb\x14\x0c\x0a%c\x88\xc6" +
"\x00\xd5\xb1p%(\xb8\x120\xf0|!\\\xd3\xcec" +
"\xd6\x9c6\xfd\xbd\xf6\x14:5\x99\xcdg\xcc<\xf7M" +
"P\x1d\xbb\x06\x16\x0b\x8e-v\x17=\x00\xc0\xf6z\x01" +
"\x07\xc4v\xf8\xe8\xd9\x8e\xf2\xa2\xb1\xd7.\x96\xfc\xfd\xe5" +
"\xa2\x00\xa0\x89\xbeH\xae\xb3^e\x04\x00W\xb0\x1e\xe5" +
"{\x00\x98d=\xca\x13d\x9c\xf5(\xdf\x06\xc0\x14\xeb" +
"Q^\x06\xc0\x95\xacG\xf99\x00\xb6\xb0^e\x12\x00" +
"\x19\xebQ^\x03\xc04\xeb\x95\xbf\xc8\xb6*'\x00P" +
"a\xbd\xcaQ\x00\\\xc5z\xe4\xf3U\xac[\xf9\x1d\x00" +
"^\xcdz\x94\x1f\x01\xe0j\xd6\xab\xfc\x1a\x00\xd7\xb0\xad" +
"\xf2\xb9\x83mS\xbe\x06\x80\x9dl\x9b\x1cw-\xdb&" +
"\xedu\xb1~\xf9|\x0d\xeb\x97~\xa9\x95\xe7\x18\xeb\x97" +
"\xfedX\xbf\xb4\xbb\x8e\xed\x90\xf6\xd6\xf7\x0d*\xdb\x11" +
"\x00\xe3l\xaf\xf2\x02\x006\xb0\xbd\xd2\x816\xb6K9" +
"\x0c\x80\xadlP:\xb6\x81\xed\x90/^\xcb\x06\xa5\xe1" +
"\x8d\x95\xdf\xee\xbeAe%\x19hf{\x95\x07\x00\xf0" +
":\xb6W\xf9#\x00n\xea\x1bU6\x90\xe0zv@" +
"\x86\xa0\xa7\xef\x90\xa2\x10\xb0\xb9\xefH\xf8\xc7'\x18W" +
"\xbe\x00\x8070._\xede\\\x06/Qynd" +
"\\9\x03\x80M\xec\x88\xfc\xbd\x91\x1d\"\x1f\x02\xc3\x99" +
"\xb5-\x87\x1b\x00\x10x\xc2\xdf\xcf\xdd\xbc@?\xcb}" +
"\xe1r+3\x98\xcb\x09\x8bp\xbd(\x84\x11\xa6\xc8\xbe" +
"\xa9\xa9\x84'\xfc\x05\xe8\x90c'){\x09\x1e\xe5\xc5" +
"\xa1R\xdc\x9d\x11R>\x14\xa65x\xc2?hz\xa6" +
"c\x0f\x95\xe6\x89\xd4\xf0\xa5\xac\x93\xcf\x0aH\xcc\x84\xe3" +
"IM%T\x95>\x91K\x83\xa1\x8fQ\xd9\xa8i\x87" +
"\xe2\x83\x00\x81+h&\xba\x80\x01\xdf7\xed\xbc\x17x" +
"|F\xe8\xc2\xf7!\x19>\x0a\x7f\x97\xcd'-\x18\x08" +
"\x87_h\xec\x80'\xa4\\\xe8\xc9\xaaXNE\x99'" +
"+\x0a\x81Fm\xf6\x8a\x9c}D\x9a\xa8\xbc\xb9\xc7\xb1" +
"\x8c\xac\xc2=_\x17\xc2\x96\xaa\xa4\x89~$\xca\x12\x1d" +
"\x11\xaa{l!8\x98KT\x02/Q%D\xf7\x9b" +
"\xd3b\xdf\xd4\x94'\xfc0\x10\xc3b\x8a\x97\xd0\xf2\xb3" +
"\xdc\x16\x13f\xc2\xf0\x0b5\x97\xb1\x1a\xb7\x8c\x0c\\@" +
"\x81!u,Y>E\xc4LP@\x08\x1d\x179'" +
">=-lC\x18Rb\xe7=Z+\xddrf\x87" +
"\x9dY\xfbv\xc7\xbdC\xcc\x85\x0ed\x934\xd9\xfa\xdc" +
"\x0f\x14\xe7I\xcdDEJs\xd7\xd5\xea\x9c\xfd\x89\x82" +
"i\x89\xa1\x02\xb7\xf3\xa6\x9d\xd7\xc5@\xa8.G\x1f\x13" +
"\xae\x87\xa6\xe7\x0b\xdb'A\xb8l9n\xe7\x845\xec" +
"\xc0@\x98\x9b\x95\xf4\x18\xf1\xa8\x0eU\x1et\x07\x92%" +
"7'\xe4\xa2\xce\xf9\xc2Uln\xd5\xc2</\x1d\xa5" +
"\x18\xab\xe2Lv\xde\x1c\xc2\xec\x1ds\xcd\x8c\xe3\x9a~" +
"\xb9\x86\xab\xa1\x19rZ\x8c\x8b\xe3%\xd3\x15\x1e\xed\x86" +
"\"\xfa\x01\xa7__/bu\xb4p9\xc6\\\xe1y" +
"\xcan\xee\xedw\x06+\x1a\xf3\xc6\x1b4\x8e\x96<\x95" +
"\xc2\x1f\xaefT\xab\x1e;\x09*~}*\xb4\xea\x8e" +
"Z\xf2kC4\xc9!\xf6\xcd\x08\xd75\x0dQW\xa4" +
"U\x1brl\xc3\xf4Mga0\xaa;P/p\xc3" +
"\x99\x1d\xe2\xae\xees\x1fe\xf4$\x84\xb3\xa3\x8e!\xac" +
"\x837\x85S\xa9`\xbb\x8b^\xd6\xc9\xf1$\x19$x" +
"\xc4s\xec1\xe4~\xe1v\xcb\xe1\x95YK\xac\x81\xfb" +
"\x85\xfdbN\xba \x01\xee\x17v:\xd1!+\xa6\xc8" +
"\x12\xadG\xd2\xb5\xb9U;,\xe2W8\x1a\xc9Yq" +
"\xa3<s\xe8\xd0\xb8\xb9z:\xb2#x\x02@\xbf\x0b" +
"U\xd4\x0bX? \x99\xc0\x11\x00\xdd \xbc\x88\xf53" +
"\x92M\xe38\x80n\x11>\x87\xf5c\x92\x95\xa4\xbeO" +
"\xf8i\xc2c\x8di\x8c\x01\xb0\x93x\x13\x80>G\xf8" +
"Y\xc2\xe3Mi\x8c\x03\xb0{\xf00\x80~\x9a\xf0\x07" +
"\x09ohNc\x03\x00\xbb\x1f\xbf\x06\xa0?H\xf89" +
"\xc2\x13-ij\"\xd8\x97\xf0\x01\x00\xfd\x1c\xe1\xe7\x09" +
"olMc#u\x14R\xff<\xe1\xcf\x12\xde\xd4\x96" +
"\xc6&\x00\xf6\x0c\xba\x00\xfa\xd3\x84_ \xbc9\x91\xc6" +
"f\x00\xf6<>\x01\xa0_ \xfc\x15Tp}\x8b\x92" +
"\xc6\x16\x00\xf6\x03\xdc\x0e\xa0\xbfH\xf8ET\x10[\xd3" +
"\xd8\x0a\xc0~\x88\x9b\x01\xf4\x97\x08~\x1b\x15\x0c<\x87" +
"\xb6\xcc\xa8\x03\x03\xb6Ci\x85M\xa0`\x13``q" +
"\xdf\xf4K\x86\xa03\xbd\x05\x14l!\xcc\xb1\xf3\x04\x02" +
"\x8a\x1a\xc6\xadEz\x19On\xa3fP\xb0\x190\x98" +
"\x14\x9c\xba\x89aPE\xbe\x06\x16\x1c\xd7<\xe1\xd8>" +
"rk0\x97+\xb9\\\xcd\x95k\xc2\x19\xe1\xfaf\x8e" +
"[\x18\x8are:\x00\xe7[\x1b\xc4\x8ah8j\xd5" +
"\x0b+e\x0e2RV\xc3K\xb69'g\xe7\xf9|" +
"\xba8jZ\x09\xcb\xf40\x0e\x0a\xc6\x01\x07\x0a\xdc\xbb" +
"\xdd\x9cC\x04\x05\x1109s\xc7\xaea\\\x018\xa6" +
"\xa24\xb0\"\xd2\xc3\xc4\xffF\x0f\xb3\xaf\xe4\xdf\xa8\xf3" +
"\xe9\xa2%(1K\x9eL\xca\xd5\xb2\x09\x1b\xdcI\x8d" +
"H\xaa\xff&\xeaCR[\xe9\xd4WS\xbd\x04\xc6R" +
"\xdd\xd4\x0b\xc4S\xeb\x8f\x02\x9c*\xd9\xc7lg\xd6\xce" +
"\xd8\xce\xee\xa2\x17\xd8\xce\x903#\\\x0ej^\x9c\xb2" +
"\x9dQ\xee\xe7\x0a\xc14\xfd+\x8c;`\xc0\x09KH" +
"\x05\xc8BX\x85?v\x1f\x9a0\xb6\xdet\xa5Ft" +
"A\xe3\xda\x7f%}\x0a\x03\xedc:5\xf6\xa9%_" +
"F\xa0\xd6\xb4~\xf5\x01\x00\xed\xeb*j\xdf\x8c4\xad" +
"OQ+z^E\xed\xd9H\xd3\xfa\xccf\x00\xed\x9b" +
"*j\xafD\x9a\xd6\x1f\x90\xe6K*j\xaf+\x881" +
"\xb9\x13S\xaf\x9e\x01\xd0.\xaa\xa8\xfd\x98\xb6!\xcam" +
"\x98zs\x12@{CE\xedm\x05k\x9d\x0c\x8e\xb9" +
"N\x9e\xea\xa4\xecOk\x84\xa1\xd2\x9fz\xd5#\x8e\xfa" +
"1P\xb0\x150Y\xe4~\xa1\x9a\x03\xedu\xc6\x06(" +
"\xb3\xa1\xe8x\xb2\xa2\x86\xfdn\x8d|U\xecY\x8eS" +
"\x1c\xe7\xbe\xc0AZ\xc0\xbc\xa8\xe7nU\x02\x89Q\xd3" +
"\xae\xa1\xd5\xa0\xaa\x7f+\xb7\xd4\x92\xaf5b\xf4\xaa\xa3" +
"\xe9h\x84\xc55\x8d\x07\xd5\xd4\x83$%_\xb0\xbb\xe8" +
"\xe9\xb4\xaf\x01\x85v\xaeV\x19\xd7\xc4v\x02\xe8\xe9\x98" +
"\x8a\xfa\xbaX\xa42v\xc5\xa8R\xac&|c,R" +
"\x19\xd7\xc7\xa8\x02\xae#\xfc\x86\x98\x82X)\x8c=1" +
"*t\xd7\x13|3\xa9\xc70,\x8c[cTx\xb7" +
"\x10~+\xe1q%,\x8c\xfd\xb1\x17\x00\xf4[\x09\xdf" +
"Cx\x83\x1a\x16\xc6]r\xd8\xdb\x08\xcf\x12\x9e\x88\x85" +
"\x85q\xaf\xb4\xbf\x87\xf0\xfd\x847\xaaaa\xd4bT" +
"\x18\xf7\x13\xfeY\xc2\x9bbaa<\x12\xa3\xc2x\x17" +
"\xe1\x05\xc2\x9b\xe3aa\x14\xb1/\x00\xe8\x05\xc2}\xc2" +
"[\x1a\xc2\xc2x<\xf6#*\xe0\x84\x9f%\xbc\xf5r" +
"X\x19\xef\x91\xfe\xdcM\xf8}\x84\xb7u\xa5\xb1\x0d\x80" +
"\xdd\x1b\xa3\x82\x7f\x9a\xf0\x07\x09_\xf1\xab4\xae\xa0\xc2" +
".\xfd\xbc\x8f\xf0/\x12\x9elLc\x12\x80=\x12{" +
"\x8d\x0a;\xe1\xe7\x09ooJc;\x15v\x19\x9f\xc7" +
"\x08\x7f\x9a\xf0Ts\x1aS\x00\xec[r^O\x13~" +
"\x81\xf0\x95\xc94\xae\xa4\xc2\x1e\x9b\x04\xd0\x9f#\xfc%" +
"\xc2YK\x1a\x19\x00\xfb~\xec{T\xc1\x09\x7f\x9d\xf0" +
"tk\x1a\xd3\x00\xec\xd5\x18\x1d(\xaf\x13\xfeS\xc2W" +
"\xb5\xa5q\x15\x00{K\xc6\xe7\xc7\x84_&\xfc\xaa\xb5" +
"i\xbc\x0a\x80\xfdR\xea_&\xfc=\xc2\xaf\xfeu\x1a" +
"\xaf\x06`\xbf\x95\xfe\xbcG\xf8\x9f\x09_\xdd\x95\xc6\xd5" +
"\x00\xecO\xb1\xa3\x00\xfa\x071\x15\xc7\xe3\x0a\xa6\xd64" +
"\xa6q\x0d\x00\xfbP\x86\xe7\xcf\xa4\x1e#\xbc#\x9e\xc6" +
"\x0e\x00\x86qr3\x16WQo'\xbc\xf3\x9a4v" +
"\x02\xb0\xb68\xb9\xd3J\xf8j\xc2\xd7f\xd2\xb8\x16\x80" +
"\xad\x8aS8\xd3\x84\xaf#\xbckc\x1a\xbb(;\xe3" +
"t>\xaf%\xfcz\xc2\xafiN\xe35\x00\xac;N" +
"\xe1\xd9H\xf8\x16\xc23-i\xcc\x10q\x8c\xd3\xb4\xb6" +
"\x10~+\xe1\xebZ\xd3\xb8\x8e\xd2P\xe2\xb7\x12\xbe\x87" +
"\xf0\xf5mi\\Oi\x18?\x03\xa0\x0f\x13>F\xf8" +
"\x06\x96\xc6\x0d\x00l4N\xe93F\xf8]\x84_\xbb" +
"6\x8d\xd7\x02\xb0CR\xffN\xc2\x0d\xc276\xa4q" +
"#\x00\xe3R\xdf \xbcHxw2\x8d\xdd\xd4_\xc4" +
")l\x16\xe1sq\x05O\xcd\xf2\xf2\x1d|ZT+" +
"\xcd\xc0,/\x8f\x8b\xa9\xeac\xe0:\xdc y\xa4\x18" +
"\x85\xa7Z\xd6\x9c\x06\xd5\xf4kU\xc3\xae\xf4\xd80\x10" +
"vt\x8b\x04\x18\xe2\xc3\xe6\x80\xe7S\xf7\\U\x18(" +
"\xf0\x13\xdc5j\xd6I\x7f\x0f?\xc1A]\x02D\xd7" +
"\x186\xe9}\xb5n \xe0\xc6\x8c\xe99n\x192\xfa" +
"\xbc\x83\x9e^\x1a4fL$a\xe8\xc2\"\x99R\x95" +
"U\xec\xe6\xb0\xee\x98c\x8b\x09^\xae\x9e\xc3\x19\x8b\xdb" +
"\xc2\xc3\x06P\xb0\x010\xf0MKd\x89\x95\xaa\xc2\xa8" +
"\xaa\xd4\"\xa3\x98\xbe^\xca\xe7\x85\xe7\x0bC\x1a\x8f4" +
"\x0c^E\x00\x03\xc6|w\x85\xe7\x9b\xd3T\xa7\x8dq" +
"\x87\x1b\x13\xa6\xa1\xfa\x85\x9a\x90\xd6\x81\x9adH\x889" +
"\x1f\x93\xf5+T@L\x02\x06\x86Y\x89\xea\xed\xae3" +
"=\xc1\xcbC\x19a\x13\xa7\xa8\xb50\x15^\x8bUb" +
"\x1b\xf1h\x9a\xe8\x98;#\x16\xc6o\x96\x97ua\x89" +
"\x1c\xd2\xd9\x12^\x88`\xb2~\x13V\x19\xb9:g4" +
"Cb\xe0G\x03R0\xf3\x85Y^\x1e\x82\xa4\xc5=" +
"\x0f\x93\xf5{\xc9\xf0\xed\xcc,/\xef5\xaa\x9dO\x90" +
"\xabp\x83\x05T\xa8\x9ey\xf2\\9( \xe3\xd2t" +
"0\x01\x0a&*\xd1\xa1F\x07\xd4\x12\x0dR;\x98*" +
".\xe6\xeb\xe7\x0f&\xeb'U]:\xea\xd8\xce~H" +
"D\xfb\xcd\x9c3],\xf9\xc2@)3i\x07\xd4d" +
"a\x93\xba[\xa0-\\\x1e\x1e\xbc5Y\xd1u\x8cR" +
"N\xb8\xa8\x0b\x8f|\x8cJ\xc4\x8c)fu\xac3\xed" +
"Z\xa8+\xb2a\xac,#,\x94\xecD\xa7d\x1b\xdc" +
"-\xef\xc5\xfa\xae\xa8\xc8& 9?\x8a\x1f\xc5Yj" +
"\x94+\xa4\xe9rY\xa95j\x94\xcdNj\xbbl\x0e" +
"\x9bv\x02\xd06\xf5\xcd\xdc\xa9\xa2ps\xc2\xf6\x97\xd3" +
"\x9bm\x8b\xf6fK\xb5\x11\x94\xe0\xf22Q\xcc\xc9\xbe" +
"\xacU\x0e\xde\x15v\xa6\xab6\xcb\xce\xb4m'\xc0\xa9" +
")W\x88Y^N\xe6L\xbf\\\xedH\x97\xe3\xc9\xd6" +
"eu\x95\x09c\xeb\xcd\xcblC?\xb5\xdc\x01\xb6-" +
"\xf7\x85\xadW\x0a\xa6\xbc\xb3D\x9b\xe2\x98\xae\xf5\xb7'" +
"\xa9k\x9dSQ;\x1b\xe9o\xef\xb9\xa9~\xd5\x8a\x95" +
"\xf6\xf6\xde\x0d\x00\xdai\x15\xb5\x07\xa9\xbdm\x0f\xdb\xdb" +
"\xfb\xe9\xed\xb3*j\x0f\xd7\x1b\xaa\xd4C\xd4\xf3>\xa8" +
"\xa2vN\xc1\xa4_.\xd2^\xaa}G\xabl\xe7)" +
"b\xdd\xd5\xdcMx\xbe[\xeb`'\x1d\xc7\xaa\x95\x85" +
"\xa3\x15\x02\x1e=U\x96\x13\x93-\xcb\x0db\xdf\x95^" +
"\xd8S)TT\xa6\xe4~\xb8EF\xe7\xf10%\xbf" +
":\"S\xf2KG%Yz\xe4&I\x96\xee\x1f\x97" +
"d\xe9^RiH\x9d\x9c\x04\xc0D\xaaL`c\xaa" +
"\xe4RIH\x1d\xa7\xf7\x9aS\xd3\xf4^K\xca\xa4\x9f" +
"\xd6\x94 \xcd\xb6\x14\x8f\x90\xac`\xda\xf1\x1dw\x96\x97" +
"\x01\xa0\xfew2k\xda\xc72\xbe[\xb2\x8f\x05\xf2\xdf" +
"\xaci\x03\x1e;Ut\xcdi\xee\x96\x83\xcao\x16\x12" +
"\xa6},\xf0\x04UO\xee\x02\x96\xeb\x7fg\xcad#" +
"\xf0\x89\xc0rW\x9a\xaf\xfd-\xcd\x07%;Gs6" +
"!9e\x0a#p\x85g\x1a\x82\xce\x19\x93[\x81e" +
"\xce\x98v\x9e\xfaxW\x08\xff\x8at\x8b\x96t\xcc1" +
"m?\xbc\x04\x89|\"\x18Y\xea\x13\xc18\x80VP" +
"Q\xf3\xa9\xc9\x8f\x85\xe9x\x9c\xc0\xa2\x8a\xda\xdd\x94\x8e" +
"\xf10\x1d\xcb'\xea\xc9\xfc\xb1/\x02r%w\x86\xfb" +
"%\x17\xea'z\xe0\x87\x17\xb1\x02\x06,\x87J\xca\"" +
"\xce\xf31v|\xdf\xc7!\x9ecUF&Kk-" +
"\x0e=\x14\x87\xebU\xd4n\x8e\xc4a+My\x8b\x8a" +
"\xda\xad\xcb\x99\xdd\x15\x1c\xd8W\xf2o\x94\xb4K^A" +
"\xca\xd5\x90\xf1\xed\xd9,\x13z}\x98\xd0]#2\xa1" +
"\xd7l\x00H\xda\x8e-\x02\x93\x9a\x06\x9b[\x94(b" +
"\xae\xf6w\"\xc7\xdd\x8f\xfd\x95HO\xfa\xdc\x17Z\x0c" +
"#\xdf\xceS\xb8!\xb1\xbb\xe8i\xa7k\xec\xef\xb8J" +
"}hQUQ\xbf[\x8d\xb0\xbf\xb2J\xf4\xecn\xc2" +
"\xefS#\xec\xef^\x95\xfa\xeb\xb3\x84?L\xb8\x8a!" +
"\xfd{H}\x02@\x7f\x98\xf0\xaf\x13\x1eSC\xfa\xf7" +
"Ui\xff\x1c\xe1\xe7\x09\x8f\xc7B\xfa\xf7\xb8J\xfd\xf2" +
"c\x84?MxC<\xa4\x7f\xdf\x92v\x9e&\xfc\x02" +
"\xe1\x89\x86\x90\xfe=\xaf\xbe\x0c\xa0_ \xfc\x15\xc2\x1b" +
"\x13!\xfd\xfb\x81J\xec\xf5E\xc2/\x12\xde\xd4\x18\xd2" +
"\xbf\x1f\xaa\xc4\x1b^!\xfc\x0d\xc2\x9b\x9bB\xfawI" +
"%\xf6\xfa:\xe1?%\xbc\xa59\xa4\x7foI?\x7f" +
"L\xf8e\xc2[[B\xfa\xf7K\x95\xfa\xf7\xcb\x84\xbf" +
"Gx[kH\xff~\xab\x12\xff\xf8\x0d\xe1\x1f\x10\xbe" +
"\xa2-\xa4\x7f\x7fP\x89&}\xa0\xaa8.\xd9\xdf\x8a" +
"\x90\xfd}(\x87\xfd+\xa97J\xf6\x97\x0c\xd9_\\" +
"\xb2\xb9\x18\xd1\xa4v\xc9\xfe\xdaC\xf6\xd7&\xd9Y+" +
"\xe1\xab%\xfbK\x85\xeco\x95d\xcduR\xce\xd4\x90" +
"\xfd\xad\x97\xacv#\xe1[$\xfb[\x19\xb2\xbf^i" +
"\xe7\x06\xc2o\x91\xec\x8f\x85\xeco\x9b\xd4\xbf\x85\xf0a" +
"\xc9\xfe\xd2!\xfb\x1b\x94,r\x98\xf01\xc9\xfeV\x85" +
"\xecoT\xda\xc9\x12~\xa7d\x7fW\x85\xec\xef@\xec" +
"\xcc<\xf6\xbd\xe6\xea\x90\xfe\x1d\x91\xac\xb6\xce\xbe;n" +
"\x0b\xe9\xdfB\xf6\xdd9\x18\xd2\xbf\xe3\xd2N\x91\xf0\xbb" +
"\x09_\xbb3\xa4\x7fe\x89\xd7Yy\xd7PH\xff\xee" +
"\x91x\x9d}_3\x1c\xd2\xbf\xfb\xa5\x9f5\xf6\x8d\x99" +
"\x90\xfd=\x12\xdb\x00\xa0?H\xf0\xb9\x98\x82\xf2^\xde" +
"\x1as\x05$=/\xd2&sk\x96\x97\xbd}6V" +
"\xbej\xed\x1a\x90\x9fj\xea\x0a\x93.?&\xb2f\x1e" +
"\x12\x05\xdf\xab\xa1\x06\xf7\x0a\x93\x0ew\xd1\x88\xdc\xd0\xd7" +
"j\x9a!\x96\x1c\xab\xc6\x14\xe4M\xbd0`\xb1$\xeb" +
"\xd8y\x92&\x96zO9(\xdcrU#\xfa\xfe)" +
"\x91sv\x0b\xee\xd6\xde\x98r\xdc\x9c\x18r8\xa8\x9e" +
"_\x03Mo\x8c\xbb\xc7\x88\x7f\xd4\x07.\xf2\x92'\xb2" +
"\xdc\x87$M~\x01\xec\xd8(\xab\x9fis\xb4\"L" +
"\xcbq\xfd\xdd\x82\x03\xd6\xc7\xf3]>5e\xe6F\xd1" +
"1\xc2o]Q&\x92\xe7\x9e\xee\xf3\xdc\xb1\xe8\xb8\xd3" +
"\x8e!h\x160o\xa2aa\x1bs!#>r\x91" +
"\x06\x07,\xcb\x99\x8dF\xa8\xbe\x1c\xbeS\xd4\xcd<\xda" +
"5\xba\x18~0Zl4\xc4\xb3\x0eV\xe3\x89\x0be" +
"\x07\x05V\xe3\xbd\xe0\xdd\xa20\xb85\xca\xe7 3." +
"\xf2\xc2\x9e\x8f\xcb\xb8\x0d\xe6|sF\xd4'\xec\x0b\xcf" +
"\xe2CC\xbb`\xc0\xce\xf3|\xc4TE@\xfcCH" +
"\x11.\x92\xdd\x81\x8e?\xe8NG\x13Ff\xf3\x1e\x9a" +
"\xf0\x90[2\xbd\xc8@2\xf7\x96\x94\x14K\x96'\x06" +
"m\x032\xbb-\xd3\x10U<\x91/z\xcb\xfcV\xbf" +
"\xe0\\\xbee\x99\x9d\xfb\xff\xbb\x92\xfeD\x85\xfeJ\xf6" +
"\x9b(\x17\xe5)\x1av)\xdb\xc2\xb6\xb0w\\\x9e\xa2" +
"=\xe1)\xda=\"\xdb\xc2\xf5\x9b\x01N\xe5J\xaeK" +
"\xa4\xa9\xe8\x0a\xc3\xcc\xf9\x02\xd0\x08\x8a\x8e\xe7\x99\x93\x96" +
"\xa8\x9e\xad\xb6!wAr\x8a\x9b\xd6\x15/\xf4\x87+" +
"W\xc7\xb5\x9b\xe3\x05\xd7\xd9\xdb\x01\xb4/\xaa\xa8=\x16" +
"i,\x1e\x1d\x8f\xdcq+\x18\xba\xfe\xd4\xe1\xc8\x1d\xb7" +
"\xaa\x84\x0d\xd63g\x00\xb4\xa7U\xd4.\xd4\xaf\xb3\x9f" +
"\xa7\xb7\x9fSQ{#r\x9d}\x89\x14_WQ\xfb" +
"\x8d\x82\x03\\f\xd7\x82t\xb5h\xae\xf5\xdcq|n" +
"\xddnZ\xa0\x0a\x0f\x1bA\xc1F\x88\\\x83\x0b\xe3v" +
"\xd3\x12\x1e\xd4$V\xe5\xfb\x1b\xa0W\xbd\xedn\xad|" +
"\xf1\xa8\x8apX\xf8\xdc\xb4<\xa8_\x87\xd7\xfesL" +
"\xe5:|9\xc4`^\x16\\)\xf8\xd5\xcf\x83\xc3\x03" +
"\xa1\x0f\x92\xba\xd6\xd6`\x175w\xc3*jc\x0aV" +
"\x97`\x94\xa2\x9dUQ\xbb3\xb2\x04\x07(\x88\xfbU" +
"\xd4>\xabD\xa6\x1c\xa1B\xcb\x8b\xda\xff\x05\x00\x00\xff" +
"\xff1<\xb37"
const schema_b526ba661d550a59 = "x\xda\x94Z}lT\xd7\x95?\xe7\xbd\xb1\xc7\x9f\x8c" +
"\x877\x04\x0c&\x13(\xd0\x98\xc6i\x8c\xf16\xa6I" +
"]c\x03\xc1\x1a\x83\xc7\xc3G@A\xea\xf5\xbc\xeb\x99" +
"\x07\xcf\xefM\xde{c{\x10\x11\x84\x82\x04YP\x93" +
"n\xd2\x92*lC\x02R\xda\x0d-\xed\x86\x15\x1bQ" +
"-AD\"Y\x90\x92\xaa\xa9\x9a4\xbbI\xa3JI" +
"\xba\x89\xd2v\x13m\xb6\xdb\xe8\xad\xce\xbd\xf3\xf1\xfc\xd1" +
"\x10\xff\xc3\xf8\xfd\xcey\xe7\x9e{\xee\xb9\xe7\x9e\xdf}" +
"\xdc\xf1\xf5\xfao\x86\xda\x1b3\x0d\xa0$\x1f\xae\xaa\xf6" +
"\xa3{v\xbccy?9\x00\xd1E\xe8\xef\xa8\xdb\xba" +
"x\xe4\xf9\x15\xe7!\x14\x06\xe8h\xae\xde\x89Z[u" +
"\x18T\xff\xa7\x1f'\xbav\xfe\xf1\xf2\x83\x93\xb5\xaa\x84" +
"Zm\xf5^\xd4\x16W\x87\x01\xb4\xe6\xeaw\x01\xfd{" +
"{/\x7fgx\xc5\x0f\x0fCt\x91RQ\x06\xec\xc0" +
"p?j\xf3\xc2\xf4R4l\x87\x00\xfd\xb6\x97\xf2\xf7" +
"\xfe\xb8\xf9\xf8#\x90l\xc3Z\xff\xb2\xf5o\xf5\xdb\xae" +
"\xdc\xf7\xdfP\x85\xa4s\xbc\xfe!\xd4\xce\xd4\xd3\x9f\xa7" +
"\xea\xdfR\x00\xfd\x7f\xfc\xcaGyc\xef\xce\xefA\xf4" +
"K\x8a\x7f\xf0\x93O\xbe\xd4\xf1_\x7f:M\x96_\x9f" +
"s\x10\xb5\x0f\xe7\x90\x0f\xef\xcf\xf9>\xa0?\xf79\xcc" +
"d\xae^\x7fb\x86iuE\x86Q\x1b\x88\xd0\xb4\xda" +
"~\x1bW\xfb\xc2\xd9S3h\xb5Fv\xa2v\xb7\xd0" +
"2\xd5\x8eo~:\\\xfd4$\x17\xe1\xb4\xd97\x93" +
"^[D\xbe\xb2\x1d\x01+~%\x17\xe1\x9c\x80:E" +
"\xa8\xe3d\xd3B\xd4\xce6}\x19\xa0\xe3\xa5\xa6\xf10" +
"\xa0\xdf\xd7{\xe9\xe5\xb3o=}fZ\xb4\x92\xcd\x07" +
"Q\xe3\xcd4'\xd6\xfc5@\x7f\xeb\xe9\xe4\xeb\xb7\x8d" +
"]?3\x83\xb7\xbcy'j\x85f\xf2\xf6'\xbfM" +
"?\xf6\xf7\xcf|\xf4O\xd3\xecmm^S\xb1\xb7\x19" +
"\xd0\xdfu\xfe\xdd\xf6\xef|\xf6\xd6Of\xb0\x97'{" +
"G\x85\xbd\xee\xb7\xfe\xf0\xc0\xc0-\xdb\xcf\xcd8\xeap" +
"i\xd4\xab\x98Xb\xa4\xad\x0b3hm%[\x86\xd0" +
"\xca^\xfc\x87GS_\xef\xfb\xc5\x0cZ\xeb\xc8\xd6\x0e" +
"\xa1uH]\xfd6\xbfk\xd9\xe5\x19\xb4:Ik\xa3" +
"\xd0\xfa\xe8\x9f\xbf\xf1\xea\x9dw\xef\xbe:uU\x14\xd2" +
"[\xde<\x17\xb5N\x9akG{s\x9cVe\xcd\x8e" +
"\xed9\xf3\xb5\x1f\xfe\xfbLC/\xa4\xa1\x17\x92\xd1W" +
"\x0e>\x99\xf9\xbf7\xbew}\xa6\xa1Ik\xa3\xd0Z" +
"|%\xfd\xe6\x9b\xde\xc1W\xa7\x85x\xf9\xc2\xb5\xa8u" +
".\x14\xc3.\x14\xc9\xd0\xd5u\xf1\xd4\xb5\x87\xfe\xe7\xd7" +
"\xe4\xa5:\xc5\xe2\x92E\xfd\xa8u.\xa2\x05i_D" +
"\x1b\xa73\xd1\xf7\xed\x13\xdb\xbf\xf7\xc6\x0c\xa3\xcfk\xd9" +
"\x89Zk\x0b\x8d\xde\xf5\xed\x97\x7f\xf0\xfd\xaa\xef\xff'" +
"\x99T\xa6\xa8U\xb5\xacEm^\x0b\x99\x8c\xb6\x8c\x03" +
"\xfa\xb7\xbe\xfa\xeb\x8fn[\xf7\xdc\x9f\xa7\xed\x98|\xcb" +
"n\xd4\x8e\x0a\xcd\xc3-\x19\xc0\xca\xe6\x9blVn\xc5" +
"\x9f\x91\xf6\x8b-\xf3\x01:\xae\xb7\x84)m\x9bO\x9e" +
"\xdcX\xfb\xc1M\x1f\xcf\xe0\xebk\xcbv\xa2\xf6\xfe2" +
"\xf2\xf5\xe9P\xee\xb3\xaf\x1f:\xfe\xbf3h\xbdHZ" +
"\xaf\x0b\xad\x95\xef-\xdf\xf4\xee}\xdf\xf8\xcb\xb4x\x9e" +
"_6\x8c\xdaK\xcb\xc8\xc9\x17\x97\xed\x07\xf4\x9fr\x07" +
"\xaf\xbe\xbc\xeb\xe9\xbfLuR,z\xd5\xf2\x83\xa85" +
"/\x17\xe1Z.\xa2\x1f\xfd\x97\xf1#\x1f\xf6\x0c\xffu" +
"\x86\xe1\x1f[1\x8c\xda\x8fV\xd0\xf0\xfbO\x9c{7" +
"u\xe2\x88?5\x93\xc4\xcc\x0f\xafx\x1e\xb5\x93+\xc8" +
"\x85\xc7W\xfc\x14\xda\xfc4w83\xbf\x9a\x0e\xe5]" +
"\xcf\x1e\xfdjZ\xfc\xdc\x9ef9+\xb7\xa6W<\x0c" +
"q\x97;c\\\xd5W\x0f\"~\xae~\xcac\xce\xa0" +
"a\xda^/sz\xbbm\xcbsls\x101\xd9\xa4" +
"\x86\x00B\x08\x10e;\x01\x92\xdfR1i*\x18E" +
"\x8c!\x81\xc6w\x01\x92\xa6\x8a\xc9\x09\x05\xa3\x8a\x12C" +
"\x05 \x9a'MO\xc5\xe4\x01\x05\xa3\xaa\x1aC\x15 " +
"\xfa@?@r\x9f\x8a\xc9#\x0a\xfa\xd9\xbc\xdeKc" +
"\x80j\x9b8\x17\x14\x9c\x0b\xe8\xbb\x1e\xe7\x8eae0" +
"a\x8c\x1a\xdeFk\x04\xed\xb2\xccbcF\x86y\x06" +
"\xa8\xb6U\x06sY\xdb\xe2\x1br.\x00`S\xa5\x80" +
"\x03b\x13|\xfel\x07XN\xdfh\xe5\xf2\xde\x96B" +
"\x8e\x03\xd0D/\x92\xebZ\x9b\xd2\x0f\x80s\xb4V\xe5" +
"\xe7\x00\x18\xd1Z\x95\xa7\xc8\xb8\xd6\xaa\xfc\x18\x00\xa3Z" +
"\xabr\x19\x00\xe7j\xad\xca\x1b\x00X\xaf\xb5)\xc3\x00" +
"\xa8i\xad\xca\xcb\x00\x18\xd3\xda\xc4/j\xed\xca^\x00" +
"T\xb46e7\x00\xce\xd3Z\xc5\xf3M\xdar\xe5\x0f" +
"\x008_kU~\x09\x80\x0b\xb46\xe5\x1d\x00l\xd6" +
"\xda\xc5\xf3B\xadS\xf9\x01\x00.\xd2:\xc5\xb8-Z" +
"\xa7\xb0\xb7X\xeb\x12\xcf7k]\xc2/\xb5\xf8\x1c\xd2" +
"\xba\x84?q\xadK\xd8\xbdE\xbb[\xd8[\xd2\xd1\xa3" +
"\xacA\x00\xac\xd26*\xcf\x03`\xb5\xb6Q8\xd0\xa8" +
"\xadSv\x02`\x83\xd6#\x1c[\xaa\xdd-^\xfc\x92" +
"\xd6#\x0c/+\xfe.\xef\xe8Q\xe6\x92\x81:m\xa3" +
"\xf2\x10\x00\xae\xd06*\x7f\x06\xc0/w\x0c(KI" +
"p\xab\xb6U\x84\xa0\xb5c\x87\xa2\x10\xb0\xb2c\x97\xfc" +
"\xe3+\x1aS\xbe\x0b\x80\xb7iL\xbc\xda\xa61\x11\xbc" +
"p\xf1\xb9Fc\xcaA\x00\xac\xd5v\x89\xdf\xdb\xb5\x1d" +
"\xe4\x83\xaf\xdb\xe3\x96i3\x1d\x00|\x97{[\x98\x93" +
"\xe1\xe8%\x98\xc7\x1df\xc6{\xd2in\x12\x9e\xcaq" +
"\xae\xcb\x14\xd9<2\x12v\xb97\x05\xed\xb5\xad\x08e" +
"/\xc1\x03,\xd7\x9b\xafr\xc6\xb8\x90\xf7\xca\xb4\x06\x97" +
"{\xdb\x0c\xd7\xb0\xad\xde\xfc$\x91*_J\xd8\x99\x04" +
"\x87\xf0\x98\x1cOh*RU\xf8D.\xf5H\x1f\x83" +
"\xb2\x01\xc3\x92\xe2m\x00\xbe\xc3i&)\x0e\xdd\x9eg" +
"X\x19\xd7w\xd9\x18Oq\xcf\x83\x88|\xe4\xde:\x8b" +
"\x0d\x9b\xd0-\x87\x9fjl\xab\xcb\x85\x9c\xa7\"%\xb1" +
"\x98\x8a2I\x96\xe3\x1c\xf5\xf2\xec\x151\xfb\x804\\" +
"|\xf3\x1e\xdb\xd4\x13\x0as\xbd\x14\xe7\x96P%M\xf4" +
"\x02Q\x16h?W\x9d=S\xc1\x9et\xb8\x18x\x81" +
"*\x12\xddb\x8c\xf2\xcd##.\xf7d \xfa\xf8\x08" +
"\xcb\xa3\xe9%\x98\xc5\xb7\x1ba\xdd\xcb\x96]\xc6R\xdc" +
"\xe2\"p>\x05\x86\xd41oz\x14\x11#L\x01!" +
"t\x88\xa7\xed\xaa\xd1Qn\xe9\\\x17\x12+\xe3\xd2Z" +
"\xa5L{\xbc\xcf\x1e\xb7\xd6\xdb\xce&>!\x1dHD" +
"h\xb2\x95\xb9o\xcdM\x92\x1a\xe1\xa2\x94\xe6\x9eRK" +
"s\xf6\xb6g\x0d\x93\xf7f\x99\x951\xacL\x8awK" +
"u1\xfa w\\4\\\x8f[\x1e\x09\xe4\xb2\xa5\x99" +
"\x95\xe6f\x9f\x0d\xdd27\x8b\xe9\xd1\xefR\x1d*>" +
"\xa4l\x88\xe4\x9d4\x17\x8b:\xe1qG\xb1\x98Y\x0e" +
"\xf3\xa4t\x14b,\x89\xe3\x89Is\x90\xd9;\xe8\x18" +
"q\xdb1\xbcB\x19W\xa5\x19r\x9a\x0f\xf1\xfb\xf3\x86" +
"\xc3]\xda\x0d9\xf4|F\xbf^*\x87\xa5\xd1\xe4r" +
"\x0c:\xdcu\x95\x0d\xcc\xddb\xf7\x145&\x8d\xd7\xa3" +
"\xef\xce\xbb*\x85_\xaefP\xab\x12;\x01*^e" +
"*\xb4\xea\xb6\x9a\xf7\xcaC\xd4\x8a!6\x8fq\xc71" +
"t^Q\xa4U\xeb\xb5-\xdd\xf0\x0c{j0J;" +
"0\x95e\xba=\xde\xcb\x9c\x94\xc7<\x14\xd1\x13\x10\x8e" +
"\x0f\xd8:7\xb7\xad\x92S)b\x1brn\xc2N\xb3" +
"\x08\x19$\xb8\xdf\xb5\xadAd^v\xbdi\xb3\xe2\xac" +
"\x05V\xcd\xbc\xec\x16>!\\\x10\x00\xf3\xb2k\xed\xe0" +
"\x90ESd\x89\xd6#\xe2X\xcc,\x1f\x16U78" +
"\x1a\xc9Y~\xbb8s\xe8\xd0X]:\x1d\xb5]\xb8" +
"\x17 u\x1f\xaa\x98\xcab\xe5\x80\xd48\xf6\x03\xa4t" +
"\xc2sX9#\xb5Q\x1c\x02H\x99\x84O`\xe5\x98" +
"\xd4\xf2B\xdf#\xfc\x00\xe1\xa1\x9a\x18\x86\x00\xb4\x07p" +
"\x15@j\x82\xf0C\x84W\xd5\xc6\xb0\x0a@{\x10w" +
"\x02\xa4\x0e\x10~\x8c\xf0\xea\xba\x18V\x03hG\xf1\x07" +
"\x00\xa9c\x84\x9f <\\\x1f\xa3&B{\x0c\x1f\x02" +
"H\x9d \xfc4\xe15\x0d1\xac\x01\xd0N\x09\xfd\xd3" +
"\x84\x9f#\xbc\xb61\x86\xb5\x00\xdaYt\x00R\xcf\x12" +
"~\x81\xf0\xbap\x0c\xeb\x00\xb4\xf3\xf8\x14@\xea\x02\xe1" +
"WP\xc1%\xf5J\x0c\xeb\x01\xb4\x17p\x0d@\xea\"" +
"\xe1WQAl\x88a\x03\xf5L\xb8\x12 u\x89\xe0" +
"7QA\xdf\xb5i\xcb\x0c\xd8\xd0m\xd9\x94VX\x0b" +
"\x0a\xd6\x02\xfa&\xf3\x0c/\xafs:\xd3\xebA\xc1z" +
"\xc2l+C /c\xcc\x9c\xa6\x17w\xc56\xaa" +
"\x03\x05\xeb\x00\xfda\xce\xa8\x9b\xe8\x03\x95g\xca`\xd6" +
"v\x8c\xbd\xb6\xe5!3{\xd2\xe9\xbc\xc3\xd4t\xa1," +
"\x1c\xe3\x8eg\xa4\x99\x89R\x94.\xd0\x018\xd9Z\x0f" +
"\x16E}A\xab\xae\xac\x94i\x88\x0bY\x19\xcf[\xc6" +
"\x84\x98\x9d\xeb\xb1\xd1\xdc\x80a\x86M\xc3\xc5*P\xb0" +
"\x0a\xb0;\xcb\xdc\xf5\xc6\x04\"(\x88\x80\x91\xb1M\xeb" +
"\xfap\x0e\xe0\xa0\x8a\xc2\xc0\x9c@\x0fS\xf57z\x98" +
"\xcdy\xef\xf6\x14\x1b\xcd\x99\x9c\x123\xef\x8a\xa4\\ " +
"\x9a\xb0\x9e\xb5\xd4\x88D\xbbVQ\x1f\x12m\xa7S_" +
"\x8d\xb6\x11\x18\x8a.\xa7^\xa0*\xbad7\xc0\xfe\xbc" +
"\xb5\xc7\xb2\xc7\xad\xb8eo\xc8\xb9\xbee\xf7\xdac\xdc" +
"a\xa0f\xf8~\xcb\x1e`^:\xeb\x8f\xd2\xbf\\\xdf" +
"\x04\xdd\xb6,!E \x01\xb2\x0a\x7f\xe1>4\xac\xb7" +
"\xaf\xbaQ#:\xa5q\xed\xba\x91>\x85\x81\xf61\x9d" +
"\x1a\x9b\xd5\xbc'\"PnZ\x1f\x7f\x08 \xf9\x84\x8a" +
"\xc9g\x02M\xeb\x19jEO\xab\x98<\x17hZ\xcf" +
"\xae\x04H>\xa3b\xf2J\xa0i}\x814/\xa9\x98" +
"\xbc\xa6 \x86\xc4N\x8c\xbet\x10 yU\xc5\xe4\xaf" +
"h\x1b\xa2\xd8\x86\xd1W\x87\x01\x92\xaf\xa8\x98|S\xc1" +
"r'\x83\x83\x8e\x9d\xa1:)\xfa\xd32a(\xf6\xa7" +
"n\xe9\x88\xa3~\x0c\x14l\x00\x8c\xe4\x98\x97-\xe5@" +
"S\x85\xb1\x01\x8al\xc8\xd9\xae\xa8\xa8\xb2\xdf-\x93\xaf" +
"\xa2=\xd3\xb6sC\xcc\xe3\xd8C\x0b\x98\xe1\x95\xdc-" +
"I <`Xe\xb4\x14T\xf5o\xe5\x96\x9a\xf7\x92" +
"5\x18\xbc\xea\xa8\xdd\x1d`q\xb5C~)\xf5 B" +
"\xc9\xe7o\xc8\xb9)\xda\xd7\x80<y\xa4\\\x19\x7f\xa4" +
"\xae\xa5\xca\xa2ReQ\x03\x95\xf1\xacJ\x95\xe2\x19\xc2" +
"\x9fS\x03\x95\xf1g*U\xc0s\x84_T\x15\xc4b" +
"a\xfcW\x95\x0a\xdd\x05\x82\xaf\x90z\x08ea|A" +
"\xa5\xc2{\x89\xf0k\x84W)\xb20\xbe\xa4>\x0f\x90" +
"\xbaF\xf8o\x08\xafVea|M\x0c\xfb\x0a\xe1o" +
"\x12\x1e\x0e\xc9\xc2\xf8\xba\xb0\xff\x1b\xc2\x7fOx\x8d*" +
"\x0b\xe3\xefT*\x8c\xbf'\xfc\x8f\x84\xd7\x86da\xfc" +
"P\xa5\xc2\xf8\x01\xe1\x9f\x12^W%\x0b\xe3'\xeaw" +
"\x01R\x9f\x12\x1e\x0a)\x18\xad\xaf\x96\x85\x11C\xbf\x04" +
"H\xd5\x84TL\xc5\x08ox[V\xc6h\x88\xfci" +
" |\x01\xe1\x8d\x8bc\xd8\x08\xa0\xcd\x0bQ\xc1o\"" +
"\xbc\x85\xf09\xbf\x8b\xe1\x1c\x00\xad9D~. |" +
"\x19\xe1\x91\x9a\x18F\x00\xb4%\xa1\x97\x01R\xb7\x12\xbe" +
"\x9a\xf0\xa6\xda\x186\x11\x9f\x0fQ|\xee \xfc.\xc2" +
"\xa3u1\x8c\x02h]!\x9a\xd7]\x84\xdfC\xf8\xdc" +
"H\x0c\xe7\x02h\xebB\xc3\x00\xa9>\xc2\x07\x09\xd7\xea" +
"c\xa8\x01h\x03\xa1\x9f\x03\xa4\x06\x09\xbf\x8f\xf0XC" +
"\x0cc\x00\xda\x8e\x10\x1d(\xf7\x11\x9e%|^c\x0c" +
"\xe7\xd1\xc1\x17\xa2\xf8\xe8\x84\xe7\x08\xbf\xa9%\x867\xd1" +
"\xc1'\xf4s\x84\xef#|\xfe;1\x9c\x0f\xa0\x15\x84" +
"?\xfb\x08?B\xf8\x82\xc51\\\x00\xa0\x1d\x0e\xed\x06" +
"H\x1d\"\xfca\xc2\x9bkb\xd8\x0c\xa0\x1d\x17\xf19" +
"B\xf8\xa3\x84/\xac\x8a\xe1B\x00\xed\x11\xe1\xe7\xa3\x84" +
"?I\xf8\xa2\x9bc\xb8\x08@;)\xfcy\x82\xf0g" +
"\x08o\x89\xc7\xb0\x05@;#\xe2y\x9a\xf0s\x84/" +
"^\x16\xc3\xc5\x94\x9e!:\xa0\x9f%\xfc\x02\xe17\xd7" +
"\xc5\xf0f:\xf8D|\x9e#\xfc\x12\xe1\xf1\xfa\x18\xc6" +
"\x01\xb4_\x88y]\"\xfc\x1a\xe1\xb74\xc4\xf0\x16\xca" +
"C\x81_#\xfc7\x84/i\x8c\xe1\x12\xca\xc3\xd0A" +
"\x80\xd4\xaf\x08\x7f;\xa4\xe0\xfeqV\xd8\xc4Fy\xa9" +
"\x14t\x8f\xb3\xc2\x10\x1f)=\xfa\x8e\xcdt\x92\x07\xaa" +
"\x85<v\x12\xc6(\xa8\x86W\xde\xd6V\xb1\x09\x86n" +
"\xd9rM\x13\xa0\xc4\xfb\x8cn\xd7\xa3\xf6\xb6\xa4\xd0\x9d" +
"e{\x99\xa3\x97\xad\x93\xfe=l/\x03u\x06\x10\x1d" +
"\xbd\xcf\xa0\xf7\xd5\x8a\x01\x9f\xe9c\x86k;\x05\x88\xa7" +
"&\x9d\xc4\xf4R\x8f>f \x09\xa5\x0b\xd3dJI" +
"V\xb4\x9b\xc6\x8ac\xb6\xc5\xb7\xb3B\xe9\xa0\x8c\x9b\xcc" +
"\xe2.V\x83\x82\xd5\x80\xbeg\x98<A\xb4Q\xe5z" +
"I\xa5\x1c\x19\xc5\xf0R\xf9L\x86\xbb\x1e\xd7\x85\xf1\xc0" +
"\x89\xee\x16\x05\xd0\xadOv\x97\xbb\x9e1J\x85T\x1f" +
"\xb2\x99\xbe\xdd\xd0U/[\x16\xd2:P\x17\x0ba>" +
"\xe1a\xa4r\xc7\x09\x88\x11@_7\x8aQ]\xef\xd8" +
"\xa3\xdbY\xa17\xce-j\xfa\xcb=F\x91xb\x89" +
"y\x06<\x1a%\xbe\xe4\x8c\xf1\xa9\xf1\x1bg\x85\x147" +
"y\x1a\xa9\xf8\xcb\x1b\x0b\x8cT\xae\xaa\x8a#\x97\xe6\x8c" +
"\x86\xec\xdc\xbd`@\xb2F&;\xce\x0a\xbd\x101\x99" +
"\xebb\xa4rq(\xdf\x8e\x8f\xb3\xc2F\xbd\xd4\x9a\xf8" +
"\xe9b\xf3>\x85\xabT2O\x14\xfem\x1c\xe2\x0eM" +
"\x07\xc3\xa0`\xb8\x18\x1d\xeaD@\xcd\xd3 \xe5\x93\xa3" +
"\xe8b\xa6r@`\xa4r\x94T\xa4\x03\xb6eo\x81" +
"p\xb0!L\xdb\xa3\xb9\xbc\xc7u\x142\x83v@Y" +
"&\xbb\xc8\x0d\x1c-\xee0y2\x96e9\xc7\xd6\xf3" +
"i\xee`\x8a\xbb\xe4cE\xf2\xb9\xbd~\x99\xaaHz" +
"+\xa2M-E\x8dh\x12\xa2kDSU\xbb\x16\x80" +
"v\x8fg\xa4\xf7\xe7\xb8\x93\xe6\x967\x9b\x9e\xa63\xd8" +
"\xd3\xcct\xfcR\xde\x89K8>!\xfa\x99\x061\xf8" +
"b\xd9\xd1\xcd[):\xba\xc6\xb5\x00\xfbG\x1c\xce\xc7" +
"Y!\x926\xbcB\xa9\x93\x9b\x8d'\xed\xb3\xea\xc6\xc2" +
"z\xfb\xeaY\xb6o_\x9b\xed\x00\x9d\xb3}\xa1\xfdF" +
"\xc1\x14w}hQ\x1cc\xe5\xbe\xf0\x01\xea\xf6&T" +
"L\x1e\x0a\xf4\x85\x0f\xae\xaa\\Qb\xb1-<\xbc\x14" +
" y@\xc5\xe41j\x0b\x9bd[x\x94\xde>\xa4" +
"b\xf2\xe1J#\x12=N\xbd\xe21\x15\x93'\x14\x8c" +
"x\x85\x1c\xa5x\xf9\xfbSq\x97\x8d\x10[-\xed\xeb" +
"\xb0\xeb9\xe5\xceo\xd8\xb6\xcd\xf2n\xdd]$\xae\xc1" +
"b?\x9b\x98\xdc1\xdb v\xdc\xe8\x85{\x8a\xf5\x83" +
"\xaa\x87\xd8\x0fw\x8a\xe8\x9c\x92)\xf9x\xbfH\xc9\xc7" +
"v\x0b\x92\xf1\xc8*A2\x8e\x0e\x09\x92q\x98T\xaa" +
"\xa3\x0f\x0c\x03`8Z \xb0&\x9awh\xa7F\xef" +
"\xa7\xf7\xea\xa2\xa3\xf4^}\xd4\xa0\x9f\x86('\xcd\xc6" +
"(\x0b\x90\x13\x7f\xd4\xf6lg\x9c\x15\x00\xa0\xf2w$" +
"aX{\xe2\x9e\x93\xb7\xf6\xf8\xe2\xdf\x84a\x01\xee\xd9" +
"\x9fs\x8cQ\xe6\x14\xfc\xe2o\x02\xc2\x86\xb5\xc7w9" +
"\x155\xe6\x00\x16*\x7f\xc7\x0bd\xc3\xf7\x88\xf81G" +
"\x98/\xff-\xcc\xfby+Ms6 2bp\xdd" +
"w\xb8k\xe8\x9c\xca\xbf\xc1L\xdf4\xc6\x0c+C\xfd" +
"\xaf\xc3\xb9wC\x9aBK:h\x1b\x96'/\x0f\x02" +
"W\xeb\xfd3]\xad\x0f\x01$\xb3*&=j\x8eC" +
"2\x1d\xef'0\xa7br\x1f\xa5c\x95L\xc7\xc2\xde" +
"J2\x7fa\x02\x9d\xce;c\xcc\xcb;P9h}" +
"O^`r\xe86m*)\xd3\xb8\xc2\x17\xd8\xf1\x1d" +
"_\x84\xb0\x0d\x96\x98\x8c(\xad\xe58\xb4R\x1cnU" +
"1\xb9:\x10\x87v\x9a\xf2\x1d*&\xef\x9a\xcd\xecn" +
"\xe0\x00\x11gAW\xc4\xd5\x9dX\x0d\x11\xdf\xd6\x95\"" +
"\xa1\x97\xc8\x84^\xdc/\x12\xbay)@\xc4\xb2-\xee" +
"\x1bt\x96[\xcc\xa4D\xe1\x13\xe5\xbf\xc3i\xe6|\xe1" +
"\xaf+\xa9\x88\xc7<\x9e\x0ca\xe0\x9bs\x14\x97\x867" +
"\xe4\xdc\xe4\x812k\xba_\xa5\xb67G4b_\x90" +
"5\x15\x04\xad\xd9G\xf8\x91 k:\xacR[z\x88" +
"\xf0\x87\x09WQ\xd2\xa6\xe3\xeaS\x00\xa9\x87\x09\x7fB" +
"\xd0&U\xd2\xa6\xc7\x85\xfd\x13\x84\x9f\x16\xb4)$i" +
"\xd3)\x95\xda\xd2'\x09\x7fV\xd0\xa6*I\x9b~$" +
"\xec<K\xf8\x05A\x9b\xaa%m:\xaf^\x9eD\xcb" +
"j\xc2\x926\xbd X\xdfE\xc2\xaf\x0a\xdaT#i" +
"\xd3\x8b\x82f]!\xfc\x15A\x9bj%m\xba.X" +
"_\x85\xae\xd5\xd7I\xda\xf4\x9a\xf0\xf3W\x84\xbfMx" +
"C\xbd\xa4M\xff!h\xd6\xdb\x84\x7f@xc\x83\xa4" +
"M\xef\xab\xd4\xb6\xbfG\xf8\xc7\x84\xcfi\x94\xb4\xe9O" +
"\x82\xc6}\xac\xaa8$X\xd3\x1c\xc9\x9a>\x13\xc3\xfe" +
"\x95\xd4k\x04k\x8aH\xd6T%\xba\xfc\x10u\xe7M" +
"\x8255I\xd6\xd4(XD\x85\xad\xcd\x8dJ\xd64" +
"/\xf4\xfc$V\xa6\xa9\x925-\x09\x91\x9b\xcb\x08\xbf" +
"C\xb0\xa6\xb9\x925\xb5\x09;\xb7\x11~\xa7`M\x9a" +
"dM\x9dB\xffN\xc2\xfb\x04k\x8aI\xd6\xd4#X" +
"M\x85\x95\xcd\x9f'Y\xd3\x80\xb0\x93 \xfc^\xc1\x9a" +
"n\x92\xaci\xab`\x17[\x08\xff\x96`M\xf3%k" +
"\xda%\xd8`\x85\xad-\xfc\xa6dM\\\x8c\x9b%\xdc" +
"\x13\xac\xa9G\xb2\xa6\xfb\x85\x9d\x0a[kY+YS" +
"A\xe0\x13\x84\x1f\x12\xac\xa9W\xb2\xa6\x07\x05~\x80\xf0" +
"c\x825\xf5I\xd6tT\xf8Yfk\x18\x97\xa4\xe9" +
"\x91\xd0R\x80\xd41\x82O\x84\x14\x14\xf7\xd9\xe6\xa0\xc3" +
"!\xe2\xba\x81\xee\x95\x99\xe3\xac\xe0n\xb6\xb0\xf85h" +
"]\xb7\xf8\xc4QQ\x18v\xd8\x1e\x9e02\x10\xcez" +
"n\x19\xd5\x99\x9b\x1d\xb6\x99\x83z\xe0f\xbb\\\xd3t" +
">\xe3X\xe5\x06^\xdcps\x1d\xa6K\x12\xb6\x95!" +
"ix\xa6\xf7\x94m\xdc)\x944\x82\xef\xef\xe7i{" +
"\x03gN\xf9\x8d\x11\xdbI\xf3^\x9b\x81\xeaze\xd0" +
"p\x07\x99\xb3\x87hAe\xe0\x1c\xcb\xbb<\xc1<\x88" +
"\xd0\xe4\xa7\xc0\xb6\x85\xa2\xfa\x19\x16C3@\x80l\xc7" +
"\xdb\xc0\x19`e<\xcfa##Fz\x00m]~" +
"#\x0a\x12\x84\x0csS\x1eK\xef\x09\x8e;j\xeb\x9c" +
"f\x01\x93&*\x0b\xdb\xa0\x03q\xfe\xb9\x8b\xd4\xd3m" +
"\x9a\xf6x0B\x95\xe5\xf0\xec\\\xca\xc8\xa0Ufq" +
"\xf2C\xcbt\xa3\x12O\xd8X\x8a'N\x95m\xe3X" +
"\x8a\xf7\x94ws\\g\xe6\x00\x9b\x80\xf8\x10\xcfpk" +
"2.\xe2\xd6\x93\xf6\x8c1^\x99\xb0\xc7]\x93\xf5\xf6" +
"\xae\x83n+\xc32\x01SEA\x1f\x1a.\x17\"\x9c" +
"&\xdb\x84\xb6\xd7\xe3\x8c\x06\x13Fd\xf3=4\xe1^" +
"'o\xb8\x81\x81D\xee\xcd(\xc9\xe5M\x97\xf7X:" +
"\xc47\x98\x86\xceKx8\x93sg\xf9\x8d{\xca\xb9" +
"|\xe7,;\xf7\xbf\xbb\x91\xfe\xf6\"+\x15\xa44\\" +
"\xc8\x89STv)\x9d\xb2-l\x1b\x12\xa7h\xab<" +
"E\x97\xf7\x8b\xb6p\xc9J\x80\xfd\xe9\xbc\xe3\x10i\xca" +
"9\\7\xd2\x1e\x07\xd4\xfd\x9c\xed\xba\xc6\xb0\xc9Kg" +
"\xab\xa5\x8b]\x10\x19a\x86y\xc3\x8b\xf0\xbe\xe2\x95k" +
"\xf9\xc6u\xca5\xf0\x1a\x80\xe4\xa3*&\x9f\x0c4\x16" +
"'\x87\x02w\xc3\x0aJ\xd7\xcf\xec\x0c\xdc\x0d\xab\x8al" +
"\xb0\xce\x1e\x04H>\xabb\xf2B\xe5\x1a\xf8<\xbd\xfd" +
"\x9c\x8a\xc9W\x02\xd7\xc0\xd7I\xf1\x9a\x8a\xc9\xf7\x14\xec" +
"f\"\xbb\xa6\xa4\xabIs\xad\xe4\x8e\xed1s\xbda" +
"\x82\xca]\xac\x01\x05k p}\xcc\xf5\xf5\x86\xc9]" +
"(K\xcc\xe2w+@\xb7tK\xdcP\xfcRP\x12" +
"a\x1f\xf7\x98a\xbaP\xb9F.\xff\xa7\x92\xe25\xf2" +
"l\x88\xc1\xa4,\xb8Q\xf0K\x9f\xd5\xfa\xba\xa5\x0f\x82" +
"\xba\x96\xd7`\x1d5w}*&\x07\x15,-\xc1\x00" +
"E;\xa1b\xf2\xde\xc0\x12l\xa5 nQ1\xf9-" +
"%0\xe5\x00\x15\x9a]\xd4\xfe?\x00\x00\xff\xff\xd6\xbd" +
"\x94\xef"
func RegisterSchema(reg *schemas.Registry) {
reg.Register(&schemas.Schema{
-1
View File
@@ -1 +0,0 @@
{"description":"Synthetic Go-produced v2 current-road and data-only preview fixture; no real road data.","event_base64":"AAAAACQAAAAAAAAAAgABAADyBSoBAAAAjwAAAAAAAAAAAAAAEAAHAAAAoEEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIABQABAAAAAAAAAAIAAAAAAHBBgAELJwEAAAAA8gUqAQAAAAEAAAAAAAAAKQAAAAAAAAAAAHBCAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAoCAABhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhAAAAAAAAAAA="}
+2 -52
View File
@@ -1,6 +1,7 @@
package cereal
import (
"bytes"
"capnproto.org/go/capnp/v3"
"encoding/base64"
"encoding/json"
@@ -60,58 +61,7 @@ func TestMapdV1HostWireFixture(t *testing.T) {
if err != nil {
t.Fatal(err)
}
actualMessage, err := capnp.Unmarshal(raw)
if err != nil {
t.Fatal(err)
}
oldMessage, err := capnp.Unmarshal(expected)
if err != nil {
t.Fatal(err)
}
actualRoot, err := actualMessage.Root()
if err != nil {
t.Fatal(err)
}
oldRoot, err := oldMessage.Root()
if err != nil {
t.Fatal(err)
}
// Appended default fields can enlarge storage without changing old values.
equal, err := capnp.Equal(actualRoot, oldRoot)
if err != nil || !equal {
if !bytes.Equal(raw, expected) {
t.Fatal("provider wire differs from shared host fixture; review both schemas")
}
}
func TestPreviewHostWireFixture(t *testing.T) {
data, err := os.ReadFile("testdata/mapd_preview_wire.json")
if err != nil {
t.Fatal(err)
}
var fixture struct {
EventBase64 string `json:"event_base64"`
}
if err := json.Unmarshal(data, &fixture); err != nil {
t.Fatal(err)
}
raw, err := base64.StdEncoding.DecodeString(fixture.EventBase64)
if err != nil {
t.Fatal(err)
}
msg, err := capnp.Unmarshal(raw)
if err != nil {
t.Fatal(err)
}
event, err := log.ReadRootEvent(msg)
if err != nil {
t.Fatal(err)
}
out, err := event.MapdOut()
if err != nil {
t.Fatal(err)
}
boundary, err := out.PreviewBoundaryId()
if err != nil || !event.Valid() || out.SampleVersion() != 2 || out.SpeedLimit() != 20 || out.PreviewSpeedLimit() != 15 || out.PreviewDistance() != 60 || len(boundary) != 64 || out.PreviewWayId() != 2 {
t.Fatal("provider preview differs from shared host fixture")
}
}
-82
View File
@@ -1,82 +0,0 @@
package main
import (
"crypto/sha256"
"fmt"
"math"
"pfeifer.dev/mapd/cereal/custom"
"pfeifer.dev/mapd/maps"
)
const previewHorizonM = 500
const previewMaxWays = 64
type LimitPreview struct {
Speed float32
Distance float32
BoundaryID string
WayID int64
}
// A preview follows only a unique legal continuation, never a guessed turn.
func (s *State) previewLimit() LimitPreview {
if !s.ShadowOnly || s.SnapshotID == "" || s.CurrentWay.SelectionType != custom.WaySelectionType_current || s.MatchedLimitMps <= 0 {
return LimitPreview{}
}
way, forward, position := s.CurrentWay.Way, s.CurrentWay.OnWay.IsForward, s.Position
distance := float32(0)
seen := map[int64]bool{way.Id(): true}
for range previewMaxWays {
_, end := way.GetStartEnd(forward)
if distance+position.DistanceTo(end) > previewHorizonM {
return LimitPreview{}
}
segment, found, err := way.DistanceToNode(position, forward, end)
if err != nil || !found || segment < 0 || math.IsNaN(float64(segment)) || math.IsInf(float64(segment), 0) {
return LimitPreview{}
}
distance += segment
box := s.Data.OverlapBox()
if distance <= 0 || distance > previewHorizonM || !box.PosInside(end) {
return LimitPreview{}
}
matches, err := way.MatchingWays(&s.Data, end)
if err != nil {
return LimitPreview{}
}
legal := map[int64]maps.Way{}
for _, next := range matches {
first, last := next.GetStartEnd(true)
if first.Equals(last) && !next.OneWay() {
return LimitPreview{} // A two-way loop has two possible entry directions.
}
if next.Id() > 0 && (next.IsForwardFrom(end) || !next.OneWay()) {
legal[next.Id()] = next
}
}
if len(legal) != 1 {
return LimitPreview{}
}
var next maps.Way
for _, candidate := range legal {
next = candidate
}
if seen[next.Id()] {
return LimitPreview{}
}
nextForward := next.IsForwardFrom(end)
road := CurrentWay{Way: next, OnWay: maps.OnWayResult{IsForward: nextForward}}
speed := road.EffectiveMaxSpeed()
if speed <= 0 || math.IsNaN(speed) || math.IsInf(speed, 0) || speed > math.MaxFloat32 {
return LimitPreview{}
}
if math.Abs(speed-float64(s.MatchedLimitMps)) > .02 {
key := fmt.Sprintf("%s:%d:%d:%t:%.7f:%.7f", s.SnapshotID, way.Id(), next.Id(), nextForward, end.Lat(), end.Lon())
return LimitPreview{float32(speed), distance, fmt.Sprintf("%x", sha256.Sum256([]byte(key))), next.Id()}
}
seen[next.Id()] = true
way, forward, position = next, nextForward, end
}
return LimitPreview{}
}
-200
View File
@@ -1,200 +0,0 @@
package main
import (
"crypto/sha256"
"fmt"
"math"
"os"
"path/filepath"
"strings"
"testing"
"time"
"capnproto.org/go/capnp/v3"
"pfeifer.dev/mapd/cereal"
"pfeifer.dev/mapd/cereal/custom"
"pfeifer.dev/mapd/cereal/offline"
"pfeifer.dev/mapd/maps"
m "pfeifer.dev/mapd/math"
)
func previewState(t *testing.T, specs []syntheticWay, index int, forward bool, latitude float64) State {
t.Helper()
tile := packedRoadTile(t, specs)
return State{ShadowOnly: true, SnapshotID: "verified-snapshot", Data: tile,
Position: m.NewPosition(latitude, -97.9), MatchedLimitMps: float32(specs[index].speed),
CurrentWay: CurrentWay{Way: tile.Ways.At(index), OnWay: maps.OnWayResult{IsForward: forward}, SelectionType: custom.WaySelectionType_current}}
}
func previewRoads() []syntheticWay {
return []syntheticWay{
{1, 20, 35.1, -97.9, 35.101, -97.9, false, 2},
{2, 20, 35.101, -97.9, 35.102, -97.9, false, 2},
{3, 10, 35.102, -97.9, 35.103, -97.9, false, 2},
}
}
func TestPreviewRawDistanceAndStableBoundaryAcrossApproachWays(t *testing.T) {
s := previewState(t, previewRoads(), 0, true, 35.1005)
a := s.previewLimit()
s = previewState(t, previewRoads(), 1, true, 35.1015)
b := s.previewLimit()
if a.Speed != 10 || a.WayID != 3 || a.BoundaryID == "" || a.BoundaryID != b.BoundaryID || math.Abs(float64(a.Distance-b.Distance)-111.2) > 1 {
t.Fatalf("unexpected road preview: %+v %+v", a, b)
}
s.Position = m.NewPosition(35.1014, -97.9)
if s.previewLimit().Distance <= b.Distance {
t.Fatal("raw distance was held at a previous minimum")
}
s.SnapshotID = "changed-snapshot"
if s.previewLimit().BoundaryID == b.BoundaryID {
t.Fatal("boundary identity survived a different snapshot")
}
}
func TestPreviewDirectionUnknownLimitsBranchesAndHorizon(t *testing.T) {
s := previewState(t, previewRoads(), 2, false, 35.1025)
if p := s.previewLimit(); p.Speed != 20 || p.WayID != 2 || p.Distance <= 0 {
t.Fatalf("reverse traversal failed: %+v", p)
}
for _, mode := range []string{"branch", "unknown", "oneway", "horizon", "unqualified", "unverified"} {
t.Run(mode, func(t *testing.T) {
roads := previewRoads()
index, forward, latitude := 0, true, 35.1005
switch mode {
case "branch":
roads = append(roads, syntheticWay{4, 30, 35.101, -97.9, 35.102, -97.899, false, 2})
case "unknown":
roads[1].speed = 0
case "oneway":
roads[1].oneWay = true
index, forward, latitude = 2, false, 35.1025
case "horizon":
roads[0].lat1 = 35.09
latitude = 35.091
}
s := previewState(t, roads, index, forward, latitude)
if mode == "unqualified" {
s.CurrentWay.SelectionType = custom.WaySelectionType_possible
}
if mode == "unverified" {
s.SnapshotID = ""
}
if p := s.previewLimit(); p != (LimitPreview{}) {
t.Fatalf("uncertain preview escaped: %+v", p)
}
})
}
}
func TestPreviewClosedWayRequiresOneLegalDirection(t *testing.T) {
for _, tc := range []struct {
name string
oneWay, branch, wantPreview bool
}{
{"one-way loop", true, false, true},
{"two-way loop", false, false, false},
{"two-way loop plus continuation", false, true, false},
} {
t.Run(tc.name, func(t *testing.T) {
roads := []syntheticWay{
{1, 20, 35.1, -97.9, 35.101, -97.9, true, 2},
{2, 10, 35.101, -97.9, 35.102, -97.9, tc.oneWay, 2},
}
if tc.branch {
roads = append(roads, syntheticWay{3, 15, 35.101, -97.9, 35.102, -97.899, false, 2})
}
decoded, err := capnp.UnmarshalPacked(packedRoadTileBytes(t, roads))
if err != nil {
t.Fatal(err)
}
root, err := decoded.Root()
if err != nil {
t.Fatal(err)
}
msg, _, err := capnp.NewMessage(capnp.SingleSegment(nil))
if err != nil {
t.Fatal(err)
}
if err := msg.SetRoot(root); err != nil {
t.Fatal(err)
}
raw, err := offline.ReadRootOffline(msg)
if err != nil {
t.Fatal(err)
}
ways, err := raw.Ways()
if err != nil {
t.Fatal(err)
}
loop := ways.At(1)
loop.SetMaxLon(-97.898)
nodes, err := loop.NewNodes(4)
if err != nil {
t.Fatal(err)
}
for i, p := range [][2]float64{{35.101, -97.9}, {35.102, -97.9}, {35.102, -97.899}, {35.101, -97.9}} {
nodes.At(i).SetLatitude(p[0])
nodes.At(i).SetLongitude(p[1])
}
data, err := msg.MarshalPacked()
if err != nil {
t.Fatal(err)
}
tile := maps.ReadOffline(data)
if !tile.Loaded {
t.Fatal("closed road tile rejected")
}
s := State{ShadowOnly: true, SnapshotID: "verified-snapshot", Data: tile,
Position: m.NewPosition(35.1005, -97.9), MatchedLimitMps: 20,
CurrentWay: CurrentWay{Way: tile.Ways.At(0), OnWay: maps.OnWayResult{IsForward: true}, SelectionType: custom.WaySelectionType_current}}
p := s.previewLimit()
if tc.wantPreview {
if p.Speed != 10 || p.WayID != 2 || p.Distance <= 0 || p.BoundaryID == "" {
t.Fatalf("legal loop preview missing: %+v", p)
}
} else if p != (LimitPreview{}) {
t.Fatalf("ambiguous loop produced a preview: %+v", p)
}
})
}
}
func TestShadowPreviewSharesFixFreshnessWithoutControlOutputs(t *testing.T) {
roads := []syntheticWay{{1, 20, 35.149, -97.9, 35.151, -97.9, false, 2}, {2, 10, 35.151, -97.9, 35.153, -97.9, false, 2}}
data := packedRoadTileBytes(t, roads)
id := strings.Repeat("a", 64)
dir := filepath.Join(t.TempDir(), id)
area := maps.Area{Box: m.Box{MinPos: m.NewPosition(35, -98), MaxPos: m.NewPosition(35.25, -97.75)}}
path := maps.GenerateBoundsFileName(area, maps.OfflineSettings{OutputDirectory: dir})
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatal(err)
}
rel, err := filepath.Rel(dir, path)
if err != nil {
t.Fatal(err)
}
load := maps.SnapshotLoader(dir, maps.SnapshotReceipt{Tiles: []maps.SnapshotTile{{Path: rel, Size: int64(len(data)), SHA256: fmt.Sprintf("%x", sha256.Sum256(data))}}})
s := State{ShadowOnly: true, SnapshotID: id}
s.Init()
boot := uint64(10_100_000_000)
s.bootNow = func() uint64 { return boot }
sample := syntheticSample(t, cereal.GpsSourceExternal, 10_000_000_000, 1, true, true)
s.ProcessGps(sample, true, load, time.Unix(0, 0))
sample.SourceChanged, sample.FixMonoTime = false, 10_050_000_000
s.ProcessGps(sample, true, load, time.Unix(0, 1))
event, output := decodedOutput(t, &s)
boundary, _ := output.PreviewBoundaryId()
if !event.Valid() || output.SpeedLimit() != 20 || output.PreviewSpeedLimit() != 10 || boundary == "" || output.PreviewDistance() <= 0 || output.NextSpeedLimit() != 0 || output.SuggestedSpeed() != 0 {
t.Fatalf("shadow preview: valid=%v status=%v way=%v limit=%v preview=%+v", event.Valid(), output.RoadStatus(), output.WaySelectionType(), output.SpeedLimit(), s.LimitPreview)
}
boot = 10_600_000_000
event, output = decodedOutput(t, &s)
boundary, _ = output.PreviewBoundaryId()
if event.Valid() || output.PreviewSpeedLimit() != 0 || boundary != "" {
t.Fatal("expired GPS retained an upcoming limit")
}
}
+1 -12
View File
@@ -11,7 +11,6 @@ import (
"os"
"reflect"
"runtime"
"slices"
"strconv"
"strings"
"syscall"
@@ -163,6 +162,7 @@ func GenerateOffline(s OfflineSettings) error {
MaxSpeedBackward: ParseMaxSpeed(tags["maxspeed:backward"]),
MaxSpeedAdvisory: ParseMaxSpeed(tags["maxspeed:advisory"]),
Lanes: uint8(lanes),
OneWay: tags["oneway"] == "yes",
Id: int64(way.ID),
HighwayClass: HighwayClassFromTag(tags["highway"]),
@@ -191,17 +191,6 @@ func GenerateOffline(s OfflineSettings) error {
tmpWay.Nodes[i].Latitude = n.Lat
tmpWay.Nodes[i].Longitude = n.Lon
}
switch tags["oneway"] {
case "yes", "1":
tmpWay.OneWay = true
case "-1":
tmpWay.OneWay = true
slices.Reverse(tmpWay.Nodes)
tmpWay.MaxSpeedForward, tmpWay.MaxSpeedBackward = tmpWay.MaxSpeedBackward, tmpWay.MaxSpeedForward
tmpWay.MaxSpeedForwardConditional, tmpWay.MaxSpeedBackwardConditional = tmpWay.MaxSpeedBackwardConditional, tmpWay.MaxSpeedForwardConditional
case "":
tmpWay.OneWay = tags["junction"] == "roundabout" || tags["highway"] == "motorway"
}
tmpWay.Box.MinPos = m.NewPosition(minLat, minLon)
tmpWay.Box.MaxPos = m.NewPosition(maxLat, maxLon)
scannedWays = append(scannedWays, tmpWay)
-78
View File
@@ -1,78 +0,0 @@
package maps
import (
"math"
"os"
"path/filepath"
"testing"
m "pfeifer.dev/mapd/math"
)
func TestGeneratorPreservesFixedOneWayDirectionAndSpeedTags(t *testing.T) {
for _, tc := range []struct {
name, oneWay, highway, junction string
wantOneWay, reverse bool
}{
{"ordinary", "", "residential", "", false, false},
{"yes", "yes", "residential", "", true, false},
{"numeric", "1", "residential", "", true, false},
{"reverse", "-1", "residential", "", true, true},
{"roundabout", "", "residential", "roundabout", true, false},
{"motorway", "", "motorway", "", true, false},
{"roundabout explicit no", "no", "residential", "roundabout", false, false},
{"roundabout explicit zero", "0", "residential", "roundabout", false, false},
{"motorway explicit no", "no", "motorway", "", false, false},
{"motorway explicit zero", "0", "motorway", "", false, false},
{"motorway link", "", "motorway_link", "", false, false},
{"reversible", "reversible", "residential", "roundabout", false, false},
{"alternating", "alternating", "motorway", "", false, false},
} {
t.Run(tc.name, func(t *testing.T) {
forwardRule, backwardRule := "15 @ (Mo-Fr 08:00-09:00)", "25 @ (Sa 10:00-11:00)"
tags := []string{"highway", tc.highway, "maxspeed", "30 mph",
"maxspeed:forward", "10", "maxspeed:backward", "20",
"maxspeed:forward:conditional", forwardRule, "maxspeed:backward:conditional", backwardRule}
if tc.oneWay != "" {
tags = append(tags, "oneway", tc.oneWay)
}
if tc.junction != "" {
tags = append(tags, "junction", tc.junction)
}
root := t.TempDir()
pbf := filepath.Join(root, "road.osm.pbf")
if err := os.WriteFile(pbf, syntheticPBF(tags...), 0600); err != nil {
t.Fatal(err)
}
output := filepath.Join(root, "tiles")
if err := GenerateOffline(OfflineSettings{InputFile: pbf, OutputDirectory: output, Overlap: .001,
Box: m.Box{MinPos: m.NewPosition(35, -98), MaxPos: m.NewPosition(35.25, -97.75)}}); err != nil {
t.Fatal(err)
}
tile, err := FindWaysAroundPositionIn(m.NewPosition(35.15, -97.9), output)
if err != nil || !tile.Loaded || tile.Ways.Len() != 1 {
t.Fatalf("generated road not loaded: %v", err)
}
way := tile.Ways.At(0)
if way.Nodes.Len() != 2 {
t.Fatalf("generated road has %d nodes", way.Nodes.Len())
}
first, last := 35.1, 35.2
forwardSpeed, backwardSpeed := 10./3.6, 20./3.6
if tc.reverse {
first, last = last, first
forwardSpeed, backwardSpeed = backwardSpeed, forwardSpeed
forwardRule, backwardRule = backwardRule, forwardRule
}
firstNode, lastNode := way.Nodes.At(0), way.Nodes.At(1)
if way.Id() != 42 || way.OneWay() != tc.wantOneWay ||
math.Abs(firstNode.Lat()-first) > 1e-9 || math.Abs(lastNode.Lat()-last) > 1e-9 ||
math.Abs(way.MaxSpeed()-13.4112) > 1e-6 ||
math.Abs(way.MaxSpeedForward()-forwardSpeed) > 1e-5 || math.Abs(way.MaxSpeedBackward()-backwardSpeed) > 1e-5 ||
way.ConditionalMaxSpeedRaw(true) != forwardRule || way.ConditionalMaxSpeedRaw(false) != backwardRule {
t.Fatalf("generated road lost direction or speed metadata: oneWay=%v forward=%v backward=%v",
way.OneWay(), way.MaxSpeedForward(), way.MaxSpeedBackward())
}
})
}
}
+4 -12
View File
@@ -309,25 +309,17 @@ func pbfBlock(kind string, raw []byte) []byte {
data = append(data, header...)
return append(data, blob...)
}
func syntheticPBF(tags ...string) []byte {
func syntheticPBF() []byte {
header := pbfBytesField(nil, 4, []byte("OsmSchema-V0.6"))
header = pbfBytesField(header, 5, []byte("LocationsOnWays"))
if len(tags) == 0 {
tags = []string{"highway", "residential", "maxspeed", "30 mph"}
}
var table []byte
for _, value := range append([]string{""}, tags...) {
for _, value := range []string{"", "highway", "residential", "maxspeed", "30 mph"} {
table = pbfBytesField(table, 1, []byte(value))
}
var keys, values []uint64
for i := 0; i < len(tags); i += 2 {
keys = append(keys, uint64(i+1))
values = append(values, uint64(i+2))
}
var way []byte
way = pbfVarintField(way, 1, 42)
way = pbfBytesField(way, 2, pbfPacked(keys...))
way = pbfBytesField(way, 3, pbfPacked(values...))
way = pbfBytesField(way, 2, pbfPacked(1, 3))
way = pbfBytesField(way, 3, pbfPacked(2, 4))
way = pbfBytesField(way, 8, pbfPacked(protowire.EncodeZigZag(1), protowire.EncodeZigZag(1)))
way = pbfBytesField(way, 9, pbfPacked(protowire.EncodeZigZag(351000000), protowire.EncodeZigZag(1000000)))
way = pbfBytesField(way, 10, pbfPacked(protowire.EncodeZigZag(-979000000), protowire.EncodeZigZag(0)))
+6 -9
View File
@@ -149,12 +149,8 @@ func (w *Way) IsForwardFrom(matchNode m.Position) bool {
return true
}
lastNode := w.Nodes.At(w.Nodes.Len() - 1)
if lastNode.Equals(matchNode) {
// A closed one-way can be entered at its repeated endpoint in stored order.
return w.OneWay() && matchNode.Equals(w.Nodes.At(0))
}
return true
lastNode := w.Nodes.At(w.Nodes.Len()-1)
return !lastNode.Equals(matchNode)
}
func IsForward(lineStart m.Position, lineEnd m.Position, bearing float64) bool {
@@ -435,7 +431,7 @@ func (w *Way) DistanceFrom(pos m.Position) (DistanceResult, error) {
minIdx := 0
for i := 0; i < w.Nodes.Len()-1; i++ {
nodeStart := w.Nodes.At(i)
nodeEnd := w.Nodes.At(i + 1)
nodeEnd := w.Nodes.At(i+1)
line := m.Line{Start: nodeStart, End: nodeEnd}
linePosition := line.NearestPosition(pos)
distance := pos.DistanceTo(linePosition.Pos)
@@ -451,7 +447,7 @@ func (w *Way) DistanceFrom(pos m.Position) (DistanceResult, error) {
onWayDistance := minNodeStart.DistanceTo(minLinePosition.Pos)
for i := range minIdx {
nodeStart := w.Nodes.At(i)
nodeEnd := w.Nodes.At(i + 1)
nodeEnd := w.Nodes.At(i+1)
onWayDistance += nodeStart.DistanceTo(nodeEnd)
}
@@ -495,7 +491,8 @@ func (w *Way) MatchingWays(offlineMaps *Offline, matchNode m.Position) ([]Way, e
continue
}
if way.Id() == w.Id() {
box := way.Box()
if box.Equals(w.Box()) {
continue
}
-45
View File
@@ -55,48 +55,3 @@ func TestBearingAlignmentIsAnUndirectedNonnegativePenalty(t *testing.T) {
}
}
}
func TestWayEntryDirectionForOpenAndClosedWays(t *testing.T) {
start, end := m.NewPosition(35.15, -97.9), m.NewPosition(35.151, -97.9)
for _, tc := range []struct {
name string
closed, oneWay, fromEnd, wantForward bool
}{
{"open start", false, true, false, true},
{"open end", false, true, true, false},
{"closed one-way", true, true, false, true},
{"closed two-way", true, false, false, false},
} {
t.Run(tc.name, func(t *testing.T) {
_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
if err != nil {
t.Fatal(err)
}
raw, err := offline.NewRootWay(seg)
if err != nil {
t.Fatal(err)
}
raw.SetOneWay(tc.oneWay)
positions := []m.Position{start, end}
if tc.closed {
positions = append(positions, start)
}
nodes, err := raw.NewNodes(int32(len(positions)))
if err != nil {
t.Fatal(err)
}
for i, p := range positions {
nodes.At(i).SetLatitude(p.Lat())
nodes.At(i).SetLongitude(p.Lon())
}
from := start
if tc.fromEnd {
from = end
}
way := NewWay(raw)
if got := way.IsForwardFrom(from); got != tc.wantForward {
t.Fatalf("entry direction: got %v want %v", got, tc.wantForward)
}
})
}
}
+1 -7
View File
@@ -24,7 +24,7 @@ type syntheticWay struct {
// The actual offline Capnp representation is packed, decoded, and passed to
// State; tests do not mock GetCurrentWay or BuildMessage.
func packedRoadTileBytes(tb testing.TB, specs []syntheticWay) []byte {
func packedRoadTile(tb testing.TB, specs []syntheticWay) maps.Offline {
tb.Helper()
msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
if err != nil {
@@ -65,12 +65,6 @@ func packedRoadTileBytes(tb testing.TB, specs []syntheticWay) []byte {
if err != nil {
tb.Fatal(err)
}
return data
}
func packedRoadTile(tb testing.TB, specs []syntheticWay) maps.Offline {
tb.Helper()
data := packedRoadTileBytes(tb, specs)
parsed := maps.ReadOffline(data)
if !parsed.Loaded {
tb.Fatal("packed tile rejected")
-9
View File
@@ -46,7 +46,6 @@ type State struct {
SourceGeneration uint64
ProducerSession uint64
MatchedLimitMps float32
LimitPreview LimitPreview
lastMapLoadAttempt time.Time
bootNow func() uint64
// The fix the current road result was computed from. The loop runs at
@@ -91,7 +90,6 @@ func (s *State) clearRoadDerived() {
s.TargetVelocities = nil
s.MapCurveSpeed = 0
s.MatchedLimitMps = 0
s.LimitPreview = LimitPreview{}
s.NextHazard.Reset()
s.NextAdvisorySpeed.Reset()
s.clearSpeedLimitEvidence()
@@ -257,7 +255,6 @@ func (s *State) ProcessGps(sample cereal.GpsSample, success bool, load MapLoader
s.MatchedLimitMps = float32(limit)
s.RoadStatus = custom.MapdOut_SampleStatus_matchedLimit
}
s.LimitPreview = s.previewLimit()
}
func (s *State) SuggestedSpeed() float32 {
@@ -364,12 +361,6 @@ func (s *State) buildMessageAt(now uint64) (*capnp.Message, error) {
output.SetRoadName(s.CurrentWay.Way.Name())
output.SetSpeedLimit(s.MatchedLimitMps)
if s.LimitPreview.BoundaryID != "" {
output.SetPreviewSpeedLimit(s.LimitPreview.Speed)
output.SetPreviewDistance(s.LimitPreview.Distance)
output.SetPreviewBoundaryId(s.LimitPreview.BoundaryID)
output.SetPreviewWayId(s.LimitPreview.WayID)
}
output.SetConditionalSpeedLimit(s.CurrentWay.ConditionalMaxSpeedRaw())
-3
View File
@@ -20,6 +20,3 @@ libmessaging.*
libmessaging_shared.*
.sconsign.dblite
.mypy_cache/
/msgq/_ipc.pyi
/msgq/visionipc/_visionipc.pyi
+3 -2
View File
@@ -1,3 +1,4 @@
include README.md pyproject.toml setup.py SConstruct SConscript
recursive-include msgq *.py py.typed *.h *.cc
global-exclude *.pyi *.so *.o *.os *.a *.pyc
recursive-include site_scons *.py
recursive-include msgq *.py *.pyx *.pxd *.h *.cc
global-exclude *.so *.o *.os *.a *.pyc
+4 -33
View File
@@ -1,29 +1,7 @@
import sys
import sysconfig
from pathlib import Path
import nanobind
Import('env', 'arch', 'common')
envNanobind = env.Clone(LIBS=[])
nb_dir = Path(nanobind.source_dir()).parent
envNanobind.Append(CPPPATH=[sysconfig.get_paths()['include'], nanobind.include_dir(), str(nb_dir / "ext/robin_map/include")],
CCFLAGS=["-fvisibility=hidden", "-fno-strict-aliasing"])
if arch == "Darwin":
envNanobind.Append(LINKFLAGS=["-bundle", "-undefined", "dynamic_lookup"])
else:
# Python symbols resolve at import time.
envNanobind["LINKFLAGS"] = [f for f in env["LINKFLAGS"] if f != "-Wl,--no-undefined"] + ["-pthread", "-shared"]
# Third-party runtime sources do not follow our shadowing warning policy.
envNanobindRuntime = envNanobind.Clone()
envNanobindRuntime['CCFLAGS'] = [flag for flag in envNanobindRuntime['CCFLAGS'] if not flag.startswith('-Wshadow')]
nanobind_lib = envNanobindRuntime.SharedObject('msgq/nb_combined.os', str(nb_dir / "src/nb_combined.cpp"))
Import('env', 'envCython', 'common')
visionipc_dir = Dir('msgq/visionipc')
python_suffix = ".so"
# Build msgq
msgq_objects = env.SharedObject([
@@ -32,7 +10,7 @@ msgq_objects = env.SharedObject([
'msgq/msgq.cc',
])
msgq = env.Library('msgq', msgq_objects)
msgq_python = envNanobind.Program(f'msgq/_ipc{python_suffix}', ['msgq/ipc_py.cc', nanobind_lib], LIBS=envNanobind["LIBS"]+[msgq]+common)
msgq_python = envCython.Program('msgq/ipc_pyx.so', 'msgq/ipc_pyx.pyx', LIBS=envCython["LIBS"]+[msgq]+common)
# Build Vision IPC
vipc_files = ['visionipc.cc', 'visionipc_server.cc', 'visionipc_client.cc']
@@ -46,17 +24,10 @@ vipc_objects = env.SharedObject(vipc_sources)
visionipc = env.Library('visionipc', vipc_objects)
vipc_libs = envNanobind["LIBS"] + [visionipc, msgq] + common
visionipc_python = envNanobind.Program(f'{visionipc_dir.abspath}/_visionipc{python_suffix}', [f'{visionipc_dir.abspath}/visionipc_py.cc', nanobind_lib],
vipc_libs = envCython["LIBS"] + [visionipc, msgq] + common
envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir.abspath}/visionipc_pyx.pyx',
LIBS=vipc_libs)
# ASan modules require a preloaded runtime before Python can import them.
if "-fsanitize=address" not in env["LINKFLAGS"]:
root = Dir('.').abspath
for module in ('msgq._ipc', 'msgq.visionipc._visionipc'):
env.Command(module.replace('.', '/') + '.pyi', [msgq_python, visionipc_python],
f'"{sys.executable}" -m nanobind.stubgen -i "{root}" -m {module} -o "$TARGET"')
if GetOption('extras'):
env.Program('msgq/test_runner', ['msgq/msgq_tests.cc'], LIBS=[msgq]+common)
+15 -1
View File
@@ -1,6 +1,7 @@
import os
import platform
import subprocess
import sysconfig
arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
if platform.system() == "Darwin":
@@ -13,6 +14,7 @@ cpppath = [
"#/",
'#msgq/',
'/usr/lib/include',
sysconfig.get_paths()['include'],
]
AddOption('--minimal',
@@ -60,9 +62,21 @@ env = Environment(
CFLAGS="-std=gnu11",
CXXFLAGS="-std=c++1z",
CPPPATH=cpppath,
tools=["default"]
CYTHONCFILESUFFIX=".cpp",
tools=["default", "cython"]
)
Export('env', 'arch', 'common')
envCython = env.Clone(LIBS=[])
envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
envCython["CCFLAGS"].remove('-Werror')
if arch == "Darwin":
envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
else:
envCython["LINKFLAGS"] = ["-pthread", "-shared"]
Export('envCython')
SConscript(['SConscript'])
+2 -2
View File
@@ -18,12 +18,12 @@ test:
ty:
run: ty check .
codespell:
run: codespell {files} -L ned,stdio,master --builtin clear,rare,informal,usage,code,names,en-GB_to_en-US -S uv.lock
run: codespell {files} -L ned,stdio,master --builtin clear,rare,informal,usage,code,names,en-GB_to_en-US -S uv.lock,*_pyx.cpp
files: git ls-tree -r HEAD --name-only
cppcheck:
run: cppcheck --error-exitcode=1 --inline-suppr --language=c++ --force --quiet -j4 --check-level=exhaustive $(git ls-files '*.cc' | grep -v -E '(msgq_tests|test_runner)\.cc')
cpplint:
run: cpplint --recursive --quiet --counting=detailed --linelength=240 --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces msgq/
run: cpplint --exclude=msgq/ipc_pyx.cpp --exclude=msgq/visionipc/visionipc_pyx.cpp --recursive --quiet --counting=detailed --linelength=240 --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces msgq/
# *** tests ***
test_runner:
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
ipc_pyx.cpp
+4 -4
View File
@@ -1,8 +1,8 @@
from msgq._ipc import Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
from msgq.ipc_pyx import Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
from msgq._ipc import MultiplePublishersError, IpcError
from msgq.ipc_pyx import MultiplePublishersError, IpcError
from typing import Optional, List
from typing import Optional, List, Union
__all__ = [
"Poller",
@@ -25,7 +25,7 @@ __all__ = [
NO_TRAVERSAL_LIMIT = 2**64-1
def fake_event_handle(endpoint: str, identifier: Optional[str] = None, override: bool = True, enable: bool = False) -> SocketEventHandle:
def fake_event_handle(endpoint: str, identifier: Optional[Union[str, bytes]] = None, override: bool = True, enable: bool = False) -> SocketEventHandle:
ident = identifier if identifier is not None else get_fake_prefix()
handle = SocketEventHandle(endpoint, ident, override)
if override:
Binary file not shown.
+61
View File
@@ -0,0 +1,61 @@
# distutils: language = c++
#cython: language_level=3
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp cimport bool
cdef extern from "msgq/event.h":
cdef cppclass Event:
@staticmethod
int wait_for_one(vector[Event], int) except +
Event()
Event(int)
void set()
int clear()
void wait(int) except +
bool peek()
int fd()
cdef cppclass SocketEventHandle:
@staticmethod
void toggle_fake_events(bool)
@staticmethod
void set_fake_prefix(string)
@staticmethod
string fake_prefix()
SocketEventHandle(string, string, bool)
bool is_enabled()
void set_enabled(bool)
Event recv_called()
Event recv_ready()
cdef extern from "msgq/ipc.h":
cdef cppclass Message:
void init(size_t)
void init(char *, size_t)
void close()
size_t getSize()
char *getData()
cdef cppclass SubSocket:
SubSocket() nogil
int connect(string, bool, size_t) nogil
Message * receive(bool) nogil
void setTimeout(int) nogil
cdef cppclass PubSocket:
PubSocket()
int connect(string, size_t)
int sendMessage(Message *)
int send(char *, size_t)
bool all_readers_updated()
cdef cppclass Poller:
Poller()
void registerSocket(SubSocket *)
vector[SubSocket*] poll(int) nogil
-113
View File
@@ -1,113 +0,0 @@
#include <cerrno>
#include <chrono>
#include <cstdint>
#include <cstring>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <vector>
#include <nanobind/nanobind.h>
#include <nanobind/stl/optional.h>
#include <nanobind/stl/string.h>
#include <nanobind/stl/vector.h>
#include "msgq/event.h"
#include "msgq/ipc.h"
namespace nb = nanobind;
namespace {
struct IpcError : std::runtime_error {
using std::runtime_error::runtime_error;
};
struct MultiplePublishersError : IpcError {
using IpcError::IpcError;
};
void check_error(bool failed, int error, const std::string &endpoint = "") {
if (!failed) return;
std::string message = "Messaging failure " + (endpoint.empty() ? "" : "with " + endpoint) + ": " + std::strerror(error);
if (error == EADDRINUSE) throw MultiplePublishersError(message);
throw IpcError(message);
}
} // namespace
NB_MODULE(_ipc, m) {
// Daemon threads can still hold sockets at exit.
nb::set_leak_warnings(false);
nb::exception<IpcError> ipc_error(m, "IpcError");
nb::exception<MultiplePublishersError>(m, "MultiplePublishersError", ipc_error);
m.def("toggle_fake_events", &SocketEventHandle::toggle_fake_events, nb::arg("enabled"));
m.def("set_fake_prefix", &SocketEventHandle::set_fake_prefix, nb::arg("prefix"));
m.def("get_fake_prefix", &SocketEventHandle::fake_prefix);
m.def("delete_fake_prefix", [] { SocketEventHandle::set_fake_prefix(""); });
m.def("wait_for_one_event", &Event::wait_for_one, nb::arg("events"), nb::arg("timeout") = -1, nb::call_guard<nb::gil_scoped_release>());
nb::class_<Event>(m, "Event")
.def(nb::init<>())
.def("set", &Event::set)
.def("clear", &Event::clear)
.def("wait", &Event::wait, nb::arg("timeout") = -1, nb::call_guard<nb::gil_scoped_release>())
.def("peek", &Event::peek)
.def_prop_ro("fd", &Event::fd)
.def_prop_ro("ptr", [](Event &event) { return reinterpret_cast<uintptr_t>(&event); });
nb::class_<SocketEventHandle>(m, "SocketEventHandle")
.def(nb::init<std::string, std::string, bool>(), nb::arg("endpoint"), nb::arg("identifier"), nb::arg("override"))
.def_prop_rw("enabled", &SocketEventHandle::is_enabled, &SocketEventHandle::set_enabled)
.def_prop_ro("recv_called_event", &SocketEventHandle::recv_called, nb::keep_alive<0, 1>())
.def_prop_ro("recv_ready_event", &SocketEventHandle::recv_ready, nb::keep_alive<0, 1>());
nb::class_<SubSocket>(m, "SubSocket")
.def(nb::init<>())
.def("connect", [](SubSocket &socket, const std::string &name, bool conflate, size_t segment_size) {
nb::gil_scoped_release release;
int result = socket.connect(name, conflate, segment_size);
check_error(result != 0, errno, name);
}, nb::arg("endpoint"), nb::arg("conflate") = false, nb::arg("segment_size") = 0)
.def("setTimeout", &SubSocket::setTimeout, nb::arg("timeout"), nb::call_guard<nb::gil_scoped_release>())
.def("receive", [](SubSocket &socket, bool non_blocking) -> std::optional<nb::bytes> {
std::unique_ptr<Message> message;
{
nb::gil_scoped_release release;
message.reset(socket.receive(non_blocking));
}
if (!message) return std::nullopt;
return nb::bytes(message->getData(), message->getSize());
}, nb::arg("non_blocking") = false);
nb::class_<Poller>(m, "Poller")
.def(nb::init<>())
.def("registerSocket", &Poller::registerSocket, nb::arg("socket"), nb::keep_alive<1, 2>())
.def("poll", &Poller::poll, nb::arg("timeout"), nb::rv_policy::reference, nb::call_guard<nb::gil_scoped_release>());
nb::class_<PubSocket>(m, "PubSocket")
.def(nb::init<>())
.def("connect", [](PubSocket &socket, const std::string &name, size_t segment_size) {
int result = socket.connect(name, segment_size);
check_error(result != 0, errno, name);
}, nb::arg("endpoint"), nb::arg("segment_size") = 0)
.def("send", [](PubSocket &socket, nb::bytes data) {
int result = socket.send(const_cast<char *>(data.c_str()), data.size());
check_error(result != static_cast<int>(data.size()), errno);
}, nb::arg("data"))
.def("all_readers_updated", &PubSocket::all_readers_updated)
.def("wait_for_readers", [](PubSocket &socket, double timeout, double interval) {
// Python's sleep releases the GIL and handles signals while waiting.
nb::object sleep = nb::module_::import_("time").attr("sleep");
auto deadline = std::chrono::steady_clock::now() + std::chrono::duration<double>(timeout);
while (std::chrono::steady_clock::now() < deadline) {
if (socket.all_readers_updated()) return;
sleep(interval);
}
PyErr_SetString(PyExc_TimeoutError, "subscriber did not connect");
throw nb::python_error();
}, nb::arg("timeout") = 1.0, nb::arg("interval") = 0.001);
}
+230
View File
@@ -0,0 +1,230 @@
# distutils: language = c++
# cython: c_string_encoding=ascii, language_level=3
import time
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp cimport bool
from libc cimport errno
from libc.string cimport strerror
from cython.operator import dereference
from .ipc cimport SubSocket as cppSubSocket
from .ipc cimport PubSocket as cppPubSocket
from .ipc cimport Poller as cppPoller
from .ipc cimport Message as cppMessage
from .ipc cimport Event as cppEvent, SocketEventHandle as cppSocketEventHandle
class IpcError(Exception):
def __init__(self, endpoint=None):
suffix = f"with {endpoint.decode('utf-8')}" if endpoint else ""
message = f"Messaging failure {suffix}: {strerror(errno.errno).decode('utf-8')}"
super().__init__(message)
class MultiplePublishersError(IpcError):
pass
def toggle_fake_events(bool enabled):
cppSocketEventHandle.toggle_fake_events(enabled)
def set_fake_prefix(string prefix):
cppSocketEventHandle.set_fake_prefix(prefix)
def get_fake_prefix():
return cppSocketEventHandle.fake_prefix()
def delete_fake_prefix():
cppSocketEventHandle.set_fake_prefix(b"")
def wait_for_one_event(list events, int timeout=-1):
cdef vector[cppEvent] items
for event in events:
items.push_back(dereference(<cppEvent*><size_t>event.ptr))
return cppEvent.wait_for_one(items, timeout)
cdef class Event:
cdef cppEvent event;
cdef setEvent(self, cppEvent event):
self.event = event
def set(self):
self.event.set()
def clear(self):
return self.event.clear()
def wait(self, int timeout=-1):
self.event.wait(timeout)
def peek(self):
return self.event.peek()
@property
def fd(self):
return self.event.fd()
@property
def ptr(self):
return <size_t><void*>&self.event
cdef class SocketEventHandle:
cdef cppSocketEventHandle * handle;
def __cinit__(self, string endpoint, string identifier, bool override):
self.handle = new cppSocketEventHandle(endpoint, identifier, override)
def __dealloc__(self):
del self.handle
@property
def enabled(self):
return self.handle.is_enabled()
@enabled.setter
def enabled(self, bool value):
self.handle.set_enabled(value)
@property
def recv_called_event(self):
e = Event()
e.setEvent(self.handle.recv_called())
return e
@property
def recv_ready_event(self):
e = Event()
e.setEvent(self.handle.recv_ready())
return e
cdef class Poller:
cdef cppPoller * poller
cdef list sub_sockets
def __cinit__(self):
self.sub_sockets = []
self.poller = new cppPoller()
def __dealloc__(self):
del self.poller
def registerSocket(self, SubSocket socket):
self.sub_sockets.append(socket)
self.poller.registerSocket(socket.socket)
def poll(self, timeout):
sockets = []
cdef int t = timeout
with nogil:
result = self.poller.poll(t)
for s in result:
socket = SubSocket()
socket.setPtr(s)
sockets.append(socket)
return sockets
cdef class SubSocket:
cdef cppSubSocket * socket
cdef bool is_owner
def __cinit__(self):
with nogil:
self.socket = new cppSubSocket()
self.is_owner = True
def __dealloc__(self):
if self.is_owner:
with nogil:
del self.socket
cdef setPtr(self, cppSubSocket * ptr):
if self.is_owner:
with nogil:
del self.socket
self.is_owner = False
self.socket = ptr
def connect(self, string endpoint, bool conflate=False, size_t segment_size=0):
cdef int r
with nogil:
r = self.socket.connect(endpoint, conflate, segment_size)
if r != 0:
if errno.errno == errno.EADDRINUSE:
raise MultiplePublishersError(endpoint)
else:
raise IpcError(endpoint)
def setTimeout(self, int timeout):
with nogil:
self.socket.setTimeout(timeout)
def receive(self, bool non_blocking=False):
cdef cppMessage *msg
with nogil:
msg = self.socket.receive(non_blocking)
if msg == NULL:
return None
else:
sz = msg.getSize()
m = msg.getData()[:sz]
with nogil:
del msg
return m
cdef class PubSocket:
cdef cppPubSocket * socket
def __cinit__(self):
self.socket = new cppPubSocket()
def __dealloc__(self):
del self.socket
def connect(self, string endpoint, size_t segment_size=0):
r = self.socket.connect(endpoint, segment_size)
if r != 0:
if errno.errno == errno.EADDRINUSE:
raise MultiplePublishersError(endpoint)
else:
raise IpcError(endpoint)
def send(self, bytes data):
length = len(data)
r = self.socket.send(<char*>data, length)
if r != length:
if errno.errno == errno.EADDRINUSE:
raise MultiplePublishersError
else:
raise IpcError
def all_readers_updated(self):
return self.socket.all_readers_updated()
def wait_for_readers(self, double timeout=1.0, double interval=0.001):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if self.all_readers_updated():
return
time.sleep(interval)
raise TimeoutError("subscriber did not connect")
BIN
View File
Binary file not shown.
-95
View File
@@ -569,98 +569,3 @@ TEST_CASE("32 active subscribers retain every message across publisher restart")
msgq_close_queue(&pub);
std::remove(file.c_str());
}
static int64_t millis_since(std::chrono::steady_clock::time_point start)
{
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start).count();
}
static std::thread send_later(msgq_queue_t *q, int delay_ms)
{
return std::thread([=]() {
std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms));
uint64_t value = 1;
msgq_msg_t msg;
msgq_msg_init_data(&msg, (char *)&value, sizeof(value));
msgq_msg_send(&msg, q);
msgq_msg_close(&msg);
});
}
TEST_CASE("msgq_poll times out")
{
cleanup_test_queue();
msgq_queue_t writer, reader;
msgq_new_queue(&writer, "test_queue", 1024);
msgq_new_queue(&reader, "test_queue", 1024);
msgq_init_publisher(&writer);
msgq_init_subscriber(&reader);
msgq_pollitem_t items[] = {{&reader, 0}};
auto start = std::chrono::steady_clock::now();
REQUIRE(msgq_poll(items, 1, 100) == 0);
REQUIRE(millis_since(start) >= 100);
REQUIRE(msgq_poll(items, 1, 0) == 0);
send_later(&writer, 0).join();
REQUIRE(msgq_poll(items, 1, 0) == 1);
msgq_close_queue(&writer);
msgq_close_queue(&reader);
}
TEST_CASE("msgq_poll wakes up on send")
{
cleanup_test_queue();
msgq_queue_t writer, reader;
msgq_new_queue(&writer, "test_queue", 1024);
msgq_new_queue(&reader, "test_queue", 1024);
msgq_init_publisher(&writer);
msgq_init_subscriber(&reader);
msgq_pollitem_t items[] = {{&reader, 0}};
std::thread sender = send_later(&writer, 50);
auto start = std::chrono::steady_clock::now();
int n = msgq_poll(items, 1, 5000);
int64_t elapsed = millis_since(start);
sender.join();
REQUIRE(n == 1);
REQUIRE(elapsed < 1000);
msgq_close_queue(&writer);
msgq_close_queue(&reader);
}
TEST_CASE("msgq_poll multiple queues")
{
cleanup_test_queue();
remove("/dev/shm/msgq_test_queue2");
remove("/tmp/msgq_test_queue2");
msgq_queue_t writer1, writer2, reader1, reader2;
msgq_new_queue(&writer1, "test_queue", 1024);
msgq_new_queue(&writer2, "test_queue2", 1024);
msgq_new_queue(&reader1, "test_queue", 1024);
msgq_new_queue(&reader2, "test_queue2", 1024);
msgq_init_publisher(&writer1);
msgq_init_publisher(&writer2);
msgq_init_subscriber(&reader1);
msgq_init_subscriber(&reader2);
msgq_pollitem_t items[] = {{&reader1, 0}, {&reader2, 0}};
std::thread sender = send_later(&writer2, 50);
auto start = std::chrono::steady_clock::now();
int n = msgq_poll(items, 2, 5000);
int64_t elapsed = millis_since(start);
sender.join();
REQUIRE(n == 1);
REQUIRE(elapsed < 1000);
REQUIRE(items[0].revents == 0);
REQUIRE(items[1].revents == 1);
msgq_close_queue(&writer1);
msgq_close_queue(&writer2);
msgq_close_queue(&reader1);
msgq_close_queue(&reader2);
}
View File
+2
View File
@@ -0,0 +1,2 @@
visionipc_pyx.cpp
*.so
+1 -1
View File
@@ -1,4 +1,4 @@
from msgq.visionipc._visionipc import VisionBuf, VisionIpcClient, VisionIpcServer, get_endpoint_name
from msgq.visionipc.visionipc_pyx import VisionBuf, VisionIpcClient, VisionIpcServer, get_endpoint_name
__all__ = [
"VisionBuf",
Binary file not shown.
+55 -106
View File
@@ -1,7 +1,6 @@
import gc
import uuid
import struct
import unittest
import uuid
from typing import Optional
from msgq.visionipc import VisionIpcServer, VisionIpcClient
@@ -11,6 +10,7 @@ class TestVisionIpc(unittest.TestCase):
client: Optional[VisionIpcClient]
def setUp(self):
# Each test needs independent listener and publisher endpoints in parallel runs.
self.name = f"test_{uuid.uuid4().hex}"
self.server = None
self.client = None
@@ -19,139 +19,88 @@ class TestVisionIpc(unittest.TestCase):
self.client = None
self.server = None
def setup_vipc(
self, name: str, stream_type: int, *other_stream_types: int,
num_buffers: int = 1, width: int = 100, height: int = 100, conflate: bool = False,
) -> tuple[VisionIpcServer, VisionIpcClient, int]:
self.server = server = VisionIpcServer(self.name)
for stream in (stream_type, *other_stream_types):
server.create_buffers(stream, num_buffers, width, height)
server.start_listener()
def setup_vipc(self, *stream_types, num_buffers=1, width=100, height=100, conflate=False):
self.server = VisionIpcServer(self.name)
for stream_type in stream_types:
self.server.create_buffers(stream_type, num_buffers, width, height)
self.server.start_listener()
self.client = client = VisionIpcClient(self.name, stream_type, conflate)
assert client.connect(True)
buffer_len = client.buffer_len
assert buffer_len is not None
return server, client, buffer_len
if len(stream_types):
self.client = VisionIpcClient(self.name, stream_types[0], conflate)
assert self.client.connect(True)
else:
self.client = None
return self.server, self.client
def test_connect(self):
_, client, _ = self.setup_vipc("camerad", 0)
assert client.is_connected()
self.setup_vipc(0)
assert self.client is not None
assert self.client.is_connected()
def test_available_streams(self):
stream_types = (0, 2)
self.setup_vipc("camerad", *stream_types)
self.setup_vipc(*stream_types)
available_streams = VisionIpcClient.available_streams(self.name, True)
assert available_streams == set(stream_types)
def test_buffers(self):
width, height, num_buffers = 100, 200, 5
_, client, buffer_len = self.setup_vipc("camerad", 0, num_buffers=num_buffers, width=width, height=height)
assert client.width == width
assert client.height == height
assert buffer_len > 0
assert client.num_buffers == num_buffers
self.setup_vipc(0, num_buffers=num_buffers, width=width, height=height)
assert self.client is not None
assert self.client.width == width
assert self.client.height == height
assert self.client.buffer_len is not None and self.client.buffer_len > 0
assert self.client.num_buffers == num_buffers
def test_send_single_buffer(self):
server, client, buffer_len = self.setup_vipc("camerad", 0)
buf = bytearray(buffer_len)
self.setup_vipc(0)
assert self.server is not None
assert self.client is not None
assert self.client.buffer_len is not None
buf = bytearray(self.client.buffer_len)
struct.pack_into("<Q", buf, 0, 1234)
server.send(0, buf, frame_id=1337)
self.server.send(0, buf, frame_id=1337)
recv_buf = client.recv()
recv_buf = self.client.recv()
assert recv_buf is not None
data = recv_buf.data
assert isinstance(data, memoryview)
assert struct.unpack_from("<Q", data, 0)[0] == 1234
assert len(data) == buffer_len
assert data[8:].nbytes == buffer_len - 8
assert client.frame_id == 1337
assert len(data) == self.client.buffer_len
assert data[8:].nbytes == self.client.buffer_len - 8
assert self.client.frame_id == 1337
assert recv_buf.frame_id == 1337
def test_no_conflate(self):
server, client, buffer_len = self.setup_vipc("camerad", 0)
buf = bytearray(buffer_len)
server.send(0, buf, frame_id=1)
server.send(0, buf, frame_id=2)
self.setup_vipc(0)
assert self.server is not None
assert self.client is not None
assert self.client.buffer_len is not None
buf = bytearray(self.client.buffer_len)
self.server.send(0, buf, frame_id=1)
self.server.send(0, buf, frame_id=2)
recv_buf = client.recv()
recv_buf = self.client.recv()
assert recv_buf is not None
assert client.frame_id == 1
assert self.client.frame_id == 1
recv_buf = client.recv()
recv_buf = self.client.recv()
assert recv_buf is not None
assert client.frame_id == 2
assert self.client.frame_id == 2
def test_conflate(self):
server, client, buffer_len = self.setup_vipc("camerad", 0, conflate=True)
buf = bytearray(buffer_len)
server.send(0, buf, frame_id=1)
server.send(0, buf, frame_id=2)
self.setup_vipc(0, conflate=True)
assert self.server is not None
assert self.client is not None
assert self.client.buffer_len is not None
buf = bytearray(self.client.buffer_len)
self.server.send(0, buf, frame_id=1)
self.server.send(0, buf, frame_id=2)
recv_buf = client.recv()
recv_buf = self.client.recv()
assert recv_buf is not None
assert client.frame_id == 2
assert self.client.frame_id == 2
recv_buf = client.recv(timeout_ms=5)
recv_buf = self.client.recv(timeout_ms=5)
assert recv_buf is None
def test_send_buffer_types(self):
server, client, buffer_len = self.setup_vipc("camerad", 0)
data = bytes(i % 256 for i in range(buffer_len))
strided = bytearray(len(data) * 2)
strided[::2] = data
for payload in (data, bytearray(data), memoryview(data), memoryview(strided)[::2], memoryview(data[::-1])[::-1]):
with self.subTest(type=type(payload)):
server.send(0, payload, frame_id=42, timestamp_sof=123, timestamp_eof=456)
buf = client.recv()
assert buf is not None
self.assertEqual(bytes(buf.data), data)
self.assertEqual(buf.frame_id, 42)
self.assertEqual(client.timestamp_sof, 123)
self.assertEqual(client.timestamp_eof, 456)
self.assertFalse(client.valid)
def test_send_wrong_size(self):
server, _, _ = self.setup_vipc("camerad", 0)
with self.assertRaises(AssertionError):
server.send(0, b"short")
def test_buffer_keeps_client_alive(self):
server, client, buffer_len = self.setup_vipc("camerad", 0)
data = b"a" * buffer_len
server.send(0, data, frame_id=42)
buf = client.recv()
assert buf is not None
self.client = None
del client
gc.collect()
self.assertEqual(buf.frame_id, 42)
view = buf.data
del buf
gc.collect()
self.assertEqual(bytes(view), data)
self.assertFalse(view.readonly)
view[0] = ord("b")
self.assertEqual(view[0], ord("b"))
def test_unconnected_properties(self):
client = VisionIpcClient("unconnected", 0, False)
for value in (client.width, client.height, client.stride, client.uv_offset, client.buffer_len):
self.assertIsNone(value)
self.assertEqual(client.num_buffers, 0)
self.assertFalse(client.is_connected())
self.assertFalse(client.connect(False))
self.assertIsNone(client.recv(timeout_ms=0))
def test_exact_uint64_timestamps_survive_receive_and_timeout(self):
server, client, size = self.setup_vipc("clock", 0, width=32, height=18)
payload = bytes(size)
for frame, sof, eof in ((0, 0, 0), (1, 2**53 + 1, 2**53 + 19),
(2**32 - 1, 2**63 + 17, 2**64 - 1)):
with self.subTest(frame=frame, sof=sof, eof=eof):
server.send(0, payload, frame_id=frame, timestamp_sof=sof, timestamp_eof=eof)
buffer = client.recv(timeout_ms=1000)
assert buffer is not None
assert (client.frame_id, client.timestamp_sof, client.timestamp_eof) == (frame, sof, eof)
assert client.recv(timeout_ms=0) is None
assert (client.frame_id, client.timestamp_sof, client.timestamp_eof) == (frame, sof, eof)
+52
View File
@@ -0,0 +1,52 @@
# distutils: language = c++
#cython: language_level=3
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp.set cimport set
from libc.stdint cimport uint32_t, uint64_t
from libcpp cimport bool, int
cdef extern from "msgq/visionipc/visionbuf.h":
ctypedef uint32_t VisionStreamType "VisionStreamType"
cdef cppclass VisionBuf:
void * addr
int fd
size_t len
size_t width
size_t height
size_t stride
size_t uv_offset
size_t idx
void set_frame_id(uint64_t id)
uint64_t get_frame_id()
cdef extern from "msgq/visionipc/visionipc.h":
struct VisionIpcBufExtra:
uint32_t frame_id
uint64_t timestamp_sof
uint64_t timestamp_eof
bool valid
cdef extern from "msgq/visionipc/visionipc_server.h":
string get_endpoint_name(string, VisionStreamType)
cdef cppclass VisionIpcServer:
VisionIpcServer(string)
void create_buffers(VisionStreamType, size_t, size_t, size_t)
void create_buffers_with_sizes(VisionStreamType, size_t, size_t, size_t, size_t, size_t, size_t)
VisionBuf * get_buffer(VisionStreamType)
void send(VisionBuf *, VisionIpcBufExtra *, bool)
void start_listener()
cdef extern from "msgq/visionipc/visionipc_client.h":
cdef cppclass VisionIpcClient:
int num_buffers
VisionBuf buffers[1]
VisionIpcClient(string, VisionStreamType, bool)
VisionBuf * recv(VisionIpcBufExtra *, int) nogil
bool connect(bool)
bool is_connected()
@staticmethod
set[VisionStreamType] getAvailableStreams(string, bool)
-109
View File
@@ -1,109 +0,0 @@
#include <cstdint>
#include <cstring>
#include <optional>
#include <string>
#include <nanobind/nanobind.h>
#include <nanobind/ndarray.h>
#include <nanobind/stl/optional.h>
#include <nanobind/stl/set.h>
#include <nanobind/stl/string.h>
#include "msgq/visionipc/visionbuf.h"
#include "msgq/visionipc/visionipc.h"
#include "msgq/visionipc/visionipc_client.h"
#include "msgq/visionipc/visionipc_server.h"
namespace nb = nanobind;
namespace {
struct PythonVisionIpcClient : VisionIpcClient {
using VisionIpcClient::VisionIpcClient;
VisionIpcBufExtra extra = {};
};
using Buffer = nb::ndarray<nb::memview, uint8_t, nb::ndim<1>, nb::device::cpu>;
using InputBuffer = nb::ndarray<const uint8_t, nb::ndim<1>, nb::device::cpu>;
} // namespace
NB_MODULE(_visionipc, m) {
// Daemon threads can still hold clients and servers at exit.
nb::set_leak_warnings(false);
m.def("get_endpoint_name", &get_endpoint_name, nb::arg("name"), nb::arg("stream"));
nb::class_<VisionBuf>(m, "VisionBuf")
.def_prop_ro("data", [](VisionBuf &buf) { return Buffer(buf.addr, {buf.len}); }, nb::rv_policy::reference_internal,
nb::sig("def data(self) -> memoryview"))
.def_ro("width", &VisionBuf::width)
.def_ro("height", &VisionBuf::height)
.def_ro("stride", &VisionBuf::stride)
.def_ro("uv_offset", &VisionBuf::uv_offset)
.def_ro("idx", &VisionBuf::idx)
.def_ro("fd", &VisionBuf::fd)
.def_prop_ro("frame_id", &VisionBuf::get_frame_id);
nb::class_<VisionIpcServer>(m, "VisionIpcServer")
.def(nb::init<std::string>(), nb::arg("name"))
.def("create_buffers", &VisionIpcServer::create_buffers, nb::arg("tp"), nb::arg("num_buffers"), nb::arg("width"), nb::arg("height"))
.def("create_buffers_with_sizes", &VisionIpcServer::create_buffers_with_sizes,
nb::arg("tp"), nb::arg("num_buffers"), nb::arg("width"), nb::arg("height"), nb::arg("size"), nb::arg("stride"), nb::arg("uv_offset"))
.def("send", [](VisionIpcServer &server, VisionStreamType tp, InputBuffer data,
uint32_t frame_id, uint64_t timestamp_sof, uint64_t timestamp_eof) {
VisionBuf *buf = server.get_buffer(tp);
if (buf->len != data.size()) {
PyErr_SetString(PyExc_AssertionError, "buffer size does not match frame size");
throw nb::python_error();
}
// The old binding accepted strided byte views as well as contiguous data.
auto *source = data.data();
if (data.stride(0) == 1) {
std::memcpy(buf->addr, source, data.size());
} else {
auto *target = static_cast<uint8_t *>(buf->addr);
for (size_t i = 0; i < data.size(); ++i) target[i] = source[static_cast<int64_t>(i) * data.stride(0)];
}
buf->set_frame_id(frame_id);
VisionIpcBufExtra extra = {frame_id, timestamp_sof, timestamp_eof, false};
server.send(buf, &extra, false);
}, nb::arg("tp"), nb::arg("data").noconvert(), nb::arg("frame_id") = 0, nb::arg("timestamp_sof") = 0, nb::arg("timestamp_eof") = 0,
nb::sig("def send(self, tp: int, data: bytes | bytearray | memoryview, frame_id: int = 0, timestamp_sof: int = 0, timestamp_eof: int = 0) -> None"))
.def("start_listener", &VisionIpcServer::start_listener);
nb::class_<PythonVisionIpcClient>(m, "VisionIpcClient")
.def(nb::init<std::string, VisionStreamType, bool>(), nb::arg("name"), nb::arg("stream"), nb::arg("conflate"))
.def_prop_ro("width", [](PythonVisionIpcClient &client) -> std::optional<size_t> {
return client.num_buffers ? std::make_optional(client.buffers[0].width) : std::nullopt;
})
.def_prop_ro("height", [](PythonVisionIpcClient &client) -> std::optional<size_t> {
return client.num_buffers ? std::make_optional(client.buffers[0].height) : std::nullopt;
})
.def_prop_ro("stride", [](PythonVisionIpcClient &client) -> std::optional<size_t> {
return client.num_buffers ? std::make_optional(client.buffers[0].stride) : std::nullopt;
})
.def_prop_ro("uv_offset", [](PythonVisionIpcClient &client) -> std::optional<size_t> {
return client.num_buffers ? std::make_optional(client.buffers[0].uv_offset) : std::nullopt;
})
.def_prop_ro("buffer_len", [](PythonVisionIpcClient &client) -> std::optional<size_t> {
return client.num_buffers ? std::make_optional(client.buffers[0].len) : std::nullopt;
})
.def_prop_ro("num_buffers", [](PythonVisionIpcClient &client) { return client.num_buffers; })
.def_prop_ro("frame_id", [](PythonVisionIpcClient &client) { return client.extra.frame_id; })
.def_prop_ro("timestamp_sof", [](PythonVisionIpcClient &client) { return client.extra.timestamp_sof; })
.def_prop_ro("timestamp_eof", [](PythonVisionIpcClient &client) { return client.extra.timestamp_eof; })
.def_prop_ro("valid", [](PythonVisionIpcClient &client) { return client.extra.valid; })
.def("recv", [](PythonVisionIpcClient &client, int timeout_ms) {
nb::gil_scoped_release release;
return client.recv(&client.extra, timeout_ms);
}, nb::arg("timeout_ms") = 100, nb::rv_policy::reference_internal,
nb::sig("def recv(self, timeout_ms: int = 100) -> VisionBuf | None"))
// Holds the GIL: connect remaps the buffers other threads may be reading.
.def("connect", [](PythonVisionIpcClient &client, bool blocking) {
return client.connect(blocking);
}, nb::arg("blocking"))
.def("is_connected", [](PythonVisionIpcClient &client) { return client.is_connected(); })
.def_static("available_streams", &VisionIpcClient::getAvailableStreams, nb::arg("name"), nb::arg("block"),
nb::call_guard<nb::gil_scoped_release>());
}
@@ -0,0 +1,10 @@
# distutils: language = c++
#cython: language_level=3
from .visionipc cimport VisionBuf as cppVisionBuf
cdef class VisionBuf:
cdef cppVisionBuf * buf
@staticmethod
cdef create(cppVisionBuf*)
+163
View File
@@ -0,0 +1,163 @@
# distutils: language = c++
# cython: c_string_encoding=ascii, language_level=3
from libc.string cimport memcpy
from libc.stdint cimport uint32_t, uint64_t
from libcpp cimport bool
from libcpp.string cimport string
from .visionipc cimport VisionIpcServer as cppVisionIpcServer
from .visionipc cimport VisionIpcClient as cppVisionIpcClient
from .visionipc cimport VisionBuf as cppVisionBuf
from .visionipc cimport VisionIpcBufExtra, VisionStreamType
from .visionipc cimport get_endpoint_name as cpp_get_endpoint_name
def get_endpoint_name(string name, VisionStreamType stream):
return cpp_get_endpoint_name(name, stream).decode('utf-8')
cdef class VisionBuf:
@staticmethod
cdef create(cppVisionBuf * cbuf):
buf = VisionBuf()
buf.buf = cbuf
return buf
@property
def data(self):
cdef unsigned char[:] data = <unsigned char[:self.buf.len]> self.buf.addr
return memoryview(data)
@property
def width(self):
return self.buf.width
@property
def height(self):
return self.buf.height
@property
def stride(self):
return self.buf.stride
@property
def uv_offset(self):
return self.buf.uv_offset
@property
def idx(self):
return self.buf.idx
@property
def fd(self):
return self.buf.fd
@property
def frame_id(self):
return self.buf.get_frame_id()
cdef class VisionIpcServer:
cdef cppVisionIpcServer * server
def __init__(self, string name):
self.server = new cppVisionIpcServer(name)
def create_buffers(self, VisionStreamType tp, size_t num_buffers, size_t width, size_t height):
self.server.create_buffers(tp, num_buffers, width, height)
def create_buffers_with_sizes(self, VisionStreamType tp, size_t num_buffers, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset):
self.server.create_buffers_with_sizes(tp, num_buffers, width, height, size, stride, uv_offset)
def send(self, VisionStreamType tp, const unsigned char[:] data, uint32_t frame_id=0, uint64_t timestamp_sof=0, uint64_t timestamp_eof=0):
cdef cppVisionBuf * buf = self.server.get_buffer(tp)
# Populate buffer
assert buf.len == len(data)
memcpy(buf.addr, &data[0], len(data))
buf.set_frame_id(frame_id)
cdef VisionIpcBufExtra extra
extra.frame_id = frame_id
extra.timestamp_sof = timestamp_sof
extra.timestamp_eof = timestamp_eof
extra.valid = False
self.server.send(buf, &extra, False)
def start_listener(self):
self.server.start_listener()
def __dealloc__(self):
del self.server
cdef class VisionIpcClient:
cdef cppVisionIpcClient * client
cdef VisionIpcBufExtra extra
def __cinit__(self, string name, VisionStreamType stream, bool conflate):
self.client = new cppVisionIpcClient(name, stream, conflate)
def __dealloc__(self):
del self.client
@property
def width(self):
return self.client.buffers[0].width if self.client.num_buffers else None
@property
def height(self):
return self.client.buffers[0].height if self.client.num_buffers else None
@property
def stride(self):
return self.client.buffers[0].stride if self.client.num_buffers else None
@property
def uv_offset(self):
return self.client.buffers[0].uv_offset if self.client.num_buffers else None
@property
def buffer_len(self):
return self.client.buffers[0].len if self.client.num_buffers else None
@property
def num_buffers(self):
return self.client.num_buffers
@property
def frame_id(self):
return self.extra.frame_id
@property
def timestamp_sof(self):
return self.extra.timestamp_sof
@property
def timestamp_eof(self):
return self.extra.timestamp_eof
@property
def valid(self):
return self.extra.valid
def recv(self, int timeout_ms=100):
cdef cppVisionBuf * buf
# release the GIL, this can block for timeout_ms
with nogil:
buf = self.client.recv(&self.extra, timeout_ms)
if not buf:
return None
return VisionBuf.create(buf)
def connect(self, bool blocking):
return self.client.connect(blocking)
def is_connected(self):
return self.client.is_connected()
@staticmethod
def available_streams(string name, bool block):
return cppVisionIpcClient.getAvailableStreams(name, block)
Binary file not shown.
+12 -12
View File
@@ -1,5 +1,5 @@
[build-system]
requires = ["setuptools>=77", "nanobind>=2.12", "scons"]
requires = ["setuptools>=77", "Cython>=3.1", "scons"]
build-backend = "setuptools.build_meta"
[project]
@@ -21,14 +21,10 @@ dependencies = []
Repository = "https://github.com/commaai/msgq"
[project.optional-dependencies]
# for building from source with SCons
build = [
"nanobind>=2.12",
"scons",
]
dev = [
"msgq-ipc[build]",
"setuptools", # for distutils
"Cython",
"scons",
"ruff",
"cppcheck",
"cpplint",
@@ -45,16 +41,13 @@ include = ["msgq", "msgq.*"]
namespaces = false
[tool.setuptools.package-data]
"msgq" = ["*.pyi", "py.typed", "*.h"]
"msgq.visionipc" = ["*.pyi", "*.h"]
"msgq" = ["*.pxd", "*.h"]
"msgq.visionipc" = ["*.pxd", "*.h"]
[tool.cibuildwheel]
build = "cp311-* cp312-* cp313-* cp314-*"
test-command = "python -m unittest discover -s msgq"
[tool.cibuildwheel.macos.environment]
MACOSX_DEPLOYMENT_TARGET = "10.13"
# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
[tool.ruff]
lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"]
@@ -63,3 +56,10 @@ lint.flake8-implicit-str-concat.allow-multiline=false
line-length = 160
target-version="py311"
[tool.ty.src]
exclude = ["site_scons/"]
[tool.ty.rules]
# Cython modules are compiled at build time, not available for static analysis
unresolved-import = "ignore"
+3 -7
View File
@@ -2,7 +2,6 @@ import os
from pathlib import Path
import subprocess
import sys
import sysconfig
from setuptools import Extension, setup
from setuptools.command.build_ext import build_ext
@@ -12,19 +11,16 @@ class SConsBuildExt(build_ext):
def build_extensions(self):
subprocess.check_call([sys.executable, "-m", "SCons", "--minimal", "-j", str(self.parallel or os.cpu_count() or 1)])
for ext in self.extensions:
module = Path(*ext.name.split("."))
source = module.with_suffix(sysconfig.get_config_var("EXT_SUFFIX"))
source = Path(*ext.name.split(".")).with_suffix(".so")
target = Path(self.get_ext_fullpath(ext.name))
target.parent.mkdir(parents=True, exist_ok=True)
self.copy_file(str(source), str(target))
stub = module.with_suffix(".pyi")
self.copy_file(str(stub), str(target.parent / stub.name))
setup(
ext_modules=[
Extension("msgq._ipc", sources=[]),
Extension("msgq.visionipc._visionipc", sources=[]),
Extension("msgq.ipc_pyx", sources=[]),
Extension("msgq.visionipc.visionipc_pyx", sources=[]),
],
cmdclass={"build_ext": SConsBuildExt},
)
+72
View File
@@ -0,0 +1,72 @@
import re
import SCons
from SCons.Action import Action
from SCons.Scanner import Scanner
pyx_from_import_re = re.compile(r'^from\s+(\S+)\s+cimport', re.M)
pyx_import_re = re.compile(r'^cimport\s+(\S+)', re.M)
cdef_import_re = re.compile(r'^cdef extern from\s+.(\S+).:', re.M)
def pyx_scan(node, env, path, arg=None):
contents = node.get_text_contents()
# from <module> cimport ...
matches = pyx_from_import_re.findall(contents)
# cimport <module>
matches += pyx_import_re.findall(contents)
# Modules can be either .pxd or .pyx files
files = [m.replace('.', '/') + '.pxd' for m in matches]
files += [m.replace('.', '/') + '.pyx' for m in matches]
# cdef extern from <file>
files += cdef_import_re.findall(contents)
# Handle relative imports
cur_dir = str(node.get_dir())
files = [cur_dir + f if f.startswith('/') else f for f in files]
# Filter out non-existing files (probably system imports)
files = [f for f in files if env.File(f).exists()]
return env.File(files)
pyxscanner = Scanner(function=pyx_scan, skeys=['.pyx', '.pxd'], recursive=True)
cythonAction = Action("$CYTHONCOM")
def create_builder(env):
try:
cython = env['BUILDERS']['Cython']
except KeyError:
cython = SCons.Builder.Builder(
action=cythonAction,
emitter={},
suffix=cython_suffix_emitter,
single_source=1
)
env.Append(SCANNERS=pyxscanner)
env['BUILDERS']['Cython'] = cython
return cython
def cython_suffix_emitter(env, source):
return "$CYTHONCFILESUFFIX"
def generate(env):
env["CYTHON"] = "cythonize"
env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE"
env["CYTHONCFILESUFFIX"] = ".cpp"
c_file, _ = SCons.Tool.createCFileBuilders(env)
c_file.suffix['.pyx'] = cython_suffix_emitter
c_file.add_action('.pyx', cythonAction)
c_file.suffix['.py'] = cython_suffix_emitter
c_file.add_action('.py', cythonAction)
create_builder(env)
def exists(env):
return True
+2 -2
View File
@@ -18,8 +18,8 @@ lefthook run test
trap 'rm -rf "$TEST_DIR"' EXIT
uv venv --python "$DIR/.venv/bin/python" "$TEST_DIR/.venv"
uv pip install --python "$TEST_DIR/.venv/bin/python" "$DIR"
cd "$("$TEST_DIR/.venv/bin/python" -c "import sysconfig; print(sysconfig.get_path('purelib'))")"
"$TEST_DIR/.venv/bin/python" -m unittest discover -t . -s msgq
cd "$TEST_DIR"
"$TEST_DIR/.venv/bin/python" -m unittest msgq.tests.test_messaging
)
# *** all done ***
+1 -1
View File
@@ -30,7 +30,7 @@ body:
attributes:
label: Provide a route where the issue occurs
description: Ensure the route is fully uploaded at https://useradmin.comma.ai. We cannot look into issues without routes, or at least a Dongle ID.
placeholder: 77611a1fac303767/2020-05-11--16-37-07
placeholder: 77611a1fac303767|2020-05-11--16-37-07
validations:
required: true
+1 -1
View File
@@ -22,7 +22,7 @@ body:
attributes:
label: Provide a route where the issue occurs if applicable
description: Ensure the route is fully uploaded at https://useradmin.comma.ai
placeholder: 77611a1fac303767/2020-05-11--16-37-07
placeholder: 77611a1fac303767|2020-05-11--16-37-07
- type: input
id: version
-4
View File
@@ -109,10 +109,6 @@ Start off by recording a route with lots of interesting events: enable LKAS and
Use the [longitudinal maneuvers](https://github.com/commaai/openpilot/tree/master/tools/longitudinal_maneuvers) report to evaluate your car's longitudinal control and tune it.
### Validation
Before submitting a port, go through the [car porting checklist](docs/CAR_PORTING_CHECKLIST.md).
## Contributing
All opendbc development is coordinated on GitHub and [Discord](https://discord.comma.ai). Check out the `#dev-opendbc-cars` channel and `Vehicle Specific` section.
-270
View File
@@ -1,270 +0,0 @@
# Car porting checklist
This checklist covers common issues we find in new car ports and many of the things we check before merging them. Use it as a guide to help you write a high-quality port and get it reviewed faster.
Not everything will apply to your car. If something can't be tested, let us know what and why. Where it applies, check each supported variant and both stock ACC and openpilot longitudinal. Some checks need the full openpilot stack, not just opendbc. Field names like `accFaulted` are defined in [`car.capnp`](../opendbc/car/car.capnp).
## Vehicle and setup
- Vehicle details
- [ ] Record make, model, year, trim, market, and relevant driver assistance options in `values.py`.
- If all variants in the US market come equipped with the necessary ADAS package, no package documentation is required.
- If a vehicle varies in features in another market, add a new [`CarDocs`](../opendbc/car/docs_definitions.py) entry.
- [ ] Add every observed ECU firmware version to `fingerprints.py`.
- [ ] Identify differences from related supported cars; verify reused signals and commands on this variant.
- Gas, gear, and brake signals often move between ICE, hybrid, and EV variants (e.g. Hyundai, Subaru, Toyota SecOC).
- Harness and CAN topology
- [ ] Confirm harness pinout, CAN bus assignments, and CAN/CAN FD configuration.
- [ ] Identify which ECUs send, receive, and forward each relevant message.
- This is useful to know which messages we need to forward and block in safety.
- Stock behavior and power lifecycle
- [ ] Record baseline stock LKAS, ACC, stop-and-go, and driver override behavior for reference when writing the port.
- Stock behavior should be matched when possible, e.g. how stock ACC commands standstill and resume.
- [ ] Verify ignition detection, startup, shutdown, and vehicle sleep with the device connected.
## Identification and parameters
- [ ] The car identifies correctly on repeated cold starts without a manually forced fingerprint or fingerprint cache.
- [ ] [Firmware queries](../opendbc/car/fw_versions.py) use the correct ECU addresses and buses without causing any DTCs on the dashboard.
- [ ] Exact and [fuzzy fingerprinting](../opendbc/car/fw_versions.py) distinguish variants that need different control or safety behavior.
- [ ] Unknown or ambiguous firmware does not silently select an incompatible platform.
- A vehicle variant we do not support that has identical firmware (such as lacking ACC or LKAS hardware) should be detected with a CAN signal in `interface.py` or `carstate.py` depending on what we can check, and set `dashcamOnly`.
- [ ] Reliable source for wheelbase, steering ratio, and other vehicle model parameters.
- [ ] `minSteerSpeed`, `minEnableSpeed`, `autoResumeSng`, and `steerAtStandstill` in `carParams` match observed behavior.
## CAN parsing and message construction
- Signal definitions and input validation
- [ ] Message addresses, lengths, buses, frequencies, endianness, signedness, scales, and units are verified.
- [ ] Checksums and counters validate against real captures, including counter wraparound.
- [ ] Missing, stale, malformed, or checksum-invalid messages produce the expected invalid CAN state.
- [ ] Variable-rate or intermittent messages have an [explicit frequency](../opendbc/can/parser.py) in `get_can_parsers` so they don't cause false CAN errors.
- For example, VW `Blinkmodi_02`, `HCA_01`, and `AWV_03` are 1 Hz when inactive and 50 Hz when active.
- [ ] Startup defaults and invalid signal values cannot look like valid engagement conditions.
- Outgoing messages and forwarding
- [ ] Copied stock messages preserve unrelated fields and use fresh source data.
- [ ] Transmitted messages have the correct bus, timing, counters, checksums, and request bits.
- [ ] Forwarding and replacement rules prevent duplicate actuation messages or bus loops.
- Diagnostics
- [ ] Diagnostic sessions, [ECU disabling](../opendbc/car/disable_ecu.py), and keepalives work across startup, restart, and shutdown, if used.
- This is for disabling an ECU, such as to gain longitudinal control. Some ports do not require this.
## Vehicle state: basics
- Seatbelt and doors
- [ ] Driver seatbelt: buckle and unbuckle both update `seatbeltUnlatched` correctly.
- [ ] Doors: open and close each door individually; `doorOpen` remains true until all monitored doors close.
- Gears
- [ ] Gear selection: ideally we parse park, reverse, neutral, drive, and alternate drive modes such as sport, B (brake) gear on hybrids, etc.
- If we cannot parse all gears, ensure we can disengage cruise on non-drive/sport gears, or anything that stock ACC would normally disengage for.
- [ ] Unknown gear values are handled explicitly instead of defaulting to drive.
- Pedals and regen
- [ ] Brake pedal: light press, firm press, hold, and release update `brakePressed` reliably.
- [ ] Accelerator pedal: press and release update `gasPressed`, including while cruise is active.
- [ ] Driver pedal signals are distinguished from braking or acceleration commanded by ACC/openpilot.
- In the past, we found some accelerator signals either activate when stock ACC accelerates, or don't activate when the user's gas was under the stock ACC request. `gasPressed` must be true whenever the user is pressing the gas pedal any amount, in any state.
- [ ] Regen paddles or other driver regen controls update `regenBraking`, if applicable.
- Parking brake and auto hold
- [ ] Parking brake engagement and release update `parkingBrake`.
- [ ] Auto hold engagement and release update `brakeHoldActive`, independently of the brake pedal.
- Speed and standstill
- [ ] Vehicle speed and individual wheel speeds have correct units and plausible values.
- [ ] Speed and `standstill` use the same signals as the safety model.
- For example, Nissan and GM use the rear wheel speeds to match safety.
- [ ] `standstill` distinguishes a complete stop from slow creep and updates promptly on drive-off.
- [ ] Cluster speed and cruise set speed agree with the car in both metric and imperial settings.
- Steering
- [ ] Steering angle has the correct sign, scale, center offset, and behavior through its full range.
- [ ] Steering torque has the correct sign and scale; driver torque and EPS torque are not confused.
- [ ] `steeringPressed` detects driver input in both directions without triggering on normal commanded steering.
- Use [`CS.update_steering_pressed`](../opendbc/car/interfaces.py) if the signal is noisy.
- Blinkers and blind spots
- [ ] Left/right blinkers are reported correctly.
- If we only have lamp state, use [`CS.update_blinker_from_lamp`](../opendbc/car/interfaces.py) so it doesn't oscillate with the blinker.
- If we only have stalk position, use [`CS.update_blinker_from_stalk`](../opendbc/car/interfaces.py) so short taps are held.
- [ ] Left/right blind spot indications are mapped correctly and clear when the stock indication clears.
- Faults and stock system state
- [ ] Temporary and permanent steering faults are logged correctly and recover only when the fault clears.
- EPS initializing states at startup are not permanent faults (e.g. VW, BYD).
- Expected low-speed steering lockouts should set `lowSpeedAlert` instead of a fault, usually with hysteresis.
- [ ] `accFaulted`, `carFaultedNonCritical`, `carNotReady`, and `vehicleSensorsInvalid` are reported correctly.
- [ ] ESP disabled/active states and stock AEB/FCW states are parsed where available.
- [ ] Stock LKAS settings (`invalidLkasSetting`) and conventional/non-adaptive cruise modes (`cruiseState.nonAdaptive`) are detected where relevant.
- For example, Nissan requires stock LKAS off, Mazda requires it on, and Tesla requires Autosteer off.
## Buttons and cruise state
- [ ] Set, resume, cancel, main cruise, speed up/down, gap, and LKAS buttons are mapped where present.
- [ ] Button press and release edges each occur once; held buttons do not cause unintended repeated engagement.
- [ ] Short presses, long presses, and simultaneous button presses are handled correctly.
- [ ] Cruise available (main), enabled, standstill, and set speed are parsed properly and match stock behavior.
- Cruise enabled should stay true during driver accelerator override and at standstill. Some dash signals fall in these states (e.g. Subaru).
- `cruiseState.standstill` should only be true if the car requires a resume press to drive off.
- [ ] Set speed changes, unit changes, and minimum/maximum set speeds behave correctly.
## Engagement and cancellation
- Engagement conditions
- [ ] The intended set/resume action engages once, only when the car is ready (`carNotReady`).
- [ ] Engagement refusal and disengagement for seatbelt, doors, gear, parking brake, and faults match the intended policy.
- Cancellation
- [ ] The physical cancel button disengages openpilot and cancels stock cruise as intended.
- [ ] If stock ACC is controlling longitudinal, verify our cancel message stops commanding acceleration.
- [ ] Software cancellation works in each supported cruise state, including standstill and driver accelerator override.
- [ ] Cancel wins over simultaneous set/resume requests; repeated cancel requests do not resume or re-enable cruise.
- If the cancel button can toggle pause/resume or the main button can engage, use `blockPcmEnable` to only allow PCM enable after a real user button press (e.g. Hyundai).
- [ ] Cruise main off disengages and prevents unintended re-engagement.
- Disengagement and recovery
- [ ] Releasing a pedal, closing a door, buckling a belt, or clearing a fault does not unexpectedly re-engage unless we are still in the soft-disable grace period.
- [ ] Stock ACC disengagement/fault causes the expected openpilot response when using stock longitudinal control.
- [ ] Lateral-only and longitudinal-active states set their respective actuation request bits correctly.
- For example, `CC.enabled` should be used when keeping the car's cruise engaged during a longitudinal override while `CC.longActive` falls.
## Lateral control
- Commands, limits, and driver override
- [ ] Small left/right commands produce the expected steering direction.
- [ ] Command magnitude and rate limits match the platform and safety implementation.
- [ ] Driver override reduces or cancels openpilot steering assistance as intended, in both directions.
- Some platforms disengage both steering and cruise on a steering override (e.g. Tesla); use `steeringDisengage` for this.
- Low-speed behavior
- [ ] Steering behaves correctly around the minimum steering speed, including crossing it in both directions.
- [ ] Check for low-speed oscillation and flag it if present. One solution is to low-pass filter the request below a certain speed.
- Faults and recovery
- [ ] Sustained steering, high steering rates, and request-bit transitions do not cause unexpected EPS faults.
- [ ] Temporary steering unavailability stops active requests and produces the expected user alert.
- Tuning and reported output
- [ ] Tuning is checked across representative speeds and curves for oscillation, lag, and saturation.
- [ ] Reported `actuatorsOutput` matches the command actually sent after clipping and rate limiting.
## Longitudinal control
- Supported control modes
- [ ] Stock ACC works correctly with openpilot lateral control, including cancel, override, and stop-and-go.
- There are two levels of support with openpilot-controlled longitudinal:
- [ ] Alpha longitudinal is chosen if we cannot maintain stock AEB or other safety features, or the car does not have a radar we can parse. openpilot longitudinal is then only available on development branches.
- [ ] openpilot longitudinal is the only mode and available in all branches if both of the above requirements are met, plus we have a good understanding of the car's longitudinal API.
- Acceleration, braking, and driver override
- [ ] Acceleration and braking command signs, scales, limits, and rate limits are verified.
- [ ] Acceleration, coasting, and braking transitions are smooth and free of conflicting requests.
- Some cars accept gas and brake commands separately; never request both at once.
- [ ] Brake press disengages while moving as well as while at a standstill.
- [ ] Accelerator override and release behave correctly, i.e. we stop requesting acceleration (inactive, or 0 m/s^2) while continuing to steer.
- Stopping, holding, and resuming
- [ ] Low-speed braking reaches a complete stop without unexpected creep or brake release.
- [ ] Stop hold works for short and extended stops; timeout and parking brake behavior are understood.
- [ ] Resume/drive-off releases hold in the correct sequence without unintended acceleration.
- [ ] Resume is sent only when requested and supported.
- [ ] Stops and starts on grades behave correctly without unexpected rollback or parking brake/gear transitions.
- [ ] `stopAccel`, `vEgoStopping`, `vEgoStarting`, `startingState`, and `startAccel` are tuned if needed.
- Powertrain states and faults
- [ ] Behavior is checked across relevant powertrain states, such as engine stop/start.
- [ ] ACC faults and transient not-ready states inhibit requests appropriately and recover cleanly.
- Tuning and stock safety features
- [ ] `longitudinalActuatorDelay` and tuning are evaluated across representative speeds, following, stops, and starts.
- [ ] Stock AEB/FCW interaction and any feature loss from ECU disabling are understood and documented.
## Radar, alerts, and stock features
- Radar
- [ ] Radar distance, lateral position, relative velocity, signs, units, and track IDs are verified, if supported.
- [ ] Radar track creation, updates, disappearance, invalid targets, and timeouts behave correctly.
- [ ] `radarDelay` is set if known.
- HUD and alerts
- [ ] HUD engaged/standby state, set speed, lane lines, lead indication, and gap bars match the requested state.
- [ ] Takeover, steering unavailable, and other required alerts appear and clear correctly.
- Stock feature interaction
- [ ] Stock LKAS and ACC do not fight openpilot commands.
- [ ] Stock safety messages are forwarded or arbitrated as intended; validate AEB interactions with replay/bench tests.
- [ ] Normal operation does not introduce persistent dash warnings or diagnostic trouble codes.
- Some cars may display warning messages on the dash or force a disengage if they detect a conflict between what their camera is sending and our request through read-back signals from the PCM or EPS. If we need to spoof a message to the camera/ADAS ECU, we need to verify it does not affect stock safety features such as LKAS or AEB.
## Safety and failure handling
- Message permissions and engagement
- [ ] The selected safety mode permits only intended transmit addresses, buses, and message lengths.
- [ ] Safety engagement/disengagement agrees with the car interface for cruise, buttons, pedals, and motion state.
- [ ] Non-zero actuation or active request bits are rejected when safety's [`controls_allowed`](../opendbc/safety/safety.h) is false.
- [ ] Any messages permitted while disengaged are limited to their intended inactive/cancel behavior.
- Actuation limits
- [ ] Steering magnitude, rate, driver torque, measured torque, and request-bit checks are covered as applicable.
- [ ] Longitudinal limits, inactive values, and cancel/resume restrictions are covered as applicable.
- [ ] All actuation-capable fields in transmitted messages are constrained, including fields unused by the controller.
- Communication failures and recovery
- [ ] Invalid counters/checksums, missing required input messages, and stale state disable control as intended.
- [ ] Relay malfunction detection and forwarding restrictions cover the actual harness topology.
- Test coverage
- [ ] Safety tests cover every new safety flag variant and reject disallowed commands.
## Regression tests
- Routes and automated checks
- [ ] Add representative routes to the [car test routes](../opendbc/car/tests/routes.py), covering relevant variants and modes.
- [ ] Add or update tests for fingerprinting, parsing, command construction, and discovered edge cases.
- [ ] Run applicable [car interface tests](../opendbc/car/tests/) and [safety tests](../opendbc/safety/tests/).
- [ ] [`test_models.py`](../opendbc/car/tests/test_models.py) passes on the new routes; it replays them to check CAN validity and that safety agrees with carstate and accepts our commands.
- [ ] Run the repository checks in [`test.sh`](../test.sh) and fix any failures.
- Regression validation
- [ ] Recheck related existing cars affected by shared DBC, controller, interface, or safety changes.
- [ ] Repeat startup, engagement, cancellation, and shutdown over multiple drives and ignition cycles.
@@ -1 +0,0 @@
.github export-ignore
@@ -1 +0,0 @@
__pycache__/
@@ -1,53 +1,35 @@
# Credits
# BluePilot angle source
The BluePilot Ford lateral library: Ford's four-signal lateral control in curvature and angle
modes, the CAN messages that carry it, and the panda safety that checks it. Everything here is
under the MIT terms in [LICENSE](LICENSE).
Adapted from BluePilotDev/bluepilot bp-7.0 at
[e1d051d7ba270261b4455068bd68f1a58db15a4a](https://github.com/BluePilotDev/bluepilot/commit/e1d051d7ba270261b4455068bd68f1a58db15a4a),
principally Alan Polk, with John Christman, Jacob Neulight, Nathan Ingraham,
Praeuner, tonesto7, Haibin Wen and other BluePilot, sunnypilot and comma contributors.
Source paths: opendbc/sunnypilot/car/ford/lateral_angle_ext.py,
fordcan_ext.py, human_turn.py and opendbc/safety/modes/ford.h.
The simultaneous upstream LICENSE and LICENSE.md notices are retained.
This software is licensed under a custom license requiring permission for use.
This project uses software from Haibin Wen and SUNNYPILOT LLC and is licensed under a custom license requiring permission for use.
## Contributors
This is a StarPilot adapter to the September 2026 proposed library shape.
It is not a released BluePilot library, tag, or claim of BluePilot validation.
The existing curvature controller remains StarPilot-owned. Pinion curvature,
curvature-primary settings and the unsafe native reset bypass are not imported.
- **Alan Polk (`alan-polk`)**: principal author of the curvature and angle controllers, the
human-turn behavior, the angle-mode side channel and its panda safety, and the lane centering
trim.
- **tonesto7**: the CAN message builders and BluePilot's early releases.
- **John Christman**: lateral integration, platform data, and the angle-mode anti-stall work.
- **Jacob Neulight**: the steering-pinion curvature measurement and its panda safety, and the
shadow-curvature measurement.
- **Blue Genie (`blue-genie`)**: the split of BluePilot's lateral control into opendbc that this
library grew from.
- **ghbarker**: the lane centering trim's confidence blend, fallback, filter and rate limit.
- **Nathan Ingraham**: the angle-mode high-speed damping adjustment.
- **Praeuner**: the angle-mode damping range.
- **OneFast440**: the early Ford safety declarations.
- **nitrobass24**: the library's structure, its inputs and settings plumbing, and the safety tests
and CI.
The exact upstream root notices are additionally retained as upstream-root-LICENSE
and upstream-root-LICENSE.md; LICENSE/ LICENSE.md retain the opendbc notices.
ORIGIN_HISTORY.tsv records the inspected path history with commit IDs/authors/dates.
The private compact upstream root clone preserves its reachable root history.
Pre-conversion opendbc submodule objects are not included in that root history.
## Third-party code
### comma.ai's opendbc
MIT, Copyright (c) 2020, Comma.ai, Inc.
- `can/fordcan_ext.py`: `create_lat_ctl_msg` and `create_lat_ctl2_msg`, comma's Ford lateral
message builders from `opendbc/car/ford/fordcan.py`, with ramp-type and precision parameters
added.
- `safety/lateral.h`: the measured-table rate check, curvature-error band and lateral-accel cap,
from comma's earlier checks in `opendbc/safety/lateral.h`, and `ford_bp_curvature_cmd_checks`,
adapted from comma's `steer_curvature_cmd_checks`.
- `safety/ford.h`: the rate-of-change shape of the path angle and path offset checks, the
curvature-error band in the shadow-curvature check, and the message-rate formula.
- `core/lateral_curv_ext.py`: `apply_ford_curvature_limits_ext`, adapted from comma's
`apply_ford_curvature_limits`, with its road-roll and lateral-accel constants.
- Small pieces: the message packing and the yaw-rate curvature measurement in
`core/carcontroller_ext.py`, and the model time-index formula in `core/values_ext.py`.
- The BluePilot safety tests start from comma's Ford safety test skeletons.
### StarPilot
MIT, Copyright (c) 2026, firestar5683 and StarPilot contributors
- `core/lane_center_trim.py`: the lane-centering core (the correction formula, lookahead clip,
lane-line probability, std and width gates, the 0.004 correction limit and the lane-line
handling), ported from StarPilot's `selfdrive/controls/lib/lane_centering.py`
(`LaneCenteringController`, originally by jc01rho). The confidence blend, fallback, filter and
rate limit are BluePilot's.
Angle lane-centering trim is separately adapted from BluePilot bp-dev
[e22afa6be9b881fa784c92ebb316db47728a3d81](https://github.com/BluePilotDev/bluepilot/commit/e22afa6be9b881fa784c92ebb316db47728a3d81),
`opendbc/sunnypilot/car/ford/lane_center_trim.py` (blob
`5eedebd79e99493624f9d6af0750d1dd5713306c`). Alan Polk authored the initial
trim, model-position fallback, correction slew and default-strength changes;
ghbarker contributed reference/NaN guards and tests. The source explicitly credits
StarPilot/u/jc01rho for curvature-domain placement/geometry. The core adaptation
adds immutable host projection and conservative stale/driver-pressure suppression.
The pinned bp-7.0 controller baseline and default-disabled policy are unchanged.
Both simultaneous root/opendbc MIT and Custom MIT notices also exist unchanged at
this trim revision; all retained notices and permission acknowledgments apply
without assigning an exclusive license or claiming a new permission grant.
@@ -1,83 +0,0 @@
# Bringing BluePilot lateral to a fork
How an openpilot fork takes BluePilot's Ford lateral control. There are two ways to do it:
1. **Use a release on your own** (preferred). Pin a release of this library and write your own hooks and settings against [HOSTS.md](HOSTS.md). BluePilot isn't in the loop and nothing waits on BluePilot: you consume lateral code that has been driven on Fords.
2. **Work with BluePilot.** BluePilot helps develop your hooks, keeps them in this repo, and tests the library on your fork in its CI, every PR and every night.
Either way the library is the same, and the same rules apply to it (below).
**This is a modified Ford safety model.** The panda safety allows signals and ranges that upstream's Ford safety blocks. Your fork decides whether to ship it, and whether BluePilot lateral is on or off by default for your users.
## The rules for either way
- **Pin a release, unmodified.** The library sits at `<pkg>/bluepilot_lateral/` in your opendbc (`<pkg>` is its package name) as one of:
- a **git submodule** pointing at a release tag, if your fork uses submodules; updating is moving the pointer;
- a **stamped copy** made with `tools/vendor.sh`, if it doesn't (see "The stamp tools").
- **Don't edit inside the library folder**, carry it as a git subtree with its history, or copy parts of it elsewhere. A fix to the library is a PR to this repo; once it's released, every fork gets it.
- **Everything else is yours:** the hooks in your opendbc, the code in your openpilot that fills the library's inputs every frame, and your settings UI.
## Option 1: use a release on your own
1. Pick a release from this repo's releases. Each one says what changed in the lateral control, the CAN messages and the panda safety, and what was validated on a car.
2. Mount it (submodule or stamped copy, above).
3. Write the hooks in your opendbc. [HOSTS.md](HOSTS.md) has the contract: the hook sites, the per-frame inputs and settings, the startup safety bits, and the host bindings to define if your safety function names differ from upstream opendbc's. Each release attaches the hook patch for sunnypilot/opendbc as a worked example.
4. Write your adapter: the small module that hands the library your fork's types (`hosts/sunnypilot/` is the example to copy). It lives in your fork, outside the library folder.
5. Run the library's tests in your tree ([HOSTS.md](HOSTS.md), "Running the tests") and drive it before you ship it.
6. To update, move to a newer release and rerun your tests.
BluePilot doesn't review or approve any of it. Questions and bugs in the library are welcome as issues in this repo.
## Option 2: work with BluePilot
For a fork that wants BluePilot to help with the hooks and keep them working. Open an issue in this repo, "Host: <fork>": where the fork's source and integration branch are, how it carries its opendbc, and how it takes outside changes. From there, this repo keeps for that fork:
- **its adapter**, `hosts/<fork>/`, plus host bindings if its safety names differ from upstream's;
- **its hooks**, `hosts/<fork>/hooks.patch`, until the fork carries them;
- **`hosts/<fork>/host.sh`**: `tree <dir>` builds the fork's opendbc at a pinned commit with the hooks and the library, placed by `vendor.sh`; `test <dir>` runs the library tests, the fork's safety suite with the BluePilot safety tests, and coverage; `make <dir>` writes `hooks.patch` back after a hook is edited in `<dir>/host`;
- **a CI job** that runs `host.sh tree` and `test` on every PR to this repo, so a library change that would break the fork fails here first;
- **a nightly check** that tries the fork's latest integration branch: a pass moves the pinned commit, a failure opens one issue saying which step broke. Once the fork carries the hooks it tests the fork's tree as it is, and runs `check_vendored.sh` on its copy.
The steps:
1. Develop and test the hooks and the fork's side in the fork, together.
2. Add the adapter, `hooks.patch`, `host.sh`, the CI job and the nightly check here, in one PR.
3. Release the library with them.
4. BluePilot opens the PR to the fork: the hooks, its settings UI and the library at that release. Later releases arrive the same way, as PRs the fork takes when it chooses.
An adapter lives in this repo only if this repo's CI tests the library on that fork.
## The stamp tools
For a fork that takes no submodules. Both run from a clone of this repo that has the release tag (`git fetch --tags`).
**`tools/vendor.sh <tag> <folder>`** makes `<folder>` exactly the library at `<tag>`:
- it empties the folder first, so files a release removed don't linger;
- it copies the release's files, without the library's own CI (`.github/`);
- it writes `<folder>/VERSION`: the tag and the commit, e.g. `v1.2.3 0a1b2c3d...`.
```bash
git clone https://github.com/BluePilotDev/bluepilot-lateral.git
cd bluepilot-lateral
tools/vendor.sh <tag> <fork checkout>/<path to its opendbc package>/bluepilot_lateral
```
Commit the folder in the fork. To update, run it again with the new tag: the diff shows exactly what changed between the two releases.
**`tools/check_vendored.sh <folder>`** reads `<folder>/VERSION` and compares the folder with that release, file by file (Python caches ignored). It lists every added, removed or changed file, and exits non-zero if there is one:
```bash
tools/check_vendored.sh <fork checkout>/<path to its opendbc package>/bluepilot_lateral
```
Run it in your CI to catch an edit to the copy. For a fork working with BluePilot, this repo's nightly check runs it too.
## Installer note: nested submodule
A fork that embeds the library as a *nested* submodule (Option 2 — e.g. `fork → opendbc → opendbc/bluepilot_lateral`) can end up with the library missing after an end-user install. Some installers run a non-recursive `git submodule update --init`, which checks out the first-level submodule but not the library nested inside it, and the build then fails to import it. Installs that use `git submodule update --init --recursive` are unaffected.
If your fork is installed that way, self-heal on launch: before the build imports the library, when its directory is empty, run a recursive submodule update scoped to the embedding submodule — guard on the library's `__init__.py`, retry, and fail soft so an offline boot is no worse than before.
## How BluePilot releases
- Lateral changes and changes to the library's safety headers are driven on a car before they're released.
- Every release says what changed in the lateral control, the CAN messages and the panda safety, and what was validated.
@@ -1,129 +0,0 @@
# Hosting the BluePilot lateral
A host is an opendbc fork with this library at `<pkg>/bluepilot_lateral/` and the hook patch applied, plus the openpilot on top of it that fills the library's inputs each frame. `<pkg>` is the fork's opendbc package: `opendbc` on sunnypilot/opendbc (ford-op/opendbc, under bp-sunnypilot), `iqdbc` on IQ.Pilot, `opendbc` on StarPilot. This page is what a host provides and what it gets back.
## Mounting the library
A host mounts this repo at `<pkg>/bluepilot_lateral/`, directly inside its opendbc package, as a git submodule (ford-op/opendbc) or, in a host that takes no submodules (IQPilot, at `artifacts/package_sources/iqdbc/iqdbc/bluepilot_lateral/`), as a stamped copy of a release. Updating is a bump of the submodule's commit, or vendoring the new release (below). The hook patch below carries every file ford-op/opendbc adds or changes outside the library, so since it mounts the library it also carries the mount's `.gitmodules` entry.
The library never names its host's package. Python imports are relative: the library's own modules by `.`/`..`, the host's opendbc by one more level (`from ...car.ford.values import CarControllerParams` in `core/`), and `tests/test_core_imports.py` fails on any absolute `opendbc`/`iqdbc`/`openpilot` import. Safety headers include each other as `../safety/<file>.h` and the host's declarations as `../../safety/declarations.h`, relative to the including file. The host includes the library's safety as `"<pkg>/bluepilot_lateral/safety/ford.h"`.
### Without submodules: a stamped copy
From a clone of this repo, `tools/vendor.sh <release tag> <host>/<pkg>/bluepilot_lateral` makes the folder exactly the library at that tag (files it no longer has are removed), without `.github/` (`.gitattributes`), plus a `VERSION` file: the tag and the commit. Nothing inside the folder is ever edited: a change goes into this repo, gets released, and the host vendors the new tag. `tools/check_vendored.sh <folder>` lists anything in the folder that differs from the release in its `VERSION` and fails if there is something; a host can run it in its CI, and this repo's IQPilot checks run it on IQPilot's copy.
### Host bindings
The safety headers are written against upstream opendbc's names. A host whose names differ defines these before it includes `safety/ford.h` (`safety/lateral_declarations.h` has the defaults):
| macro | default (upstream, sunnypilot) | IQ.Pilot |
|---|---|---|
| `BP_HOST_CURVATURE_LIMITS` | `CurvatureSteeringLimits` | `NativeCurvatureSteeringLimits` |
| `BP_HOST_CURVATURE_STATE` | `curvature_state` | `native_curvature_state` |
| `BP_HOST_RT_CURVATURE_RATE_CHECK` | `rt_curvature_rate_limit_check` | `rt_native_curvature_rate_limit_check` |
| `BP_HOST_LAT_ALLOWED()` | `(controls_allowed \|\| controls_allowed_lateral)` | `is_lat_active()` |
| `BP_HOST_SPEED_MISMATCH_CHECK()` | `speed_mismatch_check(vehicle_speed_2 ...)` | `((void)0)`: IQ's `ford_rx_hook` runs it |
The limits struct needs only `max_curvature`, `curvature_to_can`, `frequency`, `max_curvature_error` and `curvature_error_min_speed`.
## The hook patch
`tools/make_hook_patch.sh` prints it: every file outside the library that ford-op/opendbc changes from `master`, minus its own `.github/` (its CI). It fails unless the edited files are exactly the non-`.github` entries of `.github/bluepilot-hooks.txt`.
| file | what the hook does |
|---|---|
| `opendbc/car/ford/carcontroller.py` | Builds one `BluePilotLateral(CP, CP_SP, packer, CAN)`, the sunnypilot adapter (`hosts/sunnypilot/lateral.py`, below). Each frame `bp.update(CC_SP, frame)` says whether BluePilot lateral is active; if it is, `bp.step_lateral` sends the steering message at `STEER_STEP` (and returns the curvature to pass back next time) and `bp.step_lka` sends Lane_Assist_Data1 at `LKA_STEP`. Otherwise the stock path runs. |
| `opendbc/car/structs.py` | Imports `LateralInputs` from `host_types.py` and adds `CarControlSP.lateralInputs` and `CarParamsSP.lateralInputsRequired`. |
| `opendbc/sunnypilot/car/interfaces.py` | Calls `init_ford_safety_param_sp(CP, CP_SP, params_dict)` (`hosts/sunnypilot/safety_param.py`) at car init: the startup safety bits below, and `lateralInputsRequired` for every Ford. |
| `opendbc/safety/modes/ford.h` | Includes `safety/ford.h` and makes one call per hook site: counter and quality checks for the pinion message, the yaw measurement skipped while the pinion source is on, `ford_bp_rx_hook` (pinion measurement), `ford_bp_lka_tx` (angle-mode state from Lane_Assist_Data1), `ford_bp_lmc_tx` in place of the stock steering check when `BP_LATERAL` is set (the stock check runs unchanged when it isn't), and the pinion RX check list from `ford_bp_init`. |
| `opendbc/safety/tests/test_bluepilot_lateral.py` | New: hands the library's `safety/tests/` to upstream's test discovery, so upstream's `test.sh` and `mutation.py` run the BluePilot safety tests. |
## Every frame: inputs
A host fills these and the settings below for a car whose `CarParamsSP.lateralInputsRequired` is set: every Ford, BluePilot lateral on or off (`init_ford_safety_param_sp`).
`CarControlSP.lateralInputs` (`host_types.LateralInputs`):
| fields | from |
|---|---|
| `pathYawRate`, `pathX`, `pathY` | `modelV2.orientationRate.z`, `modelV2.position.x`, `.y` (33 points) |
| `laneLineLeftX/Y`, `laneLineRightX/Y` | `modelV2.laneLines[1]`, `[2]` |
| `laneLineProbs`, `laneLineStds` | `modelV2.laneLineProbs`, `.laneLineStds` |
| `laneChangeState`, `laneChangeDirection` | `modelV2.meta` |
| `steerRatio`, `stiffnessFactor`, `roll`, `angleOffsetDeg` | the learned vehicle parameters (`vehicleParameters`, paramsd) |
| `lateralDelay` | `lateralDelay.lateralDelay` |
| `valid` | true only when both the model and the vehicle parameters have been seen and every field above is filled |
Each path array has `host_types.MODEL_PATH_POINTS` (33) entries at the times in `host_types.MODEL_T_IDXS` (openpilot's `ModelConstants.T_IDXS`): the core interpolates over that grid. On a frame that isn't valid, or whose arrays aren't the full shape or contain a non-finite value, the library keeps the previous inputs.
## Every frame: settings
Every key in `opendbc.bluepilot_lateral.core.values_ext.BP_LATERAL_PARAMS` (the per-frame settings in `params/manifest.py`; hosts import the list from there), every frame, as (key, value) with the value as bytes (sunnypilot: `CarControlSP.params` entries): bools `"1"`/`"0"`, numbers as text. A missing, empty or unparsable value gives the manifest default. A host publishes its own registry default for a key nobody has set, so a host can carry its own defaults (BP7 uses 3.0 for `LC_PID_gain_UI_curv`). `python -m <pkg>.bluepilot_lateral.params.manifest` prints the keys as `params_keys.h` lines (IQ.Pilot drops `BACKUP`, which its params don't have).
## At startup: the safety bits
`values_ext.ford_safety_param(CP, params_dict)` reads the startup settings (`disable_BP_lat_UI`, `FordPrefSteerAngleCurvature`) and returns the bits. The host sends them to the panda in its second safety param (sunnypilot: the adapter ORs them into `CP_SP.safetyParam`). StarPilot has no second param: its adapter shifts them into bits 9-14 of its Ford `safetyParam` (`hosts/starpilot/safety_param.py`, clear of the bits StarPilot's Ford safety reads), and its `ford_init` hands `(param >> 9) & 0x3F` to `ford_bp_init`:
| bit | |
|---|---|
| 5 `BP_LATERAL` | the panda runs the BluePilot steering checks; the car controller sends the 4-signal messages only when it's set |
| 0 `STEER_ANGLE_CURVATURE`, 1-4 geometry index | the pinion angle replaces the yaw rate as the curvature measurement, on platforms with a row in the pinion table |
The panda keeps what it was given at init: changing `disable_BP_lat_UI` while driving takes full effect at the next init.
## Reading status
`BluePilotLateral.status()` returns `host_types.LateralStatus`, this frame's values for a BluePilot UI, under the names BP7's `controllerStateBP` publisher reads off its car controller:
| field | |
|---|---|
| `lateralUncertainty` | from whichever strategy ran |
| `angleRateLimited` | angle mode: the path angle soft rate limit bit this frame |
| `curvatureRateLimited` | angle mode: the equivalent curvature would have been rate-limited |
| `curvatureDeviationLimited` | either mode: the current-curvature error clip constrained the command |
| `humanTurnLateralPaused` | angle mode: lateral sent inactive during a manual turn |
| `stallBlipActive` | angle mode: the brief mode-0 pulse after a post-override stall |
| `disable_BP_lat_UI` | BluePilot lateral off: the setting, or the panda bit not set at init |
| `primary_lateral_control` | 0 = curvature, 1 = angle |
The four angle-mode values read false outside angle mode. bp-sunnypilot doesn't read status; it has no BluePilot UI.
## What the library uses from the host's opendbc
Every host is an opendbc fork, so the library uses these directly:
- `<pkg>.car`: `structs`, `DT_CTRL`, `ACCELERATION_DUE_TO_GRAVITY`, `scale_rot_inertia_per_mass` and `scale_tire_stiffness_per_mass` (or, on an opendbc from before the per-mass vehicle model, `scale_rot_inertia` and `scale_tire_stiffness`, at unit mass), `common.pid.PIDController`, `vehicle_model.VehicleModel` (either signature), `lateral` (`AngleSteeringLimits`, `apply_std_steer_angle_limits`, `ISO_LATERAL_ACCEL`), `ford.fordcan.CanBus`, `ford.values` (`CarControllerParams`, `FordFlags`).
- `can/fordcan_ext.py` sets LateralMotionControl2's checksum itself: upstream's packer fills Ford checksums (and overwrites it with the same value), IQ.Pilot's doesn't.
- The safety headers use `modes/ford.h`'s Ford constants and upstream's safety helpers, through the bindings above. `safety/lateral.h` and `can/fordcan_ext.py` copy four upstream functions, pinned per host package by `tests/test_bluepilot_upstream_copies.py`, so a host where one changed fails that test (a host with no pins skips it).
- The safety tests use the host's test harness: `<pkg>.safety.tests.test_ford`, `common` and `libsafety.libsafety_py`. `tests/host.py` names the host's own structs, adapter and libsafety setters for the tests that touch them.
## The core's API, and the host adapters
The core takes only the library's types:
- `core.carcontroller_ext.BluePilotLateral(CP, safety_param, packer, CAN)`: `safety_param` is the startup bits (`ford_safety_param`).
- `update(inputs, params, frame)`: `inputs` is a `host_types.LateralInputs`, `params` a `params.param_store.ParamStore` built from the frame's (key, value) settings. Then `step_lateral`, `step_lka` and `status()` as above.
Nothing under `core/`, `can/`, `params/` or `safety/` imports a host (`tests/test_core_imports.py`). `hosts/sunnypilot/`:
- `lateral.py`: a `BluePilotLateral` subclass with sunnypilot's types, `(CP, CP_SP, packer, CAN)` and `update(CC_SP, frame)`. It passes `CP_SP.safetyParam`, `CC_SP.lateralInputs` and the `CC_SP.params` entries as (key, value) pairs.
- `safety_param.py`: `init_ford_safety_param_sp(CP, CP_SP, params_dict)` ORs `ford_safety_param` into `CP_SP.safetyParam`. Its own module, so car init imports only `values_ext`.
`hosts/iqpilot/`, the same for IQ.Pilot's types:
- `lateral.py`: `BluePilotLateral(CP, CP_IQ, packer, CAN)` and `update(CC_IQ, frame)`, passing `CP_IQ.iqSafetyFlags`, `CC_IQ.lateralInputs` and the `CC_IQ.params` entries.
- `safety_param.py`: `init_ford_safety_param_iq(CP, CP_IQ, params_dict)` ORs `ford_safety_param` into `CP_IQ.iqSafetyFlags` (the panda's `current_safety_param_iq`) and sets `CP_IQ.lateralInputsRequired`.
`hosts/starpilot/`, for StarPilot, which has neither a second safety param nor CarControl / CarParams extension structs:
- `lateral.py`: `BluePilotLateral(CP, packer, CAN)`, reading the startup bits back out of the Ford `safetyParam`, and `update(frame_inputs, frame)`. `FrameInputs` carries `lateralInputs` and the `params` entries; StarPilot's openpilot side fills one each frame and hands it to the Ford CarController, in place of a CarControl extension.
- `safety_param.py`: `init_ford_safety_param_starpilot(CP, params_dict)` ORs `ford_safety_param << 9` into `CP.safetyConfigs[-1].safetyParam`; `bp_safety_param(CP)` reads them back; `lateral_inputs_required(CP)` says whether to fill `FrameInputs`. StarPilot's Transit (`FordFlags.LKA_STEERING`, steering through Lane_Assist_Data1 alone) gets neither.
Another host writes its own adapter against the same API.
## Running the tests
- Python tests: `python -m unittest discover -s <pkg>/bluepilot_lateral/tests -t .`, from the directory that holds `<pkg>`.
- Safety tests: the host's `<pkg>/safety/tests/test.sh` runs them, through a loader (ford-op/opendbc's `test_bluepilot_lateral.py`) or by naming `../../bluepilot_lateral/safety/tests` (IQ.Pilot: its cross-mode TX sweep reads `TX_MSGS` off every `Test*` class in its directory, so a loader there breaks it). Coverage of the BluePilot headers and `modes/ford.h`: `safety/tests/bluepilot_coverage.sh`, which runs the library's tests itself where the host has no loader; mutation: `safety/tests/bluepilot_mutation.py` (needs the host's `mutation.py`, which IQ.Pilot doesn't have).
- On a clean sunnypilot/opendbc checkout: `tools/test_on_host.sh <checkout>`, from a ford-op/opendbc checkout. It copies the library as committed and applies the hook patch to a temporary clone, then runs both.
- On IQPilot: `hosts/iqpilot/host.sh tree <dir>` builds iqdbc from IQ.Lvbs's repo at the pinned commit with `hosts/iqpilot/hooks.patch` (IQPilot's hooks, kept here until IQPilot carries them) and the library, vendored; `host.sh test <dir>` runs the library tests, IQPilot's safety suite with the BluePilot ones (minus `known_failures.txt`, IQPilot's own) and coverage; `host.sh make <dir>` writes the hook patch back after editing the hooks in `<dir>/host`. CI's `iqpilot` job runs tree and test. Every night `iqpilot pin check` tries IQ.Lvbs's latest `beta`: green moves the pin in a PR it merges itself (dropping `hooks.patch` once IQPilot carries the hooks), red keeps one issue open saying which step failed.
@@ -1,6 +1,4 @@
Copyright (c) 2020, Comma.ai, Inc.
Copyright (c) 2026, firestar5683 and StarPilot contributors
Copyright (c) 2024-2026, BluePilot contributors
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
@@ -0,0 +1,21 @@
# Custom MIT License
Copyright (c) 2024, Haibin Wen, SUNNYPILOT LLC
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to view and modify the Software, subject to the following conditions:
1. **Permission Required**: Permission Required for Commercial, For-Profit, or Closed Source Use: Use of the Software, in whole or in part, for any commercial purposes, for-profit projects, or in closed source projects requires explicit written permission from the original author(s).
2. **Redistribution**: Any redistribution of the Software, modified or unmodified, must retain this license notice and the following acknowledgment:
"This software is licensed under a custom license requiring permission for use."
3. **Visibility**: Any project that uses the Software must visibly mention the following acknowledgment:
"This project uses software from Haibin Wen and SUNNYPILOT LLC and is licensed under a custom license requiring permission for use."
4. **No Warranty**: THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Contact sunnypilot Support <support@sunnypilot.ai> for permission requests.
---
Haibin Wen, SUNNYPILOT LLC
@@ -0,0 +1,38 @@
9d190417bf74563abfa6f36996f1e673e9ee0a1b Praeuner 2026-08-13T22:26:15-05:00 Update dampening factor to .25
a881646ea5edc21c0db4c4e74076e0dc14671a4e alan-polk 2026-08-10T09:04:52-05:00 Merge pull request #176 from BluePilotDev/bp-no-stall
9012f76666a5c90764fcaec40832b9a607488c27 John Christman 2026-08-05T17:49:59-04:00 anti-stall
3610e3f18a0ad37d793d7ada61dbc739ab1077a3 Nathan Ingraham 2026-07-23T22:26:29-05:00 self.path_angle_gain_lowC_highV remains the platform-specific base gain. The toggle now controls separate self.user_dampening_factor. l ow_gain_calc uses: self.path_angle_gain_lowC_highV * self.user_dampening_factor UI placement, description, default, and 0.75–1.25 range remain unchanged.
1db52bc79e607ddb00214254dd4d732615e27fe4 Nathan Ingraham 2026-07-23T00:00:51-05:00 -Added “High Speed Dampening” directly below “High Speed Adjustment Factor”, range 0.75–1.25, step 0.01, and default 1.0.
5d994c24edf63f3b9004b2963f18ef4199e5d368 Jacob Neulight 2026-07-18T20:17:04-06:00 Ford safety: optional pinion-sourced angle_meas behind STEER_ANGLE_CURVATURE SP flag
699c17d9fde62c690eed9d68eed7d731744d5137 Jacob Neulight 2026-07-18T19:53:46-06:00 Ford: publish angle-mode shadow curvature from measured curvature during overrides
b7428fee4b116d4832f91c850d54cac2730de2ef alan-polk 2026-07-18T08:52:24-05:00 restore curvature safety checks to bp-6.0 logic.
97867c1eb57b7472f6fc3de62f0fef576e5a5497 alan-polk 2026-07-16T19:18:05-05:00 Human Turn Reset for Angle (#118)
7efee67e3e643d52b973895d6c6765c5cc12e314 alan-polk 2026-07-16T08:19:43-05:00 angle mode: rescale exit-detection threshold for 20Hz cadence
1958a3501ead10ae1f496f0d11e1e8cda6e2dbcc alan-polk 2026-07-14T15:22:38-05:00 Bp 7.0 variable cleanup (#111)
8f8d6d15f0a590f42b78de964ffb0d0af7f5d63d alan-polk 2026-07-14T08:53:28-05:00 Bp 7.0 panda2 (#109)
11ac3b760671b489ca449d522f1edf240a232884 alan-polk 2026-07-09T07:53:21-05:00 angle mode: fix ROC values tuned for 100Hz cadence running at actual 20Hz
dbdcea747e65f574d326a55c9bf1a9d6c9485e22 alan-polk 2026-07-08T09:21:32-05:00 Revert "ford: add current-curvature deviation check to angle mode (panda-side)"
69a9714563622729ba304ac75354ba3fd37b0b39 alan-polk 2026-07-08T09:17:09-05:00 ford: add current-curvature deviation check to angle mode (panda-side)
be7f86a5d0dc1ea3b3f0052d4b54ff0b8a70f843 John Christman 2026-07-06T21:53:55-04:00 cast on debug
5c1fbb07dd4d7d950a01cb2cd1faddd90d41e632 alan-polk 2026-07-06T14:20:49-05:00 controllerStateBP: add angleRateLimited + curvatureRateLimited diagnostics
06d654e5f48ea8fcffce6e98e48e3a30450b5097 alan-polk 2026-06-25T23:28:51-05:00 Revert "temp blend ratio change" — restore predicted-curvature blend to 0.50
540e69fcf4861a2ce5d7ca41467effc231638e27 alan-polk 2026-06-22T08:35:44-05:00 temp blend ratio change
e3819c7e630d85cfb44858840124fe7f72f1762d alan-polk 2026-06-13T20:47:09-05:00 ford.h: don't block angle mode on the inactive curvature signal
5d343738e30267688d0ffa6eebe2ec3ca57b10ee alan-polk 2026-06-13T18:28:30-05:00 ford.h: widen path_angle ROC to 2% looser than lateral_angle_ext _soft_roc
d0aac605f99d37e9da205e419f7989c1e9eaa386 alan-polk 2026-06-13T11:15:50-05:00 BluePilot: add angle-primary lateral control mode
51d5de8991508c862e7c51333e5055425b2b8d23 alan-polk 2026-04-11T12:00:10-05:00 fixing missing punctation in ford.h
8194ceb7e0bbc112c73d393a8f0add3261f6592d alan-polk 2026-04-11T11:51:49-05:00 make safety limits symetrical but control limits asymetrical. Add 1.5s duration to detect human turn
db2bdff05df103d71df62f45c2a3cb5211aba6e6 alan-polk 2026-04-09T23:47:30-05:00 move all external logic to lateral_curv_ext.py. Apply some order of operations fixes to try and eleminate panda blocking messages.
1a8ac23b5bd5d7952c15eb913c2b1e39f14ad92e Blue Genie 2026-04-08T15:10:24-05:00 Adjust panda limits
892be076daf898d806983f847c72357accff7312 Blue Genie 2026-04-08T11:45:28-05:00 Add logging functionality
ce7d27230657159332f64a253a0faea6cf8dcc95 alan-polk 2026-04-08T11:40:02-05:00 fix rate of change limits
021b5146e80ac58fe2e81dbd848bc7cd409ef08e alan-polk 2026-04-07T16:24:38-05:00 align ford.h limits with dbc and carcontroller
7f9212f7f8dc22c4a0a22443158554f4d3652a3c tonesto7 2026-04-05T21:28:37-04:00 Merge bp-6.1 upstream changes into bp-6.0-dev (#124)
dd1a9d3fd8f60462d3d8ac6de3e7860e70e1f667 alan-polk 2025-12-12T12:22:31-06:00 convert opendbc from a submodule to a subfolder
10101012d8a4cc37951a557023824cdd995443a6 alan-polk 2026-08-10T17:23:04Z BluePilot: angle-mode lane centering trim (advanced lane positioning)
560e80e9d593badca08aa60e8ddb692e58033425 alan-polk 2026-08-11T12:50:00Z BluePilot: angle-mode lane centering falls back to model position, not zero
94bf8144d4a683e9f3408f3babf892878f3b81f4 ghbarker 2026-08-12T01:31:33Z ford ALP: restore StarPilot guards (narrow lane, laneline std, NaN params) and cap applied correction at 0.0015
80825b81568ad684fe4ec37a797d3cbaa004fdcf ghbarker 2026-08-13T01:04:02Z split applied-correction ceiling into its own PR per review
e5b92cc48b34ea43ac756a96a96687c9851a188d ghbarker 2026-08-13T01:05:07Z add tests for the restored guards
af4bc410c9ef417ba723212ceaa2a77758dd8ecb alan-polk 2026-08-14T03:24:28Z BluePilot: rate-limit the lane centering correction, not just filter it
8eea9eb68e0c4126ee1c52f251e274d5865b6743 alan-polk 2026-08-19T13:33:00Z BluePilot: lower angle-mode lane centering default strength to 0.25
Can't render this file because it contains an unexpected character in line 14 and column 85.
@@ -1,14 +1,81 @@
# BluePilot lateral
# BluePilot Ford angle control
BluePilot's Ford lateral control as a library an opendbc fork mounts at `opendbc/bluepilot_lateral/`: both modes (curvature and angle), the CAN messages that carry them, the panda safety that checks them, and a settings manifest.
StarPilot adaptation of BluePilotDev/bluepilot `bp-7.0`, pinned to
[e1d051d7ba270261b4455068bd68f1a58db15a4a](https://github.com/BluePilotDev/bluepilot/commit/e1d051d7ba270261b4455068bd68f1a58db15a4a).
The layout follows Alan Polk's September 2026 library proposal. This is an
adaptation, not an upstream library release.
**This is a modified Ford safety model.** The panda safety allows signals and ranges that upstream's Ford safety blocks. A fork chooses whether to ship it.
Existing curvature control remains the default. Explicit `FordLateralMode=1`
selects angle control next drive; the saved choice persists when unavailable.
Only this selector is exposed. Fixed defaults and bounds live in
`params/manifest.json`; pinion-yaw correction and curvature-primary tuning are
outside this library. Angle lane-centering trim is available as a default-disabled
core capability, separately sourced from BluePilot bp-dev
`e22afa6be9b881fa784c92ebb316db47728a3d81`; it adds no public settings.
**Status:** `v0.2.0`. It runs on bp-sunnypilot, a sunnypilot fork, through ford-op/opendbc (sunnypilot's opendbc plus the hooks).
`core/` accepts scalar/model inputs without openpilot, Params or messaging
imports. `can/` owns encoding, `safety/ford_bp.h` owns native enforcement, and
`hosts/starpilot.py` connects the host. `params/protocol.json` generates the wire
constants. The folder is vendored without submodules, LFS or an external runtime.
- [FORKS.md](FORKS.md): how a fork takes BluePilot lateral and works with BluePilot: carrying the library (a submodule or a stamped copy), the stamp tools, and adding a fork.
- [HOSTS.md](HOSTS.md): what a host provides, the hook patch, and how to run the tests.
- [can/LANE_ASSIST_DATA1.md](can/LANE_ASSIST_DATA1.md): the Lane_Assist_Data1 bits angle mode uses.
- [CREDITS.md](CREDITS.md) and [LICENSE](LICENSE) (MIT).
## Admission
History before 2026-09-15 lives in [BluePilotDev/bluepilot](https://github.com/BluePilotDev/bluepilot).
Classic: Bronco Sport Mk1, Escape Mk4, Focus Mk4, Maverick Mk1, Explorer Mk6.
CAN FD: Escape Mk4.5, Expedition Mk4, F150 Mk14, F150 Lightning Mk1, Ranger Mk2,
Mustang Mach-E Mk1.
Exact classic safety parameters 32/33 map to angle 128/129; CAN FD 18/19 and
66/67 map to 130/131. Parameter 131 requires DEBUG; RELEASE retains stock 130.
Longitudinal, radar, fingerprint and parser policy remain host-owned. Admission
uses the final CarParams contract, not brand or steering-control enum alone.
AOL, alternative experiences, Transit, Edge, Mondeo, SecOC, passive, dashcam,
notCar and unsupported static flags are excluded. Auxiliary noOutput requires
parameter zero; production deployments with two Pandas remain unqualified until
actual startup, bus routing and public acknowledgments are verified.
## Safety design
The extension uses small Ford hooks without changing shared safety headers or
existing curvature namespaces. Active steering requires ordinary controls
permission, healthy physical RX and a fresh matching public Panda acknowledgment
of safety parameter, alternative experience, controls, RX validity, heartbeat and
faults. Cruise, gas/brake, checksum, counter, quality and freshness checks apply.
Neutral mode cannot grant permission.
The BluePilot reset-bypass latch is omitted. Rejected frames preserve accepted
actuator history and withdraw the shadow lease; accepted neutral commands or
permission withdrawal reset history. Lost counters recover through neutral
resynchronization. Version or checksum failure cannot fall through to curvature
control.
Active frames carry neutral curvature, offset and curvature rate, with checked
mode, ramp, precision, path angle, counter, checksum and cadence. Native enforcement
checks the shadow against the actual path across gains 0.95–1.3, bounds both by
`min(0.02, 2.4114/v²)`, and bounds the full actual-curvature interval against
measured curvature above 10 m/s. Actual path also obeys the 3.5886 m/s³ equivalent
jerk limit and rolling message-rate limit. Core quantization and timing margins
keep packed commands inside these bounds; an impossible intersection is neutral.
The versioned 0x3CA announcement preserves StarPilot's existing byte 4 bit 1
curvature flag and requires it clear in angle mode. It carries a counter, signed
shadow curvature, protocol version and checksum; its lease expires after 60 ms.
The shadow derives from the final path after rate limiting and saturation.
Control computes at 20 Hz, announces through 33 Hz LKA, then sends the latest
announced result at 20 Hz, adding at most one steering tick. Permission withdrawal
sends neutral immediately. Human-turn recovery uses bounded neutral blips without
a native bypass. The retained Dom factory steering delay is 0.22 s; pinned
BluePilot uses 0.20 s.
These stricter native limits and versioned metadata intentionally differ from
BluePilot 7.0. Identical bytes or safety decisions are not claimed for affected
commands. Runtime and physical ECU qualification remain pending.
## Attribution and notices
See `CREDITS.md`, `ORIGIN_HISTORY.tsv` and the project `THIRD_PARTY_NOTICES.md`.
Both upstream MIT and Custom MIT notices are retained verbatim in `LICENSE`,
`LICENSE.md`, `upstream-root-LICENSE` and `upstream-root-LICENSE.md`. The maintainer
reports author permission to use this adaptation under MIT. Both published notices
remain intact.
@@ -1 +0,0 @@
v0.2.1 2b9dc6cc784faac47a1885d0bda0561f7b76f74d
@@ -0,0 +1,34 @@
"""Adapter to the proposed BluePilot library; angle only, exact source pinned."""
from opendbc.bluepilot_lateral.can.ford import lateral, lka
from opendbc.bluepilot_lateral.core.angle import Command
from opendbc.bluepilot_lateral.hosts.starpilot import Host, qualified
class BluePilotLateral:
def __init__(self, CP, *, host=None):
if not qualified(CP):
raise ValueError("Angle mode is not configured")
self.host = host or Host(CP)
self.canfd = bool(CP.flags & 1)
self.pending = Command()
self.announced = Command()
self.announced_ns = 0
self.lka_counter = 0
self.apply_curvature_last = 0.0
def step_lateral(self, CC, CS, packer, CAN, frame, now_ns):
self.pending = self.host.update(CC, CS)
command = self.announced
# A pending permission withdrawal is immediate; an increase waits until
# the 33Hz side channel has announced the matching final actuator.
if not self.pending.active or not 0 < self.announced_ns <= now_ns <= self.announced_ns + 60_000_000:
command = Command()
self.announced = command
return [lateral(packer, CAN.main, command, canfd=self.canfd, counter=frame // 5)]
def step_lka(self, packer, CAN, now_ns):
self.announced = self.pending
self.announced_ns = now_ns
message = lka(packer, CAN.main, self.announced.shadow_curvature, self.lka_counter)
self.lka_counter = (self.lka_counter + 1) % 8
return [message]
@@ -1,60 +0,0 @@
# Lane_Assist_Data1: BluePilot's angle-mode side channel
In angle mode the curvature signal in `LateralMotionControl2` is held at 0 and path angle does the
steering, so the panda has no commanded curvature to check against the car's measured curvature.
BluePilot sends that curvature, the *shadow curvature*, plus an "angle mode engaged" flag, in bits of
`Lane_Assist_Data1` (0x3CA) that carry no DBC signal. openpilot sends this message itself at 33 Hz,
and the panda reads it in the TX hook as it goes out. A separate message was tried first: the panda
never receives its own TX, and an RX check for it blocked controls car-wide (2026-07-09).
Packed by `create_lka_msg` in `fordcan_ext.py`; read by `ford_bp_lka_tx` in `../safety/ford.h`. The
byte positions, mask and scale are named once on each side (`BP_LKA_*` in `fordcan_ext.py`,
`FORD_BP_LKA_*` in `../safety/ford_declarations.h`), and `../tests/test_lane_assist_data1.py` checks
that they agree with each other and with the DBC.
## Layout
Bit numbers are within the byte, 7 = most significant. Stock signals are from `ford_lincoln_base_pt.dbc`.
| byte | bits | stock Ford signal | BluePilot |
|---|---|---|---|
| 0 | 7-5 | `LkaActvStats_D2_Req` | |
| 0 | 4-2 | `LdwActvStats_D_Req` | |
| 0 | 1-0 | `LdwActvIntns_D_Req` | |
| 1-2 | 12 bits | `LaCurvature_No_Calc` | |
| 2-3 | 12 bits | `LaRefAng_No_Req` | |
| 4 | 7 | `LaRampType_B_Req` | |
| 4 | 6-5 | `LkaDrvOvrrd_D_Rq` | |
| 4 | 4-1 | none | free |
| 4 | 0 | none | **angle mode engaged** |
| 5-6 | all | none | **shadow curvature** |
| 7 | all | none | free |
## Fields
- **Angle mode engaged** (byte 4 bit 0): 1 while angle mode is the selected lateral mode and
BluePilot lateral is on.
- **Shadow curvature** (bytes 5-6): int16, big-endian, 1e-6 1/m per unit, clamped to
-32768..32767 (about ±0.0328 1/m). It is the curvature angle mode's path angle was derived from,
after the same error clip curvature mode applies; while the driver is pressing, inactive, or
during a human-turn or stall-blip pulse, it is the car's measured curvature instead. It is sent
negated, in Ford's sign convention, like the curvature and path angle in `LateralMotionControl2`.
## What the panda does
- It latches both fields from every `Lane_Assist_Data1` frame that goes through its TX hook,
including one it blocks for another reason.
- **Angle mode engaged widens the path-angle range.** While the flag is set, the lateral command's
path angle may use the DBC's full range (-0.5 to 0.5235 rad); otherwise it is held to ±0.25 rad
(`ford_lmc_checks`). The flag alone does it: curvature 0 without the flag keeps the tight range.
- **The shadow curvature check.** For a lateral command (`LateralMotionControl` on CAN or
`LateralMotionControl2` on CAN FD) with curvature 0, steering enabled and the flag set, the
panda scales the shadow curvature by 0.05 to curvature CAN units (2e-5 1/m, truncated toward
zero) and, above 10 m/s, blocks the frame if it is outside the measured curvature ± the curvature
error band + 1 (`ford_shadow_curvature_error_check`). It has no rate limit: path angle has its own.
- `ford_bp_init` clears the flag.
## Not here yet
A protocol-version byte and a shared bit allocation for other forks (StarPilot) are Phase 3 (the StarPilot host issue, #16):
they change bytes on the wire.
@@ -0,0 +1,37 @@
"""Ford angle codecs, adapted from BluePilot fordcan_ext.py at e1d051d7.
Core commands use planner signs; path and shadow are negated on the wire.
The v1 side channel retains byte4 bit1 for StarPilot's curvature announcement.
"""
from opendbc.bluepilot_lateral.params import protocol as p
def checksum4(dat):
return (15 - (sum(dat[:7]) & 15)) & 15
def lka(packer, bus, shadow, counter):
addr, original, _ = packer.make_can_msg("Lane_Assist_Data1", bus, {"LaRefAng_No_Req": 0., "LaCurvature_No_Calc": 0.})
dat = bytearray(original)
raw = max(-20000, min(20000, round(-shadow / p.SHADOW_SCALE))) & 0xFFFF
dat[4] |= p.ANGLE_MASK | ((counter % p.COUNTER_MODULUS) << p.COUNTER_SHIFT)
dat[5], dat[6] = raw >> 8, raw & 255
dat[7] = p.VERSION | (checksum4(dat) << p.CHECKSUM_SHIFT)
return addr, bytes(dat), bus
def lateral(packer, bus, command, *, canfd, counter):
values = {"HandsOffCnfm_B_Rq": 0, "LatCtlRampType_D_Rq": 2 if command.active else 0,
"LatCtlPrecision_D_Rq": command.precision, "LatCtlPathOffst_L_Actl": 0.0,
"LatCtlPath_An_Actl": -command.path_angle, "LatCtlCurv_No_Actl": 0.0}
if canfd:
values.update({"LatCtl_D2_Rq": int(command.active), "LatCtlCrv_NoRate2_Actl": 0.0,
"LatCtlPath_No_Cnt": counter % 16, "LatCtlPath_No_Cs": 0})
dat = packer.make_can_msg("LateralMotionControl2", bus, values)[1]
fields = ((dat[2] << 3) | (dat[3] >> 5), (dat[6] << 3) | (dat[7] >> 5),
((dat[3] & 31) << 6) | (dat[4] >> 2), ((dat[4] & 3) << 8) | dat[5])
total = int(command.active) + counter % 16 + sum(v + (v >> 8) for v in fields)
values["LatCtlPath_No_Cs"] = 255 - (total & 255)
return packer.make_can_msg("LateralMotionControl2", bus, values)
values.update({"LatCtl_D_Rq": int(command.active), "LatCtlRng_L_Max": 0, "LatCtlCurv_NoRate_Actl": 0.0})
return packer.make_can_msg("LateralMotionControl", bus, values)
@@ -1,128 +0,0 @@
"""
BluePilot Ford CAN message builder extensions (lateral only).
Extended versions of the stock fordcan.py lateral builders:
- Dynamic ramp_type and precision_type for the 4-signal lateral commands
- Lane_Assist_Data1 carries the angle-mode side channel read by bluepilot_lateral/safety/ford.h
Trimmed for the lateral proof-of-concept: the longitudinal, HUD and ICBM builders from
BluePilot's fordcan_ext.py are out of scope here.
"""
from ...car.ford.fordcan import CanBus
# BluePilot: the Lane_Assist_Data1 side channel, see LANE_ASSIST_DATA1.md. ford_declarations.h has the
# same values (tests/test_lane_assist_data1.py).
BP_LKA_ANGLE_MODE_BYTE = 4
BP_LKA_ANGLE_MODE_MASK = 0x1
BP_LKA_SHADOW_MSB_BYTE = 5 # shadow curvature: int16, big-endian, bytes 5-6
BP_LKA_SHADOW_LSB_BYTE = 6
BP_LKA_SHADOW_CURVATURE_SCALE = 1e-6 # 1/meter per raw unit
def create_lka_msg(packer, CAN: CanBus, lat_active: bool, hud_control,
angle_mode_engaged: bool = False, shadow_curvature: float = 0.0):
"""
Creates a CAN message for the Ford LKA Command.
BluePilot extension: accepts lat_active and hud_control parameters for future
lane departure warning integration (unused so far).
Also carries angle_mode_engaged + shadow_curvature (Ford sign) in bits with no DBC signal: the
layout and what the panda does with them are in LANE_ASSIST_DATA1.md.
Frequency is 33Hz.
"""
addr, dat, bus = packer.make_can_msg("Lane_Assist_Data1", CAN.main, {})
dat = bytearray(dat)
shadow_curvature_raw = int(round(shadow_curvature / BP_LKA_SHADOW_CURVATURE_SCALE))
shadow_curvature_raw = max(-32768, min(32767, shadow_curvature_raw)) & 0xFFFF
dat[BP_LKA_ANGLE_MODE_BYTE] |= BP_LKA_ANGLE_MODE_MASK if angle_mode_engaged else 0
dat[BP_LKA_SHADOW_MSB_BYTE] = (shadow_curvature_raw >> 8) & 0xFF
dat[BP_LKA_SHADOW_LSB_BYTE] = shadow_curvature_raw & 0xFF
return addr, bytes(dat), bus
def create_lat_ctl_msg(packer, CAN: CanBus, lat_active: bool, ramp_type: int, precision_type: int,
path_offset: float, path_angle: float, curvature: float, curvature_rate: float):
"""
Creates a CAN message for the Ford TJA/LCA Command (non-CAN FD).
BluePilot extension: dynamic ramp_type and precision_type parameters.
Stock hardcodes ramp_type=0 (Slow) and precision_type=1 (Precise).
Ford lane centering uses a third-order polynomial to describe the road centerline:
c0 (path_offset): lateral offset between vehicle and centerline (positive is right)
c1 (path_angle): heading angle between vehicle and centerline (positive is right)
c2 (curvature): curvature of the centerline (positive is left)
c3 (curvature_rate): rate of change of curvature
Frequency is 20Hz.
"""
values = {
"LatCtlRng_L_Max": 0, # Unknown [0|126] meter
"HandsOffCnfm_B_Rq": 0, # Unknown: 0=Inactive, 1=Active [0|1]
"LatCtl_D_Rq": 1 if lat_active else 0, # Mode: 0=None, 1=ContinuousPathFollowing, 2=InterventionLeft,
# 3=InterventionRight, 4-7=NotUsed [0|7]
"LatCtlRampType_D_Rq": ramp_type, # Ramp speed: 0=Slow, 1=Medium, 2=Fast, 3=Immediate [0|3]
"LatCtlPrecision_D_Rq": precision_type, # Precision: 0=Comfortable, 1=Precise, 2/3=NotUsed [0|3]
"LatCtlPathOffst_L_Actl": path_offset, # Path offset [-5.12|5.11] meter
"LatCtlPath_An_Actl": path_angle, # Path angle [-0.5|0.5235] radians
"LatCtlCurv_NoRate_Actl": curvature_rate, # Curvature rate [-0.001024|0.00102375] 1/meter^2
"LatCtlCurv_No_Actl": curvature, # Curvature [-0.02|0.02094] 1/meter
}
return packer.make_can_msg("LateralMotionControl", CAN.main, values)
def create_lat_ctl2_msg(packer, CAN: CanBus, mode: int, ramp_type: int, precision_type: int,
path_offset: float, path_angle: float, curvature: float,
curvature_rate: float, counter: int):
"""
Creates a CAN message for the Ford Lane Centering command (CAN FD).
BluePilot extension: dynamic ramp_type and precision_type parameters.
Stock hardcodes ramp_type=0 (Slow) and precision_type=1 (Precise).
This message replaces LateralMotionControl on CAN FD platforms and includes
counter and checksum fields.
Frequency is 20Hz.
"""
values = {
"LatCtl_D2_Rq": mode, # Mode: 0=None, 1=PathFollowingLimitedMode, 2=PathFollowingExtendedMode,
# 3=SafeRampOut, 4-7=NotUsed [0|7]
"LatCtlRampType_D_Rq": ramp_type, # 0=Slow, 1=Medium, 2=Fast, 3=Immediate [0|3]
"LatCtlPrecision_D_Rq": precision_type, # 0=Comfortable, 1=Precise, 2/3=NotUsed [0|3]
"LatCtlPathOffst_L_Actl": path_offset, # [-5.12|5.11] meter
"LatCtlPath_An_Actl": path_angle, # [-0.5|0.5235] radians
"LatCtlCurv_No_Actl": curvature, # [-0.02|0.02094] 1/meter
"LatCtlCrv_NoRate2_Actl": curvature_rate, # [-0.001024|0.001023] 1/meter^2
"HandsOffCnfm_B_Rq": 0, # 0=Inactive, 1=Active [0|1]
"LatCtlPath_No_Cnt": counter, # [0|15]
"LatCtlPath_No_Cs": 0, # [0|255]
}
# Computed here, not left to the packer: upstream's packer fills Ford checksums itself (and overwrites
# this with the same value), but a host's packer may not (IQ.Pilot's doesn't), and the PSCM drops a
# frame whose checksum is wrong.
dat = packer.make_can_msg("LateralMotionControl2", CAN.main, values)[1]
values["LatCtlPath_No_Cs"] = lat_ctl2_checksum(mode, counter, dat)
return packer.make_can_msg("LateralMotionControl2", CAN.main, values)
def lat_ctl2_checksum(mode: int, counter: int, dat: bytes) -> int:
"""LatCtlPath_No_Cs of a packed LateralMotionControl2: upstream's ford_checksum over the message's
checksum fields (mode, counter, curvature, curvature rate, path angle, path offset)."""
curvature = (dat[2] << 3) | (dat[3] >> 5)
curvature_rate = (dat[6] << 3) | (dat[7] >> 5)
path_angle = ((dat[3] & 0x1F) << 6) | (dat[4] >> 2)
path_offset = ((dat[4] & 0x3) << 8) | dat[5]
checksum = mode + counter
for sig_val in (curvature, curvature_rate, path_angle, path_offset):
checksum += sig_val + (sig_val >> 8)
return 0xFF - (checksum & 0xFF)
@@ -0,0 +1,208 @@
"""Angle-primary Ford steering adapted from Alan Polk's BluePilot bp-7.0.
Source e1d051d7ba270261b4455068bd68f1a58db15a4a lateral_angle_ext.py.
See ../LICENSE, ../LICENSE.md and ../CREDITS.md. This core has no vehicle,
openpilot, Params, messaging or model-constant imports.
"""
from dataclasses import dataclass
import math
from ..params import limits as bounds
from .trim import LaneCenterTrim, LaneGeometry, TrimConfig
def clip(value, low, high):
return max(low, min(high, value))
def interp(value, xs, ys):
if value <= xs[0]:
return ys[0]
for i in range(1, len(xs)):
if value < xs[i]:
return ys[i - 1] + (ys[i] - ys[i - 1]) * (value - xs[i - 1]) / (xs[i] - xs[i - 1])
return ys[-1]
@dataclass(frozen=True)
class Inputs:
active: bool
speed: float
measured_curvature: float
desired_curvature: float
steering_pressed: bool = False
steering_angle: float = 0.0
yaw_predictions: tuple = ()
time_indices: tuple = ()
lateral_delay: float = 0.15
lane_change: bool = False
lane_direction: int = 0
limit_status: int = 0
geometry: LaneGeometry | None = None
speed_min: float | None = None
speed_max: float | None = None
@dataclass(frozen=True)
class Command:
active: bool = False
path_angle: float = 0.0
shadow_curvature: float = 0.0
precision: int = 1
class AngleController:
def __init__(self, *, low_gain=1.0, high_gain=1.15, trim_config=None):
self.trim_config = TrimConfig() if trim_config is None else trim_config
self.trim = LaneCenterTrim()
self.low_gain = low_gain
self.high_gain = high_gain
self.path_last = 0.0
self.speed_last = 0.1
self.desired_last = 0.0
self.human_timer = 0.0
self.press_last = False
self.preturned = False
self.press_timer = 0.0
self.blip_frames = 0
self.blip_cooldown = 0.0
self.blip_hold = 0.0
self.blip_count = 0
def reset(self):
self.trim.reset()
self.path_last = 0.0
self.speed_last = 0.1
self.desired_last = 0.0
self.human_timer = 0.0
self.press_last = False
self.preturned = False
self.press_timer = 0.0
self.blip_frames = 0
self.blip_cooldown = 0.0
self.blip_hold = 0.0
self.blip_count = 0
def step(self, i: Inputs) -> Command:
values = (i.speed, i.measured_curvature, i.desired_curvature, i.steering_angle, i.lateral_delay)
if not i.active or not all(math.isfinite(v) for v in values) or i.speed < 0.0:
self.reset()
return Command()
speed_min = max(i.speed, 0.1) if i.speed_min is None else i.speed_min
speed_max = max(i.speed, 0.1) if i.speed_max is None else i.speed_max
if (not all(math.isfinite(v) for v in (speed_min, speed_max)) or
not 0 < speed_min <= max(i.speed, 0.1) <= speed_max):
self.reset()
return Command()
if i.steering_pressed and not self.press_last:
self.preturned = abs(i.steering_angle) > 45.0
self.press_last = i.steering_pressed
self.human_timer = self.human_timer + 0.05 if i.steering_pressed and abs(i.steering_angle) > 45.0 else 0.0
if self.human_timer + 1e-9 >= (3.0 if self.preturned else 1.5):
self.trim.reset()
self.path_last = 0.0
self.desired_last = i.desired_curvature
self.blip_frames = self.blip_count = 0
self.blip_hold = self.blip_cooldown = self.press_timer = 0.0
return Command()
if i.steering_pressed:
self.press_timer += 0.05
else:
if self.press_timer >= 0.5 and self.blip_cooldown <= 0.0 and self.blip_frames <= 0 and abs(self.path_last) < 0.1:
self.blip_frames = 6
self.press_timer = 0.0
if self.blip_frames > 0:
self.blip_frames -= 1
self.trim.reset()
self.path_last = 0.0
self.desired_last = i.desired_curvature
if self.blip_frames == 0:
self.blip_cooldown = 2.0
return Command()
base = clip(i.lateral_delay, 0.1, 0.15) + 0.05
predictions_valid = (len(i.yaw_predictions) == len(i.time_indices) and len(i.time_indices) >= 17 and
all(math.isfinite(v) for v in (*i.yaw_predictions, *i.time_indices)) and
all(b > a for a, b in zip(i.time_indices, i.time_indices[1:], strict=False)))
curves = tuple(y / max(0.01, i.speed) for y in i.yaw_predictions) if predictions_valid else ()
entering = bool(curves) and abs(interp(base, i.time_indices, curves)) > abs(i.desired_curvature)
factor = 1.0 if entering else interp(abs(i.desired_curvature), (0.005, 0.020), (1.0, 0.0))
lookup = base + bounds.VLT_EXTRA * interp(i.speed, (11.176, 24.5872), (1.0, 0.0)) * factor
predicted = interp(lookup, i.time_indices, curves) if curves else i.desired_curvature
saturated = i.limit_status >= 2 or self.path_last >= 0.5235 * 0.9 or self.path_last <= -0.5 * 0.9
falling = abs(i.desired_curvature) < abs(self.desired_last) - 0.010
blend = bounds.BLEND * 0.25 if not entering and (i.limit_status >= 1 or saturated or falling) else bounds.BLEND
requested = predicted * blend + i.desired_curvature * (1.0 - blend)
self.desired_last = i.desired_curvature
precision = 0 if i.lane_change and ((i.lane_direction == 1 and requested < 0) or
(i.lane_direction == 2 and requested > 0)) else 1
# BluePilot's default lane-change factors are both 1.0.
# Trim requested curvature before every existing actuator/safety limiter.
requested = self.trim.update(requested, i.speed, i.geometry, self.trim_config,
active=i.active, lane_change=i.lane_change, steering_pressed=i.steering_pressed)
kappa = clip(requested, i.measured_curvature - 0.002, i.measured_curvature + 0.002) if i.speed > 9.0 else requested
low = interp(i.speed, (13.5, 26.82), (1.0, self.low_gain))
high = interp(i.speed, (13.5, 26.82), (1.3, self.high_gain))
gain = interp(abs(kappa), (0.0007, 0.001), (low, high))
path = kappa * i.speed * gain
if saturated:
last_mag, magnitude = abs(self.path_last), abs(path)
if magnitude > last_mag:
path = self.path_last
elif last_mag - magnitude > 0.02:
path = math.copysign(last_mag - 0.02, self.path_last)
elif i.limit_status >= 1:
path = clip(path, -abs(self.path_last), abs(self.path_last))
roc = interp(i.speed, (10.0, 15.0, 25.0), (0.055, 0.0425, 0.009))
path = clip(clip(path, -0.5235, 0.5), self.path_last - roc, self.path_last + roc)
# Tightening relative to bp-7.0: the shadow describes the final actuator,
# including ROC/saturation. A truthful derived curvature must fit both the
# measured-curvature interval and the modern ISO envelope. If those bands
# cannot meet the previous actuator ROC, send a neutral mode-0 frame.
effective_speed = max(i.speed, 0.1)
cap = min(bounds.CURVATURE_MAX, bounds.LATERAL_ACCEL / max(speed_max, 1.0) ** 2)
low_kappa, high_kappa = -cap, cap
if i.speed > 9.0:
low_kappa = max(low_kappa, i.measured_curvature - 0.0018)
high_kappa = min(high_kappa, i.measured_curvature + 0.0018)
# Native cannot trust a host-selected feel gain. Intersect every admitted
# gain (0.95..1.3), so the worst equivalent curvature remains bounded.
lower_gain = bounds.GAIN_MAX if low_kappa >= 0.0 else bounds.GAIN_MIN
upper_gain = bounds.GAIN_MIN if high_kappa >= 0.0 else bounds.GAIN_MAX
lower_speed = speed_max if low_kappa >= 0.0 else speed_min
upper_speed = speed_min if high_kappa >= 0.0 else speed_max
lower = max(-bounds.PATH_MAX, low_kappa * lower_speed * lower_gain, self.path_last - roc)
upper = min(-bounds.PATH_MIN, high_kappa * upper_speed * upper_gain, self.path_last + roc)
# Bound the actual-angle equivalent derivative at the worst admitted gain.
# 45ms core margin remains inside the native elapsed-time/50ms cap.
jerk_delta = bounds.LATERAL_JERK / max(i.speed - 1.0, 1.0) ** 2 * bounds.CORE_STEER_PERIOD_S * bounds.GAIN_MIN * effective_speed
jerk_center = self.path_last / self.speed_last * effective_speed
lower = max(lower, jerk_center - jerk_delta)
upper = min(upper, jerk_center + jerk_delta)
lower += bounds.PATH_ROUNDING_MARGIN
upper -= bounds.PATH_ROUNDING_MARGIN
if lower > upper:
self.trim.reset()
self.path_last = 0.0
return Command()
path = clip(path, lower, upper)
self.path_last = path
self.speed_last = effective_speed
shadow = path / (effective_speed * gain)
self.blip_cooldown = max(0.0, self.blip_cooldown - 0.05)
gap = i.desired_curvature - i.measured_curvature
stalled = (not i.steering_pressed and not i.lane_change and i.speed > 9.0 and abs(gap) > 0.004 and
abs(i.desired_curvature) > abs(i.measured_curvature))
if stalled:
if abs(kappa - requested) > 1e-9 and self.blip_cooldown <= 0.0:
self.blip_hold += 0.05
if self.blip_hold >= 0.5 and self.blip_count < 3 and abs(path) < 0.1:
self.blip_frames = 6
self.blip_hold = 0.0
self.blip_count += 1
else:
self.blip_hold = 0.0
if i.steering_pressed or abs(gap) < 0.002:
self.blip_count = 0
return Command(True, path, shadow, precision)

Some files were not shown because too many files have changed in this diff Show More