mirror of
https://gitlvb.teallvbs.xyz/IQ.Lvbs/IQ.Pilot.git
synced 2026-06-08 10:55:03 +08:00
IQ.Pilot Release Commit @ e6f5952
This commit is contained in:
27
.gitattributes
vendored
Normal file
27
.gitattributes
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
* text=auto
|
||||
|
||||
# to move existing files into LFS:
|
||||
# git add --renormalize .
|
||||
|
||||
# Keep this icon in regular git (not LFS) for lightweight branding iteration.
|
||||
selfdrive/assets/icons_mici/experimental_mode_mici.png -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/experimental_mode_tizi.png -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/settings/konn3kt_icon.png -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/buttons/toggle_dot_enabled.png -filter -diff -merge -text
|
||||
iqpilot/selfdrive/assets -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/buttons/toggle_pill_enabled.png -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/offroad_alerts/green_wheel.png -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/setup/green_button.png -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/setup/green_button_pressed.png -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/setup/green_dm.png -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/setup/green_info.png -filter -diff -merge -text
|
||||
selfdrive/assets/icons_mici/setup/small_slider/slider_green_rounded_rectangle.png -filter -diff -merge -text
|
||||
selfdrive/assets/images/spinner_comma.png -filter -diff -merge -text
|
||||
selfdrive/assets/images/k3_spinner.png -filter -diff -merge -text
|
||||
selfdrive/assets/fonts/Syncopate-Regular.ttf -filter -diff -merge -text
|
||||
selfdrive/assets/fonts/Tektur-Variable.ttf -filter -diff -merge -text
|
||||
|
||||
|
||||
|
||||
# IQ assets should stay in normal git (not LFS), including nested PNGs/SVGs.
|
||||
iqpilot/selfdrive/assets/** -filter -diff -merge -text
|
||||
113
.gitignore
vendored
Normal file
113
.gitignore
vendored
Normal file
@@ -0,0 +1,113 @@
|
||||
venv/
|
||||
.venv/
|
||||
.ci_cache
|
||||
.env
|
||||
.clang-format
|
||||
.DS_Store
|
||||
.tags
|
||||
.ipynb_checkpoints
|
||||
.idea
|
||||
.overlay_init
|
||||
.overlay_consistent
|
||||
.sconsign.dblite
|
||||
a.out
|
||||
.hypothesis
|
||||
.cache/
|
||||
|
||||
/docs_site/
|
||||
|
||||
*.mp4
|
||||
*.dylib
|
||||
*.DSYM
|
||||
*.d
|
||||
*.pyc
|
||||
!artifacts/iqpilot_*_private/**/*.pyc
|
||||
*.pyo
|
||||
.*.swp
|
||||
.*.swo
|
||||
.*.un~
|
||||
*.tmp
|
||||
*.o
|
||||
*.o-*
|
||||
*.os
|
||||
*.os-*
|
||||
*.so
|
||||
!artifacts/iqpilot_*_private/**/*.so
|
||||
artifacts/iqpilot_private/
|
||||
artifacts/iqpilot_*_private/python/iqpilot_private/**/*.cpython-*-darwin.so
|
||||
*.a
|
||||
*.clb
|
||||
*.class
|
||||
*.pyxbldc
|
||||
*.vcd
|
||||
*.mo
|
||||
*_pyx.cpp
|
||||
*.stats
|
||||
config.json
|
||||
clcache
|
||||
compile_commands.json
|
||||
compare_runtime*.html
|
||||
|
||||
selfdrive/pandad/pandad
|
||||
cereal/services.h
|
||||
cereal/gen
|
||||
cereal/messaging/bridge
|
||||
selfdrive/ui/translations/tmp
|
||||
selfdrive/car/tests/cars_dump
|
||||
system/camerad/camerad
|
||||
system/camerad/test/ae_gray_test
|
||||
|
||||
.coverage*
|
||||
coverage.xml
|
||||
htmlcov
|
||||
pandaextra
|
||||
|
||||
.mypy_cache/
|
||||
flycheck_*
|
||||
|
||||
cppcheck_report.txt
|
||||
comma*.sh
|
||||
|
||||
selfdrive/modeld/models/*.pkl
|
||||
iqpilot/modeld*/thneed/compile
|
||||
iqpilot/modeld*/models/*.thneed
|
||||
iqpilot/modeld*/models/*.pkl
|
||||
|
||||
# openpilot log files
|
||||
*.bz2
|
||||
*.zst
|
||||
|
||||
build/
|
||||
|
||||
!**/.gitkeep
|
||||
|
||||
poetry.toml
|
||||
Pipfile
|
||||
|
||||
### VisualStudioCode ###
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
!.vscode/*.code-snippets
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
# Built Visual Studio Code Extensions
|
||||
*.vsix
|
||||
|
||||
### VisualStudioCode Patch ###
|
||||
# Ignore all local history of files
|
||||
.history
|
||||
.ionide
|
||||
|
||||
.claude/
|
||||
PLAN.md
|
||||
TASK.md
|
||||
|
||||
### JetBrains ###
|
||||
!.idea/customTargets.xml
|
||||
!.idea/tools/*
|
||||
!.run/*
|
||||
4
.lfsconfig
Normal file
4
.lfsconfig
Normal file
@@ -0,0 +1,4 @@
|
||||
[lfs]
|
||||
url = https://gitlab.com/tealtwo/iq.lvbs-lfs.git/info/lfs
|
||||
pushurl = ssh://git@gitlab.com/tealtwo/iq.lvbs-lfs.git
|
||||
locksverify = false
|
||||
4
.lfsconfig-comma
Normal file
4
.lfsconfig-comma
Normal file
@@ -0,0 +1,4 @@
|
||||
[lfs]
|
||||
url = https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
|
||||
pushurl = ssh://git@gitlab.com/commaai/openpilot-lfs.git
|
||||
locksverify = false
|
||||
56
CHANGELOG.md
Normal file
56
CHANGELOG.md
Normal file
@@ -0,0 +1,56 @@
|
||||
# IQ.Pilot User Changelog
|
||||
|
||||
This changelog is written for everyday drivers and focuses on what you will notice on the road.
|
||||
|
||||
## February 9, 2026 - IQ.Pilot Launch
|
||||
- IQ.Pilot v1.0a launched.
|
||||
- You got first-release IQ.Pilot + Konn3kt integration.
|
||||
- The UI was rebranded and cleaned up for a more consistent look.
|
||||
- Unused legacy UI/debug elements were removed.
|
||||
- Early setup and reliability issues were fixed (better error handling and cleaner setup flow).
|
||||
|
||||
## February 10, 2026 - Device Support and Stability
|
||||
- Better mici and tizi device support.
|
||||
- You can now use manual QR registration.
|
||||
- Volkswagen PQ support was enabled and expanded.
|
||||
- A major bugfix pass improved stability and controls behavior.
|
||||
- Cruise/button behavior was refined for more consistent response.
|
||||
|
||||
## February 11, 2026 - Major Vehicle Expansion
|
||||
- Tesla control support was expanded.
|
||||
- Honda MVL tuning (lateral + longitudinal) was added.
|
||||
- Volkswagen support grew across PQ and MLB with multiple fixes.
|
||||
- PQ bring-up continued, including Passat NMS-focused improvements.
|
||||
- More runtime fixes landed (including joystick and icon issues).
|
||||
|
||||
## February 12, 2026 - Sensor/Data Fixes
|
||||
- Steering-angle offset tolerance was increased, helping with temporary steering sensor misalignment.
|
||||
- Fuel level handling was fixed and improved.
|
||||
- Additional Volkswagen PQ follow-up fixes were added.
|
||||
|
||||
## February 13, 2026 - Driver Monitoring Rollback
|
||||
- Driver Monitoring was rolled back to previous behavior to reduce over aggressive alerts from a regression caused by a new comma DM model.
|
||||
|
||||
## February 15-16, 2026 - Volkswagen PQ Maturity
|
||||
- Volkswagen PQ received heavy control tuning and bugfixes.
|
||||
- You should see more consistent engagement and smoother overall behavior.
|
||||
- Lateral/longitudinal interaction and cruise response were improved.
|
||||
|
||||
## February 17, 2026 - Volkswagen Stopping Behavior Tuning
|
||||
- Volkswagen stopping and braking behavior received additional tuning.
|
||||
- Comfort and stop-response were further refined for real-world driving.
|
||||
|
||||
## February 27, 2026 - Major Refactor
|
||||
- IQ.Pilot completed a major refactor and is now independent of any other fork.
|
||||
- Driving logic cleanup removed legacy compatibility paths that could cause inconsistent behavior between vehicles and branches.
|
||||
- New (way better) Always On Lateral logic!
|
||||
- Better longitudinal controls for clearer choices for everyday use: `IQ.Pilot`, `IQ.Dynamic`, `IQ.Standard`, and your vehicles `Stock ACC`.
|
||||
- Device/System controls were reworked for better offroad management, including a timed Force On-Road workflow for diagnostics/testing.
|
||||
- Volkswagen MEB/MQBEvo platform support!
|
||||
- Offroad UI and settings UI rework!
|
||||
|
||||
## Summary
|
||||
- IQ.Pilot completed a major refactor and is now independent of any other fork.
|
||||
- Volkswagen MEB/MQBEvo platform support!
|
||||
- Offroad UI and settings UI rework!
|
||||
|
||||
269
Jenkinsfile
vendored
Normal file
269
Jenkinsfile
vendored
Normal file
@@ -0,0 +1,269 @@
|
||||
def retryWithDelay(int maxRetries, int delay, Closure body) {
|
||||
for (int i = 0; i < maxRetries; i++) {
|
||||
try {
|
||||
return body()
|
||||
} catch (Exception e) {
|
||||
sleep(delay)
|
||||
}
|
||||
}
|
||||
throw Exception("Failed after ${maxRetries} retries")
|
||||
}
|
||||
|
||||
def device(String ip, String step_label, String cmd) {
|
||||
withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
|
||||
def ssh_cmd = """
|
||||
ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
|
||||
|
||||
set -e
|
||||
|
||||
export TERM=xterm-256color
|
||||
|
||||
shopt -s huponexit # kill all child processes when the shell exits
|
||||
|
||||
export CI=1
|
||||
export PYTHONWARNINGS=error
|
||||
#export LOGPRINT=debug # this has gotten too spammy...
|
||||
export TEST_DIR=${env.TEST_DIR}
|
||||
export SOURCE_DIR=${env.SOURCE_DIR}
|
||||
export GIT_BRANCH=${env.GIT_BRANCH}
|
||||
export GIT_COMMIT=${env.GIT_COMMIT}
|
||||
export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
|
||||
export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN}
|
||||
export AZURE_TOKEN='${env.AZURE_TOKEN}'
|
||||
# only use 1 thread for tici tests since most require HIL
|
||||
export PYTEST_ADDOPTS="-n0 -s"
|
||||
|
||||
|
||||
export GIT_SSH_COMMAND="ssh -i /data/gitkey"
|
||||
|
||||
source ~/.bash_profile
|
||||
if [ -f /TICI ]; then
|
||||
source /etc/profile
|
||||
|
||||
rm -rf /tmp/tmp*
|
||||
rm -rf ~/.commacache
|
||||
rm -rf /dev/shm/*
|
||||
rm -rf /dev/tmp/tmp*
|
||||
|
||||
if ! systemctl is-active --quiet systemd-resolved; then
|
||||
echo "restarting resolved"
|
||||
sudo systemctl start systemd-resolved
|
||||
sleep 3
|
||||
fi
|
||||
|
||||
# restart aux USB
|
||||
if [ -e /sys/bus/usb/drivers/hub/3-0:1.0 ]; then
|
||||
echo "restarting aux usb"
|
||||
echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/unbind
|
||||
sleep 0.5
|
||||
echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/bind
|
||||
fi
|
||||
fi
|
||||
if [ -f /data/openpilot/launch_env.sh ]; then
|
||||
source /data/openpilot/launch_env.sh
|
||||
fi
|
||||
|
||||
ln -snf ${env.TEST_DIR} /data/pythonpath
|
||||
|
||||
cd ${env.TEST_DIR} || true
|
||||
time ${cmd}
|
||||
END"""
|
||||
|
||||
sh script: ssh_cmd, label: step_label
|
||||
}
|
||||
}
|
||||
|
||||
def deviceStage(String stageName, String deviceType, List extra_env, def steps) {
|
||||
stage(stageName) {
|
||||
if (currentBuild.result != null) {
|
||||
return
|
||||
}
|
||||
|
||||
if (isReplay()) {
|
||||
error("REPLAYING TESTS IS NOT ALLOWED. FIX THEM INSTEAD.")
|
||||
}
|
||||
|
||||
def extra = extra_env.collect { "export ${it}" }.join('\n');
|
||||
def branch = env.BRANCH_NAME ?: 'master';
|
||||
def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename || echo "/"', label: 'Getting changes'
|
||||
|
||||
lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') {
|
||||
docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
|
||||
timeout(time: 35, unit: 'MINUTES') {
|
||||
retry (3) {
|
||||
def date = sh(script: 'date', returnStdout: true).trim();
|
||||
device(device_ip, "set time", "date -s '" + date + "'")
|
||||
device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
|
||||
}
|
||||
steps.each { item ->
|
||||
def name = item[0]
|
||||
def cmd = item[1]
|
||||
|
||||
def args = item[2]
|
||||
def diffPaths = args.diffPaths ?: []
|
||||
def cmdTimeout = args.timeout ?: 9999
|
||||
|
||||
if (branch != "master" && !branch.contains("__jenkins_loop_") && diffPaths && !hasPathChanged(gitDiff, diffPaths)) {
|
||||
println "Skipping ${name}: no changes in ${diffPaths}."
|
||||
return
|
||||
} else {
|
||||
timeout(time: cmdTimeout, unit: 'SECONDS') {
|
||||
device(device_ip, name, cmd)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def hasPathChanged(String gitDiff, List<String> paths) {
|
||||
for (path in paths) {
|
||||
if (gitDiff.contains(path)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
def isReplay() {
|
||||
def replayClass = "org.jenkinsci.plugins.workflow.cps.replay.ReplayCause"
|
||||
return currentBuild.rawBuild.getCauses().any{ cause -> cause.toString().contains(replayClass) }
|
||||
}
|
||||
|
||||
def setupCredentials() {
|
||||
withCredentials([
|
||||
string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
|
||||
]) {
|
||||
env.AZURE_TOKEN = "${AZURE_TOKEN}"
|
||||
}
|
||||
|
||||
withCredentials([
|
||||
string(credentialsId: 'ci_artifacts_pat', variable: 'CI_ARTIFACTS_TOKEN'),
|
||||
]) {
|
||||
env.CI_ARTIFACTS_TOKEN = "${CI_ARTIFACTS_TOKEN}"
|
||||
}
|
||||
|
||||
withCredentials([
|
||||
string(credentialsId: 'post_comments_github_pat', variable: 'GITHUB_COMMENTS_TOKEN'),
|
||||
]) {
|
||||
env.GITHUB_COMMENTS_TOKEN = "${GITHUB_COMMENTS_TOKEN}"
|
||||
}
|
||||
}
|
||||
|
||||
def step(String name, String cmd, Map args = [:]) {
|
||||
return [name, cmd, args]
|
||||
}
|
||||
|
||||
node {
|
||||
env.CI = "1"
|
||||
env.PYTHONWARNINGS = "error"
|
||||
env.TEST_DIR = "/data/openpilot"
|
||||
env.SOURCE_DIR = "/data/openpilot_source/"
|
||||
setupCredentials()
|
||||
|
||||
env.GIT_BRANCH = checkout(scm).GIT_BRANCH
|
||||
env.GIT_COMMIT = checkout(scm).GIT_COMMIT
|
||||
|
||||
def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
|
||||
'release-tici', 'release-tizi', 'release-tizi-staging', 'testing-closet*', 'hotfix-*']
|
||||
def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
|
||||
|
||||
if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
|
||||
properties([
|
||||
disableConcurrentBuilds(abortPrevious: true)
|
||||
])
|
||||
}
|
||||
|
||||
try {
|
||||
if (env.BRANCH_NAME == 'devel-staging') {
|
||||
deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
|
||||
step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh"),
|
||||
])
|
||||
}
|
||||
|
||||
if (env.BRANCH_NAME == '__nightly') {
|
||||
parallel (
|
||||
'nightly': {
|
||||
deviceStage("build nightly", "tizi-needs-can", [], [
|
||||
step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
|
||||
])
|
||||
},
|
||||
'nightly-dev': {
|
||||
deviceStage("build nightly-dev", "tizi-needs-can", [], [
|
||||
step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
|
||||
])
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
if (!env.BRANCH_NAME.matches(excludeRegex)) {
|
||||
parallel (
|
||||
'onroad tests': {
|
||||
deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
|
||||
step("build openpilot", "cd system/manager && ./build.py"),
|
||||
step("check dirty", "release/check-dirty.sh"),
|
||||
step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
|
||||
])
|
||||
},
|
||||
'HW + Unit Tests': {
|
||||
deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
|
||||
step("build", "cd system/manager && ./build.py"),
|
||||
step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
|
||||
step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
|
||||
step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
|
||||
step("test manager", "pytest system/manager/test/test_manager.py"),
|
||||
])
|
||||
},
|
||||
'loopback': {
|
||||
deviceStage("loopback", "tizi-loopback", ["UNSAFE=1"], [
|
||||
step("build openpilot", "cd system/manager && ./build.py"),
|
||||
step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
|
||||
])
|
||||
},
|
||||
'camerad OX03C10': {
|
||||
deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
|
||||
step("build", "cd system/manager && ./build.py"),
|
||||
step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
|
||||
step("test exposure", "pytest system/camerad/test/test_exposure.py"),
|
||||
])
|
||||
},
|
||||
'camerad OS04C10': {
|
||||
deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
|
||||
step("build", "cd system/manager && ./build.py"),
|
||||
step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
|
||||
step("test exposure", "pytest system/camerad/test/test_exposure.py"),
|
||||
])
|
||||
},
|
||||
'sensord': {
|
||||
deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
|
||||
step("build", "cd system/manager && ./build.py"),
|
||||
step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
|
||||
])
|
||||
},
|
||||
'replay': {
|
||||
deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
|
||||
step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
|
||||
step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
|
||||
])
|
||||
},
|
||||
'tizi': {
|
||||
deviceStage("tizi", "tizi", ["UNSAFE=1"], [
|
||||
step("build openpilot", "cd system/manager && ./build.py"),
|
||||
step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
|
||||
step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
|
||||
step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
|
||||
// TODO: enable once new AGNOS is available
|
||||
// step("test esim", "pytest system/hardware/tici/tests/test_esim.py"),
|
||||
step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
|
||||
])
|
||||
},
|
||||
|
||||
)
|
||||
}
|
||||
} catch (Exception e) {
|
||||
currentBuild.result = 'FAILED'
|
||||
throw e
|
||||
}
|
||||
}
|
||||
98
LICENSE
Normal file
98
LICENSE
Normal file
@@ -0,0 +1,98 @@
|
||||
IQ.Lvbs License v0.1a
|
||||
|
||||
Copyright (c) 2026 IQ.Lvbs LLC, a part of Project Teal Lvbs Inc. All Rights Reserved.
|
||||
|
||||
DEFINITIONS
|
||||
|
||||
"Software" refers to IQ.Pilot, konn3kt, and all associated source code,
|
||||
documentation, and assets owned by the Copyright Holder.
|
||||
|
||||
"Open Components" refers to portions of the Software explicitly marked as
|
||||
open source.
|
||||
|
||||
"Proprietary Components" refers to all portions of the Software not made
|
||||
available to the public in source form.
|
||||
|
||||
"Copyright Holder" refers to IQ.Lvbs LLC, a part of Project Teal Lvbs Inc.
|
||||
|
||||
GRANT OF LICENSE
|
||||
|
||||
Subject to the terms of this license, you are granted a limited,
|
||||
non-exclusive, revocable license to:
|
||||
|
||||
1. View, study, and learn from the Open Components
|
||||
2. Modify the Open Components for personal, internal, or open-source public use
|
||||
3. Run the Software for personal, non-commercial purposes
|
||||
|
||||
RESTRICTIONS
|
||||
|
||||
You may NOT:
|
||||
|
||||
1. Claim ownership of any part of the Software, excluding your own
|
||||
modifications that do not incorporate Proprietary Components.
|
||||
|
||||
2. Reverse engineer, decompile, disassemble, or in any way attempt to
|
||||
circumvent the obfuscation of the Proprietary Components.
|
||||
|
||||
3. Use the Software or any derivative for commercial purposes without
|
||||
explicit written permission from the Copyright Holder.
|
||||
|
||||
4. Remove or alter any copyright notices or this license.
|
||||
|
||||
5. Sublicense, sell, or transfer rights to the Software.
|
||||
|
||||
6. Use the Software and/or its source code to compete with or create a
|
||||
substantially similar product.
|
||||
|
||||
7. Use the Software in closed source software not licensed by IQ.Lvbs LLC.
|
||||
|
||||
CONSEQUENCES OF VIOLATION
|
||||
|
||||
In the event any Restriction is violated, any product created using inspiration from, or source code from, IQ.Pilot or Konn3kt shall be subject to a licensing fee determined solely by the Copyright Holder. Additionally, the violating party hereby grants IQ.Lvbs LLC an exclusive, irrevocable, worldwide, royalty-free license to use any and all assets from the infringing product on IQ.Lvbs webpages, advertising materials, and in any other manner IQ.Lvbs sees fit.
|
||||
|
||||
OWNERSHIP
|
||||
|
||||
All rights, title, and interest in the Software remain exclusively with the
|
||||
Copyright Holder. Any modifications, improvements, or derivative works you
|
||||
create based on the Software are owned by the Copyright Holder. By
|
||||
contributing modifications, you irrevocably assign all rights to the
|
||||
Copyright Holder.
|
||||
|
||||
PROPRIETARY COMPONENTS
|
||||
|
||||
The Proprietary Components are provided in binary or obfuscated form only.
|
||||
Reverse engineering, decompilation, or disassembly of Proprietary Components
|
||||
is strictly prohibited. Violation of this provision entitles IQ.Lvbs LLC to
|
||||
pursue all available legal remedies to protect its intellectual property and
|
||||
trade secrets.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT
|
||||
HOLDER DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
||||
LIMITED TO MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
|
||||
NON-INFRINGEMENT.
|
||||
|
||||
LIMITATION OF LIABILITY
|
||||
|
||||
IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES, OR
|
||||
OTHER LIABILITY ARISING FROM THE USE OF THE SOFTWARE. THE USER ACCEPTS FULL
|
||||
RESPONSIBILITY FOR ANY AND ALL LIABILITIES WHEN USING IQ.LVBS SOFTWARE.
|
||||
|
||||
TERMINATION
|
||||
|
||||
This license terminates automatically if you violate any of its terms. Upon
|
||||
termination, you must destroy all copies of the Software in your possession.
|
||||
|
||||
The Copyright Holder reserves the right to revoke this license at any time
|
||||
for any reason.
|
||||
|
||||
GOVERNING LAW
|
||||
|
||||
This license shall be governed by the laws of the State of Illinois, United
|
||||
States of America. Any disputes arising under this license shall be subject
|
||||
to the exclusive jurisdiction of the courts located in The State of Illinois, United States.
|
||||
|
||||
---
|
||||
|
||||
For commercial licensing inquiries, contact: support@iqlvbs.com
|
||||
98
LICENSE.md
Normal file
98
LICENSE.md
Normal file
@@ -0,0 +1,98 @@
|
||||
IQ.Lvbs License v0.1a
|
||||
|
||||
Copyright (c) 2026 IQ.Lvbs LLC, a part of Project Teal Lvbs Inc. All Rights Reserved.
|
||||
|
||||
DEFINITIONS
|
||||
|
||||
"Software" refers to IQ.Pilot, konn3kt, and all associated source code,
|
||||
documentation, and assets owned by the Copyright Holder.
|
||||
|
||||
"Open Components" refers to portions of the Software explicitly marked as
|
||||
open source.
|
||||
|
||||
"Proprietary Components" refers to all portions of the Software not made
|
||||
available to the public in source form.
|
||||
|
||||
"Copyright Holder" refers to IQ.Lvbs LLC, a part of Project Teal Lvbs Inc.
|
||||
|
||||
GRANT OF LICENSE
|
||||
|
||||
Subject to the terms of this license, you are granted a limited,
|
||||
non-exclusive, revocable license to:
|
||||
|
||||
1. View, study, and learn from the Open Components
|
||||
2. Modify the Open Components for personal, internal, or open-source public use
|
||||
3. Run the Software for personal, non-commercial purposes
|
||||
|
||||
RESTRICTIONS
|
||||
|
||||
You may NOT:
|
||||
|
||||
1. Claim ownership of any part of the Software, excluding your own
|
||||
modifications that do not incorporate Proprietary Components.
|
||||
|
||||
2. Reverse engineer, decompile, disassemble, or in any way attempt to
|
||||
circumvent the obfuscation of the Proprietary Components.
|
||||
|
||||
3. Use the Software or any derivative for commercial purposes without
|
||||
explicit written permission from the Copyright Holder.
|
||||
|
||||
4. Remove or alter any copyright notices or this license.
|
||||
|
||||
5. Sublicense, sell, or transfer rights to the Software.
|
||||
|
||||
6. Use the Software and/or its source code to compete with or create a
|
||||
substantially similar product.
|
||||
|
||||
7. Use the Software in closed source software not licensed by IQ.Lvbs LLC.
|
||||
|
||||
CONSEQUENCES OF VIOLATION
|
||||
|
||||
In the event any Restriction is violated, any product created using inspiration from, or source code from, IQ.Pilot or Konn3kt shall be subject to a licensing fee determined solely by the Copyright Holder. Additionally, the violating party hereby grants IQ.Lvbs LLC an exclusive, irrevocable, worldwide, royalty-free license to use any and all assets from the infringing product on IQ.Lvbs webpages, advertising materials, and in any other manner IQ.Lvbs sees fit.
|
||||
|
||||
OWNERSHIP
|
||||
|
||||
All rights, title, and interest in the Software remain exclusively with the
|
||||
Copyright Holder. Any modifications, improvements, or derivative works you
|
||||
create based on the Software are owned by the Copyright Holder. By
|
||||
contributing modifications, you irrevocably assign all rights to the
|
||||
Copyright Holder.
|
||||
|
||||
PROPRIETARY COMPONENTS
|
||||
|
||||
The Proprietary Components are provided in binary or obfuscated form only.
|
||||
Reverse engineering, decompilation, or disassembly of Proprietary Components
|
||||
is strictly prohibited. Violation of this provision entitles IQ.Lvbs LLC to
|
||||
pursue all available legal remedies to protect its intellectual property and
|
||||
trade secrets.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT
|
||||
HOLDER DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
||||
LIMITED TO MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
|
||||
NON-INFRINGEMENT.
|
||||
|
||||
LIMITATION OF LIABILITY
|
||||
|
||||
IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES, OR
|
||||
OTHER LIABILITY ARISING FROM THE USE OF THE SOFTWARE. THE USER ACCEPTS FULL
|
||||
RESPONSIBILITY FOR ANY AND ALL LIABILITIES WHEN USING IQ.LVBS SOFTWARE.
|
||||
|
||||
TERMINATION
|
||||
|
||||
This license terminates automatically if you violate any of its terms. Upon
|
||||
termination, you must destroy all copies of the Software in your possession.
|
||||
|
||||
The Copyright Holder reserves the right to revoke this license at any time
|
||||
for any reason.
|
||||
|
||||
GOVERNING LAW
|
||||
|
||||
This license shall be governed by the laws of the State of Illinois, United
|
||||
States of America. Any disputes arising under this license shall be subject
|
||||
to the exclusive jurisdiction of the courts located in Henry County, Illinois.
|
||||
|
||||
---
|
||||
|
||||
For commercial licensing inquiries, contact: support@iqlvbs.com
|
||||
56
README.md
Normal file
56
README.md
Normal file
@@ -0,0 +1,56 @@
|
||||
# IQ.Pilot:
|
||||
|
||||
|
||||
|
||||
## Join Our Public Beta at: https://discord.iqlvbs.com
|
||||
|
||||
## Running IQ.Pilot
|
||||
* A modern comma device to run this software (3, 3x, 4)
|
||||
* One of [the supported cars](https://gitlvb.teallvbs.xyz/IQ.Lvbs/IQ.Pilot/src/branch/release/opendbc_repo/docs/CARS.md).
|
||||
* A [car harness](https://comma.ai/shop/products/car-harness) to connect to your car
|
||||
#### Side Note: Volkswagen Group, and Tesla vehicles are the ONLY fully supported vehicles for use with IQ.Pilot, other manufacturers are supported but not a top priority while we are in beta.
|
||||
## Installation
|
||||
### Preparation:
|
||||
Before Installing IQ.Pilot, you will need to ensure that SSH access is setup for your comma device.
|
||||
|
||||
#### To do so, follow the instructions below, this should work on all forks:
|
||||
* In your comma device's settings, go into "Developer Settings"
|
||||
* Enable the "Enable SSH" toggle if it is not already on.
|
||||
* Next to "SSH Keys", click on "Add" and then enter your GitHub username.
|
||||
* Run the command below (replace your_email@example.com with your GitHub account email, then paste the output [here](https://github.com/settings/keys) after clicking on "New SSH Key", then reboot your comma device.
|
||||
#### SSH Key Command: `ssh-keygen -t ed25519 -C "your_email@example.com" -f ~/.ssh/id_ed25519 -N "" && cat ~/.ssh/id_ed25519.pub`
|
||||
#### Side Note: If you have already setup SSH and can SSH into your comma device, skip this section.
|
||||
|
||||
### Installing:
|
||||
#### Installing Via SSH:
|
||||
#### Once you are connected to your device via SSH, you can paste the following command below to install IQ.Pilot:
|
||||
`cd .. && git clone https://gitlvb.teallvbs.xyz/IQ.Lvbs/IQ.Pilot.git && mv IQ.Pilot openpilot && cd openpilot && git submodule update --init && sudo reboot`
|
||||
#### If you'd like to backup your previous installation as well, paste the following command below to install IQ.Pilot:
|
||||
`cd .. && mv openpilot openpilot_backup_X && git clone https://gitlvb.teallvbs.xyz/IQ.Lvbs/IQ.Pilot.git && mv IQ.Pilot openpilot && cd openpilot && git submodule update --init && sudo reboot`
|
||||
#### Alternatively, you can use your existing fork's built in tools to switch your branch as well:
|
||||
`git remote add iqpilot https://gitlvb.teallvbs.xyz/IQ.Lvbs/IQ.Pilot.git && op switch iqpilot release`
|
||||
#### Installing Via Installer URL:
|
||||
#### Enter the following into your comma device custom URL box to install IQ.Pilot:
|
||||
`IQ-Lvbs/release`
|
||||
|
||||
## Releases:
|
||||
#### IQ.Pilot's Beta Is Currently At Release 1.0b
|
||||
| Version | Installation URL |
|
||||
|:---------:|:-----------------:|
|
||||
| `release` | `IQ-Lvbs/release` |
|
||||
|
||||
## 📊 User Data
|
||||
### IQ.Pilot uploads your data to Konn3kt, by IQ.Lvbs.
|
||||
#### Konn3kt is a secure, encrypted, feature rich management experience for your IQ.Pilot device. Konn3kt has dual end-to-end encryption, your data is encrypted in transit, and at rest, you, and only you have access to your device, and your data.
|
||||
|
||||
#### IQ.Pilot allows users to disable uploading entirely if they wish.
|
||||
#### Konn3kt doesn't collect driver camera, microphone data, and parses your logs with dual end-to-end encryption, ensuring that your data is accessible to you, and only you, not even accessible to IQ.Lvbs.
|
||||
|
||||
## Terms of Service / Privacy Policy / Licensing
|
||||
#### IQ.Pilot is subject to the License found in this repository, [Terms of Service](https://konn3kt.com/tos), and, [Privacy Policy](https://konn3kt.com/privacy).
|
||||
|
||||
|
||||
## Support IQ.Pilot?
|
||||
Sorry, I have better things to do than ask my users for donations. - Teal
|
||||
|
||||
<span>-</span> IQ.Lvbs, by Project Teal Lvbs
|
||||
3
RELEASES.md
Normal file
3
RELEASES.md
Normal file
@@ -0,0 +1,3 @@
|
||||
IQ.Pilot 1.0b
|
||||
========================
|
||||
* Major Refactor!
|
||||
224
SConstruct
Normal file
224
SConstruct
Normal file
@@ -0,0 +1,224 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import sysconfig
|
||||
import platform
|
||||
import shlex
|
||||
import numpy as np
|
||||
|
||||
import SCons.Errors
|
||||
|
||||
SCons.Warnings.warningAsException(True)
|
||||
|
||||
Decider('MD5-timestamp')
|
||||
|
||||
SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
|
||||
|
||||
AddOption('--asan', action='store_true', help='turn on ASAN')
|
||||
AddOption('--ubsan', action='store_true', help='turn on UBSan')
|
||||
AddOption('--mutation', action='store_true', help='generate mutation-ready code')
|
||||
AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
|
||||
AddOption('--minimal',
|
||||
action='store_false',
|
||||
dest='extras',
|
||||
default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
|
||||
help='the minimum build to run openpilot. no tests, tools, etc.')
|
||||
|
||||
# Detect platform
|
||||
arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
|
||||
if platform.system() == "Darwin":
|
||||
arch = "Darwin"
|
||||
brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
|
||||
elif arch == "aarch64" and os.path.isfile('/TICI'):
|
||||
arch = "larch64"
|
||||
assert arch in [
|
||||
"larch64", # linux tici arm64
|
||||
"aarch64", # linux pc arm64
|
||||
"x86_64", # linux pc x64
|
||||
"Darwin", # macOS arm64 (x86 not supported)
|
||||
]
|
||||
|
||||
env = Environment(
|
||||
ENV={
|
||||
"PATH": os.environ['PATH'],
|
||||
"PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
|
||||
"ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
|
||||
"ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
|
||||
"TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
|
||||
},
|
||||
CC='clang',
|
||||
CXX='clang++',
|
||||
CCFLAGS=[
|
||||
"-g",
|
||||
"-fPIC",
|
||||
"-O2",
|
||||
"-Wunused",
|
||||
"-Werror",
|
||||
"-Wshadow",
|
||||
"-Wno-unknown-warning-option",
|
||||
"-Wno-inconsistent-missing-override",
|
||||
"-Wno-c99-designator",
|
||||
"-Wno-reorder-init-list",
|
||||
"-Wno-vla-cxx-extension",
|
||||
],
|
||||
CFLAGS=["-std=gnu11"],
|
||||
CXXFLAGS=["-std=c++1z"],
|
||||
CPPPATH=[
|
||||
"#",
|
||||
"#msgq",
|
||||
"#third_party",
|
||||
"#third_party/json11",
|
||||
"#third_party/linux/include",
|
||||
"#third_party/acados/include",
|
||||
"#third_party/acados/include/blasfeo/include",
|
||||
"#third_party/acados/include/hpipm/include",
|
||||
"#third_party/catch2/include",
|
||||
"#third_party/libyuv/include",
|
||||
],
|
||||
LIBPATH=[
|
||||
"#common",
|
||||
"#msgq_repo",
|
||||
"#third_party",
|
||||
"#selfdrive/pandad",
|
||||
"#rednose/helpers",
|
||||
f"#third_party/libyuv/{arch}/lib",
|
||||
f"#third_party/acados/{arch}/lib",
|
||||
],
|
||||
RPATH=[],
|
||||
CYTHONCFILESUFFIX=".cpp",
|
||||
COMPILATIONDB_USE_ABSPATH=True,
|
||||
REDNOSE_ROOT="#",
|
||||
tools=["default", "cython", "compilation_db", "rednose_filter"],
|
||||
toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
|
||||
)
|
||||
|
||||
# Arch-specific flags and paths
|
||||
if arch == "larch64":
|
||||
env.Append(CPPPATH=["#third_party/opencl/include"])
|
||||
env.Append(LIBPATH=[
|
||||
"/usr/local/lib",
|
||||
"/system/vendor/lib64",
|
||||
"/usr/lib/aarch64-linux-gnu",
|
||||
])
|
||||
arch_flags = ["-D__TICI__", "-mcpu=cortex-a57", "-DQCOM2"]
|
||||
env.Append(CCFLAGS=arch_flags)
|
||||
env.Append(CXXFLAGS=arch_flags)
|
||||
elif arch == "Darwin":
|
||||
env.Append(LIBPATH=[
|
||||
f"{brew_prefix}/lib",
|
||||
f"{brew_prefix}/opt/openssl@3.0/lib",
|
||||
f"{brew_prefix}/opt/llvm/lib/c++",
|
||||
"/System/Library/Frameworks/OpenGL.framework/Libraries",
|
||||
])
|
||||
env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
|
||||
env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
|
||||
env.Append(CPPPATH=[
|
||||
f"{brew_prefix}/include",
|
||||
f"{brew_prefix}/opt/openssl@3.0/include",
|
||||
])
|
||||
else:
|
||||
env.Append(LIBPATH=[
|
||||
"/usr/lib",
|
||||
"/usr/local/lib",
|
||||
])
|
||||
|
||||
# Sanitizers and extra CCFLAGS from CLI
|
||||
if GetOption('asan'):
|
||||
env.Append(CCFLAGS=["-fsanitize=address", "-fno-omit-frame-pointer"])
|
||||
env.Append(LINKFLAGS=["-fsanitize=address"])
|
||||
elif GetOption('ubsan'):
|
||||
env.Append(CCFLAGS=["-fsanitize=undefined"])
|
||||
env.Append(LINKFLAGS=["-fsanitize=undefined"])
|
||||
|
||||
_extra_cc = shlex.split(GetOption('ccflags') or '')
|
||||
if _extra_cc:
|
||||
env.Append(CCFLAGS=_extra_cc)
|
||||
|
||||
# no --as-needed on mac linker
|
||||
if arch != "Darwin":
|
||||
env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
|
||||
|
||||
# progress output
|
||||
node_interval = 5
|
||||
node_count = 0
|
||||
def progress_function(node):
|
||||
global node_count
|
||||
node_count += node_interval
|
||||
sys.stderr.write("progress: %d\n" % node_count)
|
||||
if os.environ.get('SCONS_PROGRESS'):
|
||||
Progress(progress_function, interval=node_interval)
|
||||
|
||||
# ********** Cython build environment **********
|
||||
py_include = sysconfig.get_paths()['include']
|
||||
envCython = env.Clone()
|
||||
envCython["CPPPATH"] += [py_include, np.get_include()]
|
||||
envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
|
||||
envCython["CCFLAGS"].remove("-Werror")
|
||||
|
||||
envCython["LIBS"] = []
|
||||
if arch == "Darwin":
|
||||
envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"]
|
||||
else:
|
||||
envCython["LINKFLAGS"] = ["-pthread", "-shared"]
|
||||
|
||||
np_version = SCons.Script.Value(np.__version__)
|
||||
Export('envCython', 'np_version')
|
||||
|
||||
Export('env', 'arch')
|
||||
|
||||
# Setup cache dir
|
||||
default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
|
||||
cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
|
||||
CacheDir(cache_dir)
|
||||
Clean(["."], cache_dir)
|
||||
|
||||
# ********** start building stuff **********
|
||||
|
||||
# Build common module
|
||||
SConscript(['common/SConscript'])
|
||||
Import('_common')
|
||||
common = [_common, 'json11', 'zmq']
|
||||
Export('common')
|
||||
|
||||
# Build messaging (cereal + msgq + socketmaster + their dependencies)
|
||||
# Enable swaglog include in submodules
|
||||
env_swaglog = env.Clone()
|
||||
env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
|
||||
SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
|
||||
SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
|
||||
|
||||
SConscript(['cereal/SConscript'])
|
||||
|
||||
Import('socketmaster', 'msgq')
|
||||
messaging = [socketmaster, msgq, 'capnp', 'kj',]
|
||||
Export('messaging')
|
||||
|
||||
|
||||
# Build other submodules
|
||||
SConscript(['panda/SConscript'])
|
||||
|
||||
# Build rednose library
|
||||
SConscript(['rednose/SConscript'])
|
||||
|
||||
# Build system services
|
||||
SConscript([
|
||||
'system/loggerd/SConscript',
|
||||
])
|
||||
|
||||
if arch == "larch64":
|
||||
SConscript(['system/camerad/SConscript'])
|
||||
|
||||
# Build openpilot
|
||||
SConscript(['third_party/SConscript'])
|
||||
|
||||
SConscript(['selfdrive/SConscript'])
|
||||
|
||||
SConscript(['iqpilot/SConscript'])
|
||||
|
||||
if Dir('#tools/cabana/').exists() and GetOption('extras'):
|
||||
SConscript(['tools/replay/SConscript'])
|
||||
if arch != "larch64":
|
||||
SConscript(['tools/cabana/SConscript'])
|
||||
|
||||
|
||||
env.CompilationDatabase('compile_commands.json')
|
||||
52
SECURITY.md
Normal file
52
SECURITY.md
Normal file
@@ -0,0 +1,52 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
| Component | Version | Supported |
|
||||
| --------- | ------- | --------- |
|
||||
| IQ.Pilot | latest | TRUE |
|
||||
| konn3kt | latest | TRUE |
|
||||
|
||||
## Scope
|
||||
|
||||
This security policy covers the following components, but applies to all IQ.Lvbs software:
|
||||
|
||||
- **IQ.Pilot**
|
||||
- **konn3kt**
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you discover a security vulnerability in IQ.Pilot or konn3kt, please report it responsibly, abuse is against the terms of service for IQ.Lvbs software.
|
||||
|
||||
**Email:** security@iqlvbs.com
|
||||
|
||||
Please include:
|
||||
|
||||
- A description of the vulnerability and its impact, and potential impact
|
||||
- Steps to reproduce the issue
|
||||
- Affected component(s)
|
||||
- Any relevant logs, screenshots, or proof of concept code that can assist with reproduction.
|
||||
|
||||
## Response Timeline
|
||||
|
||||
- **Acknowledgment:** Within 168 hours of receipt
|
||||
- **Initial Assessment:** Within 1-14 days
|
||||
- **Resolution Target:** Depends on severity; critical issues are prioritized immediately
|
||||
|
||||
## Responsible Disclosure
|
||||
|
||||
We ask that you:
|
||||
|
||||
- Allow reasonable time for us to investigate and address the issue.
|
||||
- Avoid accessing or modifying other users' data, this is against IQ.Lvbs policy, and illegal in many jurisdictions.
|
||||
- Do not exploit vulnerabilities beyond what is necessary to demonstrate the issue.
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- Issues in third-party dependencies unless the vulnerability is triggered by our specific usage
|
||||
|
||||
## Recognition
|
||||
|
||||
We appreciate and recognize security researchers who help keep IQ.Lvbs software safe. With your permission, we will credit you in our release notes when a reported vulnerability is fixed.
|
||||
|
||||
- IQ.Lvbs, a part of Project Teal Lvbs Inc ©️
|
||||
22
artifacts/iqpilot_hephaestusd_private/manifest.json
Normal file
22
artifacts/iqpilot_hephaestusd_private/manifest.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"python/iqpilot_private/__init__.py": {
|
||||
"mode": 420,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"size": 0
|
||||
},
|
||||
"python/iqpilot_private/konn3kt/__init__.py": {
|
||||
"mode": 420,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"size": 0
|
||||
},
|
||||
"python/iqpilot_private/konn3kt/hephaestus/__init__.py": {
|
||||
"mode": 420,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"size": 0
|
||||
},
|
||||
"python/iqpilot_private/konn3kt/hephaestus/hephaestusd.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "7b051542fd1dced2f4bc5752b183283dbe0ef2b8b918a2a58ad8380b8d0a0bc7",
|
||||
"size": 1395688
|
||||
}
|
||||
}
|
||||
Binary file not shown.
37
artifacts/iqpilot_model_selector_private/manifest.json
Normal file
37
artifacts/iqpilot_model_selector_private/manifest.json
Normal file
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"python/iqpilot_private/__init__.py": {
|
||||
"mode": 420,
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"size": 0
|
||||
},
|
||||
"python/iqpilot_private/models/__init__.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "7888179fd64163ddcfa367653b8163a189ce8e40f5c2946d00b9b7195a6588b6",
|
||||
"size": 67920
|
||||
},
|
||||
"python/iqpilot_private/models/__init__.py": {
|
||||
"mode": 420,
|
||||
"sha256": "5128931b54d4f1ff131185e7711d56a788b5e2cb71e3c8154a2f79fef184b452",
|
||||
"size": 118
|
||||
},
|
||||
"python/iqpilot_private/models/fetcher.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "2565fc49f7b38cbef30d93320703903aca318271f1d416b7f1c101bbe5692be5",
|
||||
"size": 136272
|
||||
},
|
||||
"python/iqpilot_private/models/git_auth.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "3b54389098393aaf6d6eafc68467f3f0bf1935d7469ce6f28ed1d11f5fbde6bc",
|
||||
"size": 68168
|
||||
},
|
||||
"python/iqpilot_private/models/helpers.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "3bab3b2480d36b9d76de3811d4fad77b371b8403578548640038e925891d7678",
|
||||
"size": 138744
|
||||
},
|
||||
"python/iqpilot_private/models/manager.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "1ef51a05e18e0d08d817cdc16733568b8b8743743a7e47c19ff98dfc2dab44ee",
|
||||
"size": 205984
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
"""
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
47
artifacts/iqpilot_navd_private/manifest.json
Normal file
47
artifacts/iqpilot_navd_private/manifest.json
Normal file
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"python/iqpilot_private/__init__.py": {
|
||||
"mode": 420,
|
||||
"sha256": "5128931b54d4f1ff131185e7711d56a788b5e2cb71e3c8154a2f79fef184b452",
|
||||
"size": 118
|
||||
},
|
||||
"python/iqpilot_private/navd/__init__.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "ca2f71b555f915a666a3f72d2219cdcf8ce050d0375e111968cf4d8642dea71f",
|
||||
"size": 105616
|
||||
},
|
||||
"python/iqpilot_private/navd/__init__.py": {
|
||||
"mode": 420,
|
||||
"sha256": "5128931b54d4f1ff131185e7711d56a788b5e2cb71e3c8154a2f79fef184b452",
|
||||
"size": 118
|
||||
},
|
||||
"python/iqpilot_private/navd/event_builder.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "84a7e30fee9706b2c968931437c1de97fd1c33e7ee3504c231fe36f849672455",
|
||||
"size": 545376
|
||||
},
|
||||
"python/iqpilot_private/navd/gps_lane_tracker.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "3ad02893601d3af784d43489998375f8b3d2b7997f5ee94fad793a6ee9f143f9",
|
||||
"size": 764928
|
||||
},
|
||||
"python/iqpilot_private/navd/helpers.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "f3e0a1a4f9fd960d8a3a6cabd425f5ecb40ea3a97b9fef7ecae9f0bd66dca968",
|
||||
"size": 970856
|
||||
},
|
||||
"python/iqpilot_private/navd/navd.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "49153e1f81cfdf39fefd4ded1b48e60cecd56be222d4a72b35ebdb0271b90f6b",
|
||||
"size": 1644696
|
||||
},
|
||||
"python/iqpilot_private/navd/route_manager.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "52e83c32284c658d9206eae4aa2363fa8ab16425987b85d12a726c05b63d78c0",
|
||||
"size": 2642968
|
||||
},
|
||||
"python/iqpilot_private/navd/web_server.cpython-312-aarch64-linux-gnu.so": {
|
||||
"mode": 493,
|
||||
"sha256": "251bfaa77faceb85efa132023704bccee84367e5d8a8d958909d646fa7975412",
|
||||
"size": 1446232
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
"""
|
||||
Binary file not shown.
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
|
||||
"""
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
133
artifacts/runtime/ensure_private_installed.sh
Normal file
133
artifacts/runtime/ensure_private_installed.sh
Normal file
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." >/dev/null && pwd )"
|
||||
INSTALL_ROOT="${ROOT_DIR}/.iqpilot"
|
||||
VERIFY_SCRIPT="${ROOT_DIR}/artifacts/runtime/verify_proprietary_bundle.py"
|
||||
|
||||
manifest_hash() {
|
||||
local manifest_path="$1"
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "${manifest_path}" | awk '{print $1}'
|
||||
else
|
||||
shasum -a 256 "${manifest_path}" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
NAVD_BUNDLE="${ROOT_DIR}/artifacts/iqpilot_navd_private"
|
||||
HEPHA_BUNDLE="${ROOT_DIR}/artifacts/iqpilot_hephaestusd_private"
|
||||
MODEL_SELECTOR_BUNDLE="${ROOT_DIR}/artifacts/iqpilot_model_selector_private"
|
||||
NAVD_STATE_FILE="${INSTALL_ROOT}/.installed_navd_manifest.sha256"
|
||||
HEPHA_STATE_FILE="${INSTALL_ROOT}/.installed_hephaestusd_manifest.sha256"
|
||||
MODEL_SELECTOR_STATE_FILE="${INSTALL_ROOT}/.installed_model_selector_manifest.sha256"
|
||||
NAVD_SENTINEL_BASE="${INSTALL_ROOT}/python/iqpilot_private/navd/navd"
|
||||
HEPHA_SENTINEL_BASE="${INSTALL_ROOT}/python/iqpilot_private/konn3kt/hephaestus/hephaestusd"
|
||||
MODEL_SELECTOR_SENTINEL_BASE="${INSTALL_ROOT}/python/iqpilot_private/models/manager"
|
||||
|
||||
NAVD_HASH=""
|
||||
HEPHA_HASH=""
|
||||
MODEL_SELECTOR_HASH=""
|
||||
NEED_INSTALL=0
|
||||
HAVE_BUNDLE=0
|
||||
|
||||
module_present() {
|
||||
local base_path="$1"
|
||||
if [ -f "${base_path}.pyc" ]; then
|
||||
return 0
|
||||
fi
|
||||
if compgen -G "${base_path}".*.so >/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
if [ -f "${NAVD_BUNDLE}/manifest.json" ]; then
|
||||
HAVE_BUNDLE=1
|
||||
NAVD_HASH="$(manifest_hash "${NAVD_BUNDLE}/manifest.json")"
|
||||
NAVD_DST_HASH=""
|
||||
if [ -f "${NAVD_STATE_FILE}" ]; then
|
||||
NAVD_DST_HASH="$(cat "${NAVD_STATE_FILE}" 2>/dev/null || true)"
|
||||
fi
|
||||
if ! module_present "${NAVD_SENTINEL_BASE}" || [ "${NAVD_HASH}" != "${NAVD_DST_HASH}" ]; then
|
||||
NEED_INSTALL=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -f "${HEPHA_BUNDLE}/manifest.json" ]; then
|
||||
HAVE_BUNDLE=1
|
||||
HEPHA_HASH="$(manifest_hash "${HEPHA_BUNDLE}/manifest.json")"
|
||||
HEPHA_DST_HASH=""
|
||||
if [ -f "${HEPHA_STATE_FILE}" ]; then
|
||||
HEPHA_DST_HASH="$(cat "${HEPHA_STATE_FILE}" 2>/dev/null || true)"
|
||||
fi
|
||||
if ! module_present "${HEPHA_SENTINEL_BASE}" || [ "${HEPHA_HASH}" != "${HEPHA_DST_HASH}" ]; then
|
||||
NEED_INSTALL=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -f "${MODEL_SELECTOR_BUNDLE}/manifest.json" ]; then
|
||||
HAVE_BUNDLE=1
|
||||
MODEL_SELECTOR_HASH="$(manifest_hash "${MODEL_SELECTOR_BUNDLE}/manifest.json")"
|
||||
MODEL_SELECTOR_DST_HASH=""
|
||||
if [ -f "${MODEL_SELECTOR_STATE_FILE}" ]; then
|
||||
MODEL_SELECTOR_DST_HASH="$(cat "${MODEL_SELECTOR_STATE_FILE}" 2>/dev/null || true)"
|
||||
fi
|
||||
if ! module_present "${MODEL_SELECTOR_SENTINEL_BASE}" || [ "${MODEL_SELECTOR_HASH}" != "${MODEL_SELECTOR_DST_HASH}" ]; then
|
||||
NEED_INSTALL=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${HAVE_BUNDLE}" -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${NEED_INSTALL}" -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TMP_ROOT="${INSTALL_ROOT}.tmp.$$"
|
||||
echo "Installing bundled private artifacts..."
|
||||
rm -rf "${TMP_ROOT}"
|
||||
mkdir -p "${TMP_ROOT}"
|
||||
|
||||
if [ -d "${INSTALL_ROOT}" ]; then
|
||||
cp -a "${INSTALL_ROOT}"/. "${TMP_ROOT}/"
|
||||
fi
|
||||
|
||||
if [ -f "${NAVD_BUNDLE}/manifest.json" ]; then
|
||||
python3 "${VERIFY_SCRIPT}" "${NAVD_BUNDLE}"
|
||||
cp -a "${NAVD_BUNDLE}"/. "${TMP_ROOT}/"
|
||||
fi
|
||||
|
||||
if [ -f "${HEPHA_BUNDLE}/manifest.json" ]; then
|
||||
python3 "${VERIFY_SCRIPT}" "${HEPHA_BUNDLE}"
|
||||
cp -a "${HEPHA_BUNDLE}"/. "${TMP_ROOT}/"
|
||||
fi
|
||||
|
||||
if [ -f "${MODEL_SELECTOR_BUNDLE}/manifest.json" ]; then
|
||||
python3 "${VERIFY_SCRIPT}" "${MODEL_SELECTOR_BUNDLE}"
|
||||
cp -a "${MODEL_SELECTOR_BUNDLE}"/. "${TMP_ROOT}/"
|
||||
fi
|
||||
|
||||
if [ -d "${INSTALL_ROOT}" ]; then
|
||||
rm -rf "${INSTALL_ROOT}.bak"
|
||||
mv "${INSTALL_ROOT}" "${INSTALL_ROOT}.bak"
|
||||
fi
|
||||
|
||||
mv "${TMP_ROOT}" "${INSTALL_ROOT}"
|
||||
|
||||
if [ -d "${INSTALL_ROOT}.bak" ]; then
|
||||
rm -rf "${INSTALL_ROOT}.bak"
|
||||
fi
|
||||
|
||||
if [ -n "${NAVD_HASH}" ]; then
|
||||
printf '%s\n' "${NAVD_HASH}" > "${NAVD_STATE_FILE}"
|
||||
fi
|
||||
if [ -n "${HEPHA_HASH}" ]; then
|
||||
printf '%s\n' "${HEPHA_HASH}" > "${HEPHA_STATE_FILE}"
|
||||
fi
|
||||
if [ -n "${MODEL_SELECTOR_HASH}" ]; then
|
||||
printf '%s\n' "${MODEL_SELECTOR_HASH}" > "${MODEL_SELECTOR_STATE_FILE}"
|
||||
fi
|
||||
|
||||
echo "Private artifacts installed."
|
||||
61
artifacts/runtime/verify_proprietary_bundle.py
Normal file
61
artifacts/runtime/verify_proprietary_bundle.py
Normal file
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Verify IQ Pilot proprietary bundle manifest")
|
||||
p.add_argument("bundle_root", help="Bundle directory containing manifest.json")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
while True:
|
||||
b = f.read(1024 * 1024)
|
||||
if not b:
|
||||
break
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
root = Path(args.bundle_root).resolve()
|
||||
manifest_path = root / "manifest.json"
|
||||
if not manifest_path.exists():
|
||||
raise SystemExit(f"manifest not found: {manifest_path}")
|
||||
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
failures: list[str] = []
|
||||
|
||||
for rel, meta in manifest.items():
|
||||
f = root / rel
|
||||
if not f.exists():
|
||||
failures.append(f"missing: {rel}")
|
||||
continue
|
||||
|
||||
got = sha256(f)
|
||||
exp = str(meta["sha256"])
|
||||
if got != exp:
|
||||
failures.append(f"sha256 mismatch: {rel}")
|
||||
|
||||
got_size = f.stat().st_size
|
||||
exp_size = int(meta["size"])
|
||||
if got_size != exp_size:
|
||||
failures.append(f"size mismatch: {rel} ({got_size} != {exp_size})")
|
||||
|
||||
if failures:
|
||||
print("FAILED")
|
||||
for x in failures:
|
||||
print(x)
|
||||
raise SystemExit(1)
|
||||
|
||||
print("OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
95
cereal/README.md
Normal file
95
cereal/README.md
Normal file
@@ -0,0 +1,95 @@
|
||||
# What is cereal?
|
||||
|
||||
cereal is the messaging system for openpilot. It uses [msgq](https://github.com/commaai/msgq) as a pub/sub backend, and [Cap'n proto](https://capnproto.org/capnp-tool.html) for serialization of the structs.
|
||||
|
||||
|
||||
## Messaging Spec
|
||||
|
||||
You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`.
|
||||
|
||||
All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type.
|
||||
|
||||
### Best Practices
|
||||
|
||||
- **All fields must describe quantities in SI units**, unless otherwise specified in the field name.
|
||||
- In the context of the message they are in, field names should be completely unambiguous.
|
||||
- All values should be easy to plot and be human-readable with minimal parsing.
|
||||
|
||||
### Maintaining backwards-compatibility
|
||||
|
||||
When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can
|
||||
be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other
|
||||
things are not. Read more details [here](https://capnproto.org/language.html).
|
||||
|
||||
### Custom forks
|
||||
|
||||
Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
|
||||
spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
|
||||
then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
|
||||
[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
|
||||
fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
|
||||
|
||||
An example of compatible changes:
|
||||
```diff
|
||||
diff --git a/cereal/custom.capnp b/cereal/custom.capnp
|
||||
index 3348e859e..3365c7b98 100644
|
||||
--- a/cereal/custom.capnp
|
||||
+++ b/cereal/custom.capnp
|
||||
@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
|
||||
# DO rename the structs
|
||||
# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
|
||||
|
||||
-struct CustomReserved0 @0x81c2f05a394cf4af {
|
||||
+struct SteeringInfo @0x81c2f05a394cf4af {
|
||||
+ active @0 :Bool;
|
||||
+ steeringAngleDeg @1 :Float32;
|
||||
+ steeringRateDeg @2 :Float32;
|
||||
+ steeringAccelDeg @3 :Float32;
|
||||
}
|
||||
|
||||
struct CustomReserved1 @0xaedffd8f31e7b55d {
|
||||
diff --git a/cereal/log.capnp b/cereal/log.capnp
|
||||
index 1209f3fd9..b189f58b6 100644
|
||||
--- a/cereal/log.capnp
|
||||
+++ b/cereal/log.capnp
|
||||
@@ -2558,14 +2558,14 @@ struct Event {
|
||||
|
||||
# DO change the name of the field
|
||||
# DON'T change anything after the "@"
|
||||
- customReservedRawData0 @124 :Data;
|
||||
+ rawCanData @124 :Data;
|
||||
customReservedRawData1 @125 :Data;
|
||||
customReservedRawData2 @126 :Data;
|
||||
|
||||
# DO change the name of the field and struct
|
||||
# DON'T change the ID (e.g. @107)
|
||||
# DON'T change which struct it points to
|
||||
- customReserved0 @107 :Custom.CustomReserved0;
|
||||
+ steeringInfo @107 :Custom.SteeringInfo;
|
||||
customReserved1 @108 :Custom.CustomReserved1;
|
||||
customReserved2 @109 :Custom.CustomReserved2;
|
||||
customReserved3 @110 :Custom.CustomReserved3;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Example
|
||||
---
|
||||
```python
|
||||
import cereal.messaging as messaging
|
||||
|
||||
# in subscriber
|
||||
sm = messaging.SubMaster(['sensorEvents'])
|
||||
while 1:
|
||||
sm.update()
|
||||
print(sm['sensorEvents'])
|
||||
|
||||
```
|
||||
|
||||
```python
|
||||
# in publisher
|
||||
pm = messaging.PubMaster(['sensorEvents'])
|
||||
dat = messaging.new_message('sensorEvents', size=1)
|
||||
dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
|
||||
pm.send('sensorEvents', dat)
|
||||
```
|
||||
20
cereal/SConscript
Normal file
20
cereal/SConscript
Normal file
@@ -0,0 +1,20 @@
|
||||
Import('env', 'common', 'msgq')
|
||||
|
||||
cereal_dir = Dir('.')
|
||||
gen_dir = Dir('gen')
|
||||
|
||||
# Build cereal
|
||||
schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
|
||||
env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
|
||||
schema_files,
|
||||
f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
|
||||
|
||||
cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
|
||||
|
||||
# Build messaging
|
||||
services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
|
||||
env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
|
||||
|
||||
socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
|
||||
|
||||
Export('cereal', 'socketmaster')
|
||||
11
cereal/__init__.py
Normal file
11
cereal/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
import capnp
|
||||
from importlib.resources import as_file, files
|
||||
|
||||
capnp.remove_import_hook()
|
||||
|
||||
with as_file(files("cereal")) as fspath:
|
||||
CEREAL_PATH = fspath.as_posix()
|
||||
log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
|
||||
car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
|
||||
custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))
|
||||
1
cereal/car.capnp
Symbolic link
1
cereal/car.capnp
Symbolic link
@@ -0,0 +1 @@
|
||||
../opendbc_repo/opendbc/car/car.capnp
|
||||
550
cereal/custom.capnp
Normal file
550
cereal/custom.capnp
Normal file
@@ -0,0 +1,550 @@
|
||||
using Cxx = import "./include/c++.capnp";
|
||||
$Cxx.namespace("cereal");
|
||||
|
||||
@0xb526ba661d550a59;
|
||||
|
||||
# custom.capnp: a home for empty structs reserved for custom forks
|
||||
# These structs are guaranteed to remain reserved and empty in mainline
|
||||
# cereal, so use these if you want custom events in your fork.
|
||||
|
||||
# DO rename the structs
|
||||
# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
|
||||
|
||||
struct AlwaysOnLateral {
|
||||
state @0 :AlwaysOnLateralState;
|
||||
enabled @1 :Bool;
|
||||
active @2 :Bool;
|
||||
available @3 :Bool;
|
||||
|
||||
enum AlwaysOnLateralState {
|
||||
disabled @0;
|
||||
paused @1;
|
||||
enabled @2;
|
||||
softDisabling @3;
|
||||
overriding @4;
|
||||
}
|
||||
}
|
||||
|
||||
# Same struct as Log.RadarState.LeadData
|
||||
struct LeadData {
|
||||
dRel @0 :Float32;
|
||||
yRel @1 :Float32;
|
||||
vRel @2 :Float32;
|
||||
aRel @3 :Float32;
|
||||
vLead @4 :Float32;
|
||||
dPath @6 :Float32;
|
||||
vLat @7 :Float32;
|
||||
vLeadK @8 :Float32;
|
||||
aLeadK @9 :Float32;
|
||||
fcw @10 :Bool;
|
||||
status @11 :Bool;
|
||||
aLeadTau @12 :Float32;
|
||||
modelProb @13 :Float32;
|
||||
radar @14 :Bool;
|
||||
radarTrackId @15 :Int32 = -1;
|
||||
|
||||
aLeadDEPRECATED @5 :Float32;
|
||||
}
|
||||
|
||||
struct IQState @0x81c2f05a394cf4af {
|
||||
aol @0 :AlwaysOnLateral;
|
||||
|
||||
enum AudibleAlert {
|
||||
none @0;
|
||||
|
||||
engage @1;
|
||||
disengage @2;
|
||||
refuse @3;
|
||||
|
||||
warningSoft @4;
|
||||
warningImmediate @5;
|
||||
|
||||
prompt @6;
|
||||
promptRepeat @7;
|
||||
promptDistracted @8;
|
||||
|
||||
# unused, these are reserved for upstream events so we don't collide
|
||||
reserved9 @9;
|
||||
reserved10 @10;
|
||||
reserved11 @11;
|
||||
reserved12 @12;
|
||||
reserved13 @13;
|
||||
reserved14 @14;
|
||||
reserved15 @15;
|
||||
reserved16 @16;
|
||||
reserved17 @17;
|
||||
reserved18 @18;
|
||||
reserved19 @19;
|
||||
reserved20 @20;
|
||||
reserved21 @21;
|
||||
reserved22 @22;
|
||||
reserved23 @23;
|
||||
reserved24 @24;
|
||||
reserved25 @25;
|
||||
reserved26 @26;
|
||||
reserved27 @27;
|
||||
reserved28 @28;
|
||||
reserved29 @29;
|
||||
reserved30 @30;
|
||||
|
||||
promptSingleLow @31;
|
||||
promptSingleHigh @32;
|
||||
}
|
||||
}
|
||||
|
||||
struct IQModelManager @0xaedffd8f31e7b55d {
|
||||
activeBundle @0 :ModelBundle;
|
||||
selectedBundle @1 :ModelBundle;
|
||||
availableBundles @2 :List(ModelBundle);
|
||||
|
||||
struct DownloadUri {
|
||||
uri @0 :Text;
|
||||
sha256 @1 :Text;
|
||||
}
|
||||
|
||||
enum DownloadStatus {
|
||||
notDownloading @0;
|
||||
downloading @1;
|
||||
downloaded @2;
|
||||
cached @3;
|
||||
failed @4;
|
||||
}
|
||||
|
||||
struct DownloadProgress {
|
||||
status @0 :DownloadStatus;
|
||||
progress @1 :Float32;
|
||||
eta @2 :UInt32;
|
||||
}
|
||||
|
||||
struct Artifact {
|
||||
fileName @0 :Text;
|
||||
downloadUri @1 :DownloadUri;
|
||||
downloadProgress @2 :DownloadProgress;
|
||||
}
|
||||
|
||||
struct Model {
|
||||
type @0 :Type;
|
||||
artifact @1 :Artifact; # Main artifact
|
||||
metadata @2 :Artifact; # Metadata artifact
|
||||
|
||||
enum Type {
|
||||
supercombo @0;
|
||||
navigation @1;
|
||||
vision @2;
|
||||
policy @3;
|
||||
offPolicy @4;
|
||||
}
|
||||
}
|
||||
|
||||
enum Runner {
|
||||
snpe @0;
|
||||
tinygrad @1;
|
||||
stock @2;
|
||||
}
|
||||
|
||||
struct Override {
|
||||
key @0 :Text;
|
||||
value @1 :Text;
|
||||
}
|
||||
|
||||
struct ModelBundle {
|
||||
index @0 :UInt32;
|
||||
internalName @1 :Text;
|
||||
displayName @2 :Text;
|
||||
models @3 :List(Model);
|
||||
status @4 :DownloadStatus;
|
||||
generation @5 :UInt32;
|
||||
environment @6 :Text;
|
||||
runner @7 :Runner;
|
||||
is20hz @8 :Bool;
|
||||
ref @9 :Text;
|
||||
minimumSelectorVersion @10 :UInt32;
|
||||
overrides @11 :List(Override);
|
||||
}
|
||||
}
|
||||
|
||||
struct IQPlan @0xf35cc4560bbf6ec2 {
|
||||
iqDynamic @0 :IQDynamicControl;
|
||||
longitudinalPlanSource @1 :LongitudinalPlanSource;
|
||||
smartCruiseControl @2 :SmartCruiseControl;
|
||||
speedLimit @3 :SpeedLimit;
|
||||
vTarget @4 :Float32;
|
||||
aTarget @5 :Float32;
|
||||
events @6 :List(IQOnroadEvent.Event);
|
||||
e2eAlerts @7 :E2eAlerts;
|
||||
|
||||
struct IQDynamicControl {
|
||||
state @0 :IQDynamicControlState;
|
||||
enabled @1 :Bool;
|
||||
active @2 :Bool;
|
||||
|
||||
enum IQDynamicControlState {
|
||||
acc @0;
|
||||
blended @1;
|
||||
}
|
||||
}
|
||||
|
||||
struct SmartCruiseControl {
|
||||
vision @0 :Vision;
|
||||
map @1 :Map;
|
||||
|
||||
struct Vision {
|
||||
state @0 :VisionState;
|
||||
vTarget @1 :Float32;
|
||||
aTarget @2 :Float32;
|
||||
currentLateralAccel @3 :Float32;
|
||||
maxPredictedLateralAccel @4 :Float32;
|
||||
enabled @5 :Bool;
|
||||
active @6 :Bool;
|
||||
}
|
||||
|
||||
struct Map {
|
||||
state @0 :MapState;
|
||||
vTarget @1 :Float32;
|
||||
aTarget @2 :Float32;
|
||||
enabled @3 :Bool;
|
||||
active @4 :Bool;
|
||||
}
|
||||
|
||||
enum VisionState {
|
||||
disabled @0; # System disabled or inactive.
|
||||
enabled @1; # No predicted substantial turn on vision range.
|
||||
entering @2; # A substantial turn is predicted ahead, adapting speed to turn comfort levels.
|
||||
turning @3; # Actively turning. Managing acceleration to provide a roll on turn feeling.
|
||||
leaving @4; # Road ahead straightens. Start to allow positive acceleration.
|
||||
overriding @5; # System overriding with manual control.
|
||||
}
|
||||
|
||||
enum MapState {
|
||||
disabled @0; # System disabled or inactive.
|
||||
enabled @1; # No predicted substantial turn on map range.
|
||||
turning @2; # Actively turning. Managing acceleration to provide a roll on turn feeling.
|
||||
overriding @3; # System overriding with manual control.
|
||||
}
|
||||
}
|
||||
|
||||
struct SpeedLimit {
|
||||
resolver @0 :Resolver;
|
||||
assist @1 :Assist;
|
||||
|
||||
struct Resolver {
|
||||
speedLimit @0 :Float32;
|
||||
distToSpeedLimit @1 :Float32;
|
||||
source @2 :Source;
|
||||
speedLimitOffset @3 :Float32;
|
||||
speedLimitLast @4 :Float32;
|
||||
speedLimitFinal @5 :Float32;
|
||||
speedLimitFinalLast @6 :Float32;
|
||||
speedLimitValid @7 :Bool;
|
||||
speedLimitLastValid @8 :Bool;
|
||||
}
|
||||
|
||||
struct Assist {
|
||||
state @0 :AssistState;
|
||||
enabled @1 :Bool;
|
||||
active @2 :Bool;
|
||||
vTarget @3 :Float32;
|
||||
aTarget @4 :Float32;
|
||||
}
|
||||
|
||||
enum Source {
|
||||
none @0;
|
||||
car @1;
|
||||
map @2;
|
||||
}
|
||||
|
||||
enum AssistState {
|
||||
disabled @0;
|
||||
inactive @1; # No speed limit set or not enabled by parameter.
|
||||
preActive @2;
|
||||
pending @3; # Awaiting new speed limit.
|
||||
adapting @4; # Reducing speed to match new speed limit.
|
||||
active @5; # Cruising at speed limit.
|
||||
}
|
||||
}
|
||||
|
||||
enum LongitudinalPlanSource {
|
||||
cruise @0;
|
||||
sccVision @1;
|
||||
sccMap @2;
|
||||
speedLimitAssist @3;
|
||||
}
|
||||
|
||||
struct E2eAlerts {
|
||||
greenLightAlert @0 :Bool;
|
||||
leadDepartAlert @1 :Bool;
|
||||
}
|
||||
}
|
||||
|
||||
struct IQOnroadEvent @0xda96579883444c35 {
|
||||
events @0 :List(Event);
|
||||
|
||||
struct Event {
|
||||
name @0 :EventName;
|
||||
|
||||
# event types
|
||||
enable @1 :Bool;
|
||||
noEntry @2 :Bool;
|
||||
warning @3 :Bool; # alerts presented only when enabled or soft disabling
|
||||
userDisable @4 :Bool;
|
||||
softDisable @5 :Bool;
|
||||
immediateDisable @6 :Bool;
|
||||
preEnable @7 :Bool;
|
||||
permanent @8 :Bool; # alerts presented regardless of openpilot state
|
||||
overrideLateral @10 :Bool;
|
||||
overrideLongitudinal @9 :Bool;
|
||||
}
|
||||
|
||||
enum EventName {
|
||||
lkasEnable @0;
|
||||
lkasDisable @1;
|
||||
manualSteeringRequired @2;
|
||||
manualLongitudinalRequired @3;
|
||||
silentLkasEnable @4;
|
||||
silentLkasDisable @5;
|
||||
silentBrakeHold @6;
|
||||
silentWrongGear @7;
|
||||
silentReverseGear @8;
|
||||
silentDoorOpen @9;
|
||||
silentSeatbeltNotLatched @10;
|
||||
silentParkBrake @11;
|
||||
controlsMismatchLateral @12;
|
||||
hyundaiRadarTracksConfirmed @13;
|
||||
experimentalModeSwitched @14;
|
||||
wrongCarModeAlertOnly @15;
|
||||
pedalPressedAlertOnly @16;
|
||||
laneTurnLeft @17;
|
||||
laneTurnRight @18;
|
||||
speedLimitPreActive @19;
|
||||
speedLimitActive @20;
|
||||
speedLimitChanged @21;
|
||||
speedLimitPending @22;
|
||||
e2eChime @23;
|
||||
}
|
||||
}
|
||||
|
||||
struct IQCarParams @0x80ae746ee2596b11 {
|
||||
flags @0 :UInt32; # flags for car specific quirks in iqpilot
|
||||
safetyParam @1 : Int16; # flags for iqpilot's custom safety flags
|
||||
pcmCruiseSpeed @3 :Bool;
|
||||
enableGasInterceptor @4 :Bool;
|
||||
|
||||
neuralNetworkLateralControl @2 :NeuralNetworkLateralControl;
|
||||
|
||||
struct NeuralNetworkLateralControl {
|
||||
model @0 :Model;
|
||||
fuzzyFingerprint @1 :Bool;
|
||||
|
||||
struct Model {
|
||||
path @0 :Text;
|
||||
name @1 :Text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct IQCarControl @0xa5cd762cd951a455 {
|
||||
aol @0 :AlwaysOnLateral;
|
||||
params @1 :List(Param);
|
||||
leadOne @2 :LeadData;
|
||||
leadTwo @3 :LeadData;
|
||||
|
||||
struct Param {
|
||||
key @0 :Text;
|
||||
type @2 :ParamType;
|
||||
value @3 :Data;
|
||||
|
||||
valueDEPRECATED @1 :Text; # The data type change may cause issues with backwards compatibility.
|
||||
}
|
||||
|
||||
enum ParamType {
|
||||
string @0;
|
||||
bool @1;
|
||||
int @2;
|
||||
float @3;
|
||||
time @4;
|
||||
json @5;
|
||||
bytes @6;
|
||||
}
|
||||
}
|
||||
|
||||
struct IQBackupManager @0xf98d843bfd7004a3 {
|
||||
backupStatus @0 :Status;
|
||||
restoreStatus @1 :Status;
|
||||
backupProgress @2 :Float32;
|
||||
restoreProgress @3 :Float32;
|
||||
lastError @4 :Text;
|
||||
currentBackup @5 :BackupInfo;
|
||||
backupHistory @6 :List(BackupInfo);
|
||||
|
||||
enum Status {
|
||||
idle @0;
|
||||
inProgress @1;
|
||||
completed @2;
|
||||
failed @3;
|
||||
}
|
||||
|
||||
struct Version {
|
||||
major @0 :UInt16;
|
||||
minor @1 :UInt16;
|
||||
patch @2 :UInt16;
|
||||
build @3 :UInt16;
|
||||
branch @4 :Text;
|
||||
}
|
||||
|
||||
struct MetadataEntry {
|
||||
key @0 :Text;
|
||||
value @1 :Text;
|
||||
tags @2 :List(Text);
|
||||
}
|
||||
|
||||
struct BackupInfo {
|
||||
deviceId @0 :Text;
|
||||
version @1 :UInt32;
|
||||
config @2 :Text;
|
||||
isEncrypted @3 :Bool;
|
||||
createdAt @4 :Text; # ISO timestamp
|
||||
updatedAt @5 :Text; # ISO timestamp
|
||||
iqpilotVersion @6 :Version;
|
||||
backupMetadata @7 :List(MetadataEntry);
|
||||
}
|
||||
}
|
||||
|
||||
struct IQCarState @0xb86e6369214c01c8 {
|
||||
speedLimit @0 :Float32;
|
||||
accelPressed @1 :Bool;
|
||||
decelPressed @2 :Bool;
|
||||
}
|
||||
|
||||
struct IQLiveMapData @0xf416ec09499d9d19 {
|
||||
speedLimitValid @0 :Bool;
|
||||
speedLimit @1 :Float32;
|
||||
speedLimitAheadValid @2 :Bool;
|
||||
speedLimitAhead @3 :Float32;
|
||||
speedLimitAheadDistance @4 :Float32;
|
||||
roadName @5 :Text;
|
||||
}
|
||||
|
||||
struct IQModelDataV2 @0xa1680744031fdb2d {
|
||||
laneTurnDirection @0 :TurnDirection;
|
||||
|
||||
enum TurnDirection {
|
||||
none @0;
|
||||
turnLeft @1;
|
||||
turnRight @2;
|
||||
}
|
||||
}
|
||||
|
||||
enum NavDirection {
|
||||
none @0;
|
||||
left @1;
|
||||
right @2;
|
||||
}
|
||||
|
||||
struct IQNavState @0xcb9fd56c7057593a {
|
||||
# Navigation state and guidance information
|
||||
active @0 :Bool; # Whether navigation is currently active
|
||||
destinationValid @1 :Bool; # Whether we have a valid destination
|
||||
|
||||
# Current position and route info
|
||||
distanceRemaining @2 :Float32; # Total distance remaining to destination (m)
|
||||
timeRemaining @3 :Float32; # Estimated time remaining to destination (s)
|
||||
currentSegmentIndex @4 :UInt32; # Index of current route segment
|
||||
totalSegments @5 :UInt32; # Total number of segments in route
|
||||
|
||||
# Next maneuver information
|
||||
nextManeuverValid @6 :Bool; # Whether next maneuver data is valid
|
||||
nextManeuverDistance @7 :Float32; # Distance to next maneuver (m)
|
||||
nextManeuverType @8 :ManeuverType; # Type of next maneuver
|
||||
nextManeuverDirection @9 :IQModelDataV2.TurnDirection; # Direction for next maneuver
|
||||
nextManeuverDescription @10 :Text; # Human-readable maneuver description
|
||||
nextManeuverAngle @21 :Float32; # Turn angle in degrees (for angle-adaptive enforcement)
|
||||
|
||||
# Turn desire control for lateral planning
|
||||
shouldSendTurnDesire @11 :Bool; # Whether to send turn desires to model
|
||||
turnDesireDirection @12 :IQModelDataV2.TurnDirection; # Direction for turn desire
|
||||
|
||||
# Lane change desire control for highway exits/ramps (>45 mph)
|
||||
shouldSendLaneChangeDesire @22 :Bool; # Whether to send lane change desires for high-speed exits
|
||||
laneChangeDesireDirection @23 :IQModelDataV2.TurnDirection; # Direction for lane change desire
|
||||
|
||||
# Speed guidance for longitudinal planning
|
||||
targetSpeed @13 :Float32; # Target speed for upcoming maneuver (m/s)
|
||||
targetSpeedValid @14 :Bool; # Whether target speed is valid
|
||||
|
||||
# Destination info
|
||||
destinationLatitude @15 :Float64;
|
||||
destinationLongitude @16 :Float64;
|
||||
destinationName @17 :Text;
|
||||
|
||||
# Lane positioning guidance for exits/turns
|
||||
shouldSendLanePositioning @18 :Bool; # Whether to send lane positioning desires (keepLeft/keepRight)
|
||||
lanePositioningDirection @19 :IQModelDataV2.TurnDirection; # Direction for lane positioning
|
||||
|
||||
# Lane tracking debug info (model vs GPS comparison for testing)
|
||||
laneDebugInfo @20 :LaneDebugInfo;
|
||||
|
||||
# Navigation-specific UI event fields (separate from model/desire system)
|
||||
navTurnDesireDirection @24 :NavDirection; # For "Navigation: Turning Left/Right" UI alerts
|
||||
navLaneChangeDesireDirection @25 :NavDirection; # For "Navigation: Initiating Lane Change" UI alerts
|
||||
navLanePositioningDirection @26 :NavDirection; # For future lane positioning UI alerts
|
||||
navSpeedTargetActive @27 :Bool; # For "Navigation: Reducing Speed" UI alert
|
||||
|
||||
# Second next maneuver information (for "Then" section in navigation banner UI)
|
||||
secondNextManeuverValid @28 :Bool; # Whether second next maneuver data is valid
|
||||
secondNextManeuverType @29 :ManeuverType; # Type of second next maneuver
|
||||
secondNextManeuverDirection @30 :NavDirection; # Direction for second next maneuver
|
||||
secondNextManeuverDistance @31 :Float32; # Distance to second next maneuver (m)
|
||||
nextManeuverModifier @32 :Text; # Raw Mapbox modifier for next maneuver (slight_left, sharp_right, etc.)
|
||||
secondNextManeuverModifier @33 :Text; # Raw Mapbox modifier for second next maneuver
|
||||
|
||||
enum ManeuverType {
|
||||
none @0;
|
||||
turn @1; # Regular turn at intersection
|
||||
exit @2; # Highway exit
|
||||
merge @3; # Merge onto highway
|
||||
fork @4; # Road fork
|
||||
continueStraight @5; # Continue straight
|
||||
arrive @6; # Arrive at destination
|
||||
roundabout @7; # Enter/exit roundabout
|
||||
}
|
||||
|
||||
struct LaneDebugInfo {
|
||||
modelLane @0 :Text; # "left", "middle", "right", "unknown"
|
||||
modelConfidence @1 :Float32; # 0.0-1.0
|
||||
gpsLane @2 :Text; # "left", "middle", "right", "unknown"
|
||||
gpsConfidence @3 :Float32; # 0.0-1.0
|
||||
lateralOffset @4 :Float32; # Meters from road centerline (negative=left, positive=right)
|
||||
gpsAccuracy @5 :Float32; # GPS position accuracy (meters)
|
||||
agreement @6 :Bool; # Do model and GPS agree?
|
||||
}
|
||||
}
|
||||
|
||||
struct CustomReserved10 @0xa8e2e4a8c6f4d3b2 {
|
||||
}
|
||||
|
||||
struct CustomReserved11 @0xc2243c65e0340384 {
|
||||
}
|
||||
|
||||
struct CustomReserved12 @0x9ccdc8676701b412 {
|
||||
}
|
||||
|
||||
struct CustomReserved13 @0xcd96dafb67a082d0 {
|
||||
}
|
||||
|
||||
struct CustomReserved14 @0xb057204d7deadf3f {
|
||||
}
|
||||
|
||||
struct CustomReserved15 @0xbd443b539493bc68 {
|
||||
}
|
||||
|
||||
struct CustomReserved16 @0xfc6241ed8877b611 {
|
||||
}
|
||||
|
||||
struct CustomReserved17 @0xa30662f84033036c {
|
||||
}
|
||||
|
||||
struct CustomReserved18 @0xc86a3d38d13eb3ef {
|
||||
}
|
||||
|
||||
struct CustomReserved19 @0xa4f1eb3323f5f582 {
|
||||
}
|
||||
26
cereal/include/c++.capnp
Normal file
26
cereal/include/c++.capnp
Normal file
@@ -0,0 +1,26 @@
|
||||
# Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
|
||||
# Licensed under the MIT License:
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
@0xbdf87d7bb8304e81;
|
||||
$namespace("capnp::annotations");
|
||||
|
||||
annotation namespace(file): Text;
|
||||
annotation name(field, enumerant, struct, enum, interface, method, param, group, union): Text;
|
||||
574
cereal/legacy.capnp
Normal file
574
cereal/legacy.capnp
Normal file
@@ -0,0 +1,574 @@
|
||||
using Cxx = import "./include/c++.capnp";
|
||||
$Cxx.namespace("cereal");
|
||||
|
||||
@0x80ef1ec4889c2a63;
|
||||
|
||||
# legacy.capnp: a home for deprecated structs
|
||||
|
||||
struct LogRotate @0x9811e1f38f62f2d1 {
|
||||
segmentNum @0 :Int32;
|
||||
path @1 :Text;
|
||||
}
|
||||
|
||||
struct LiveUI @0xc08240f996aefced {
|
||||
rearViewCam @0 :Bool;
|
||||
alertText1 @1 :Text;
|
||||
alertText2 @2 :Text;
|
||||
awarenessStatus @3 :Float32;
|
||||
}
|
||||
|
||||
struct UiLayoutState @0x88dcce08ad29dda0 {
|
||||
activeApp @0 :App;
|
||||
sidebarCollapsed @1 :Bool;
|
||||
mapEnabled @2 :Bool;
|
||||
mockEngaged @3 :Bool;
|
||||
|
||||
enum App @0x9917470acf94d285 {
|
||||
home @0;
|
||||
music @1;
|
||||
nav @2;
|
||||
settings @3;
|
||||
none @4;
|
||||
}
|
||||
}
|
||||
|
||||
struct OrbslamCorrection @0x8afd33dc9b35e1aa {
|
||||
correctionMonoTime @0 :UInt64;
|
||||
prePositionECEF @1 :List(Float64);
|
||||
postPositionECEF @2 :List(Float64);
|
||||
prePoseQuatECEF @3 :List(Float32);
|
||||
postPoseQuatECEF @4 :List(Float32);
|
||||
numInliers @5 :UInt32;
|
||||
}
|
||||
|
||||
struct EthernetPacket @0xa99a9d5b33cf5859 {
|
||||
pkt @0 :Data;
|
||||
ts @1 :Float32;
|
||||
}
|
||||
|
||||
struct CellInfo @0xcff7566681c277ce {
|
||||
timestamp @0 :UInt64;
|
||||
repr @1 :Text; # android toString() for now
|
||||
}
|
||||
|
||||
struct WifiScan @0xd4df5a192382ba0b {
|
||||
bssid @0 :Text;
|
||||
ssid @1 :Text;
|
||||
capabilities @2 :Text;
|
||||
frequency @3 :Int32;
|
||||
level @4 :Int32;
|
||||
timestamp @5 :Int64;
|
||||
|
||||
centerFreq0 @6 :Int32;
|
||||
centerFreq1 @7 :Int32;
|
||||
channelWidth @8 :ChannelWidth;
|
||||
operatorFriendlyName @9 :Text;
|
||||
venueName @10 :Text;
|
||||
is80211mcResponder @11 :Bool;
|
||||
passpoint @12 :Bool;
|
||||
|
||||
distanceCm @13 :Int32;
|
||||
distanceSdCm @14 :Int32;
|
||||
|
||||
enum ChannelWidth @0xcb6a279f015f6b51 {
|
||||
w20Mhz @0;
|
||||
w40Mhz @1;
|
||||
w80Mhz @2;
|
||||
w160Mhz @3;
|
||||
w80Plus80Mhz @4;
|
||||
}
|
||||
}
|
||||
|
||||
struct LiveEventData @0x94b7baa90c5c321e {
|
||||
name @0 :Text;
|
||||
value @1 :Int32;
|
||||
}
|
||||
|
||||
struct ModelData @0xb8aad62cffef28a9 {
|
||||
frameId @0 :UInt32;
|
||||
frameAge @12 :UInt32;
|
||||
frameDropPerc @13 :Float32;
|
||||
timestampEof @9 :UInt64;
|
||||
modelExecutionTime @14 :Float32;
|
||||
gpuExecutionTime @16 :Float32;
|
||||
rawPred @15 :Data;
|
||||
|
||||
path @1 :PathData;
|
||||
leftLane @2 :PathData;
|
||||
rightLane @3 :PathData;
|
||||
lead @4 :LeadData;
|
||||
freePath @6 :List(Float32);
|
||||
|
||||
settings @5 :ModelSettings;
|
||||
leadFuture @7 :LeadData;
|
||||
speed @8 :List(Float32);
|
||||
meta @10 :MetaData;
|
||||
longitudinal @11 :LongitudinalData;
|
||||
|
||||
struct PathData @0x8817eeea389e9f08 {
|
||||
points @0 :List(Float32);
|
||||
prob @1 :Float32;
|
||||
std @2 :Float32;
|
||||
stds @3 :List(Float32);
|
||||
poly @4 :List(Float32);
|
||||
validLen @5 :Float32;
|
||||
}
|
||||
|
||||
struct LeadData @0xd1c9bef96d26fa91 {
|
||||
dist @0 :Float32;
|
||||
prob @1 :Float32;
|
||||
std @2 :Float32;
|
||||
relVel @3 :Float32;
|
||||
relVelStd @4 :Float32;
|
||||
relY @5 :Float32;
|
||||
relYStd @6 :Float32;
|
||||
relA @7 :Float32;
|
||||
relAStd @8 :Float32;
|
||||
}
|
||||
|
||||
struct ModelSettings @0xa26e3710efd3e914 {
|
||||
bigBoxX @0 :UInt16;
|
||||
bigBoxY @1 :UInt16;
|
||||
bigBoxWidth @2 :UInt16;
|
||||
bigBoxHeight @3 :UInt16;
|
||||
boxProjection @4 :List(Float32);
|
||||
yuvCorrection @5 :List(Float32);
|
||||
inputTransform @6 :List(Float32);
|
||||
}
|
||||
|
||||
struct MetaData @0x9744f25fb60f2bf8 {
|
||||
engagedProb @0 :Float32;
|
||||
desirePrediction @1 :List(Float32);
|
||||
brakeDisengageProb @2 :Float32;
|
||||
gasDisengageProb @3 :Float32;
|
||||
steerOverrideProb @4 :Float32;
|
||||
desireState @5 :List(Float32);
|
||||
}
|
||||
|
||||
struct LongitudinalData @0xf98f999c6a071122 {
|
||||
distances @2 :List(Float32);
|
||||
speeds @0 :List(Float32);
|
||||
accelerations @1 :List(Float32);
|
||||
}
|
||||
}
|
||||
|
||||
struct ECEFPoint @0xc25bbbd524983447 {
|
||||
x @0 :Float64;
|
||||
y @1 :Float64;
|
||||
z @2 :Float64;
|
||||
}
|
||||
|
||||
struct ECEFPointDEPRECATED @0xe10e21168db0c7f7 {
|
||||
x @0 :Float32;
|
||||
y @1 :Float32;
|
||||
z @2 :Float32;
|
||||
}
|
||||
|
||||
struct GPSPlannerPoints @0xab54c59699f8f9f3 {
|
||||
curPosDEPRECATED @0 :ECEFPointDEPRECATED;
|
||||
pointsDEPRECATED @1 :List(ECEFPointDEPRECATED);
|
||||
curPos @6 :ECEFPoint;
|
||||
points @7 :List(ECEFPoint);
|
||||
valid @2 :Bool;
|
||||
trackName @3 :Text;
|
||||
speedLimit @4 :Float32;
|
||||
accelTarget @5 :Float32;
|
||||
}
|
||||
|
||||
struct GPSPlannerPlan @0xf5ad1d90cdc1dd6b {
|
||||
valid @0 :Bool;
|
||||
poly @1 :List(Float32);
|
||||
trackName @2 :Text;
|
||||
speed @3 :Float32;
|
||||
acceleration @4 :Float32;
|
||||
pointsDEPRECATED @5 :List(ECEFPointDEPRECATED);
|
||||
points @6 :List(ECEFPoint);
|
||||
xLookahead @7 :Float32;
|
||||
}
|
||||
|
||||
struct UiNavigationEvent @0x90c8426c3eaddd3b {
|
||||
type @0: Type;
|
||||
status @1: Status;
|
||||
distanceTo @2: Float32;
|
||||
endRoadPointDEPRECATED @3: ECEFPointDEPRECATED;
|
||||
endRoadPoint @4: ECEFPoint;
|
||||
|
||||
enum Type @0xe8db07dcf8fcea05 {
|
||||
none @0;
|
||||
laneChangeLeft @1;
|
||||
laneChangeRight @2;
|
||||
mergeLeft @3;
|
||||
mergeRight @4;
|
||||
turnLeft @5;
|
||||
turnRight @6;
|
||||
}
|
||||
|
||||
enum Status @0xb9aa88c75ef99a1f {
|
||||
none @0;
|
||||
passive @1;
|
||||
approaching @2;
|
||||
active @3;
|
||||
}
|
||||
}
|
||||
|
||||
struct LiveLocationData @0xb99b2bc7a57e8128 {
|
||||
status @0 :UInt8;
|
||||
|
||||
# 3D fix
|
||||
lat @1 :Float64;
|
||||
lon @2 :Float64;
|
||||
alt @3 :Float32; # m
|
||||
|
||||
# speed
|
||||
speed @4 :Float32; # m/s
|
||||
|
||||
# NED velocity components
|
||||
vNED @5 :List(Float32);
|
||||
|
||||
# roll, pitch, heading (x,y,z)
|
||||
roll @6 :Float32; # WRT to center of earth?
|
||||
pitch @7 :Float32; # WRT to center of earth?
|
||||
heading @8 :Float32; # WRT to north?
|
||||
|
||||
# what are these?
|
||||
wanderAngle @9 :Float32;
|
||||
trackAngle @10 :Float32;
|
||||
|
||||
# car frame -- https://upload.wikimedia.org/wikipedia/commons/f/f5/RPY_angles_of_cars.png
|
||||
|
||||
# gyro, in car frame, deg/s
|
||||
gyro @11 :List(Float32);
|
||||
|
||||
# accel, in car frame, m/s^2
|
||||
accel @12 :List(Float32);
|
||||
|
||||
accuracy @13 :Accuracy;
|
||||
|
||||
source @14 :SensorSource;
|
||||
# if we are fixing a location in the past
|
||||
fixMonoTime @15 :UInt64;
|
||||
|
||||
gpsWeek @16 :Int32;
|
||||
timeOfWeek @17 :Float64;
|
||||
|
||||
positionECEF @18 :List(Float64);
|
||||
poseQuatECEF @19 :List(Float32);
|
||||
pitchCalibration @20 :Float32;
|
||||
yawCalibration @21 :Float32;
|
||||
imuFrame @22 :List(Float32);
|
||||
|
||||
struct Accuracy @0x943dc4625473b03f {
|
||||
pNEDError @0 :List(Float32);
|
||||
vNEDError @1 :List(Float32);
|
||||
rollError @2 :Float32;
|
||||
pitchError @3 :Float32;
|
||||
headingError @4 :Float32;
|
||||
ellipsoidSemiMajorError @5 :Float32;
|
||||
ellipsoidSemiMinorError @6 :Float32;
|
||||
ellipsoidOrientationError @7 :Float32;
|
||||
}
|
||||
|
||||
enum SensorSource @0xc871d3cc252af657 {
|
||||
applanix @0;
|
||||
kalman @1;
|
||||
orbslam @2;
|
||||
timing @3;
|
||||
dummy @4;
|
||||
}
|
||||
}
|
||||
|
||||
struct OrbOdometry @0xd7700859ed1f5b76 {
|
||||
# timing first
|
||||
startMonoTime @0 :UInt64;
|
||||
endMonoTime @1 :UInt64;
|
||||
|
||||
# fundamental matrix and error
|
||||
f @2: List(Float64);
|
||||
err @3: Float64;
|
||||
|
||||
# number of inlier points
|
||||
inliers @4: Int32;
|
||||
|
||||
# for debug only
|
||||
# indexed by endMonoTime features
|
||||
# value is startMonoTime feature match
|
||||
# -1 if no match
|
||||
matches @5: List(Int16);
|
||||
}
|
||||
|
||||
struct OrbFeatures @0xcd60164a8a0159ef {
|
||||
timestampEof @0 :UInt64;
|
||||
# transposed arrays of normalized image coordinates
|
||||
# len(xs) == len(ys) == len(descriptors) * 32
|
||||
xs @1 :List(Float32);
|
||||
ys @2 :List(Float32);
|
||||
descriptors @3 :Data;
|
||||
octaves @4 :List(Int8);
|
||||
|
||||
# match index to last OrbFeatures
|
||||
# -1 if no match
|
||||
timestampLastEof @5 :UInt64;
|
||||
matches @6: List(Int16);
|
||||
}
|
||||
|
||||
struct OrbFeaturesSummary @0xd500d30c5803fa4f {
|
||||
timestampEof @0 :UInt64;
|
||||
timestampLastEof @1 :UInt64;
|
||||
|
||||
featureCount @2 :UInt16;
|
||||
matchCount @3 :UInt16;
|
||||
computeNs @4 :UInt64;
|
||||
}
|
||||
|
||||
struct OrbKeyFrame @0xc8233c0345e27e24 {
|
||||
# this is a globally unique id for the KeyFrame
|
||||
id @0: UInt64;
|
||||
|
||||
# this is the location of the KeyFrame
|
||||
pos @1: ECEFPoint;
|
||||
|
||||
# these are the features in the world
|
||||
# len(dpos) == len(descriptors) * 32
|
||||
dpos @2 :List(ECEFPoint);
|
||||
descriptors @3 :Data;
|
||||
}
|
||||
|
||||
struct KalmanOdometry @0x92e21bb7ea38793a {
|
||||
trans @0 :List(Float32); # m/s in device frame
|
||||
rot @1 :List(Float32); # rad/s in device frame
|
||||
transStd @2 :List(Float32); # std m/s in device frame
|
||||
rotStd @3 :List(Float32); # std rad/s in device frame
|
||||
}
|
||||
|
||||
struct OrbObservation @0x9b326d4e436afec7 {
|
||||
observationMonoTime @0 :UInt64;
|
||||
normalizedCoordinates @1 :List(Float32);
|
||||
locationECEF @2 :List(Float64);
|
||||
matchDistance @3: UInt32;
|
||||
}
|
||||
|
||||
struct CalibrationFeatures @0x8fdfadb254ea867a {
|
||||
frameId @0 :UInt32;
|
||||
|
||||
p0 @1 :List(Float32);
|
||||
p1 @2 :List(Float32);
|
||||
status @3 :List(Int8);
|
||||
}
|
||||
|
||||
struct NavStatus @0xbd8822120928120c {
|
||||
isNavigating @0 :Bool;
|
||||
currentAddress @1 :Address;
|
||||
|
||||
struct Address @0xce7cd672cacc7814 {
|
||||
title @0 :Text;
|
||||
lat @1 :Float64;
|
||||
lng @2 :Float64;
|
||||
house @3 :Text;
|
||||
address @4 :Text;
|
||||
street @5 :Text;
|
||||
city @6 :Text;
|
||||
state @7 :Text;
|
||||
country @8 :Text;
|
||||
}
|
||||
}
|
||||
|
||||
struct NavUpdate @0xdb98be6565516acb {
|
||||
isNavigating @0 :Bool;
|
||||
curSegment @1 :Int32;
|
||||
segments @2 :List(Segment);
|
||||
|
||||
struct LatLng @0x9eaef9187cadbb9b {
|
||||
lat @0 :Float64;
|
||||
lng @1 :Float64;
|
||||
}
|
||||
|
||||
struct Segment @0xa5b39b4fc4d7da3f {
|
||||
from @0 :LatLng;
|
||||
to @1 :LatLng;
|
||||
updateTime @2 :Int32;
|
||||
distance @3 :Int32;
|
||||
crossTime @4 :Int32;
|
||||
exitNo @5 :Int32;
|
||||
instruction @6 :Instruction;
|
||||
|
||||
parts @7 :List(LatLng);
|
||||
|
||||
enum Instruction @0xc5417a637451246f {
|
||||
turnLeft @0;
|
||||
turnRight @1;
|
||||
keepLeft @2;
|
||||
keepRight @3;
|
||||
straight @4;
|
||||
roundaboutExitNumber @5;
|
||||
roundaboutExit @6;
|
||||
roundaboutTurnLeft @7;
|
||||
unkn8 @8;
|
||||
roundaboutStraight @9;
|
||||
unkn10 @10;
|
||||
roundaboutTurnRight @11;
|
||||
unkn12 @12;
|
||||
roundaboutUturn @13;
|
||||
unkn14 @14;
|
||||
arrive @15;
|
||||
exitLeft @16;
|
||||
exitRight @17;
|
||||
unkn18 @18;
|
||||
uturn @19;
|
||||
# ...
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TrafficEvent @0xacfa74a094e62626 {
|
||||
type @0 :Type;
|
||||
distance @1 :Float32;
|
||||
action @2 :Action;
|
||||
resuming @3 :Bool;
|
||||
|
||||
enum Type @0xd85d75253435bf4b {
|
||||
stopSign @0;
|
||||
lightRed @1;
|
||||
lightYellow @2;
|
||||
lightGreen @3;
|
||||
stopLight @4;
|
||||
}
|
||||
|
||||
enum Action @0xa6f6ce72165ccb49 {
|
||||
none @0;
|
||||
yield @1;
|
||||
stop @2;
|
||||
resumeReady @3;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
struct AndroidGnss @0xdfdf30d03fc485bd {
|
||||
union {
|
||||
measurements @0 :Measurements;
|
||||
navigationMessage @1 :NavigationMessage;
|
||||
}
|
||||
|
||||
struct Measurements @0xa20710d4f428d6cd {
|
||||
clock @0 :Clock;
|
||||
measurements @1 :List(Measurement);
|
||||
|
||||
struct Clock @0xa0e27b453a38f450 {
|
||||
timeNanos @0 :Int64;
|
||||
hardwareClockDiscontinuityCount @1 :Int32;
|
||||
|
||||
hasTimeUncertaintyNanos @2 :Bool;
|
||||
timeUncertaintyNanos @3 :Float64;
|
||||
|
||||
hasLeapSecond @4 :Bool;
|
||||
leapSecond @5 :Int32;
|
||||
|
||||
hasFullBiasNanos @6 :Bool;
|
||||
fullBiasNanos @7 :Int64;
|
||||
|
||||
hasBiasNanos @8 :Bool;
|
||||
biasNanos @9 :Float64;
|
||||
|
||||
hasBiasUncertaintyNanos @10 :Bool;
|
||||
biasUncertaintyNanos @11 :Float64;
|
||||
|
||||
hasDriftNanosPerSecond @12 :Bool;
|
||||
driftNanosPerSecond @13 :Float64;
|
||||
|
||||
hasDriftUncertaintyNanosPerSecond @14 :Bool;
|
||||
driftUncertaintyNanosPerSecond @15 :Float64;
|
||||
}
|
||||
|
||||
struct Measurement @0xd949bf717d77614d {
|
||||
svId @0 :Int32;
|
||||
constellation @1 :Constellation;
|
||||
|
||||
timeOffsetNanos @2 :Float64;
|
||||
state @3 :Int32;
|
||||
receivedSvTimeNanos @4 :Int64;
|
||||
receivedSvTimeUncertaintyNanos @5 :Int64;
|
||||
cn0DbHz @6 :Float64;
|
||||
pseudorangeRateMetersPerSecond @7 :Float64;
|
||||
pseudorangeRateUncertaintyMetersPerSecond @8 :Float64;
|
||||
accumulatedDeltaRangeState @9 :Int32;
|
||||
accumulatedDeltaRangeMeters @10 :Float64;
|
||||
accumulatedDeltaRangeUncertaintyMeters @11 :Float64;
|
||||
|
||||
hasCarrierFrequencyHz @12 :Bool;
|
||||
carrierFrequencyHz @13 :Float32;
|
||||
hasCarrierCycles @14 :Bool;
|
||||
carrierCycles @15 :Int64;
|
||||
hasCarrierPhase @16 :Bool;
|
||||
carrierPhase @17 :Float64;
|
||||
hasCarrierPhaseUncertainty @18 :Bool;
|
||||
carrierPhaseUncertainty @19 :Float64;
|
||||
hasSnrInDb @20 :Bool;
|
||||
snrInDb @21 :Float64;
|
||||
|
||||
multipathIndicator @22 :MultipathIndicator;
|
||||
|
||||
enum Constellation @0x9ef1f3ff0deb5ffb {
|
||||
unknown @0;
|
||||
gps @1;
|
||||
sbas @2;
|
||||
glonass @3;
|
||||
qzss @4;
|
||||
beidou @5;
|
||||
galileo @6;
|
||||
}
|
||||
|
||||
enum State @0xcbb9490adce12d72 {
|
||||
unknown @0;
|
||||
codeLock @1;
|
||||
bitSync @2;
|
||||
subframeSync @3;
|
||||
towDecoded @4;
|
||||
msecAmbiguous @5;
|
||||
symbolSync @6;
|
||||
gloStringSync @7;
|
||||
gloTodDecoded @8;
|
||||
bdsD2BitSync @9;
|
||||
bdsD2SubframeSync @10;
|
||||
galE1bcCodeLock @11;
|
||||
galE1c2ndCodeLock @12;
|
||||
galE1bPageSync @13;
|
||||
sbasSync @14;
|
||||
}
|
||||
|
||||
enum MultipathIndicator @0xc04e7b6231d4caa8 {
|
||||
unknown @0;
|
||||
detected @1;
|
||||
notDetected @2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct NavigationMessage @0xe2517b083095fd4e {
|
||||
type @0 :Int32;
|
||||
svId @1 :Int32;
|
||||
messageId @2 :Int32;
|
||||
submessageId @3 :Int32;
|
||||
data @4 :Data;
|
||||
status @5 :Status;
|
||||
|
||||
enum Status @0xec1ff7996b35366f {
|
||||
unknown @0;
|
||||
parityPassed @1;
|
||||
parityRebuilt @2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct LidarPts @0xe3d6685d4e9d8f7a {
|
||||
r @0 :List(UInt16); # uint16 m*500.0
|
||||
theta @1 :List(UInt16); # uint16 deg*100.0
|
||||
reflect @2 :List(UInt8); # uint8 0-255
|
||||
|
||||
# For storing out of file.
|
||||
idx @3 :UInt64;
|
||||
|
||||
# For storing in file
|
||||
pkt @4 :Data;
|
||||
}
|
||||
|
||||
|
||||
2710
cereal/log.capnp
Normal file
2710
cereal/log.capnp
Normal file
File diff suppressed because it is too large
Load Diff
269
cereal/messaging/__init__.py
Normal file
269
cereal/messaging/__init__.py
Normal file
@@ -0,0 +1,269 @@
|
||||
# must be built with scons
|
||||
from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
|
||||
Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
|
||||
set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
|
||||
import msgq
|
||||
import os
|
||||
import capnp
|
||||
import time
|
||||
|
||||
from typing import Optional, List, Union, Dict
|
||||
|
||||
from cereal import log
|
||||
from cereal.services import SERVICE_LIST
|
||||
from openpilot.common.utils import MovingAverage
|
||||
|
||||
NO_TRAVERSAL_LIMIT = 2**64-1
|
||||
|
||||
|
||||
def pub_sock(endpoint: str) -> PubSocket:
|
||||
service = SERVICE_LIST.get(endpoint)
|
||||
segment_size = service.queue_size if service else 0
|
||||
return msgq.pub_sock(endpoint, segment_size)
|
||||
|
||||
|
||||
def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
|
||||
conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
|
||||
service = SERVICE_LIST.get(endpoint)
|
||||
segment_size = service.queue_size if service else 0
|
||||
return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
|
||||
timeout=timeout, segment_size=segment_size)
|
||||
|
||||
|
||||
def reset_context():
|
||||
msgq.context = Context()
|
||||
|
||||
|
||||
def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
|
||||
with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
|
||||
return msg
|
||||
|
||||
|
||||
def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
|
||||
args = {
|
||||
'valid': False,
|
||||
'logMonoTime': int(time.monotonic() * 1e9),
|
||||
**kwargs
|
||||
}
|
||||
dat = log.Event.new_message(**args)
|
||||
if service is not None:
|
||||
if size is None:
|
||||
dat.init(service)
|
||||
else:
|
||||
dat.init(service, size)
|
||||
return dat
|
||||
|
||||
|
||||
def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]:
|
||||
"""Receive all message currently available on the queue"""
|
||||
msgs = drain_sock_raw(sock, wait_for_one=wait_for_one)
|
||||
return [log_from_bytes(m) for m in msgs]
|
||||
|
||||
|
||||
# TODO: print when we drop packets?
|
||||
def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]:
|
||||
"""Same as drain sock, but only returns latest message. Consider using conflate instead."""
|
||||
dat = None
|
||||
|
||||
while 1:
|
||||
if wait and dat is None:
|
||||
recv = sock.receive()
|
||||
else:
|
||||
recv = sock.receive(non_blocking=True)
|
||||
|
||||
if recv is None: # Timeout hit
|
||||
break
|
||||
|
||||
dat = recv
|
||||
|
||||
if dat is not None:
|
||||
dat = log_from_bytes(dat)
|
||||
|
||||
return dat
|
||||
|
||||
|
||||
def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
|
||||
dat = sock.receive()
|
||||
if dat is not None:
|
||||
dat = log_from_bytes(dat)
|
||||
return dat
|
||||
|
||||
|
||||
def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
|
||||
dat = sock.receive(non_blocking=True)
|
||||
if dat is not None:
|
||||
dat = log_from_bytes(dat)
|
||||
return dat
|
||||
|
||||
|
||||
def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
|
||||
"""Keep receiving until we get a message"""
|
||||
while True:
|
||||
dat = sock.receive()
|
||||
if dat is not None:
|
||||
return log_from_bytes(dat)
|
||||
|
||||
|
||||
class FrequencyTracker:
|
||||
def __init__(self, service_freq: float, update_freq: float, is_poll: bool):
|
||||
freq = max(min(service_freq, update_freq), 1.)
|
||||
if is_poll:
|
||||
min_freq = max_freq = freq
|
||||
else:
|
||||
max_freq = min(freq, update_freq)
|
||||
if service_freq >= 2 * update_freq:
|
||||
min_freq = update_freq
|
||||
elif update_freq >= 2* service_freq:
|
||||
min_freq = freq
|
||||
else:
|
||||
min_freq = min(freq, freq / 2.)
|
||||
|
||||
self.min_freq = min_freq * 0.8
|
||||
self.max_freq = max_freq * 1.2
|
||||
self.avg_dt = MovingAverage(int(10 * freq))
|
||||
self.recent_avg_dt = MovingAverage(int(freq))
|
||||
self.prev_time = 0.0
|
||||
|
||||
def record_recv_time(self, cur_time: float) -> None:
|
||||
# TODO: Handle case where cur_time is less than prev_time
|
||||
if self.prev_time > 1e-5:
|
||||
dt = cur_time - self.prev_time
|
||||
|
||||
self.avg_dt.add_value(dt)
|
||||
self.recent_avg_dt.add_value(dt)
|
||||
|
||||
self.prev_time = cur_time
|
||||
|
||||
@property
|
||||
def valid(self) -> bool:
|
||||
if self.avg_dt.count == 0:
|
||||
return False
|
||||
|
||||
avg_freq = 1.0 / self.avg_dt.get_average()
|
||||
if self.min_freq <= avg_freq <= self.max_freq:
|
||||
return True
|
||||
|
||||
avg_freq_recent = 1.0 / self.recent_avg_dt.get_average()
|
||||
return self.min_freq <= avg_freq_recent <= self.max_freq
|
||||
|
||||
|
||||
class SubMaster:
|
||||
def __init__(self, services: List[str], poll: Optional[str] = None,
|
||||
ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
|
||||
ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
|
||||
self.frame = -1
|
||||
self.services = services
|
||||
self.seen = {s: False for s in services}
|
||||
self.updated = {s: False for s in services}
|
||||
self.recv_time = {s: 0. for s in services}
|
||||
self.recv_frame = {s: 0 for s in services}
|
||||
self.sock = {}
|
||||
self.data = {}
|
||||
self.logMonoTime = {s: 0 for s in services}
|
||||
|
||||
# zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
|
||||
on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
|
||||
self.static_freq_services = set(s for s in services if not on_demand[s])
|
||||
self.alive = {s: on_demand[s] for s in services}
|
||||
self.freq_ok = {s: on_demand[s] for s in services}
|
||||
self.valid = {s: on_demand[s] for s in services}
|
||||
|
||||
self.freq_tracker: Dict[str, FrequencyTracker] = {}
|
||||
self.poller = Poller()
|
||||
polled_services = set([poll, ] if poll is not None else services)
|
||||
self.non_polled_services = set(services) - polled_services
|
||||
|
||||
self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq
|
||||
self.ignore_alive = [] if ignore_alive is None else ignore_alive
|
||||
self.ignore_valid = [] if ignore_valid is None else ignore_valid
|
||||
|
||||
self.simulation = bool(int(os.getenv("SIMULATION", "0")))
|
||||
|
||||
# if freq and poll aren't specified, assume the max to be conservative
|
||||
assert frequency is None or poll is None, "Do not specify 'frequency' - frequency of the polled service will be used."
|
||||
self.update_freq = frequency or max([SERVICE_LIST[s].frequency for s in polled_services])
|
||||
|
||||
for s in services:
|
||||
p = self.poller if s not in self.non_polled_services else None
|
||||
self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True)
|
||||
|
||||
try:
|
||||
data = new_message(s)
|
||||
except capnp.lib.capnp.KjException:
|
||||
data = new_message(s, 0) # lists
|
||||
|
||||
self.data[s] = getattr(data.as_reader(), s)
|
||||
self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll)
|
||||
|
||||
def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
|
||||
return self.data[s]
|
||||
|
||||
def _check_avg_freq(self, s: str) -> bool:
|
||||
return SERVICE_LIST[s].frequency > 0.99 and (s not in self.ignore_average_freq) and (s not in self.ignore_alive)
|
||||
|
||||
def update(self, timeout: int = 100) -> None:
|
||||
msgs = []
|
||||
for sock in self.poller.poll(timeout):
|
||||
msgs.append(recv_one_or_none(sock))
|
||||
|
||||
# non-blocking receive for non-polled sockets
|
||||
for s in self.non_polled_services:
|
||||
msgs.append(recv_one_or_none(self.sock[s]))
|
||||
self.update_msgs(time.monotonic(), msgs)
|
||||
|
||||
def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
|
||||
self.frame += 1
|
||||
self.updated = dict.fromkeys(self.services, False)
|
||||
for msg in msgs:
|
||||
if msg is None:
|
||||
continue
|
||||
|
||||
s = msg.which()
|
||||
self.seen[s] = True
|
||||
self.updated[s] = True
|
||||
|
||||
self.freq_tracker[s].record_recv_time(cur_time)
|
||||
self.recv_time[s] = cur_time
|
||||
self.recv_frame[s] = self.frame
|
||||
self.data[s] = getattr(msg, s)
|
||||
self.logMonoTime[s] = msg.logMonoTime
|
||||
self.valid[s] = msg.valid
|
||||
|
||||
for s in self.static_freq_services:
|
||||
# alive if delay is within 10x the expected frequency; checks relaxed in simulator
|
||||
self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
|
||||
self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
|
||||
|
||||
def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
|
||||
return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
|
||||
|
||||
def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
|
||||
return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
|
||||
|
||||
def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
|
||||
return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
|
||||
|
||||
def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
|
||||
return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
|
||||
|
||||
|
||||
class PubMaster:
|
||||
def __init__(self, services: List[str]):
|
||||
self.sock = {}
|
||||
for s in services:
|
||||
self.sock[s] = pub_sock(s)
|
||||
|
||||
def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder]) -> None:
|
||||
if not isinstance(dat, bytes):
|
||||
dat = dat.to_bytes()
|
||||
self.sock[s].send(dat)
|
||||
|
||||
def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
|
||||
for _ in range(int(timeout*(1./dt))):
|
||||
if self.sock[s].all_readers_updated():
|
||||
return True
|
||||
time.sleep(dt)
|
||||
return False
|
||||
|
||||
def all_readers_updated(self, s: str) -> bool:
|
||||
return self.sock[s].all_readers_updated() # type: ignore
|
||||
72
cereal/messaging/bridge.cc
Normal file
72
cereal/messaging/bridge.cc
Normal file
@@ -0,0 +1,72 @@
|
||||
#include <cassert>
|
||||
|
||||
#include "cereal/messaging/msgq_to_zmq.h"
|
||||
#include "cereal/services.h"
|
||||
#include "common/util.h"
|
||||
|
||||
ExitHandler do_exit;
|
||||
|
||||
static std::vector<std::string> get_services(const std::string &whitelist_str, bool zmq_to_msgq) {
|
||||
std::vector<std::string> service_list;
|
||||
for (const auto& it : services) {
|
||||
std::string name = it.second.name;
|
||||
bool in_whitelist = whitelist_str.find(name) != std::string::npos;
|
||||
if (zmq_to_msgq && !in_whitelist) {
|
||||
continue;
|
||||
}
|
||||
service_list.push_back(name);
|
||||
}
|
||||
return service_list;
|
||||
}
|
||||
|
||||
void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &ip) {
|
||||
MsgqToZmq bridge;
|
||||
bridge.run(endpoints, ip);
|
||||
}
|
||||
|
||||
void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
|
||||
auto poller = std::make_unique<ZMQPoller>();
|
||||
auto pub_context = std::make_unique<MSGQContext>();
|
||||
auto sub_context = std::make_unique<ZMQContext>();
|
||||
std::map<SubSocket *, PubSocket *> sub2pub;
|
||||
|
||||
for (auto endpoint : endpoints) {
|
||||
auto pub_sock = new MSGQPubSocket();
|
||||
auto sub_sock = new ZMQSubSocket();
|
||||
size_t queue_size = services.at(endpoint).queue_size;
|
||||
pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
|
||||
sub_sock->connect(sub_context.get(), endpoint, ip, false);
|
||||
|
||||
poller->registerSocket(sub_sock);
|
||||
sub2pub[sub_sock] = pub_sock;
|
||||
}
|
||||
|
||||
while (!do_exit) {
|
||||
for (auto sub_sock : poller->poll(100)) {
|
||||
std::unique_ptr<Message> msg(sub_sock->receive(true));
|
||||
if (msg) {
|
||||
sub2pub[sub_sock]->sendMessage(msg.get());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up allocated sockets
|
||||
for (auto &[sub_sock, pub_sock] : sub2pub) {
|
||||
delete sub_sock;
|
||||
delete pub_sock;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
bool is_zmq_to_msgq = argc > 2;
|
||||
std::string ip = is_zmq_to_msgq ? argv[1] : "127.0.0.1";
|
||||
std::string whitelist_str = is_zmq_to_msgq ? std::string(argv[2]) : "";
|
||||
std::vector<std::string> endpoints = get_services(whitelist_str, is_zmq_to_msgq);
|
||||
|
||||
if (is_zmq_to_msgq) {
|
||||
zmq_to_msgq(endpoints, ip);
|
||||
} else {
|
||||
msgq_to_zmq(endpoints, ip);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
102
cereal/messaging/messaging.h
Normal file
102
cereal/messaging/messaging.h
Normal file
@@ -0,0 +1,102 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <utility>
|
||||
|
||||
#include <capnp/serialize.h>
|
||||
|
||||
#include "cereal/gen/cpp/log.capnp.h"
|
||||
#include "common/timing.h"
|
||||
#include "msgq/ipc.h"
|
||||
|
||||
class SubMaster {
|
||||
public:
|
||||
SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll = {},
|
||||
const char *address = nullptr, const std::vector<const char *> &ignore_alive = {});
|
||||
void update(int timeout = 1000);
|
||||
void update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages);
|
||||
inline bool allAlive(const std::vector<const char *> &service_list = {}) { return all_(service_list, false, true); }
|
||||
inline bool allValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, false); }
|
||||
inline bool allAliveAndValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, true); }
|
||||
void drain();
|
||||
~SubMaster();
|
||||
|
||||
uint64_t frame = 0;
|
||||
bool updated(const char *name) const;
|
||||
bool alive(const char *name) const;
|
||||
bool valid(const char *name) const;
|
||||
uint64_t rcv_frame(const char *name) const;
|
||||
uint64_t rcv_time(const char *name) const;
|
||||
cereal::Event::Reader &operator[](const char *name) const;
|
||||
|
||||
private:
|
||||
bool all_(const std::vector<const char *> &service_list, bool valid, bool alive);
|
||||
Poller *poller_ = nullptr;
|
||||
struct SubMessage;
|
||||
std::map<SubSocket *, SubMessage *> messages_;
|
||||
std::map<std::string, SubMessage *> services_;
|
||||
};
|
||||
|
||||
class MessageBuilder : public capnp::MallocMessageBuilder {
|
||||
public:
|
||||
MessageBuilder() = default;
|
||||
|
||||
cereal::Event::Builder initEvent(bool valid = true) {
|
||||
cereal::Event::Builder event = initRoot<cereal::Event>();
|
||||
event.setLogMonoTime(nanos_since_boot());
|
||||
event.setValid(valid);
|
||||
return event;
|
||||
}
|
||||
|
||||
kj::ArrayPtr<capnp::byte> toBytes() {
|
||||
heapArray_ = capnp::messageToFlatArray(*this);
|
||||
return heapArray_.asBytes();
|
||||
}
|
||||
|
||||
size_t getSerializedSize() {
|
||||
return capnp::computeSerializedSizeInWords(*this) * sizeof(capnp::word);
|
||||
}
|
||||
|
||||
int serializeToBuffer(unsigned char *buffer, size_t buffer_size) {
|
||||
size_t serialized_size = getSerializedSize();
|
||||
if (serialized_size > buffer_size) { return -1; }
|
||||
kj::ArrayOutputStream out(kj::ArrayPtr<capnp::byte>(buffer, buffer_size));
|
||||
capnp::writeMessage(out, *this);
|
||||
return serialized_size;
|
||||
}
|
||||
|
||||
private:
|
||||
kj::Array<capnp::word> heapArray_;
|
||||
};
|
||||
|
||||
class PubMaster {
|
||||
public:
|
||||
PubMaster(const std::vector<const char *> &service_list);
|
||||
inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); }
|
||||
int send(const char *name, MessageBuilder &msg);
|
||||
~PubMaster();
|
||||
|
||||
private:
|
||||
std::map<std::string, PubSocket *> sockets_;
|
||||
};
|
||||
|
||||
class AlignedBuffer {
|
||||
public:
|
||||
kj::ArrayPtr<const capnp::word> align(const char *data, const size_t size) {
|
||||
words_size = size / sizeof(capnp::word) + 1;
|
||||
if (aligned_buf.size() < words_size) {
|
||||
aligned_buf = kj::heapArray<capnp::word>(words_size < 512 ? 512 : words_size);
|
||||
}
|
||||
memcpy(aligned_buf.begin(), data, size);
|
||||
return aligned_buf.slice(0, words_size);
|
||||
}
|
||||
inline kj::ArrayPtr<const capnp::word> align(Message *m) {
|
||||
return align(m->getData(), m->getSize());
|
||||
}
|
||||
private:
|
||||
kj::Array<capnp::word> aligned_buf;
|
||||
size_t words_size;
|
||||
};
|
||||
146
cereal/messaging/msgq_to_zmq.cc
Normal file
146
cereal/messaging/msgq_to_zmq.cc
Normal file
@@ -0,0 +1,146 @@
|
||||
#include "cereal/messaging/msgq_to_zmq.h"
|
||||
|
||||
#include <cassert>
|
||||
|
||||
#include "cereal/services.h"
|
||||
#include "common/util.h"
|
||||
|
||||
extern ExitHandler do_exit;
|
||||
|
||||
// Max messages to process per socket per poll
|
||||
constexpr int MAX_MESSAGES_PER_SOCKET = 50;
|
||||
|
||||
static std::string recv_zmq_msg(void *sock) {
|
||||
zmq_msg_t msg;
|
||||
zmq_msg_init(&msg);
|
||||
std::string ret;
|
||||
if (zmq_msg_recv(&msg, sock, 0) > 0) {
|
||||
ret.assign((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
|
||||
}
|
||||
zmq_msg_close(&msg);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
|
||||
zmq_context = std::make_unique<ZMQContext>();
|
||||
msgq_context = std::make_unique<MSGQContext>();
|
||||
|
||||
// Create ZMQPubSockets for each endpoint
|
||||
for (const auto &endpoint : endpoints) {
|
||||
auto &socket_pair = socket_pairs.emplace_back();
|
||||
socket_pair.endpoint = endpoint;
|
||||
socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
|
||||
int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
|
||||
if (ret != 0) {
|
||||
printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Start ZMQ monitoring thread to monitor socket events
|
||||
std::thread thread(&MsgqToZmq::zmqMonitorThread, this);
|
||||
|
||||
// Main loop for processing messages
|
||||
while (!do_exit) {
|
||||
{
|
||||
std::unique_lock lk(mutex);
|
||||
cv.wait(lk, [this]() { return do_exit || !sub2pub.empty(); });
|
||||
if (do_exit) break;
|
||||
|
||||
for (auto sub_sock : msgq_poller->poll(100)) {
|
||||
// Process messages for each socket
|
||||
ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
|
||||
for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
|
||||
auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
|
||||
if (!msg) break;
|
||||
|
||||
while (pub_sock->sendMessage(msg.get()) == -1) {
|
||||
if (errno != EINTR) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
util::sleep_for(1); // Give zmqMonitorThread a chance to acquire the mutex
|
||||
}
|
||||
|
||||
thread.join();
|
||||
}
|
||||
|
||||
void MsgqToZmq::zmqMonitorThread() {
|
||||
std::vector<zmq_pollitem_t> pollitems;
|
||||
|
||||
// Set up ZMQ monitor for each pub socket
|
||||
for (int i = 0; i < socket_pairs.size(); ++i) {
|
||||
std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
|
||||
zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
|
||||
|
||||
void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
|
||||
zmq_connect(monitor_socket, addr.c_str());
|
||||
pollitems.emplace_back(zmq_pollitem_t{.socket = monitor_socket, .events = ZMQ_POLLIN});
|
||||
}
|
||||
|
||||
while (!do_exit) {
|
||||
int ret = zmq_poll(pollitems.data(), pollitems.size(), 1000);
|
||||
if (ret < 0) {
|
||||
if (errno == EINTR) {
|
||||
// Due to frequent EINTR signals from msgq, introduce a brief delay (200 ms)
|
||||
// to reduce CPU usage during retry attempts.
|
||||
util::sleep_for(200);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
for (int i = 0; i < pollitems.size(); ++i) {
|
||||
if (pollitems[i].revents & ZMQ_POLLIN) {
|
||||
// First frame in message contains event number and value
|
||||
std::string frame = recv_zmq_msg(pollitems[i].socket);
|
||||
if (frame.empty()) continue;
|
||||
|
||||
uint16_t event_type = *(uint16_t *)(frame.data());
|
||||
|
||||
// Second frame in message contains event address
|
||||
frame = recv_zmq_msg(pollitems[i].socket);
|
||||
if (frame.empty()) continue;
|
||||
|
||||
std::unique_lock lk(mutex);
|
||||
auto &pair = socket_pairs[i];
|
||||
if (event_type & ZMQ_EVENT_ACCEPTED) {
|
||||
printf("socket [%s] connected\n", pair.endpoint.c_str());
|
||||
if (++pair.connected_clients == 1) {
|
||||
// Create new MSGQ subscriber socket and map to ZMQ publisher
|
||||
pair.sub_sock = std::make_unique<MSGQSubSocket>();
|
||||
size_t queue_size = services.at(pair.endpoint).queue_size;
|
||||
pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
|
||||
sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
|
||||
registerSockets();
|
||||
}
|
||||
} else if (event_type & ZMQ_EVENT_DISCONNECTED) {
|
||||
printf("socket [%s] disconnected\n", pair.endpoint.c_str());
|
||||
if (pair.connected_clients == 0 || --pair.connected_clients == 0) {
|
||||
// Remove MSGQ subscriber socket from mapping and reset it
|
||||
sub2pub.erase(pair.sub_sock.get());
|
||||
pair.sub_sock.reset(nullptr);
|
||||
registerSockets();
|
||||
}
|
||||
}
|
||||
cv.notify_one();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up monitor sockets
|
||||
for (int i = 0; i < pollitems.size(); ++i) {
|
||||
zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
|
||||
zmq_close(pollitems[i].socket);
|
||||
}
|
||||
cv.notify_one();
|
||||
}
|
||||
|
||||
void MsgqToZmq::registerSockets() {
|
||||
msgq_poller = std::make_unique<MSGQPoller>();
|
||||
for (const auto &socket_pair : socket_pairs) {
|
||||
if (socket_pair.sub_sock) {
|
||||
msgq_poller->registerSocket(socket_pair.sub_sock.get());
|
||||
}
|
||||
}
|
||||
}
|
||||
37
cereal/messaging/msgq_to_zmq.h
Normal file
37
cereal/messaging/msgq_to_zmq.h
Normal file
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#define private public
|
||||
#include "msgq/impl_msgq.h"
|
||||
#include "msgq/impl_zmq.h"
|
||||
|
||||
class MsgqToZmq {
|
||||
public:
|
||||
MsgqToZmq() {}
|
||||
void run(const std::vector<std::string> &endpoints, const std::string &ip);
|
||||
|
||||
protected:
|
||||
void registerSockets();
|
||||
void zmqMonitorThread();
|
||||
|
||||
struct SocketPair {
|
||||
std::string endpoint;
|
||||
std::unique_ptr<ZMQPubSocket> pub_sock;
|
||||
std::unique_ptr<MSGQSubSocket> sub_sock;
|
||||
int connected_clients = 0;
|
||||
};
|
||||
|
||||
std::unique_ptr<MSGQContext> msgq_context;
|
||||
std::unique_ptr<ZMQContext> zmq_context;
|
||||
std::mutex mutex;
|
||||
std::condition_variable cv;
|
||||
std::unique_ptr<MSGQPoller> msgq_poller;
|
||||
std::map<SubSocket *, ZMQPubSocket *> sub2pub;
|
||||
std::vector<SocketPair> socket_pairs;
|
||||
};
|
||||
211
cereal/messaging/socketmaster.cc
Normal file
211
cereal/messaging/socketmaster.cc
Normal file
@@ -0,0 +1,211 @@
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
|
||||
#include "cereal/services.h"
|
||||
#include "cereal/messaging/messaging.h"
|
||||
|
||||
const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
|
||||
|
||||
static inline bool inList(const std::vector<const char *> &list, const char *value) {
|
||||
for (auto &v : list) {
|
||||
if (strcmp(value, v) == 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
class MessageContext {
|
||||
public:
|
||||
MessageContext() : ctx_(nullptr) {}
|
||||
~MessageContext() { delete ctx_; }
|
||||
inline Context *context() {
|
||||
std::call_once(init_flag, [=]() { ctx_ = Context::create(); });
|
||||
return ctx_;
|
||||
}
|
||||
private:
|
||||
Context *ctx_;
|
||||
std::once_flag init_flag;
|
||||
};
|
||||
|
||||
MessageContext message_context;
|
||||
|
||||
struct SubMaster::SubMessage {
|
||||
std::string name;
|
||||
SubSocket *socket = nullptr;
|
||||
float freq = 0.0f;
|
||||
bool updated = false, alive = false, valid = false, ignore_alive;
|
||||
uint64_t rcv_time = 0, rcv_frame = 0;
|
||||
void *allocated_msg_reader = nullptr;
|
||||
bool is_polled = false;
|
||||
capnp::FlatArrayMessageReader *msg_reader = nullptr;
|
||||
AlignedBuffer aligned_buf;
|
||||
cereal::Event::Reader event;
|
||||
};
|
||||
|
||||
SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll,
|
||||
const char *address, const std::vector<const char *> &ignore_alive) {
|
||||
poller_ = Poller::create();
|
||||
for (auto name : service_list) {
|
||||
if (services.count(std::string(name)) == 0) {
|
||||
fprintf(stderr, "SubMaster: unknown service '%s', skipping subscription\n", name);
|
||||
continue;
|
||||
}
|
||||
|
||||
service serv = services.at(std::string(name));
|
||||
SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
|
||||
assert(socket != 0);
|
||||
bool is_polled = inList(poll, name) || poll.empty();
|
||||
if (is_polled) poller_->registerSocket(socket);
|
||||
SubMessage *m = new SubMessage{
|
||||
.name = name,
|
||||
.socket = socket,
|
||||
.freq = serv.frequency,
|
||||
.ignore_alive = inList(ignore_alive, name),
|
||||
.allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
|
||||
.is_polled = is_polled};
|
||||
m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader({});
|
||||
messages_[socket] = m;
|
||||
services_[name] = m;
|
||||
}
|
||||
}
|
||||
|
||||
void SubMaster::update(int timeout) {
|
||||
for (auto &kv : messages_) kv.second->updated = false;
|
||||
|
||||
auto sockets = poller_->poll(timeout);
|
||||
|
||||
// add non-polled sockets for non-blocking receive
|
||||
for (auto &kv : messages_) {
|
||||
SubMessage *m = kv.second;
|
||||
SubSocket *s = kv.first;
|
||||
if (!m->is_polled) sockets.push_back(s);
|
||||
}
|
||||
|
||||
uint64_t current_time = nanos_since_boot();
|
||||
|
||||
std::vector<std::pair<std::string, cereal::Event::Reader>> messages;
|
||||
|
||||
for (auto s : sockets) {
|
||||
Message *msg = s->receive(true);
|
||||
if (msg == nullptr) continue;
|
||||
|
||||
SubMessage *m = messages_.at(s);
|
||||
|
||||
m->msg_reader->~FlatArrayMessageReader();
|
||||
capnp::ReaderOptions options;
|
||||
options.traversalLimitInWords = kj::maxValue; // Don't limit
|
||||
m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(m->aligned_buf.align(msg), options);
|
||||
delete msg;
|
||||
messages.push_back({m->name, m->msg_reader->getRoot<cereal::Event>()});
|
||||
}
|
||||
|
||||
update_msgs(current_time, messages);
|
||||
}
|
||||
|
||||
void SubMaster::update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages){
|
||||
if (++frame == UINT64_MAX) frame = 1;
|
||||
|
||||
for (auto &kv : messages) {
|
||||
auto m_find = services_.find(kv.first);
|
||||
if (m_find == services_.end()){
|
||||
continue;
|
||||
}
|
||||
SubMessage *m = m_find->second;
|
||||
m->event = kv.second;
|
||||
m->updated = true;
|
||||
m->rcv_time = current_time;
|
||||
m->rcv_frame = frame;
|
||||
m->valid = m->event.getValid();
|
||||
if (SIMULATION) m->alive = true;
|
||||
}
|
||||
|
||||
if (!SIMULATION) {
|
||||
for (auto &kv : messages_) {
|
||||
SubMessage *m = kv.second;
|
||||
m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool SubMaster::all_(const std::vector<const char *> &service_list, bool valid, bool alive) {
|
||||
int found = 0;
|
||||
for (auto &kv : messages_) {
|
||||
SubMessage *m = kv.second;
|
||||
if (service_list.size() == 0 || inList(service_list, m->name.c_str())) {
|
||||
found += (!valid || m->valid) && (!alive || (m->alive || m->ignore_alive));
|
||||
}
|
||||
}
|
||||
return service_list.size() == 0 ? found == messages_.size() : found == service_list.size();
|
||||
}
|
||||
|
||||
void SubMaster::drain() {
|
||||
while (true) {
|
||||
auto polls = poller_->poll(0);
|
||||
if (polls.size() == 0)
|
||||
break;
|
||||
|
||||
for (auto sock : polls) {
|
||||
Message *msg = sock->receive(true);
|
||||
delete msg;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool SubMaster::updated(const char *name) const {
|
||||
return services_.at(name)->updated;
|
||||
}
|
||||
|
||||
bool SubMaster::alive(const char *name) const {
|
||||
return services_.at(name)->alive;
|
||||
}
|
||||
|
||||
bool SubMaster::valid(const char *name) const {
|
||||
return services_.at(name)->valid;
|
||||
}
|
||||
|
||||
uint64_t SubMaster::rcv_frame(const char *name) const {
|
||||
return services_.at(name)->rcv_frame;
|
||||
}
|
||||
|
||||
uint64_t SubMaster::rcv_time(const char *name) const {
|
||||
return services_.at(name)->rcv_time;
|
||||
}
|
||||
|
||||
cereal::Event::Reader &SubMaster::operator[](const char *name) const {
|
||||
return services_.at(name)->event;
|
||||
}
|
||||
|
||||
SubMaster::~SubMaster() {
|
||||
delete poller_;
|
||||
for (auto &kv : messages_) {
|
||||
SubMessage *m = kv.second;
|
||||
m->msg_reader->~FlatArrayMessageReader();
|
||||
free(m->allocated_msg_reader);
|
||||
delete m->socket;
|
||||
delete m;
|
||||
}
|
||||
}
|
||||
|
||||
PubMaster::PubMaster(const std::vector<const char *> &service_list) {
|
||||
for (auto name : service_list) {
|
||||
if (services.count(name) == 0) {
|
||||
fprintf(stderr, "PubMaster: unknown service '%s', skipping publisher\n", name);
|
||||
continue;
|
||||
}
|
||||
service serv = services.at(std::string(name));
|
||||
PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
|
||||
assert(socket);
|
||||
sockets_[name] = socket;
|
||||
}
|
||||
}
|
||||
|
||||
int PubMaster::send(const char *name, MessageBuilder &msg) {
|
||||
auto bytes = msg.toBytes();
|
||||
return send(name, bytes.begin(), bytes.size());
|
||||
}
|
||||
|
||||
PubMaster::~PubMaster() {
|
||||
for (auto s : sockets_) delete s.second;
|
||||
}
|
||||
0
cereal/messaging/tests/__init__.py
Normal file
0
cereal/messaging/tests/__init__.py
Normal file
185
cereal/messaging/tests/test_messaging.py
Normal file
185
cereal/messaging/tests/test_messaging.py
Normal file
@@ -0,0 +1,185 @@
|
||||
import os
|
||||
import capnp
|
||||
import multiprocessing
|
||||
import numbers
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from parameterized import parameterized
|
||||
import pytest
|
||||
|
||||
from cereal import log, car
|
||||
import cereal.messaging as messaging
|
||||
from cereal.services import SERVICE_LIST
|
||||
|
||||
events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
|
||||
|
||||
def random_sock():
|
||||
return random.choice(events)
|
||||
|
||||
def random_socks(num_socks=10):
|
||||
return list({random_sock() for _ in range(num_socks)})
|
||||
|
||||
def random_bytes(length=1000):
|
||||
return bytes([random.randrange(0xFF) for _ in range(length)])
|
||||
|
||||
def zmq_sleep(t=1):
|
||||
if "ZMQ" in os.environ:
|
||||
time.sleep(t)
|
||||
|
||||
|
||||
# TODO: this should take any capnp struct and returrn a msg with random populated data
|
||||
def random_carstate():
|
||||
fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
|
||||
msg = messaging.new_message("carState")
|
||||
cs = msg.carState
|
||||
for f in fields:
|
||||
setattr(cs, f, random.random() * 10)
|
||||
return msg
|
||||
|
||||
# TODO: this should compare any capnp structs
|
||||
def assert_carstate(cs1, cs2):
|
||||
for f in car.CarState.schema.non_union_fields:
|
||||
# TODO: check all types
|
||||
val1, val2 = getattr(cs1, f), getattr(cs2, f)
|
||||
if isinstance(val1, numbers.Number):
|
||||
assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'"
|
||||
|
||||
def delayed_send(delay, sock, dat):
|
||||
def send_func():
|
||||
sock.send(dat)
|
||||
threading.Timer(delay, send_func).start()
|
||||
|
||||
|
||||
class TestMessaging:
|
||||
def setUp(self):
|
||||
# TODO: ZMQ tests are too slow; all sleeps will need to be
|
||||
# replaced with logic to block on the necessary condition
|
||||
if "ZMQ" in os.environ:
|
||||
pytest.skip()
|
||||
|
||||
# ZMQ pub socket takes too long to die
|
||||
# sleep to prevent multiple publishers error between tests
|
||||
zmq_sleep()
|
||||
|
||||
@parameterized.expand(events)
|
||||
def test_new_message(self, evt):
|
||||
try:
|
||||
msg = messaging.new_message(evt)
|
||||
except capnp.lib.capnp.KjException:
|
||||
msg = messaging.new_message(evt, random.randrange(200))
|
||||
assert (time.monotonic() - msg.logMonoTime) < 0.1
|
||||
assert not msg.valid
|
||||
assert evt == msg.which()
|
||||
|
||||
@parameterized.expand(events)
|
||||
def test_pub_sock(self, evt):
|
||||
messaging.pub_sock(evt)
|
||||
|
||||
@parameterized.expand(events)
|
||||
def test_sub_sock(self, evt):
|
||||
messaging.sub_sock(evt)
|
||||
|
||||
@parameterized.expand([
|
||||
(messaging.drain_sock, capnp._DynamicStructReader),
|
||||
(messaging.drain_sock_raw, bytes),
|
||||
])
|
||||
def test_drain_sock(self, func, expected_type):
|
||||
sock = "carState"
|
||||
pub_sock = messaging.pub_sock(sock)
|
||||
sub_sock = messaging.sub_sock(sock, timeout=1000)
|
||||
zmq_sleep()
|
||||
|
||||
# no wait and no msgs in queue
|
||||
msgs = func(sub_sock)
|
||||
assert isinstance(msgs, list)
|
||||
assert len(msgs) == 0
|
||||
|
||||
# no wait but msgs are queued up
|
||||
num_msgs = random.randrange(3, 10)
|
||||
for _ in range(num_msgs):
|
||||
pub_sock.send(messaging.new_message(sock).to_bytes())
|
||||
time.sleep(0.1)
|
||||
msgs = func(sub_sock)
|
||||
assert isinstance(msgs, list)
|
||||
assert all(isinstance(msg, expected_type) for msg in msgs)
|
||||
assert len(msgs) == num_msgs
|
||||
|
||||
def test_recv_sock(self):
|
||||
sock = "carState"
|
||||
pub_sock = messaging.pub_sock(sock)
|
||||
sub_sock = messaging.sub_sock(sock, timeout=100)
|
||||
zmq_sleep()
|
||||
|
||||
# no wait and no msg in queue, socket should timeout
|
||||
recvd = messaging.recv_sock(sub_sock)
|
||||
assert recvd is None
|
||||
|
||||
# no wait and one msg in queue
|
||||
msg = random_carstate()
|
||||
pub_sock.send(msg.to_bytes())
|
||||
time.sleep(0.01)
|
||||
recvd = messaging.recv_sock(sub_sock)
|
||||
assert isinstance(recvd, capnp._DynamicStructReader)
|
||||
# https://github.com/python/mypy/issues/13038
|
||||
assert_carstate(msg.carState, recvd.carState)
|
||||
|
||||
def test_recv_one(self):
|
||||
sock = "carState"
|
||||
pub_sock = messaging.pub_sock(sock)
|
||||
sub_sock = messaging.sub_sock(sock, timeout=1000)
|
||||
zmq_sleep()
|
||||
|
||||
# no msg in queue, socket should timeout
|
||||
recvd = messaging.recv_one(sub_sock)
|
||||
assert recvd is None
|
||||
|
||||
# one msg in queue
|
||||
msg = random_carstate()
|
||||
pub_sock.send(msg.to_bytes())
|
||||
recvd = messaging.recv_one(sub_sock)
|
||||
assert isinstance(recvd, capnp._DynamicStructReader)
|
||||
assert_carstate(msg.carState, recvd.carState)
|
||||
|
||||
@pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
|
||||
def test_recv_one_or_none(self):
|
||||
sock = "carState"
|
||||
pub_sock = messaging.pub_sock(sock)
|
||||
sub_sock = messaging.sub_sock(sock)
|
||||
zmq_sleep()
|
||||
|
||||
# no msg in queue, socket shouldn't block
|
||||
recvd = messaging.recv_one_or_none(sub_sock)
|
||||
assert recvd is None
|
||||
|
||||
# one msg in queue
|
||||
msg = random_carstate()
|
||||
pub_sock.send(msg.to_bytes())
|
||||
recvd = messaging.recv_one_or_none(sub_sock)
|
||||
assert isinstance(recvd, capnp._DynamicStructReader)
|
||||
assert_carstate(msg.carState, recvd.carState)
|
||||
|
||||
def test_recv_one_retry(self):
|
||||
sock = "carState"
|
||||
sock_timeout = 0.1
|
||||
pub_sock = messaging.pub_sock(sock)
|
||||
sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
|
||||
zmq_sleep()
|
||||
|
||||
# this test doesn't work with ZMQ since multiprocessing interrupts it
|
||||
if "ZMQ" not in os.environ:
|
||||
# wait 5 socket timeouts and make sure it's still retrying
|
||||
p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
|
||||
p.start()
|
||||
time.sleep(sock_timeout*5)
|
||||
assert p.is_alive()
|
||||
p.terminate()
|
||||
|
||||
# wait 5 socket timeouts before sending
|
||||
msg = random_carstate()
|
||||
start_time = time.monotonic()
|
||||
delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
|
||||
recvd = messaging.recv_one_retry(sub_sock)
|
||||
assert (time.monotonic() - start_time) >= sock_timeout*5
|
||||
assert isinstance(recvd, capnp._DynamicStructReader)
|
||||
assert_carstate(msg.carState, recvd.carState)
|
||||
160
cereal/messaging/tests/test_pub_sub_master.py
Normal file
160
cereal/messaging/tests/test_pub_sub_master.py
Normal file
@@ -0,0 +1,160 @@
|
||||
import random
|
||||
import time
|
||||
from typing import Sized, cast
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
|
||||
random_bytes, random_carstate, assert_carstate, \
|
||||
zmq_sleep
|
||||
from cereal.services import SERVICE_LIST
|
||||
|
||||
|
||||
class TestSubMaster:
|
||||
|
||||
def setup_method(self):
|
||||
# ZMQ pub socket takes too long to die
|
||||
# sleep to prevent multiple publishers error between tests
|
||||
zmq_sleep(3)
|
||||
|
||||
def test_init(self):
|
||||
sm = messaging.SubMaster(events)
|
||||
for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
|
||||
sm.sock, sm.data, sm.logMonoTime, sm.valid]:
|
||||
assert len(cast(Sized, p)) == len(events)
|
||||
|
||||
def test_init_state(self):
|
||||
socks = random_socks()
|
||||
sm = messaging.SubMaster(socks)
|
||||
assert sm.frame == -1
|
||||
assert not any(sm.updated.values())
|
||||
assert not any(sm.seen.values())
|
||||
on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in sm.services}
|
||||
assert all(sm.alive[s] == sm.valid[s] == sm.freq_ok[s] == on_demand[s] for s in sm.services)
|
||||
assert all(t == 0. for t in sm.recv_time.values())
|
||||
assert all(f == 0 for f in sm.recv_frame.values())
|
||||
assert all(t == 0 for t in sm.logMonoTime.values())
|
||||
|
||||
for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
|
||||
sm.sock, sm.data, sm.logMonoTime, sm.valid]:
|
||||
assert len(cast(Sized, p)) == len(socks)
|
||||
|
||||
def test_getitem(self):
|
||||
sock = "carState"
|
||||
pub_sock = messaging.pub_sock(sock)
|
||||
sm = messaging.SubMaster([sock,])
|
||||
zmq_sleep()
|
||||
|
||||
msg = random_carstate()
|
||||
pub_sock.send(msg.to_bytes())
|
||||
sm.update(1000)
|
||||
assert_carstate(msg.carState, sm[sock])
|
||||
|
||||
# TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs
|
||||
def test_update(self):
|
||||
sock = "carState"
|
||||
pub_sock = messaging.pub_sock(sock)
|
||||
sm = messaging.SubMaster([sock,])
|
||||
zmq_sleep()
|
||||
|
||||
for i in range(10):
|
||||
msg = messaging.new_message(sock)
|
||||
pub_sock.send(msg.to_bytes())
|
||||
sm.update(1000)
|
||||
assert sm.frame == i
|
||||
assert all(sm.updated.values())
|
||||
|
||||
def test_update_timeout(self):
|
||||
sock = random_sock()
|
||||
sm = messaging.SubMaster([sock,])
|
||||
timeout = random.randrange(1000, 3000)
|
||||
start_time = time.monotonic()
|
||||
sm.update(timeout)
|
||||
t = time.monotonic() - start_time
|
||||
assert t >= timeout/1000.
|
||||
assert t < 3
|
||||
assert not any(sm.updated.values())
|
||||
|
||||
def test_avg_frequency_checks(self):
|
||||
for poll in (True, False):
|
||||
sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"],
|
||||
poll=("modelV2" if poll else None),
|
||||
frequency=(20. if not poll else None))
|
||||
|
||||
checks = {
|
||||
"carState": (20, 20),
|
||||
"modelV2": (20, 20 if poll else 10),
|
||||
"cameraOdometry": (20, 10),
|
||||
"liveCalibration": (4, 4),
|
||||
"carParams": (None, None),
|
||||
"userBookmark": (None, None),
|
||||
}
|
||||
|
||||
for service, (max_freq, min_freq) in checks.items():
|
||||
if max_freq is not None:
|
||||
assert sm._check_avg_freq(service)
|
||||
assert sm.freq_tracker[service].max_freq == max_freq*1.2
|
||||
assert sm.freq_tracker[service].min_freq == min_freq*0.8
|
||||
else:
|
||||
assert not sm._check_avg_freq(service)
|
||||
|
||||
def test_alive(self):
|
||||
pass
|
||||
|
||||
def test_ignore_alive(self):
|
||||
pass
|
||||
|
||||
def test_valid(self):
|
||||
pass
|
||||
|
||||
# SubMaster should always conflate
|
||||
def test_conflate(self):
|
||||
sock = "carState"
|
||||
pub_sock = messaging.pub_sock(sock)
|
||||
sm = messaging.SubMaster([sock,])
|
||||
|
||||
n = 10
|
||||
for i in range(n+1):
|
||||
msg = messaging.new_message(sock)
|
||||
msg.carState.vEgo = i
|
||||
pub_sock.send(msg.to_bytes())
|
||||
time.sleep(0.01)
|
||||
sm.update(1000)
|
||||
assert sm[sock].vEgo == n
|
||||
|
||||
|
||||
class TestPubMaster:
|
||||
|
||||
def setup_method(self):
|
||||
# ZMQ pub socket takes too long to die
|
||||
# sleep to prevent multiple publishers error between tests
|
||||
zmq_sleep(3)
|
||||
|
||||
def test_init(self):
|
||||
messaging.PubMaster(events)
|
||||
|
||||
def test_send(self):
|
||||
socks = random_socks()
|
||||
pm = messaging.PubMaster(socks)
|
||||
sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
|
||||
zmq_sleep()
|
||||
|
||||
# PubMaster accepts either a capnp msg builder or bytes
|
||||
for capnp in [True, False]:
|
||||
for i in range(100):
|
||||
sock = socks[i % len(socks)]
|
||||
|
||||
if capnp:
|
||||
try:
|
||||
msg = messaging.new_message(sock)
|
||||
except Exception:
|
||||
msg = messaging.new_message(sock, random.randrange(50))
|
||||
else:
|
||||
msg = random_bytes()
|
||||
|
||||
pm.send(sock, msg)
|
||||
recvd = sub_socks[sock].receive()
|
||||
|
||||
if capnp:
|
||||
msg.clear_write_flag()
|
||||
msg = msg.to_bytes()
|
||||
assert msg == recvd, i
|
||||
27
cereal/messaging/tests/test_services.py
Normal file
27
cereal/messaging/tests/test_services.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Dict
|
||||
from parameterized import parameterized
|
||||
|
||||
from cereal import log
|
||||
import cereal.services as services
|
||||
from cereal.services import SERVICE_LIST
|
||||
|
||||
|
||||
class TestServices:
|
||||
|
||||
@parameterized.expand(SERVICE_LIST.keys())
|
||||
def test_services(self, s):
|
||||
service = SERVICE_LIST[s]
|
||||
assert service.frequency <= 104
|
||||
assert service.decimation != 0
|
||||
|
||||
def test_generated_header(self):
|
||||
with tempfile.NamedTemporaryFile(suffix=".h") as f:
|
||||
ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
|
||||
assert ret == 0, "generated services header is not valid C"
|
||||
|
||||
def test_all_services_exist_in_log_union(self):
|
||||
event_fields = set(log.Event.schema.union_fields)
|
||||
missing = sorted(s for s in SERVICE_LIST if s not in event_fields)
|
||||
assert not missing, f"services missing from log.capnp Event union: {missing}"
|
||||
222
cereal/messaging/tests/validate_sp_cereal_upstream.py
Executable file
222
cereal/messaging/tests/validate_sp_cereal_upstream.py
Executable file
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import sys
|
||||
from typing import Any, List, Tuple
|
||||
|
||||
DEBUG = False
|
||||
|
||||
|
||||
def print_debug(string: str) -> None:
|
||||
if DEBUG:
|
||||
print(string)
|
||||
|
||||
|
||||
def create_schema_instance(struct: Any, prop: Tuple[str, Any]) -> Any:
|
||||
"""
|
||||
Create a new instance of a schema type, handling different field types.
|
||||
|
||||
Args:
|
||||
struct: The Cap'n Proto schema structure
|
||||
prop: A tuple containing the field name and field metadata
|
||||
|
||||
Returns:
|
||||
A new initialized schema instance
|
||||
"""
|
||||
struct_instance = struct.new_message()
|
||||
field_name, field_metadata = prop
|
||||
|
||||
try:
|
||||
field_type = field_metadata.proto.slot.type.which()
|
||||
|
||||
# Initialize different types of fields
|
||||
if field_type in ('list', 'text', 'data'):
|
||||
struct_instance.init(field_name, 1)
|
||||
print_debug(f"Initialized list/text/data field: {field_name}")
|
||||
elif field_type in ('struct', 'object'):
|
||||
struct_instance.init(field_name)
|
||||
print_debug(f"Initialized struct/object field: {field_name}")
|
||||
|
||||
return struct_instance
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating instance for {field_name}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def get_schema_fields(schema_struct: Any) -> List[Tuple[str, Any]]:
|
||||
"""
|
||||
Retrieve all fields from a given schema structure.
|
||||
|
||||
Args:
|
||||
schema_struct: The Cap'n Proto schema structure
|
||||
|
||||
Returns:
|
||||
A list of field names and their metadata
|
||||
"""
|
||||
try:
|
||||
# Get all fields from the schema
|
||||
schema_fields = list(schema_struct.schema.fields.items())
|
||||
|
||||
print_debug("Discovered schema fields:")
|
||||
for field_name, field_metadata in schema_fields:
|
||||
print_debug(f"- {field_name}")
|
||||
|
||||
return schema_fields
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error retrieving schema fields: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def generate_schema_instances(schema_struct: Any) -> List[Any]:
|
||||
"""
|
||||
Generate instances for all fields in a given schema.
|
||||
|
||||
Args:
|
||||
schema_struct: The Cap'n Proto schema structure
|
||||
|
||||
Returns:
|
||||
A list of schema instances
|
||||
"""
|
||||
schema_fields = get_schema_fields(schema_struct)
|
||||
instances = []
|
||||
|
||||
for field_prop in schema_fields:
|
||||
try:
|
||||
instance = create_schema_instance(schema_struct, field_prop)
|
||||
if instance is not None:
|
||||
instances.append(instance)
|
||||
except Exception as e:
|
||||
print(f"Skipping field due to error: {e}")
|
||||
|
||||
print(f"Generated {len(instances)} schema instances")
|
||||
return instances
|
||||
|
||||
|
||||
def persist_instances(instances: List[Any], filename: str) -> None:
|
||||
"""
|
||||
Write schema instances to a binary file.
|
||||
|
||||
Args:
|
||||
instances: List of schema instances
|
||||
filename: Output file path
|
||||
"""
|
||||
try:
|
||||
with open(filename, 'wb') as f:
|
||||
for instance in instances:
|
||||
f.write(instance.to_bytes())
|
||||
|
||||
print(f"Successfully wrote {len(instances)} instances to {filename}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error persisting instances: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def read_instances(filename: str, schema_type: Any) -> List[Any]:
|
||||
"""
|
||||
Read schema instances from a binary file.
|
||||
|
||||
Args:
|
||||
filename: Input file path
|
||||
schema_type: The schema type to use for reading
|
||||
|
||||
Returns:
|
||||
A list of read schema instances
|
||||
"""
|
||||
try:
|
||||
with open(filename, 'rb') as f:
|
||||
data = f.read()
|
||||
|
||||
instances = list(schema_type.read_multiple_bytes(data))
|
||||
|
||||
print(f"Read {len(instances)} instances from {filename}")
|
||||
return instances
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error reading instances: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def compare_schemas(original_instances: List[Any], read_instances: List[Any]) -> bool:
|
||||
"""
|
||||
Compare original and read-back instances to detect potential breaking changes.
|
||||
|
||||
Args:
|
||||
original_instances: List of originally generated instances
|
||||
read_instances: List of instances read back from file
|
||||
|
||||
Returns:
|
||||
Boolean indicating whether schemas appear compatible
|
||||
"""
|
||||
if len(original_instances) != len(read_instances):
|
||||
print("❌ Schema Compatibility Warning: Instance count mismatch")
|
||||
return False
|
||||
|
||||
compatible = True
|
||||
for struct in read_instances:
|
||||
try:
|
||||
getattr(struct, struct.which()) # Attempting to access the field to validate readability
|
||||
except Exception as e:
|
||||
print(f"❌ Structural change detected: {struct.which()} is not readable.\nFull error: {e}")
|
||||
compatible = False
|
||||
|
||||
return compatible
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
CLI entry point for schema compatibility testing.
|
||||
"""
|
||||
# Setup argument parser
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Cap\'n Proto Schema Compatibility Testing Tool',
|
||||
epilog='Test schema compatibility by generating and reading back instances.'
|
||||
)
|
||||
|
||||
# Add mutually exclusive group for generation or reading mode
|
||||
mode_group = parser.add_mutually_exclusive_group(required=True)
|
||||
mode_group.add_argument('-g', '--generate', action='store_true',
|
||||
help='Generate schema instances')
|
||||
mode_group.add_argument('-r', '--read', action='store_true',
|
||||
help='Read and validate schema instances')
|
||||
|
||||
# Common arguments
|
||||
parser.add_argument('-f', '--file',
|
||||
default='schema_instances.bin',
|
||||
help='Output/input binary file (default: schema_instances.bin)')
|
||||
|
||||
# Parse arguments
|
||||
args = parser.parse_args()
|
||||
|
||||
# Import the schema dynamically
|
||||
try:
|
||||
from cereal import log
|
||||
schema_type = log.Event
|
||||
except ImportError:
|
||||
print("Error: Unable to import schema. Ensure 'cereal' is installed.")
|
||||
sys.exit(1)
|
||||
|
||||
# Execute based on mode
|
||||
if args.generate:
|
||||
print("🔧 Generating Schema Instances")
|
||||
instances = generate_schema_instances(schema_type)
|
||||
persist_instances(instances, args.file)
|
||||
print("✅ Instance generation complete")
|
||||
|
||||
elif args.read:
|
||||
print("🔍 Reading and Validating Schema Instances")
|
||||
generated_instances = generate_schema_instances(schema_type)
|
||||
read_back_instances = read_instances(args.file, schema_type)
|
||||
|
||||
# Compare schemas
|
||||
if compare_schemas(generated_instances, read_back_instances):
|
||||
print("✅ Schema Compatibility: No breaking changes detected")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("❌ Potential Schema Breaking Changes Detected")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
147
cereal/services.py
Executable file
147
cereal/services.py
Executable file
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
from enum import IntEnum
|
||||
from typing import Optional
|
||||
|
||||
|
||||
# TODO: this should be automatically determined using the capnp schema
|
||||
class QueueSize(IntEnum):
|
||||
BIG = 10 * 1024 * 1024 # 10MB - video frames, large AI outputs
|
||||
MEDIUM = 2 * 1024 * 1024 # 2MB - high freq (CAN), livestream
|
||||
SMALL = 250 * 1024 # 250KB - most services
|
||||
|
||||
|
||||
class Service:
|
||||
def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
|
||||
queue_size: QueueSize = QueueSize.SMALL):
|
||||
self.should_log = should_log
|
||||
self.frequency = frequency
|
||||
self.decimation = decimation
|
||||
self.queue_size = queue_size
|
||||
|
||||
|
||||
_services: dict[str, tuple] = {
|
||||
# service: (should_log, frequency, qlog decimation (optional))
|
||||
# note: the "EncodeIdx" packets will still be in the log
|
||||
"gyroscope": (True, 104., 104),
|
||||
"accelerometer": (True, 104., 104),
|
||||
"magnetometer": (True, 25.),
|
||||
"lightSensor": (True, 100., 100),
|
||||
"temperatureSensor": (True, 2., 200),
|
||||
"gpsNMEA": (True, 9.),
|
||||
"deviceState": (True, 2., 1),
|
||||
"touch": (True, 20., 1),
|
||||
"can": (True, 100., 2053, QueueSize.BIG), # decimation gives ~3 msgs in a full segment
|
||||
"controlsState": (True, 100., 10, QueueSize.MEDIUM),
|
||||
"selfdriveState": (True, 100., 10),
|
||||
"pandaStates": (True, 10., 1),
|
||||
"peripheralState": (True, 2., 1),
|
||||
"radarState": (True, 20., 5),
|
||||
"roadEncodeIdx": (False, 20., 1),
|
||||
"liveTracks": (True, 20.),
|
||||
"sendcan": (True, 100., 139, QueueSize.MEDIUM),
|
||||
"logMessage": (True, 0.),
|
||||
"errorLogMessage": (True, 0., 1),
|
||||
"liveCalibration": (True, 4., 4),
|
||||
"liveTorqueParameters": (True, 4., 1),
|
||||
"liveDelay": (True, 4., 1),
|
||||
"androidLog": (True, 0.),
|
||||
"carState": (True, 100., 10),
|
||||
"carControl": (True, 100., 10),
|
||||
"carOutput": (True, 100., 10),
|
||||
"longitudinalPlan": (True, 20., 10),
|
||||
"driverAssistance": (True, 20., 20),
|
||||
"procLog": (True, 0.5, 15, QueueSize.BIG),
|
||||
"gpsLocationExternal": (True, 10., 10),
|
||||
"gpsLocation": (True, 1., 1),
|
||||
"ubloxGnss": (True, 10.),
|
||||
"qcomGnss": (True, 2.),
|
||||
"gnssMeasurements": (True, 10., 10),
|
||||
"clocks": (True, 0.1, 1),
|
||||
"ubloxRaw": (True, 20.),
|
||||
"livePose": (True, 20., 4),
|
||||
"liveParameters": (True, 20., 5),
|
||||
"cameraOdometry": (True, 20., 10),
|
||||
"thumbnail": (True, 1 / 60., 1),
|
||||
"onroadEvents": (True, 1., 1),
|
||||
"carParams": (True, 0.02, 1),
|
||||
"roadCameraState": (True, 20., 20),
|
||||
"driverCameraState": (True, 20., 20),
|
||||
"driverEncodeIdx": (False, 20., 1),
|
||||
"driverStateV2": (True, 20., 10),
|
||||
"driverMonitoringState": (True, 20., 10),
|
||||
"wideRoadEncodeIdx": (False, 20., 1),
|
||||
"wideRoadCameraState": (True, 20., 20),
|
||||
"drivingModelData": (True, 20., 10),
|
||||
"modelV2": (True, 20., None, QueueSize.BIG),
|
||||
"managerState": (True, 2., 1),
|
||||
"uploaderState": (True, 0., 1),
|
||||
"navInstruction": (True, 1., 10),
|
||||
"navRoute": (True, 0.),
|
||||
"navThumbnail": (True, 0.),
|
||||
"qRoadEncodeIdx": (False, 20.),
|
||||
"userBookmark": (True, 0., 1),
|
||||
"soundPressure": (True, 10., 10),
|
||||
"rawAudioData": (False, 20.),
|
||||
"bookmarkButton": (True, 0., 1),
|
||||
"audioFeedback": (True, 0., 1),
|
||||
"roadEncodeData": (False, 20., None, QueueSize.BIG),
|
||||
"driverEncodeData": (False, 20., None, QueueSize.BIG),
|
||||
"wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
|
||||
"qRoadEncodeData": (False, 20., None, QueueSize.BIG),
|
||||
|
||||
# iqpilot
|
||||
"iqModelManager": (False, 1., 1, QueueSize.BIG),
|
||||
"backupManagerK3": (False, 1., 1),
|
||||
"iqCarParams": (True, 0.02, 1),
|
||||
"iqCarControl": (True, 100., 10),
|
||||
"iqCarState": (True, 100., 10),
|
||||
"iqLiveMapData": (True, 1., 1),
|
||||
"iqNavState": (True, 5., 10),
|
||||
"iqState": (True, 100., 10),
|
||||
"iqPlan": (True, 20., 10),
|
||||
"iqOnroadEvents": (True, 1., 1),
|
||||
"iqModelDataV2": (True, 20., None, QueueSize.BIG),
|
||||
"liveLocationKalman": (True, 20.),
|
||||
|
||||
# debug
|
||||
"uiDebug": (True, 0., 1),
|
||||
"testJoystick": (True, 0.),
|
||||
"alertDebug": (True, 20., 5),
|
||||
"livestreamWideRoadEncodeIdx": (False, 20.),
|
||||
"livestreamRoadEncodeIdx": (False, 20.),
|
||||
"livestreamDriverEncodeIdx": (False, 20.),
|
||||
"livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
|
||||
"livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
|
||||
"livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
|
||||
"customReservedRawData0": (True, 0.),
|
||||
"customReservedRawData1": (True, 0.),
|
||||
"customReservedRawData2": (True, 0.),
|
||||
}
|
||||
SERVICE_LIST = {name: Service(*vals) for
|
||||
idx, (name, vals) in enumerate(_services.items())}
|
||||
|
||||
|
||||
def build_header():
|
||||
h = ""
|
||||
h += "/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */\n"
|
||||
h += "#ifndef __SERVICES_H\n"
|
||||
h += "#define __SERVICES_H\n"
|
||||
|
||||
h += "#include <map>\n"
|
||||
h += "#include <string>\n"
|
||||
|
||||
h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
|
||||
h += "static std::map<std::string, service> services = {\n"
|
||||
for k, v in SERVICE_LIST.items():
|
||||
should_log = "true" if v.should_log else "false"
|
||||
decimation = -1 if v.decimation is None else v.decimation
|
||||
h += ' { "%s", {"%s", %s, %f, %d, %d}},\n' % \
|
||||
(k, k, should_log, v.frequency, decimation, v.queue_size)
|
||||
h += "};\n"
|
||||
|
||||
h += "#endif\n"
|
||||
return h
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(build_header())
|
||||
1
common/.gitignore
vendored
Normal file
1
common/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
*.cpp
|
||||
24
common/SConscript
Normal file
24
common/SConscript
Normal file
@@ -0,0 +1,24 @@
|
||||
Import('env', 'envCython', 'arch')
|
||||
|
||||
common_libs = [
|
||||
'params.cc',
|
||||
'swaglog.cc',
|
||||
'util.cc',
|
||||
'ratekeeper.cc',
|
||||
'clutil.cc',
|
||||
]
|
||||
|
||||
_common = env.Library('common', common_libs, LIBS="json11")
|
||||
Export('_common')
|
||||
|
||||
if GetOption('extras'):
|
||||
env.Program('tests/test_common',
|
||||
['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
|
||||
LIBS=[_common, 'json11', 'zmq', 'pthread'])
|
||||
|
||||
# Cython bindings
|
||||
params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
|
||||
|
||||
common_python = [params_python]
|
||||
|
||||
Export('common_python')
|
||||
0
common/__init__.py
Normal file
0
common/__init__.py
Normal file
26
common/api/__init__.py
Normal file
26
common/api/__init__.py
Normal file
@@ -0,0 +1,26 @@
|
||||
import openpilot.common.api.comma_connect
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, dongle_id):
|
||||
self.service = openpilot.common.api.comma_connect.CommaConnectApi(dongle_id)
|
||||
|
||||
def request(self, method, endpoint, **params):
|
||||
return self.service.request(method, endpoint, **params)
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
return self.service.get(*args, **kwargs)
|
||||
|
||||
def post(self, *args, **kwargs):
|
||||
return self.service.post(*args, **kwargs)
|
||||
|
||||
def get_token(self, payload_extra=None, expiry_hours=1):
|
||||
return self.service.get_token(payload_extra, expiry_hours)
|
||||
|
||||
|
||||
def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
|
||||
return openpilot.common.api.comma_connect.CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
|
||||
|
||||
|
||||
def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
|
||||
return openpilot.common.api.comma_connect.CommaConnectApi(None).get_key_pair()
|
||||
72
common/api/base.py
Normal file
72
common/api/base.py
Normal file
@@ -0,0 +1,72 @@
|
||||
import jwt
|
||||
import os
|
||||
import requests
|
||||
import unicodedata
|
||||
from datetime import datetime, timedelta, UTC
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from openpilot.system.version import get_version
|
||||
|
||||
# name: jwt signature algorithm
|
||||
KEYS = {"id_rsa": "RS256",
|
||||
"id_ecdsa": "ES256"}
|
||||
|
||||
|
||||
class BaseApi:
|
||||
def __init__(self, dongle_id, api_host, user_agent="openpilot-"):
|
||||
self.dongle_id = dongle_id
|
||||
self.api_host = api_host
|
||||
self.user_agent = user_agent
|
||||
self.jwt_algorithm, self.private_key, _ = self.get_key_pair()
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
return self.request('GET', *args, **kwargs)
|
||||
|
||||
def post(self, *args, **kwargs):
|
||||
return self.request('POST', *args, **kwargs)
|
||||
|
||||
def request(self, method, endpoint, timeout=None, access_token=None, **params):
|
||||
return self.api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
|
||||
|
||||
def _get_token(self, payload_extra=None, expiry_hours=1, **extra_payload):
|
||||
now = datetime.now(UTC).replace(tzinfo=None)
|
||||
payload = {
|
||||
'identity': self.dongle_id,
|
||||
'nbf': now,
|
||||
'iat': now,
|
||||
'exp': now + timedelta(hours=expiry_hours),
|
||||
**extra_payload
|
||||
}
|
||||
if payload_extra is not None:
|
||||
payload.update(payload_extra)
|
||||
token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
|
||||
if isinstance(token, bytes):
|
||||
token = token.decode('utf8')
|
||||
return token
|
||||
|
||||
def get_token(self, payload_extra=None, expiry_hours=1):
|
||||
return self._get_token(payload_extra, expiry_hours)
|
||||
|
||||
def remove_non_ascii_chars(self, text):
|
||||
normalized_text = unicodedata.normalize('NFD', text)
|
||||
ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
|
||||
return ascii_encoded_text.decode()
|
||||
|
||||
def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
|
||||
headers = {}
|
||||
if access_token is not None:
|
||||
headers['Authorization'] = "JWT " + access_token
|
||||
|
||||
version = self.remove_non_ascii_chars(get_version())
|
||||
headers['User-Agent'] = self.user_agent + version
|
||||
|
||||
# TODO: add session to Api
|
||||
req = requests if session is None else session
|
||||
return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
|
||||
|
||||
@staticmethod
|
||||
def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
|
||||
for key in KEYS:
|
||||
if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
|
||||
with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
|
||||
return KEYS[key], private.read(), public.read()
|
||||
return None, None, None
|
||||
11
common/api/comma_connect.py
Normal file
11
common/api/comma_connect.py
Normal file
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
|
||||
from openpilot.common.api.base import BaseApi
|
||||
|
||||
API_HOST = os.getenv('API_HOST', 'https://api-konn3kt.teallvbs.xyz')
|
||||
|
||||
|
||||
class CommaConnectApi(BaseApi):
|
||||
def __init__(self, dongle_id):
|
||||
super().__init__(dongle_id, API_HOST)
|
||||
self.user_agent = "openpilot-"
|
||||
4
common/basedir.py
Normal file
4
common/basedir.py
Normal file
@@ -0,0 +1,4 @@
|
||||
import os
|
||||
|
||||
|
||||
BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
|
||||
98
common/clutil.cc
Normal file
98
common/clutil.cc
Normal file
@@ -0,0 +1,98 @@
|
||||
#include "common/clutil.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
|
||||
#include "common/util.h"
|
||||
#include "common/swaglog.h"
|
||||
|
||||
namespace { // helper functions
|
||||
|
||||
template <typename Func, typename Id, typename Name>
|
||||
std::string get_info(Func get_info_func, Id id, Name param_name) {
|
||||
size_t size = 0;
|
||||
CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
|
||||
std::string info(size, '\0');
|
||||
CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
|
||||
return info;
|
||||
}
|
||||
inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
|
||||
inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
|
||||
|
||||
void cl_print_info(cl_platform_id platform, cl_device_id device) {
|
||||
size_t work_group_size = 0;
|
||||
cl_device_type device_type = 0;
|
||||
clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
|
||||
clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
|
||||
const char *type_str = "Other...";
|
||||
switch (device_type) {
|
||||
case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
|
||||
case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
|
||||
case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
|
||||
}
|
||||
|
||||
LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
|
||||
LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
|
||||
LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
|
||||
LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
|
||||
LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
|
||||
LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
|
||||
LOGD("max work group size: %zu", work_group_size);
|
||||
LOGD("type = %d, %s", (int)device_type, type_str);
|
||||
}
|
||||
|
||||
void cl_print_build_errors(cl_program program, cl_device_id device) {
|
||||
cl_build_status status;
|
||||
clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
|
||||
size_t log_size;
|
||||
clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
|
||||
std::string log(log_size, '\0');
|
||||
clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
|
||||
|
||||
LOGE("build failed; status=%d, log: %s", status, log.c_str());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
cl_device_id cl_get_device_id(cl_device_type device_type) {
|
||||
cl_uint num_platforms = 0;
|
||||
CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
|
||||
std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
|
||||
CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
|
||||
|
||||
for (size_t i = 0; i < num_platforms; ++i) {
|
||||
LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
|
||||
|
||||
// Get first device
|
||||
if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
|
||||
cl_print_info(platform_ids[i], device_id);
|
||||
return device_id;
|
||||
}
|
||||
}
|
||||
LOGE("No valid openCL platform found");
|
||||
assert(0);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
cl_context cl_create_context(cl_device_id device_id) {
|
||||
return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
|
||||
}
|
||||
|
||||
void cl_release_context(cl_context context) {
|
||||
clReleaseContext(context);
|
||||
}
|
||||
|
||||
cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
|
||||
return cl_program_from_source(ctx, device_id, util::read_file(path), args);
|
||||
}
|
||||
|
||||
cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
|
||||
const char *csrc = src.c_str();
|
||||
cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
|
||||
if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
|
||||
cl_print_build_errors(prg, device_id);
|
||||
assert(0);
|
||||
}
|
||||
return prg;
|
||||
}
|
||||
28
common/clutil.h
Normal file
28
common/clutil.h
Normal file
@@ -0,0 +1,28 @@
|
||||
#pragma once
|
||||
|
||||
#ifdef __APPLE__
|
||||
#include <OpenCL/cl.h>
|
||||
#else
|
||||
#include <CL/cl.h>
|
||||
#endif
|
||||
|
||||
#include <string>
|
||||
|
||||
#define CL_CHECK(_expr) \
|
||||
do { \
|
||||
assert(CL_SUCCESS == (_expr)); \
|
||||
} while (0)
|
||||
|
||||
#define CL_CHECK_ERR(_expr) \
|
||||
({ \
|
||||
cl_int err = CL_INVALID_VALUE; \
|
||||
__typeof__(_expr) _ret = _expr; \
|
||||
assert(_ret&& err == CL_SUCCESS); \
|
||||
_ret; \
|
||||
})
|
||||
|
||||
cl_device_id cl_get_device_id(cl_device_type device_type);
|
||||
cl_context cl_create_context(cl_device_id device_id);
|
||||
void cl_release_context(cl_context context);
|
||||
cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
|
||||
cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);
|
||||
23
common/constants.py
Normal file
23
common/constants.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import numpy as np
|
||||
|
||||
# conversions
|
||||
class CV:
|
||||
# Speed
|
||||
MPH_TO_KPH = 1.609344
|
||||
KPH_TO_MPH = 1. / MPH_TO_KPH
|
||||
MS_TO_KPH = 3.6
|
||||
KPH_TO_MS = 1. / MS_TO_KPH
|
||||
MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
|
||||
MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
|
||||
MS_TO_KNOTS = 1.9438
|
||||
KNOTS_TO_MS = 1. / MS_TO_KNOTS
|
||||
|
||||
# Angle
|
||||
DEG_TO_RAD = np.pi / 180.
|
||||
RAD_TO_DEG = 1. / DEG_TO_RAD
|
||||
|
||||
# Mass
|
||||
LB_TO_KG = 0.453592
|
||||
|
||||
|
||||
ACCELERATION_DUE_TO_GRAVITY = 9.81 # m/s^2
|
||||
1
common/file_helpers.py
Normal file
1
common/file_helpers.py
Normal file
@@ -0,0 +1 @@
|
||||
from openpilot.common.utils import CallbackReader, get_upload_stream
|
||||
34
common/filter_simple.py
Normal file
34
common/filter_simple.py
Normal file
@@ -0,0 +1,34 @@
|
||||
class FirstOrderFilter:
|
||||
def __init__(self, x0, rc, dt, initialized=True):
|
||||
self.x = x0
|
||||
self.dt = dt
|
||||
self.update_alpha(rc)
|
||||
self.initialized = initialized
|
||||
|
||||
def update_alpha(self, rc):
|
||||
self.alpha = self.dt / (rc + self.dt)
|
||||
|
||||
def update(self, x):
|
||||
if self.initialized:
|
||||
self.x = (1. - self.alpha) * self.x + self.alpha * x
|
||||
else:
|
||||
self.initialized = True
|
||||
self.x = x
|
||||
return self.x
|
||||
|
||||
|
||||
class BounceFilter(FirstOrderFilter):
|
||||
def __init__(self, x0, rc, dt, initialized=True, bounce=2):
|
||||
self.velocity = FirstOrderFilter(0.0, 0.15, dt)
|
||||
self.bounce = bounce
|
||||
super().__init__(x0, rc, dt, initialized)
|
||||
|
||||
def update(self, x):
|
||||
super().update(x)
|
||||
scale = self.dt / (1.0 / 60.0) # tuned at 60 fps
|
||||
self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
|
||||
self.velocity.update(0.0)
|
||||
if abs(self.velocity.x) < 1e-5:
|
||||
self.velocity.x = 0.0
|
||||
self.x += self.velocity.x
|
||||
return self.x
|
||||
42
common/git.py
Normal file
42
common/git.py
Normal file
@@ -0,0 +1,42 @@
|
||||
from functools import cache
|
||||
import subprocess
|
||||
from openpilot.common.utils import run_cmd, run_cmd_default
|
||||
|
||||
|
||||
@cache
|
||||
def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
|
||||
return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
|
||||
|
||||
|
||||
@cache
|
||||
def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
|
||||
return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
|
||||
|
||||
|
||||
@cache
|
||||
def get_short_branch(cwd: str | None = None) -> str:
|
||||
return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
|
||||
|
||||
|
||||
@cache
|
||||
def get_branch(cwd: str | None = None) -> str:
|
||||
return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
|
||||
|
||||
|
||||
@cache
|
||||
def get_origin(cwd: str | None = None) -> str:
|
||||
try:
|
||||
local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
|
||||
tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
|
||||
return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd)
|
||||
except subprocess.CalledProcessError: # Not on a branch, fallback
|
||||
return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd)
|
||||
|
||||
|
||||
@cache
|
||||
def get_normalized_origin(cwd: str | None = None) -> str:
|
||||
return get_origin(cwd) \
|
||||
.replace("git@", "", 1) \
|
||||
.replace(".git", "", 1) \
|
||||
.replace("https://", "", 1) \
|
||||
.replace(":", "/", 1)
|
||||
89
common/gpio.py
Normal file
89
common/gpio.py
Normal file
@@ -0,0 +1,89 @@
|
||||
import os
|
||||
import fcntl
|
||||
import ctypes
|
||||
from functools import cache
|
||||
|
||||
def gpio_init(pin: int, output: bool) -> None:
|
||||
try:
|
||||
with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
|
||||
f.write(b"out" if output else b"in")
|
||||
except Exception as e:
|
||||
print(f"Failed to set gpio {pin} direction: {e}")
|
||||
|
||||
def gpio_set(pin: int, high: bool) -> None:
|
||||
try:
|
||||
with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
|
||||
f.write(b"1" if high else b"0")
|
||||
except Exception as e:
|
||||
print(f"Failed to set gpio {pin} value: {e}")
|
||||
|
||||
def gpio_read(pin: int) -> bool | None:
|
||||
val = None
|
||||
try:
|
||||
with open(f"/sys/class/gpio/gpio{pin}/value", 'rb') as f:
|
||||
val = bool(int(f.read().strip()))
|
||||
except Exception as e:
|
||||
print(f"Failed to set gpio {pin} value: {e}")
|
||||
|
||||
return val
|
||||
|
||||
def gpio_export(pin: int) -> None:
|
||||
if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
|
||||
return
|
||||
|
||||
try:
|
||||
with open("/sys/class/gpio/export", 'w') as f:
|
||||
f.write(str(pin))
|
||||
except Exception:
|
||||
print(f"Failed to export gpio {pin}")
|
||||
|
||||
@cache
|
||||
def get_irq_action(irq: int) -> list[str]:
|
||||
try:
|
||||
with open(f"/sys/kernel/irq/{irq}/actions") as f:
|
||||
actions = f.read().strip().split(',')
|
||||
return actions
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
|
||||
def get_irqs_for_action(action: str) -> list[str]:
|
||||
ret = []
|
||||
with open("/proc/interrupts") as f:
|
||||
for l in f.readlines():
|
||||
irq = l.split(':')[0].strip()
|
||||
if irq.isdigit() and action in get_irq_action(irq):
|
||||
ret.append(irq)
|
||||
return ret
|
||||
|
||||
# *** gpiochip ***
|
||||
|
||||
class gpioevent_data(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("timestamp", ctypes.c_uint64),
|
||||
("id", ctypes.c_uint32),
|
||||
]
|
||||
|
||||
class gpioevent_request(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("lineoffset", ctypes.c_uint32),
|
||||
("handleflags", ctypes.c_uint32),
|
||||
("eventflags", ctypes.c_uint32),
|
||||
("label", ctypes.c_char * 32),
|
||||
("fd", ctypes.c_int)
|
||||
]
|
||||
|
||||
def gpiochip_get_ro_value_fd(label: str, gpiochip_id: int, pin: int) -> int:
|
||||
GPIOEVENT_REQUEST_BOTH_EDGES = 0x3
|
||||
GPIOHANDLE_REQUEST_INPUT = 0x1
|
||||
GPIO_GET_LINEEVENT_IOCTL = 0xc030b404
|
||||
|
||||
rq = gpioevent_request()
|
||||
rq.lineoffset = pin
|
||||
rq.handleflags = GPIOHANDLE_REQUEST_INPUT
|
||||
rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES
|
||||
rq.label = label.encode('utf-8')[:31] + b'\0'
|
||||
|
||||
fd = os.open(f"/dev/gpiochip{gpiochip_id}", os.O_RDONLY)
|
||||
fcntl.ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, rq)
|
||||
os.close(fd)
|
||||
return int(rq.fd)
|
||||
8
common/gps.py
Normal file
8
common/gps.py
Normal file
@@ -0,0 +1,8 @@
|
||||
from openpilot.common.params import Params
|
||||
|
||||
|
||||
def get_gps_location_service(params: Params) -> str:
|
||||
if params.get_bool("UbloxAvailable"):
|
||||
return "gpsLocationExternal"
|
||||
else:
|
||||
return "gpsLocation"
|
||||
81
common/i2c.py
Normal file
81
common/i2c.py
Normal file
@@ -0,0 +1,81 @@
|
||||
import os
|
||||
import fcntl
|
||||
import ctypes
|
||||
|
||||
# I2C constants from /usr/include/linux/i2c-dev.h
|
||||
I2C_SLAVE = 0x0703
|
||||
I2C_SLAVE_FORCE = 0x0706
|
||||
I2C_SMBUS = 0x0720
|
||||
|
||||
# SMBus transfer types
|
||||
I2C_SMBUS_READ = 1
|
||||
I2C_SMBUS_WRITE = 0
|
||||
I2C_SMBUS_BYTE_DATA = 2
|
||||
I2C_SMBUS_I2C_BLOCK_DATA = 8
|
||||
|
||||
I2C_SMBUS_BLOCK_MAX = 32
|
||||
|
||||
|
||||
class _I2cSmbusData(ctypes.Union):
|
||||
_fields_ = [
|
||||
("byte", ctypes.c_uint8),
|
||||
("word", ctypes.c_uint16),
|
||||
("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
|
||||
]
|
||||
|
||||
|
||||
class _I2cSmbusIoctlData(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("read_write", ctypes.c_uint8),
|
||||
("command", ctypes.c_uint8),
|
||||
("size", ctypes.c_uint32),
|
||||
("data", ctypes.POINTER(_I2cSmbusData)),
|
||||
]
|
||||
|
||||
|
||||
class SMBus:
|
||||
def __init__(self, bus: int):
|
||||
self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
|
||||
|
||||
def __enter__(self) -> 'SMBus':
|
||||
return self
|
||||
|
||||
def __exit__(self, *args) -> None:
|
||||
self.close()
|
||||
|
||||
def close(self) -> None:
|
||||
if hasattr(self, '_fd') and self._fd >= 0:
|
||||
os.close(self._fd)
|
||||
self._fd = -1
|
||||
|
||||
def _set_address(self, addr: int, force: bool = False) -> None:
|
||||
ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
|
||||
fcntl.ioctl(self._fd, ioctl_arg, addr)
|
||||
|
||||
def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
|
||||
ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
|
||||
fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
|
||||
|
||||
def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
|
||||
self._set_address(addr, force)
|
||||
data = _I2cSmbusData()
|
||||
self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
|
||||
return int(data.byte)
|
||||
|
||||
def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
|
||||
self._set_address(addr, force)
|
||||
data = _I2cSmbusData()
|
||||
data.byte = value & 0xFF
|
||||
self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
|
||||
|
||||
def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
|
||||
self._set_address(addr, force)
|
||||
if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
|
||||
raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
|
||||
|
||||
data = _I2cSmbusData()
|
||||
data.block[0] = length
|
||||
self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
|
||||
read_len = int(data.block[0]) or length
|
||||
read_len = min(read_len, length)
|
||||
return [int(b) for b in data.block[1 : read_len + 1]]
|
||||
249
common/logging_extra.py
Normal file
249
common/logging_extra.py
Normal file
@@ -0,0 +1,249 @@
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import copy
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
import socket
|
||||
import logging
|
||||
import traceback
|
||||
import numpy as np
|
||||
from threading import local
|
||||
from collections import OrderedDict
|
||||
from contextlib import contextmanager
|
||||
|
||||
LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
|
||||
|
||||
def json_handler(obj):
|
||||
if isinstance(obj, np.bool_):
|
||||
return bool(obj)
|
||||
# if isinstance(obj, (datetime.date, datetime.time)):
|
||||
# return obj.isoformat()
|
||||
return repr(obj)
|
||||
|
||||
def json_robust_dumps(obj):
|
||||
return json.dumps(obj, default=json_handler)
|
||||
|
||||
class NiceOrderedDict(OrderedDict):
|
||||
def __str__(self):
|
||||
return json_robust_dumps(self)
|
||||
|
||||
class SwagFormatter(logging.Formatter):
|
||||
def __init__(self, swaglogger):
|
||||
logging.Formatter.__init__(self, None, '%a %b %d %H:%M:%S %Z %Y')
|
||||
|
||||
self.swaglogger = swaglogger
|
||||
self.host = socket.gethostname()
|
||||
|
||||
def format_dict(self, record):
|
||||
record_dict = NiceOrderedDict()
|
||||
|
||||
if isinstance(record.msg, dict):
|
||||
record_dict['msg'] = record.msg
|
||||
else:
|
||||
try:
|
||||
record_dict['msg'] = record.getMessage()
|
||||
except (ValueError, TypeError):
|
||||
record_dict['msg'] = [record.msg]+record.args
|
||||
|
||||
record_dict['ctx'] = self.swaglogger.get_ctx()
|
||||
|
||||
if record.exc_info:
|
||||
record_dict['exc_info'] = self.formatException(record.exc_info)
|
||||
|
||||
record_dict['level'] = record.levelname
|
||||
record_dict['levelnum'] = record.levelno
|
||||
record_dict['name'] = record.name
|
||||
record_dict['filename'] = record.filename
|
||||
record_dict['lineno'] = record.lineno
|
||||
record_dict['pathname'] = record.pathname
|
||||
record_dict['module'] = record.module
|
||||
record_dict['funcName'] = record.funcName
|
||||
record_dict['host'] = self.host
|
||||
record_dict['process'] = record.process
|
||||
record_dict['thread'] = record.thread
|
||||
record_dict['threadName'] = record.threadName
|
||||
record_dict['created'] = record.created
|
||||
|
||||
return record_dict
|
||||
|
||||
def format(self, record):
|
||||
if self.swaglogger is None:
|
||||
raise Exception("must set swaglogger before calling format()")
|
||||
return json_robust_dumps(self.format_dict(record))
|
||||
|
||||
class SwagLogFileFormatter(SwagFormatter):
|
||||
def fix_kv(self, k, v):
|
||||
# append type to names to preserve legacy naming in logs
|
||||
# avoids overlapping key namespaces with different types
|
||||
# e.g. log.info() creates 'msg' -> 'msg$s'
|
||||
# log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i'
|
||||
# because overlapping namespace 'msg' caused problems
|
||||
if isinstance(v, (str, bytes)):
|
||||
k += "$s"
|
||||
elif isinstance(v, float):
|
||||
k += "$f"
|
||||
elif isinstance(v, bool):
|
||||
k += "$b"
|
||||
elif isinstance(v, int):
|
||||
k += "$i"
|
||||
elif isinstance(v, dict):
|
||||
nv = {}
|
||||
for ik, iv in v.items():
|
||||
ik, iv = self.fix_kv(ik, iv)
|
||||
nv[ik] = iv
|
||||
v = nv
|
||||
elif isinstance(v, list):
|
||||
k += "$a"
|
||||
return k, v
|
||||
|
||||
def format(self, record):
|
||||
if isinstance(record, str):
|
||||
v = json.loads(record)
|
||||
else:
|
||||
v = self.format_dict(record)
|
||||
|
||||
mk, mv = self.fix_kv('msg', v['msg'])
|
||||
del v['msg']
|
||||
v[mk] = mv
|
||||
v['id'] = uuid.uuid4().hex
|
||||
|
||||
return json_robust_dumps(v)
|
||||
|
||||
class SwagErrorFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
return record.levelno < logging.ERROR
|
||||
|
||||
def _tmpfunc():
|
||||
return 0
|
||||
|
||||
def _srcfile():
|
||||
return os.path.normcase(_tmpfunc.__code__.co_filename)
|
||||
|
||||
class SwagLogger(logging.Logger):
|
||||
def __init__(self):
|
||||
logging.Logger.__init__(self, "swaglog")
|
||||
|
||||
self.global_ctx = {}
|
||||
|
||||
self.log_local = local()
|
||||
self.log_local.ctx = {}
|
||||
|
||||
def local_ctx(self):
|
||||
try:
|
||||
return self.log_local.ctx
|
||||
except AttributeError:
|
||||
self.log_local.ctx = {}
|
||||
return self.log_local.ctx
|
||||
|
||||
def get_ctx(self):
|
||||
return dict(self.local_ctx(), **self.global_ctx)
|
||||
|
||||
@contextmanager
|
||||
def ctx(self, **kwargs):
|
||||
old_ctx = self.local_ctx()
|
||||
self.log_local.ctx = copy.copy(old_ctx) or {}
|
||||
self.log_local.ctx.update(kwargs)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.log_local.ctx = old_ctx
|
||||
|
||||
def bind(self, **kwargs):
|
||||
self.local_ctx().update(kwargs)
|
||||
|
||||
def bind_global(self, **kwargs):
|
||||
self.global_ctx.update(kwargs)
|
||||
|
||||
def event(self, event, *args, **kwargs):
|
||||
evt = NiceOrderedDict()
|
||||
evt['event'] = event
|
||||
if args:
|
||||
evt['args'] = args
|
||||
evt.update(kwargs)
|
||||
if 'error' in kwargs:
|
||||
self.error(evt)
|
||||
elif 'debug' in kwargs:
|
||||
self.debug(evt)
|
||||
else:
|
||||
self.info(evt)
|
||||
|
||||
def timestamp(self, event_name):
|
||||
if LOG_TIMESTAMPS:
|
||||
t = time.monotonic()
|
||||
tstp = NiceOrderedDict()
|
||||
tstp['timestamp'] = NiceOrderedDict()
|
||||
tstp['timestamp']["event"] = event_name
|
||||
tstp['timestamp']["time"] = t*1e9
|
||||
self.debug(tstp)
|
||||
|
||||
def findCaller(self, stack_info=False, stacklevel=1):
|
||||
"""
|
||||
Find the stack frame of the caller so that we can note the source
|
||||
file name, line number and function name.
|
||||
"""
|
||||
f = sys._getframe(3)
|
||||
#On some versions of IronPython, currentframe() returns None if
|
||||
#IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
orig_f = f
|
||||
while f and stacklevel > 1:
|
||||
f = f.f_back
|
||||
stacklevel -= 1
|
||||
if not f:
|
||||
f = orig_f
|
||||
rv = "(unknown file)", 0, "(unknown function)", None
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
|
||||
if filename == _srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
sinfo = None
|
||||
if stack_info:
|
||||
sio = io.StringIO()
|
||||
sio.write('Stack (most recent call last):\n')
|
||||
traceback.print_stack(f, file=sio)
|
||||
sinfo = sio.getvalue()
|
||||
if sinfo[-1] == '\n':
|
||||
sinfo = sinfo[:-1]
|
||||
sio.close()
|
||||
rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
|
||||
break
|
||||
return rv
|
||||
|
||||
if __name__ == "__main__":
|
||||
log = SwagLogger()
|
||||
|
||||
stdout_handler = logging.StreamHandler(sys.stdout)
|
||||
stdout_handler.setLevel(logging.INFO)
|
||||
stdout_handler.addFilter(SwagErrorFilter())
|
||||
log.addHandler(stdout_handler)
|
||||
|
||||
stderr_handler = logging.StreamHandler(sys.stderr)
|
||||
stderr_handler.setLevel(logging.ERROR)
|
||||
log.addHandler(stderr_handler)
|
||||
|
||||
log.info("asdasd %s", "a")
|
||||
log.info({'wut': 1})
|
||||
log.warning("warning")
|
||||
log.error("error")
|
||||
log.critical("critical")
|
||||
log.event("test", x="y")
|
||||
|
||||
with log.ctx():
|
||||
stdout_handler.setFormatter(SwagFormatter(log))
|
||||
stderr_handler.setFormatter(SwagFormatter(log))
|
||||
log.bind(user="some user")
|
||||
log.info("in req")
|
||||
print("")
|
||||
log.warning("warning")
|
||||
print("")
|
||||
log.error("error")
|
||||
print("")
|
||||
log.critical("critical")
|
||||
print("")
|
||||
log.event("do_req", a=1, b="c")
|
||||
45
common/markdown.py
Normal file
45
common/markdown.py
Normal file
@@ -0,0 +1,45 @@
|
||||
HTML_REPLACEMENTS = [
|
||||
(r'&', r'&'),
|
||||
(r'"', r'"'),
|
||||
]
|
||||
|
||||
def parse_markdown(text: str, tab_length: int = 2) -> str:
|
||||
lines = text.split("\n")
|
||||
output: list[str] = []
|
||||
list_level = 0
|
||||
|
||||
def end_outstanding_lists(level: int, end_level: int) -> int:
|
||||
while level > end_level:
|
||||
level -= 1
|
||||
output.append("</ul>")
|
||||
if level > 0:
|
||||
output.append("</li>")
|
||||
return end_level
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if i + 1 < len(lines) and lines[i + 1].startswith("==="): # heading
|
||||
output.append(f"<h1>{line}</h1>")
|
||||
elif line.startswith("==="):
|
||||
pass
|
||||
elif line.lstrip().startswith("* "): # list
|
||||
line_level = 1 + line.count(" " * tab_length, 0, line.index("*"))
|
||||
if list_level >= line_level:
|
||||
list_level = end_outstanding_lists(list_level, line_level)
|
||||
else:
|
||||
list_level += 1
|
||||
if list_level > 1:
|
||||
output[-1] = output[-1].replace("</li>", "")
|
||||
output.append("<ul>")
|
||||
output.append(f"<li>{line.replace('*', '', 1).lstrip()}</li>")
|
||||
else:
|
||||
list_level = end_outstanding_lists(list_level, 0)
|
||||
if len(line) > 0:
|
||||
output.append(line)
|
||||
|
||||
end_outstanding_lists(list_level, 0)
|
||||
output_str = "\n".join(output) + "\n"
|
||||
|
||||
for (fr, to) in HTML_REPLACEMENTS:
|
||||
output_str = output_str.replace(fr, to)
|
||||
|
||||
return output_str
|
||||
85
common/mat.h
Normal file
85
common/mat.h
Normal file
@@ -0,0 +1,85 @@
|
||||
#pragma once
|
||||
|
||||
typedef struct vec3 {
|
||||
float v[3];
|
||||
} vec3;
|
||||
|
||||
typedef struct vec4 {
|
||||
float v[4];
|
||||
} vec4;
|
||||
|
||||
typedef struct mat3 {
|
||||
float v[3*3];
|
||||
} mat3;
|
||||
|
||||
typedef struct mat4 {
|
||||
float v[4*4];
|
||||
} mat4;
|
||||
|
||||
static inline mat3 matmul3(const mat3 &a, const mat3 &b) {
|
||||
mat3 ret = {{0.0}};
|
||||
for (int r=0; r<3; r++) {
|
||||
for (int c=0; c<3; c++) {
|
||||
float v = 0.0;
|
||||
for (int k=0; k<3; k++) {
|
||||
v += a.v[r*3+k] * b.v[k*3+c];
|
||||
}
|
||||
ret.v[r*3+c] = v;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline vec3 matvecmul3(const mat3 &a, const vec3 &b) {
|
||||
vec3 ret = {{0.0}};
|
||||
for (int r=0; r<3; r++) {
|
||||
for (int c=0; c<3; c++) {
|
||||
ret.v[r] += a.v[r*3+c] * b.v[c];
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline mat4 matmul(const mat4 &a, const mat4 &b) {
|
||||
mat4 ret = {{0.0}};
|
||||
for (int r=0; r<4; r++) {
|
||||
for (int c=0; c<4; c++) {
|
||||
float v = 0.0;
|
||||
for (int k=0; k<4; k++) {
|
||||
v += a.v[r*4+k] * b.v[k*4+c];
|
||||
}
|
||||
ret.v[r*4+c] = v;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline vec4 matvecmul(const mat4 &a, const vec4 &b) {
|
||||
vec4 ret = {{0.0}};
|
||||
for (int r=0; r<4; r++) {
|
||||
for (int c=0; c<4; c++) {
|
||||
ret.v[r] += a.v[r*4+c] * b.v[c];
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
// scales the input and output space of a transformation matrix
|
||||
// that assumes pixel-center origin.
|
||||
static inline mat3 transform_scale_buffer(const mat3 &in, float s) {
|
||||
// in_pt = ( transform(out_pt/s + 0.5) - 0.5) * s
|
||||
|
||||
mat3 transform_out = {{
|
||||
1.0f/s, 0.0f, 0.5f,
|
||||
0.0f, 1.0f/s, 0.5f,
|
||||
0.0f, 0.0f, 1.0f,
|
||||
}};
|
||||
|
||||
mat3 transform_in = {{
|
||||
s, 0.0f, -0.5f*s,
|
||||
0.0f, s, -0.5f*s,
|
||||
0.0f, 0.0f, 1.0f,
|
||||
}};
|
||||
|
||||
return matmul3(transform_in, matmul3(in, transform_out));
|
||||
}
|
||||
50
common/mock/__init__.py
Normal file
50
common/mock/__init__.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Utilities for generating mock messages for testing.
|
||||
example in common/tests/test_mock.py
|
||||
"""
|
||||
|
||||
|
||||
import functools
|
||||
import threading
|
||||
from cereal.messaging import PubMaster
|
||||
from cereal.services import SERVICE_LIST
|
||||
from openpilot.common.mock.generators import generate_livePose
|
||||
from openpilot.common.realtime import Ratekeeper
|
||||
|
||||
|
||||
MOCK_GENERATOR = {
|
||||
"livePose": generate_livePose
|
||||
}
|
||||
|
||||
|
||||
def generate_messages_loop(services: list[str], done: threading.Event):
|
||||
pm = PubMaster(services)
|
||||
rk = Ratekeeper(100)
|
||||
i = 0
|
||||
while not done.is_set():
|
||||
for s in services:
|
||||
should_send = i % (100/SERVICE_LIST[s].frequency) == 0
|
||||
if should_send:
|
||||
message = MOCK_GENERATOR[s]()
|
||||
pm.send(s, message)
|
||||
i += 1
|
||||
rk.keep_time()
|
||||
|
||||
|
||||
def mock_messages(services: list[str] | str):
|
||||
if isinstance(services, str):
|
||||
services = [services]
|
||||
|
||||
def decorator(func):
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
done = threading.Event()
|
||||
t = threading.Thread(target=generate_messages_loop, args=(services, done))
|
||||
t.start()
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
finally:
|
||||
done.set()
|
||||
t.join()
|
||||
return wrapper
|
||||
return decorator
|
||||
14
common/mock/generators.py
Normal file
14
common/mock/generators.py
Normal file
@@ -0,0 +1,14 @@
|
||||
from cereal import messaging
|
||||
|
||||
|
||||
def generate_livePose():
|
||||
msg = messaging.new_message('livePose')
|
||||
meas = {'x': 0.0, 'y': 0.0, 'z': 0.0, 'xStd': 0.0, 'yStd': 0.0, 'zStd': 0.0, 'valid': True}
|
||||
msg.livePose.orientationNED = meas
|
||||
msg.livePose.velocityDevice = meas
|
||||
msg.livePose.angularVelocityDevice = meas
|
||||
msg.livePose.accelerationDevice = meas
|
||||
msg.livePose.inputsOK = True
|
||||
msg.livePose.posenetOK = True
|
||||
msg.livePose.sensorsOK = True
|
||||
return msg
|
||||
1
common/model.h
Normal file
1
common/model.h
Normal file
@@ -0,0 +1 @@
|
||||
#define DEFAULT_MODEL "CD210 (Default)"
|
||||
244
common/params.cc
Normal file
244
common/params.cc
Normal file
@@ -0,0 +1,244 @@
|
||||
#include "common/params.h"
|
||||
|
||||
#include <dirent.h>
|
||||
#include <sys/file.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <csignal>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "common/params_keys.h"
|
||||
#include "common/queue.h"
|
||||
#include "common/swaglog.h"
|
||||
#include "common/util.h"
|
||||
#include "system/hardware/hw.h"
|
||||
|
||||
namespace {
|
||||
|
||||
volatile sig_atomic_t params_do_exit = 0;
|
||||
void params_sig_handler(int signal) {
|
||||
params_do_exit = 1;
|
||||
}
|
||||
|
||||
int fsync_dir(const std::string &path) {
|
||||
int result = -1;
|
||||
int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
|
||||
if (fd >= 0) {
|
||||
result = HANDLE_EINTR(fsync(fd));
|
||||
HANDLE_EINTR(close(fd));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool create_params_path(const std::string ¶m_path, const std::string &key_path) {
|
||||
// Make sure params path exists
|
||||
if (!util::file_exists(param_path) && !util::create_directories(param_path, 0775)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// See if the symlink exists, otherwise create it
|
||||
if (!util::file_exists(key_path)) {
|
||||
// 1) Create temp folder
|
||||
// 2) Symlink it to temp link
|
||||
// 3) Move symlink to <params>/d
|
||||
|
||||
std::string tmp_path = param_path + "/.tmp_XXXXXX";
|
||||
// this should be OK since mkdtemp just replaces characters in place
|
||||
char *tmp_dir = mkdtemp((char *)tmp_path.c_str());
|
||||
if (tmp_dir == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string link_path = std::string(tmp_dir) + ".link";
|
||||
if (symlink(tmp_dir, link_path.c_str()) != 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// don't return false if it has been created by other
|
||||
if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
|
||||
std::string params_path = path.empty() ? Path::params() : path;
|
||||
if (!create_params_path(params_path, params_path + prefix)) {
|
||||
throw std::runtime_error(util::string_format(
|
||||
"Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
|
||||
errno, params_path.c_str(), prefix.c_str()));
|
||||
}
|
||||
return params_path;
|
||||
}
|
||||
|
||||
class FileLock {
|
||||
public:
|
||||
FileLock(const std::string &fn) {
|
||||
fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
|
||||
if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
|
||||
LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
|
||||
}
|
||||
}
|
||||
~FileLock() { close(fd_); }
|
||||
|
||||
private:
|
||||
int fd_ = -1;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
Params::Params(const std::string &path) {
|
||||
params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
|
||||
params_path = ensure_params_path(params_prefix, path);
|
||||
}
|
||||
|
||||
Params::~Params() {
|
||||
if (future.valid()) {
|
||||
future.wait();
|
||||
}
|
||||
assert(queue.empty());
|
||||
}
|
||||
|
||||
std::vector<std::string> Params::allKeys(ParamKeyFlag flag) const {
|
||||
std::vector<std::string> ret;
|
||||
for (auto &p : keys) {
|
||||
if (flag == ALL || (p.second.flags & flag)) {
|
||||
ret.push_back(p.first);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool Params::checkKey(const std::string &key) {
|
||||
return keys.find(key) != keys.end();
|
||||
}
|
||||
|
||||
ParamKeyFlag Params::getKeyFlag(const std::string &key) {
|
||||
return static_cast<ParamKeyFlag>(keys[key].flags);
|
||||
}
|
||||
|
||||
ParamKeyType Params::getKeyType(const std::string &key) {
|
||||
return keys[key].type;
|
||||
}
|
||||
|
||||
std::optional<std::string> Params::getKeyDefaultValue(const std::string &key) {
|
||||
return keys[key].default_value;
|
||||
}
|
||||
|
||||
int Params::put(const char* key, const char* value, size_t value_size) {
|
||||
// Information about safely and atomically writing a file: https://lwn.net/Articles/457667/
|
||||
// 1) Create temp file
|
||||
// 2) Write data to temp file
|
||||
// 3) fsync() the temp file
|
||||
// 4) rename the temp file to the real name
|
||||
// 5) fsync() the containing directory
|
||||
std::string tmp_path = params_path + "/.tmp_value_XXXXXX";
|
||||
int tmp_fd = mkstemp((char*)tmp_path.c_str());
|
||||
if (tmp_fd < 0) return -1;
|
||||
|
||||
int result = -1;
|
||||
do {
|
||||
// Write value to temp.
|
||||
ssize_t bytes_written = HANDLE_EINTR(write(tmp_fd, value, value_size));
|
||||
if (bytes_written < 0 || (size_t)bytes_written != value_size) {
|
||||
result = -20;
|
||||
break;
|
||||
}
|
||||
|
||||
// fsync to force persist the changes.
|
||||
if ((result = HANDLE_EINTR(fsync(tmp_fd))) < 0) break;
|
||||
|
||||
FileLock file_lock(params_path + "/.lock");
|
||||
|
||||
// Move temp into place.
|
||||
if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break;
|
||||
|
||||
// fsync parent directory
|
||||
result = fsync_dir(getParamPath());
|
||||
} while (false);
|
||||
|
||||
close(tmp_fd);
|
||||
if (result != 0) {
|
||||
::unlink(tmp_path.c_str());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
int Params::remove(const std::string &key) {
|
||||
FileLock file_lock(params_path + "/.lock");
|
||||
int result = unlink(getParamPath(key).c_str());
|
||||
if (result != 0) {
|
||||
return result;
|
||||
}
|
||||
return fsync_dir(getParamPath());
|
||||
}
|
||||
|
||||
std::string Params::get(const std::string &key, bool block) {
|
||||
if (!block) {
|
||||
return util::read_file(getParamPath(key));
|
||||
} else {
|
||||
// blocking read until successful
|
||||
params_do_exit = 0;
|
||||
void (*prev_handler_sigint)(int) = std::signal(SIGINT, params_sig_handler);
|
||||
void (*prev_handler_sigterm)(int) = std::signal(SIGTERM, params_sig_handler);
|
||||
|
||||
std::string value;
|
||||
while (!params_do_exit) {
|
||||
if (value = util::read_file(getParamPath(key)); !value.empty()) {
|
||||
break;
|
||||
}
|
||||
util::sleep_for(100); // 0.1 s
|
||||
}
|
||||
|
||||
std::signal(SIGINT, prev_handler_sigint);
|
||||
std::signal(SIGTERM, prev_handler_sigterm);
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
std::map<std::string, std::string> Params::readAll() {
|
||||
FileLock file_lock(params_path + "/.lock");
|
||||
return util::read_files_in_dir(getParamPath());
|
||||
}
|
||||
|
||||
void Params::clearAll(ParamKeyFlag key_flag) {
|
||||
FileLock file_lock(params_path + "/.lock");
|
||||
|
||||
// 1) delete params of key_flag
|
||||
// 2) delete files that are not defined in the keys.
|
||||
if (DIR *d = opendir(getParamPath().c_str())) {
|
||||
struct dirent *de = NULL;
|
||||
while ((de = readdir(d))) {
|
||||
if (de->d_type != DT_DIR) {
|
||||
auto it = keys.find(de->d_name);
|
||||
if (it == keys.end() || (it->second.flags & key_flag)) {
|
||||
unlink(getParamPath(de->d_name).c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
closedir(d);
|
||||
}
|
||||
|
||||
fsync_dir(getParamPath());
|
||||
}
|
||||
|
||||
void Params::putNonBlocking(const std::string &key, const std::string &val) {
|
||||
queue.push(std::make_pair(key, val));
|
||||
// start thread on demand
|
||||
if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
|
||||
future = std::async(std::launch::async, &Params::asyncWriteThread, this);
|
||||
}
|
||||
}
|
||||
|
||||
void Params::asyncWriteThread() {
|
||||
// TODO: write the latest one if a key has multiple values in the queue.
|
||||
std::pair<std::string, std::string> p;
|
||||
while (queue.try_pop(p, 0)) {
|
||||
// Params::put is Thread-Safe
|
||||
put(p.first, p.second);
|
||||
}
|
||||
}
|
||||
91
common/params.h
Normal file
91
common/params.h
Normal file
@@ -0,0 +1,91 @@
|
||||
#pragma once
|
||||
|
||||
#include <future>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "common/queue.h"
|
||||
|
||||
enum ParamKeyFlag {
|
||||
PERSISTENT = 0x02,
|
||||
CLEAR_ON_MANAGER_START = 0x04,
|
||||
CLEAR_ON_ONROAD_TRANSITION = 0x08,
|
||||
CLEAR_ON_OFFROAD_TRANSITION = 0x10,
|
||||
DONT_LOG = 0x20,
|
||||
DEVELOPMENT_ONLY = 0x40,
|
||||
CLEAR_ON_IGNITION_ON = 0x80,
|
||||
BACKUP = 0x100,
|
||||
ALL = 0xFFFFFFFF
|
||||
};
|
||||
|
||||
enum ParamKeyType {
|
||||
STRING = 0, // must be utf-8 decodable
|
||||
BOOL = 1,
|
||||
INT = 2,
|
||||
FLOAT = 3,
|
||||
TIME = 4, // ISO 8601
|
||||
JSON = 5,
|
||||
BYTES = 6
|
||||
};
|
||||
|
||||
struct ParamKeyAttributes {
|
||||
uint32_t flags;
|
||||
ParamKeyType type;
|
||||
std::optional<std::string> default_value = std::nullopt;
|
||||
};
|
||||
|
||||
class Params {
|
||||
public:
|
||||
explicit Params(const std::string &path = {});
|
||||
~Params();
|
||||
// Not copyable.
|
||||
Params(const Params&) = delete;
|
||||
Params& operator=(const Params&) = delete;
|
||||
|
||||
std::vector<std::string> allKeys(ParamKeyFlag flag = ALL) const;
|
||||
bool checkKey(const std::string &key);
|
||||
ParamKeyFlag getKeyFlag(const std::string &key);
|
||||
ParamKeyType getKeyType(const std::string &key);
|
||||
std::optional<std::string> getKeyDefaultValue(const std::string &key);
|
||||
inline std::string getParamPath(const std::string &key = {}) {
|
||||
return params_path + params_prefix + (key.empty() ? "" : "/" + key);
|
||||
}
|
||||
|
||||
// Delete a value
|
||||
int remove(const std::string &key);
|
||||
void clearAll(ParamKeyFlag flag);
|
||||
|
||||
// helpers for reading values
|
||||
std::string get(const std::string &key, bool block = false);
|
||||
inline bool getBool(const std::string &key, bool block = false) {
|
||||
return get(key, block) == "1";
|
||||
}
|
||||
std::map<std::string, std::string> readAll();
|
||||
|
||||
// helpers for writing values
|
||||
int put(const char *key, const char *val, size_t value_size);
|
||||
inline int put(const std::string &key, const std::string &val) {
|
||||
return put(key.c_str(), val.data(), val.size());
|
||||
}
|
||||
inline int putBool(const std::string &key, bool val) {
|
||||
return put(key.c_str(), val ? "1" : "0", 1);
|
||||
}
|
||||
void putNonBlocking(const std::string &key, const std::string &val);
|
||||
inline void putBoolNonBlocking(const std::string &key, bool val) {
|
||||
putNonBlocking(key, val ? "1" : "0");
|
||||
}
|
||||
|
||||
private:
|
||||
void asyncWriteThread();
|
||||
|
||||
std::string params_path;
|
||||
std::string params_prefix;
|
||||
|
||||
// for nonblocking write
|
||||
std::future<void> future;
|
||||
SafeQueue<std::pair<std::string, std::string>> queue;
|
||||
};
|
||||
19
common/params.py
Normal file
19
common/params.py
Normal file
@@ -0,0 +1,19 @@
|
||||
from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
|
||||
assert Params
|
||||
assert ParamKeyFlag
|
||||
assert ParamKeyType
|
||||
assert UnknownKeyName
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
params = Params()
|
||||
key = sys.argv[1]
|
||||
assert params.check_key(key), f"unknown param: {key}"
|
||||
|
||||
if len(sys.argv) == 3:
|
||||
val = sys.argv[2]
|
||||
print(f"SET: {key} = {val}")
|
||||
params.put(key, val)
|
||||
elif len(sys.argv) == 2:
|
||||
print(f"GET: {key} = {params.get(key)}")
|
||||
317
common/params_keys.h
Normal file
317
common/params_keys.h
Normal file
@@ -0,0 +1,317 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "cereal/gen/cpp/log.capnp.h"
|
||||
|
||||
inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
{"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
|
||||
{"AdbEnabled", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"AlwaysOnDM", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"ApiCache_Device", {PERSISTENT, STRING}},
|
||||
{"ApiCache_FirehoseStats", {PERSISTENT, JSON}},
|
||||
{"AssistNowToken", {PERSISTENT, STRING}},
|
||||
{"AthenadPid", {PERSISTENT, INT}},
|
||||
{"AthenadUploadQueue", {PERSISTENT, JSON}},
|
||||
{"AthenadRecentlyViewedRoutes", {PERSISTENT, STRING}},
|
||||
{"BackupManagerK3_CreateBackup", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"BackupManagerK3_RestoreVersion", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"BootCount", {PERSISTENT, INT}},
|
||||
{"CalibrationParams", {PERSISTENT, BYTES}},
|
||||
{"CameraDebugExpGain", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"CameraDebugExpTime", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"CarBatteryCapacity", {PERSISTENT, INT}},
|
||||
{"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
|
||||
{"CarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
|
||||
{"CarParamsPersistent", {PERSISTENT, BYTES}},
|
||||
{"CarParamsPrevRoute", {PERSISTENT, BYTES}},
|
||||
{"CompletedTrainingVersion", {PERSISTENT, STRING, "0"}},
|
||||
{"ControlsReady", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
|
||||
{"CurrentBootlog", {PERSISTENT, STRING}},
|
||||
{"CurrentRoute", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
|
||||
{"DisableLogging", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
|
||||
{"DisablePowerDown", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"DisableUpdates", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"DisengageOnAccelerator", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"DongleId", {PERSISTENT, STRING}},
|
||||
{"DoReboot", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
|
||||
{"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY | BACKUP, BOOL}},
|
||||
{"ExperimentalMode", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"ExperimentalModeConfirmed", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"FirmwareQueryDone", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
|
||||
{"ForcePowerDown", {PERSISTENT, BOOL}},
|
||||
{"GitBranch", {PERSISTENT, STRING}},
|
||||
{"GitCommit", {PERSISTENT, STRING}},
|
||||
{"GitCommitDate", {PERSISTENT, STRING}},
|
||||
{"GitDiff", {PERSISTENT, STRING}},
|
||||
{"GithubSshKeys", {PERSISTENT | BACKUP, STRING}},
|
||||
{"GithubUsername", {PERSISTENT | BACKUP, STRING}},
|
||||
{"GitRemote", {PERSISTENT, STRING}},
|
||||
{"GsmApn", {PERSISTENT | BACKUP, STRING}},
|
||||
{"GsmMetered", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"GsmRoaming", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"HardwareSerial", {PERSISTENT, STRING}},
|
||||
{"HasAcceptedTerms", {PERSISTENT, STRING, "0"}},
|
||||
{"HephaestusdPid", {PERSISTENT, INT}},
|
||||
{"InstallDate", {PERSISTENT, TIME}},
|
||||
{"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"IsEngaged", {PERSISTENT, BOOL}},
|
||||
{"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"IsMetric", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"IsOnroad", {PERSISTENT, BOOL}},
|
||||
{"IsRhdDetected", {PERSISTENT, BOOL}},
|
||||
{"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
|
||||
{"Konn3ktSshKeys", {PERSISTENT, STRING}},
|
||||
{"LanguageSetting", {PERSISTENT | BACKUP, STRING, "en"}},
|
||||
{"LastAthenaPingTime", {CLEAR_ON_MANAGER_START, INT}},
|
||||
{"LastGPSPosition", {PERSISTENT, STRING}},
|
||||
{"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
|
||||
{"LastAgnosPowerMonitorShutdown", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
|
||||
{"LastUpdateTime", {PERSISTENT, TIME}},
|
||||
{"LastUpdateUptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
|
||||
{"LiveParameters", {PERSISTENT, JSON}},
|
||||
{"LiveParametersV2", {PERSISTENT, BYTES}},
|
||||
{"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
|
||||
{"LocationFilterInitialState", {PERSISTENT, BYTES}},
|
||||
{"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
|
||||
{"LongitudinalPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
|
||||
{"NetworkMetered", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
|
||||
{"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
|
||||
{"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
|
||||
{"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
|
||||
{"Offroad_IsTakingSnapshot", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
|
||||
{"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
|
||||
{"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"Offroad_DriverMonitoringUncertain", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
|
||||
{"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
|
||||
{"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
|
||||
{"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
|
||||
{"PrimeType", {PERSISTENT, INT}},
|
||||
{"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"RecordFront", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"RecordFrontLock", {PERSISTENT, BOOL}}, // for the internal fleet
|
||||
{"SecOCKey", {PERSISTENT | DONT_LOG | BACKUP, STRING}},
|
||||
{"ShowDebugInfo", {PERSISTENT, BOOL}},
|
||||
{"RouteCount", {PERSISTENT, INT, "0"}},
|
||||
{"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
|
||||
{"SshEnabled", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"TermsVersion", {PERSISTENT, STRING}},
|
||||
{"TorqueBar", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"TrainingVersion", {PERSISTENT, STRING}},
|
||||
{"UbloxAvailable", {PERSISTENT, BOOL}},
|
||||
{"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
|
||||
{"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}},
|
||||
{"UpdaterAvailableBranches", {PERSISTENT, STRING}},
|
||||
{"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
|
||||
{"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
|
||||
{"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"UpdaterLastFetchTime", {PERSISTENT, TIME}},
|
||||
{"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"Version", {PERSISTENT, STRING}},
|
||||
|
||||
// --- iqpilot params --- //
|
||||
{"ApiCache_DriveStats", {PERSISTENT, JSON}},
|
||||
{"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"BlinkerMinLateralControlSpeed", {PERSISTENT | BACKUP, INT, "20"}}, // MPH or km/h
|
||||
{"BlinkerPauseLateralControl", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"Brightness", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"CarList", {PERSISTENT, JSON}},
|
||||
{"IQCarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
|
||||
{"IQCarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
|
||||
{"IQCarParamsPersistent", {PERSISTENT, BYTES}},
|
||||
{"CarPlatformBundle", {PERSISTENT | BACKUP, JSON}},
|
||||
{"ChevronInfo", {PERSISTENT | BACKUP, INT, "4"}},
|
||||
{"CustomAccIncrementsEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"CustomAccLongPressIncrement", {PERSISTENT | BACKUP, INT, "5"}},
|
||||
{"CustomAccShortPressIncrement", {PERSISTENT | BACKUP, INT, "1"}},
|
||||
{"DeviceBootMode", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"DevUIInfo", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"EnableCopyparty", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"EnableEsimProvisioning", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"EnableGithubRunner", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"GreenLightAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"GithubRunnerSufficientVoltage", {CLEAR_ON_MANAGER_START , BOOL}},
|
||||
{"HideVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"InteractivityTimeout", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"IsDevelopmentBranch", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"IsReleaseIqBranch", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"LastGPSPositionLLK", {PERSISTENT, STRING}},
|
||||
{"LeadDepartAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"MaxTimeOffroad", {PERSISTENT | BACKUP, INT, "1800"}},
|
||||
{"ModelRunnerTypeCache", {CLEAR_ON_ONROAD_TRANSITION, INT}},
|
||||
{"ForceOnroadUntil", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
|
||||
{"OffroadMode", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"Offroad_TiciSupport", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"OnroadScreenOffBrightness", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"OnroadScreenOffTimer", {PERSISTENT | BACKUP, INT, "15"}},
|
||||
{"OnroadUploads", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"TrueVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
|
||||
// AOL (Always On Lateral) params
|
||||
{"AolEnabled", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"AolMainCruiseAllowed", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"AolSteeringMode", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"AolUnifiedEngagementMode", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
|
||||
// Model Manager params
|
||||
{"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
|
||||
{"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT, "0"}},
|
||||
{"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
|
||||
{"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
|
||||
{"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}},
|
||||
|
||||
// Neural Network Lateral Control
|
||||
{"NeuralNetworkLateralControl", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
|
||||
// Backup Manager params
|
||||
{"BackupManager_CreateBackup", {PERSISTENT, BOOL}},
|
||||
{"BackupManager_RestoreVersion", {PERSISTENT, STRING}},
|
||||
|
||||
// iqpilot car specific params
|
||||
{"HyundaiLongitudinalTuning", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"pqhca5or7Toggle", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"AllowLateralWhenLongUnavailable", {PERSISTENT | BACKUP, BOOL}},
|
||||
|
||||
{"IQDynamicMode", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"BlindSpot", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
|
||||
// iqpilot model params
|
||||
{"CameraOffset", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
|
||||
{"LagdToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"LagdToggleDelay", {PERSISTENT | BACKUP, FLOAT, "0.2"}},
|
||||
{"LagdValueCache", {PERSISTENT, FLOAT, "0.2"}},
|
||||
{"LaneTurnDesire", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"LaneTurnValue", {PERSISTENT | BACKUP, FLOAT, "19.0"}},
|
||||
{"PlanplusControl", {PERSISTENT | BACKUP, FLOAT, "1.0"}},
|
||||
|
||||
// mapd
|
||||
{"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}},
|
||||
{"MapdVersion", {PERSISTENT, STRING}},
|
||||
{"MapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT, "0.0"}},
|
||||
{"NextMapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, JSON}},
|
||||
{"Offroad_OSMUpdateRequired", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"OsmDbUpdatesCheck", {CLEAR_ON_MANAGER_START, BOOL}}, // mapd database update happens with device ON, reset on boot
|
||||
{"OSMDownloadBounds", {PERSISTENT, STRING}},
|
||||
{"OsmDownloadedDate", {PERSISTENT, STRING, "0.0"}},
|
||||
{"OSMDownloadLocations", {PERSISTENT, JSON}},
|
||||
|
||||
// navigation / hephaestusd bridge params
|
||||
{"AthenaNavigationRoute", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"NavigationActive", {CLEAR_ON_MANAGER_START, BOOL, "0"}},
|
||||
{"NavigationDestination", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"NavigationManeuvers", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"ScreenRecording", {PERSISTENT, BOOL, "0"}},
|
||||
{"OSMDownloadProgress", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"OsmLocal", {PERSISTENT, BOOL}},
|
||||
{"OsmLocationName", {PERSISTENT, STRING}},
|
||||
{"OsmLocationTitle", {PERSISTENT, STRING}},
|
||||
{"OsmLocationUrl", {PERSISTENT, STRING}},
|
||||
{"OsmStateName", {PERSISTENT, STRING, "All"}},
|
||||
{"OsmStateTitle", {PERSISTENT, STRING}},
|
||||
{"OsmWayTest", {PERSISTENT, STRING}},
|
||||
{"RoadName", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
|
||||
{"RoadNameToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
|
||||
// Speed Limit
|
||||
{"SpeedLimitMode", {PERSISTENT | BACKUP, INT, "1"}},
|
||||
{"SpeedLimitOffsetType", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"SpeedLimitPolicy", {PERSISTENT | BACKUP, INT, "3"}},
|
||||
{"SpeedLimitValueOffset", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
// Legacy IQ.Pilot SLC controls (ported from master-release)
|
||||
{"SpeedLimitController", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"ShowSpeedLimits", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"SLCPriority", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"speed_limit_offset1", {PERSISTENT | BACKUP, FLOAT, "0"}},
|
||||
{"speed_limit_offset2", {PERSISTENT | BACKUP, FLOAT, "0"}},
|
||||
{"speed_limit_offset3", {PERSISTENT | BACKUP, FLOAT, "0"}},
|
||||
{"speed_limit_offset4", {PERSISTENT | BACKUP, FLOAT, "0"}},
|
||||
{"speed_limit_offset5", {PERSISTENT | BACKUP, FLOAT, "0"}},
|
||||
{"speed_limit_offset6", {PERSISTENT | BACKUP, FLOAT, "0"}},
|
||||
{"speed_limit_offset7", {PERSISTENT | BACKUP, FLOAT, "0"}},
|
||||
{"SpeedLimitConfirmationHigher", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"SpeedLimitConfirmationLower", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"MapSpeedLookaheadHigher", {PERSISTENT | BACKUP, FLOAT, "5.0"}},
|
||||
{"MapSpeedLookaheadLower", {PERSISTENT | BACKUP, FLOAT, "5.0"}},
|
||||
{"SLCFallbackExperimentalMode", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"SLCFallbackSetSpeed", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"SLCFallbackPreviousSpeedLimit", {PERSISTENT | BACKUP, BOOL, "1"}},
|
||||
{"SLCOverrideMethod", {PERSISTENT | BACKUP, INT, "0"}},
|
||||
{"SLCMapboxFiller", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"SLCDataCollection", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"MapBoxRequests", {PERSISTENT, JSON}},
|
||||
{"OverpassRequests", {PERSISTENT, JSON}},
|
||||
{"SpeedLimits", {PERSISTENT, JSON}},
|
||||
{"SpeedLimitsFiltered", {PERSISTENT, JSON}},
|
||||
{"PreviousSpeedLimit", {PERSISTENT, FLOAT, "0"}},
|
||||
{"UpdateSpeedLimits", {CLEAR_ON_IGNITION_ON}},
|
||||
{"UpdateSpeedLimitsStatus", {CLEAR_ON_IGNITION_ON, STRING}},
|
||||
{"SLCMapboxSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT, "0"}},
|
||||
|
||||
// Smart Cruise Control
|
||||
{"MapTargetVelocities", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
|
||||
{"SmartCruiseControlMap", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"SmartCruiseControlVision", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
|
||||
// IQ.Pilot Parameters:
|
||||
{"pqhca5or7Toggle", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"AllowLateralWhenLongUnavailable", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"EnableCurvatureController", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"EnableSpeedLimitControl", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"EnableLongComfortMode", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"EnableSpeedLimitPredicative", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"EnableSLPredReactToSL", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"EnableSLPredReactToCurves", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"ForceRHDForBSM", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"NavDestination", {PERSISTENT, STRING}},
|
||||
{"eBrakeActive", {CLEAR_ON_MANAGER_START | BACKUP, BOOL, "0"}},
|
||||
{"Konn3ktAllowOffroadExternalCanTx", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"MapboxToken", {PERSISTENT | BACKUP, STRING}},
|
||||
{"CustomTorqueParams", {PERSISTENT | BACKUP , BOOL}},
|
||||
{"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
|
||||
{"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
|
||||
{"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
|
||||
{"TorqueParamsOverrideEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
|
||||
{"TorqueParamsOverrideFriction", {PERSISTENT | BACKUP, FLOAT, "0.1"}},
|
||||
{"TorqueParamsOverrideLatAccelFactor", {PERSISTENT | BACKUP, FLOAT, "2.5"}},
|
||||
};
|
||||
197
common/params_pyx.pyx
Normal file
197
common/params_pyx.pyx
Normal file
@@ -0,0 +1,197 @@
|
||||
# distutils: language = c++
|
||||
# cython: language_level = 3
|
||||
import builtins
|
||||
import datetime
|
||||
import json
|
||||
from libcpp cimport bool
|
||||
from libcpp.string cimport string
|
||||
from libcpp.vector cimport vector
|
||||
from libcpp.optional cimport optional
|
||||
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
|
||||
cdef extern from "common/params.h":
|
||||
cpdef enum ParamKeyFlag:
|
||||
PERSISTENT
|
||||
CLEAR_ON_MANAGER_START
|
||||
CLEAR_ON_ONROAD_TRANSITION
|
||||
CLEAR_ON_OFFROAD_TRANSITION
|
||||
DEVELOPMENT_ONLY
|
||||
CLEAR_ON_IGNITION_ON
|
||||
BACKUP
|
||||
ALL
|
||||
|
||||
cpdef enum ParamKeyType:
|
||||
STRING
|
||||
BOOL
|
||||
INT
|
||||
FLOAT
|
||||
TIME
|
||||
JSON
|
||||
BYTES
|
||||
|
||||
cdef cppclass c_Params "Params":
|
||||
c_Params(string) except + nogil
|
||||
string get(string, bool) nogil
|
||||
bool getBool(string, bool) nogil
|
||||
int remove(string) nogil
|
||||
int put(string, string) nogil
|
||||
void putNonBlocking(string, string) nogil
|
||||
void putBoolNonBlocking(string, bool) nogil
|
||||
int putBool(string, bool) nogil
|
||||
bool checkKey(string) nogil
|
||||
ParamKeyType getKeyType(string) nogil
|
||||
optional[string] getKeyDefaultValue(string) nogil
|
||||
string getParamPath(string) nogil
|
||||
void clearAll(ParamKeyFlag)
|
||||
vector[string] allKeys(ParamKeyFlag)
|
||||
|
||||
PYTHON_2_CPP = {
|
||||
(str, STRING): lambda v: v,
|
||||
(builtins.bool, BOOL): lambda v: "1" if v else "0",
|
||||
(int, INT): str,
|
||||
(float, FLOAT): str,
|
||||
(datetime.datetime, TIME): lambda v: v.isoformat(),
|
||||
(dict, JSON): json.dumps,
|
||||
(list, JSON): json.dumps,
|
||||
(bytes, BYTES): lambda v: v,
|
||||
}
|
||||
CPP_2_PYTHON = {
|
||||
STRING: lambda v: v.decode("utf-8"),
|
||||
BOOL: lambda v: v == b"1",
|
||||
INT: int,
|
||||
FLOAT: float,
|
||||
TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
|
||||
JSON: json.loads,
|
||||
BYTES: lambda v: v,
|
||||
}
|
||||
|
||||
def ensure_bytes(v):
|
||||
return v.encode() if isinstance(v, str) else v
|
||||
|
||||
class UnknownKeyName(Exception):
|
||||
pass
|
||||
|
||||
cdef class Params:
|
||||
cdef c_Params* p
|
||||
cdef str d
|
||||
|
||||
def __cinit__(self, d=""):
|
||||
cdef string path = <string>d.encode()
|
||||
with nogil:
|
||||
self.p = new c_Params(path)
|
||||
self.d = d
|
||||
|
||||
def __reduce__(self):
|
||||
return (type(self), (self.d,))
|
||||
|
||||
def __dealloc__(self):
|
||||
del self.p
|
||||
|
||||
def clear_all(self, tx_flag=ParamKeyFlag.ALL):
|
||||
self.p.clearAll(tx_flag)
|
||||
|
||||
def check_key(self, key):
|
||||
key = ensure_bytes(key)
|
||||
if not self.p.checkKey(key):
|
||||
raise UnknownKeyName(key)
|
||||
return key
|
||||
|
||||
def python2cpp(self, proposed_type, expected_type, value, key):
|
||||
cast = PYTHON_2_CPP.get((proposed_type, expected_type))
|
||||
if cast:
|
||||
return cast(value)
|
||||
raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
|
||||
|
||||
def _cpp2python(self, t, value, default, key):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return CPP_2_PYTHON[t](value)
|
||||
except (KeyError, TypeError, ValueError):
|
||||
cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
|
||||
return self._cpp2python(t, default, None, key)
|
||||
|
||||
def get(self, key, bool block=False, bool return_default=False, encoding=None):
|
||||
cdef string k = self.check_key(key)
|
||||
cdef ParamKeyType t = self.p.getKeyType(k)
|
||||
cdef optional[string] default = self.p.getKeyDefaultValue(k)
|
||||
cdef string val
|
||||
with nogil:
|
||||
val = self.p.get(k, block)
|
||||
|
||||
default_val = (default.value() if default.has_value() else None) if return_default else None
|
||||
if val == b"":
|
||||
if block:
|
||||
# If we got no value while running in blocked mode
|
||||
# it means we got an interrupt while waiting
|
||||
raise KeyboardInterrupt
|
||||
else:
|
||||
return self._cpp2python(t, default_val, None, key)
|
||||
return self._cpp2python(t, val, default_val, key)
|
||||
|
||||
def get_bool(self, key, bool block=False):
|
||||
cdef string k = self.check_key(key)
|
||||
cdef bool r
|
||||
with nogil:
|
||||
r = self.p.getBool(k, block)
|
||||
return r
|
||||
|
||||
def _put_cast(self, key, dat):
|
||||
cdef string k = self.check_key(key)
|
||||
cdef ParamKeyType t = self.p.getKeyType(k)
|
||||
return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
|
||||
|
||||
def put(self, key, dat):
|
||||
"""
|
||||
Warning: This function blocks until the param is written to disk!
|
||||
In very rare cases this can take over a second, and your code will hang.
|
||||
Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
|
||||
in general try to avoid writing params as much as possible.
|
||||
"""
|
||||
cdef string k = self.check_key(key)
|
||||
cdef string dat_bytes = self._put_cast(key, dat)
|
||||
with nogil:
|
||||
self.p.put(k, dat_bytes)
|
||||
|
||||
def put_bool(self, key, bool val):
|
||||
cdef string k = self.check_key(key)
|
||||
with nogil:
|
||||
self.p.putBool(k, val)
|
||||
|
||||
def put_nonblocking(self, key, dat):
|
||||
cdef string k = self.check_key(key)
|
||||
cdef string dat_bytes = self._put_cast(key, dat)
|
||||
with nogil:
|
||||
self.p.putNonBlocking(k, dat_bytes)
|
||||
|
||||
def put_bool_nonblocking(self, key, bool val):
|
||||
cdef string k = self.check_key(key)
|
||||
with nogil:
|
||||
self.p.putBoolNonBlocking(k, val)
|
||||
|
||||
def remove(self, key):
|
||||
cdef string k = self.check_key(key)
|
||||
with nogil:
|
||||
self.p.remove(k)
|
||||
|
||||
def get_param_path(self, key=""):
|
||||
cdef string key_bytes = ensure_bytes(key)
|
||||
return self.p.getParamPath(key_bytes).decode("utf-8")
|
||||
|
||||
def get_type(self, key):
|
||||
return self.p.getKeyType(self.check_key(key))
|
||||
|
||||
def all_keys(self, flag=ParamKeyFlag.ALL):
|
||||
return self.p.allKeys(flag)
|
||||
|
||||
def get_default_value(self, key):
|
||||
cdef string k = self.check_key(key)
|
||||
cdef ParamKeyType t = self.p.getKeyType(k)
|
||||
cdef optional[string] default = self.p.getKeyDefaultValue(k)
|
||||
return self._cpp2python(t, default.value(), None, key) if default.has_value() else None
|
||||
|
||||
def cpp2python(self, key, value):
|
||||
cdef string k = self.check_key(key)
|
||||
cdef ParamKeyType t = self.p.getKeyType(k)
|
||||
return self._cpp2python(t, value, None, key)
|
||||
57
common/pid.py
Normal file
57
common/pid.py
Normal file
@@ -0,0 +1,57 @@
|
||||
import numpy as np
|
||||
from numbers import Number
|
||||
|
||||
class PIDController:
|
||||
def __init__(self, k_p, k_i, k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
|
||||
self._k_p: list[list[float]] = [[0], [k_p]] if isinstance(k_p, Number) else k_p
|
||||
self._k_i: list[list[float]] = [[0], [k_i]] if isinstance(k_i, Number) else k_i
|
||||
self._k_d: list[list[float]] = [[0], [k_d]] if isinstance(k_d, Number) else k_d
|
||||
|
||||
self.set_limits(pos_limit, neg_limit)
|
||||
|
||||
self.i_dt = 1.0 / rate
|
||||
self.speed = 0.0
|
||||
|
||||
self.reset()
|
||||
|
||||
@property
|
||||
def k_p(self):
|
||||
return np.interp(self.speed, self._k_p[0], self._k_p[1])
|
||||
|
||||
@property
|
||||
def k_i(self):
|
||||
return np.interp(self.speed, self._k_i[0], self._k_i[1])
|
||||
|
||||
@property
|
||||
def k_d(self):
|
||||
return np.interp(self.speed, self._k_d[0], self._k_d[1])
|
||||
|
||||
def reset(self):
|
||||
self.p = 0.0
|
||||
self.i = 0.0
|
||||
self.d = 0.0
|
||||
self.f = 0.0
|
||||
self.control = 0
|
||||
|
||||
def set_limits(self, pos_limit, neg_limit):
|
||||
self.pos_limit = pos_limit
|
||||
self.neg_limit = neg_limit
|
||||
|
||||
def update(self, error, error_rate=0.0, speed=0.0, feedforward=0., freeze_integrator=False):
|
||||
self.speed = speed
|
||||
self.p = self.k_p * float(error)
|
||||
self.d = self.k_d * error_rate
|
||||
self.f = feedforward
|
||||
|
||||
if not freeze_integrator:
|
||||
i = self.i + self.k_i * self.i_dt * error
|
||||
|
||||
# Don't allow windup if already clipping
|
||||
test_control = self.p + i + self.d + self.f
|
||||
i_upperbound = self.i if test_control > self.pos_limit else self.pos_limit
|
||||
i_lowerbound = self.i if test_control < self.neg_limit else self.neg_limit
|
||||
self.i = np.clip(i, i_lowerbound, i_upperbound)
|
||||
|
||||
control = self.p + self.i + self.d + self.f
|
||||
self.control = np.clip(control, self.neg_limit, self.pos_limit)
|
||||
return self.control
|
||||
43
common/prefix.h
Normal file
43
common/prefix.h
Normal file
@@ -0,0 +1,43 @@
|
||||
#pragma once
|
||||
|
||||
#include <cassert>
|
||||
#include <string>
|
||||
|
||||
#include "common/params.h"
|
||||
#include "common/util.h"
|
||||
#include "system/hardware/hw.h"
|
||||
|
||||
class OpenpilotPrefix {
|
||||
public:
|
||||
OpenpilotPrefix(std::string prefix = {}) {
|
||||
if (prefix.empty()) {
|
||||
prefix = util::random_string(15);
|
||||
}
|
||||
#ifdef __APPLE__
|
||||
msgq_path = "/tmp/msgq_" + prefix;
|
||||
#else
|
||||
msgq_path = "/dev/shm/msgq_" + prefix;
|
||||
#endif
|
||||
bool ret = util::create_directories(msgq_path, 0777);
|
||||
assert(ret);
|
||||
setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
|
||||
}
|
||||
|
||||
~OpenpilotPrefix() {
|
||||
auto param_path = Params().getParamPath();
|
||||
if (util::file_exists(param_path)) {
|
||||
std::string real_path = util::readlink(param_path);
|
||||
system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
|
||||
unlink(param_path.c_str());
|
||||
}
|
||||
if (getenv("COMMA_CACHE") == nullptr) {
|
||||
system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()).c_str());
|
||||
}
|
||||
system(util::string_format("rm %s -rf", Path::comma_home().c_str()).c_str());
|
||||
system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
|
||||
unsetenv("OPENPILOT_PREFIX");
|
||||
}
|
||||
|
||||
private:
|
||||
std::string msgq_path;
|
||||
};
|
||||
61
common/prefix.py
Normal file
61
common/prefix.py
Normal file
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import uuid
|
||||
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.hardware import PC
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
|
||||
|
||||
class OpenpilotPrefix:
|
||||
def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
|
||||
self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
|
||||
shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm"
|
||||
self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix)
|
||||
self.create_dirs_on_enter = create_dirs_on_enter
|
||||
self.clean_dirs_on_exit = clean_dirs_on_exit
|
||||
self.shared_download_cache = shared_download_cache
|
||||
|
||||
def __enter__(self):
|
||||
self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
|
||||
os.environ['OPENPILOT_PREFIX'] = self.prefix
|
||||
|
||||
if self.create_dirs_on_enter:
|
||||
self.create_dirs()
|
||||
|
||||
if self.shared_download_cache:
|
||||
os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
|
||||
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_obj, exc_tb):
|
||||
if self.clean_dirs_on_exit:
|
||||
self.clean_dirs()
|
||||
try:
|
||||
del os.environ['OPENPILOT_PREFIX']
|
||||
if self.original_prefix is not None:
|
||||
os.environ['OPENPILOT_PREFIX'] = self.original_prefix
|
||||
except KeyError:
|
||||
pass
|
||||
return False
|
||||
|
||||
def create_dirs(self):
|
||||
try:
|
||||
os.mkdir(self.msgq_path)
|
||||
except FileExistsError:
|
||||
pass
|
||||
os.makedirs(Paths.log_root(), exist_ok=True)
|
||||
|
||||
def clean_dirs(self):
|
||||
symlink_path = Params().get_param_path()
|
||||
if os.path.exists(symlink_path):
|
||||
shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True)
|
||||
os.remove(symlink_path)
|
||||
shutil.rmtree(self.msgq_path, ignore_errors=True)
|
||||
if PC:
|
||||
shutil.rmtree(Paths.log_root(), ignore_errors=True)
|
||||
if not os.environ.get("COMMA_CACHE", False):
|
||||
shutil.rmtree(Paths.download_cache_root(), ignore_errors=True)
|
||||
shutil.rmtree(Paths.comma_home(), ignore_errors=True)
|
||||
52
common/queue.h
Normal file
52
common/queue.h
Normal file
@@ -0,0 +1,52 @@
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <mutex>
|
||||
#include <queue>
|
||||
|
||||
template <class T>
|
||||
class SafeQueue {
|
||||
public:
|
||||
SafeQueue() = default;
|
||||
|
||||
void push(const T& v) {
|
||||
{
|
||||
std::unique_lock lk(m);
|
||||
q.push(v);
|
||||
}
|
||||
cv.notify_one();
|
||||
}
|
||||
|
||||
T pop() {
|
||||
std::unique_lock lk(m);
|
||||
cv.wait(lk, [this] { return !q.empty(); });
|
||||
T v = q.front();
|
||||
q.pop();
|
||||
return v;
|
||||
}
|
||||
|
||||
bool try_pop(T& v, int timeout_ms = 0) {
|
||||
std::unique_lock lk(m);
|
||||
if (!cv.wait_for(lk, std::chrono::milliseconds(timeout_ms), [this] { return !q.empty(); })) {
|
||||
return false;
|
||||
}
|
||||
v = q.front();
|
||||
q.pop();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool empty() const {
|
||||
std::scoped_lock lk(m);
|
||||
return q.empty();
|
||||
}
|
||||
|
||||
size_t size() const {
|
||||
std::scoped_lock lk(m);
|
||||
return q.size();
|
||||
}
|
||||
|
||||
private:
|
||||
mutable std::mutex m;
|
||||
std::condition_variable cv;
|
||||
std::queue<T> q;
|
||||
};
|
||||
40
common/ratekeeper.cc
Normal file
40
common/ratekeeper.cc
Normal file
@@ -0,0 +1,40 @@
|
||||
#include "common/ratekeeper.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "common/swaglog.h"
|
||||
#include "common/timing.h"
|
||||
#include "common/util.h"
|
||||
|
||||
RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
|
||||
: name(name),
|
||||
print_delay_threshold(std::max(0.f, print_delay_threshold)) {
|
||||
interval = 1 / rate;
|
||||
last_monitor_time = seconds_since_boot();
|
||||
next_frame_time = last_monitor_time + interval;
|
||||
}
|
||||
|
||||
bool RateKeeper::keepTime() {
|
||||
bool lagged = monitorTime();
|
||||
if (remaining_ > 0) {
|
||||
util::sleep_for(remaining_ * 1000);
|
||||
}
|
||||
return lagged;
|
||||
}
|
||||
|
||||
bool RateKeeper::monitorTime() {
|
||||
++frame_;
|
||||
last_monitor_time = seconds_since_boot();
|
||||
remaining_ = next_frame_time - last_monitor_time;
|
||||
|
||||
bool lagged = remaining_ < 0;
|
||||
if (lagged) {
|
||||
if (print_delay_threshold > 0 && remaining_ < -print_delay_threshold) {
|
||||
LOGW("%s lagging by %.2f ms", name.c_str(), -remaining_ * 1000);
|
||||
}
|
||||
next_frame_time = last_monitor_time + interval;
|
||||
} else {
|
||||
next_frame_time += interval;
|
||||
}
|
||||
return lagged;
|
||||
}
|
||||
23
common/ratekeeper.h
Normal file
23
common/ratekeeper.h
Normal file
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
class RateKeeper {
|
||||
public:
|
||||
RateKeeper(const std::string &name, float rate, float print_delay_threshold = 0);
|
||||
~RateKeeper() {}
|
||||
bool keepTime();
|
||||
bool monitorTime();
|
||||
inline uint64_t frame() const { return frame_; }
|
||||
inline double remaining() const { return remaining_; }
|
||||
|
||||
private:
|
||||
double interval;
|
||||
double next_frame_time;
|
||||
double last_monitor_time;
|
||||
double remaining_ = 0;
|
||||
float print_delay_threshold = 0;
|
||||
uint64_t frame_ = 0;
|
||||
std::string name;
|
||||
};
|
||||
96
common/realtime.py
Normal file
96
common/realtime.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""Utilities for reading real time clocks and keeping soft real time constraints."""
|
||||
import gc
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from setproctitle import getproctitle
|
||||
|
||||
from openpilot.common.utils import MovingAverage
|
||||
from openpilot.system.hardware import PC
|
||||
|
||||
|
||||
# time step for each process
|
||||
DT_CTRL = 0.01 # controlsd
|
||||
DT_MDL = 0.05 # model
|
||||
DT_HW = 0.5 # hardwared and manager
|
||||
DT_DMON = 0.05 # driver monitoring
|
||||
|
||||
|
||||
class Priority:
|
||||
# CORE 2
|
||||
# - modeld = 55
|
||||
# - camerad = 54
|
||||
CTRL_LOW = 51 # plannerd & radard
|
||||
|
||||
# CORE 3
|
||||
# - pandad = 55
|
||||
CTRL_HIGH = 53
|
||||
|
||||
|
||||
def set_core_affinity(cores: list[int]) -> None:
|
||||
if sys.platform == 'linux' and not PC:
|
||||
os.sched_setaffinity(0, cores)
|
||||
|
||||
|
||||
def config_realtime_process(cores: int | list[int], priority: int) -> None:
|
||||
gc.disable()
|
||||
if sys.platform == 'linux' and not PC:
|
||||
os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(priority))
|
||||
c = cores if isinstance(cores, list) else [cores, ]
|
||||
set_core_affinity(c)
|
||||
|
||||
|
||||
class Ratekeeper:
|
||||
def __init__(self, rate: float, print_delay_threshold: float | None = 0.0) -> None:
|
||||
"""Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
|
||||
self._interval = 1. / rate
|
||||
self._print_delay_threshold = print_delay_threshold
|
||||
self._frame = 0
|
||||
self._remaining = 0.0
|
||||
self._process_name = getproctitle()
|
||||
self._last_monitor_time = -1.
|
||||
self._next_frame_time = -1.
|
||||
|
||||
self.avg_dt = MovingAverage(100)
|
||||
self.avg_dt.add_value(self._interval)
|
||||
|
||||
@property
|
||||
def frame(self) -> int:
|
||||
return self._frame
|
||||
|
||||
@property
|
||||
def remaining(self) -> float:
|
||||
return self._remaining
|
||||
|
||||
@property
|
||||
def lagging(self) -> bool:
|
||||
expected_dt = self._interval * (1 / 0.9)
|
||||
return self.avg_dt.get_average() > expected_dt
|
||||
|
||||
# Maintain loop rate by calling this at the end of each loop
|
||||
def keep_time(self) -> bool:
|
||||
lagged = self.monitor_time()
|
||||
if self._remaining > 0:
|
||||
time.sleep(self._remaining)
|
||||
return lagged
|
||||
|
||||
# Monitors the cumulative lag, but does not enforce a rate
|
||||
def monitor_time(self) -> bool:
|
||||
if self._last_monitor_time < 0:
|
||||
self._next_frame_time = time.monotonic() + self._interval
|
||||
self._last_monitor_time = time.monotonic()
|
||||
|
||||
prev = self._last_monitor_time
|
||||
self._last_monitor_time = time.monotonic()
|
||||
self.avg_dt.add_value(self._last_monitor_time - prev)
|
||||
|
||||
lagged = False
|
||||
remaining = self._next_frame_time - time.monotonic()
|
||||
self._next_frame_time += self._interval
|
||||
if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
|
||||
print(f"{self._process_name} lagging by {-remaining * 1000:.2f} ms")
|
||||
lagged = True
|
||||
self._frame += 1
|
||||
self._remaining = remaining
|
||||
return lagged
|
||||
54
common/simple_kalman.py
Normal file
54
common/simple_kalman.py
Normal file
@@ -0,0 +1,54 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
def get_kalman_gain(dt, A, C, Q, R, iterations=100):
|
||||
P = np.zeros_like(Q)
|
||||
for _ in range(iterations):
|
||||
P = A.dot(P).dot(A.T) + dt * Q
|
||||
S = C.dot(P).dot(C.T) + R
|
||||
K = P.dot(C.T).dot(np.linalg.inv(S))
|
||||
P = (np.eye(len(P)) - K.dot(C)).dot(P)
|
||||
return K
|
||||
|
||||
|
||||
class KF1D:
|
||||
# this EKF assumes constant covariance matrix, so calculations are much simpler
|
||||
# the Kalman gain also needs to be precomputed using the control module
|
||||
|
||||
def __init__(self, x0, A, C, K):
|
||||
self.x0_0 = x0[0][0]
|
||||
self.x1_0 = x0[1][0]
|
||||
self.A0_0 = A[0][0]
|
||||
self.A0_1 = A[0][1]
|
||||
self.A1_0 = A[1][0]
|
||||
self.A1_1 = A[1][1]
|
||||
self.C0_0 = C[0]
|
||||
self.C0_1 = C[1]
|
||||
self.K0_0 = K[0][0]
|
||||
self.K1_0 = K[1][0]
|
||||
|
||||
self.A_K_0 = self.A0_0 - self.K0_0 * self.C0_0
|
||||
self.A_K_1 = self.A0_1 - self.K0_0 * self.C0_1
|
||||
self.A_K_2 = self.A1_0 - self.K1_0 * self.C0_0
|
||||
self.A_K_3 = self.A1_1 - self.K1_0 * self.C0_1
|
||||
|
||||
# K matrix needs to be pre-computed as follow:
|
||||
# import control
|
||||
# (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
|
||||
# self.K = np.transpose(K)
|
||||
|
||||
def update(self, meas):
|
||||
#self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
|
||||
x0_0 = self.A_K_0 * self.x0_0 + self.A_K_1 * self.x1_0 + self.K0_0 * meas
|
||||
x1_0 = self.A_K_2 * self.x0_0 + self.A_K_3 * self.x1_0 + self.K1_0 * meas
|
||||
self.x0_0 = x0_0
|
||||
self.x1_0 = x1_0
|
||||
return [self.x0_0, self.x1_0]
|
||||
|
||||
@property
|
||||
def x(self):
|
||||
return [[self.x0_0], [self.x1_0]]
|
||||
|
||||
def set_x(self, x):
|
||||
self.x0_0 = x[0][0]
|
||||
self.x1_0 = x[1][0]
|
||||
52
common/spinner.py
Executable file
52
common/spinner.py
Executable file
@@ -0,0 +1,52 @@
|
||||
import os
|
||||
import subprocess
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
|
||||
|
||||
class Spinner:
|
||||
def __init__(self):
|
||||
try:
|
||||
self.spinner_proc = subprocess.Popen(["./spinner.py"],
|
||||
stdin=subprocess.PIPE,
|
||||
cwd=os.path.join(BASEDIR, "system", "ui"),
|
||||
close_fds=True)
|
||||
except OSError:
|
||||
self.spinner_proc = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def update(self, spinner_text: str):
|
||||
if self.spinner_proc is not None:
|
||||
self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
|
||||
try:
|
||||
self.spinner_proc.stdin.flush()
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
|
||||
def update_progress(self, cur: float, total: float):
|
||||
self.update(str(round(100 * cur / total)))
|
||||
|
||||
def close(self):
|
||||
if self.spinner_proc is not None:
|
||||
self.spinner_proc.kill()
|
||||
try:
|
||||
self.spinner_proc.communicate(timeout=2.)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("WARNING: failed to kill spinner")
|
||||
self.spinner_proc = None
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import time
|
||||
with Spinner() as s:
|
||||
s.update("Spinner text")
|
||||
time.sleep(5.0)
|
||||
print("gone")
|
||||
time.sleep(5.0)
|
||||
73
common/stat_live.py
Normal file
73
common/stat_live.py
Normal file
@@ -0,0 +1,73 @@
|
||||
import numpy as np
|
||||
|
||||
class RunningStat:
|
||||
# tracks realtime mean and standard deviation without storing any data
|
||||
def __init__(self, priors=None, max_trackable=-1):
|
||||
self.max_trackable = max_trackable
|
||||
if priors is not None:
|
||||
# initialize from history
|
||||
self.M = priors[0]
|
||||
self.S = priors[1]
|
||||
self.n = priors[2]
|
||||
self.M_last = self.M
|
||||
self.S_last = self.S
|
||||
|
||||
else:
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.M = 0.
|
||||
self.S = 0.
|
||||
self.M_last = 0.
|
||||
self.S_last = 0.
|
||||
self.n = 0
|
||||
|
||||
def push_data(self, new_data):
|
||||
# short term memory hack
|
||||
if self.max_trackable < 0 or self.n < self.max_trackable:
|
||||
self.n += 1
|
||||
if self.n == 0:
|
||||
self.M_last = new_data
|
||||
self.M = self.M_last
|
||||
self.S_last = 0.
|
||||
else:
|
||||
self.M = self.M_last + (new_data - self.M_last) / self.n
|
||||
self.S = self.S_last + (new_data - self.M_last) * (new_data - self.M)
|
||||
self.M_last = self.M
|
||||
self.S_last = self.S
|
||||
|
||||
def mean(self):
|
||||
return self.M
|
||||
|
||||
def variance(self):
|
||||
if self.n >= 2:
|
||||
return self.S / (self.n - 1.)
|
||||
else:
|
||||
return 0
|
||||
|
||||
def std(self):
|
||||
return np.sqrt(self.variance())
|
||||
|
||||
def params_to_save(self):
|
||||
return [self.M, self.S, self.n]
|
||||
|
||||
class RunningStatFilter:
|
||||
def __init__(self, raw_priors=None, filtered_priors=None, max_trackable=-1):
|
||||
self.raw_stat = RunningStat(raw_priors, -1)
|
||||
self.filtered_stat = RunningStat(filtered_priors, max_trackable)
|
||||
|
||||
def reset(self):
|
||||
self.raw_stat.reset()
|
||||
self.filtered_stat.reset()
|
||||
|
||||
def push_and_update(self, new_data):
|
||||
_std_last = self.raw_stat.std()
|
||||
self.raw_stat.push_data(new_data)
|
||||
_delta_std = self.raw_stat.std() - _std_last
|
||||
if _delta_std <= 0:
|
||||
self.filtered_stat.push_data(new_data)
|
||||
else:
|
||||
pass
|
||||
# self.filtered_stat.push_data(self.filtered_stat.mean())
|
||||
|
||||
# class SequentialBayesian():
|
||||
157
common/swaglog.cc
Normal file
157
common/swaglog.cc
Normal file
@@ -0,0 +1,157 @@
|
||||
#ifndef _GNU_SOURCE
|
||||
#define _GNU_SOURCE
|
||||
#endif
|
||||
|
||||
#include "common/swaglog.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <limits>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
#include <zmq.h>
|
||||
#include <stdarg.h>
|
||||
#include "third_party/json11/json11.hpp"
|
||||
#include "common/version.h"
|
||||
#include "system/hardware/hw.h"
|
||||
|
||||
#include "iqpilot/common/version.h"
|
||||
|
||||
class SwaglogState {
|
||||
public:
|
||||
SwaglogState() {
|
||||
zctx = zmq_ctx_new();
|
||||
sock = zmq_socket(zctx, ZMQ_PUSH);
|
||||
|
||||
// Timeout on shutdown for messages to be received by the logging process
|
||||
int timeout = 100;
|
||||
zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
|
||||
zmq_connect(sock, Path::swaglog_ipc().c_str());
|
||||
|
||||
// workaround for https://github.com/dropbox/json11/issues/38
|
||||
setlocale(LC_NUMERIC, "C");
|
||||
|
||||
print_level = CLOUDLOG_WARNING;
|
||||
if (const char* print_lvl = getenv("LOGPRINT")) {
|
||||
if (strcmp(print_lvl, "debug") == 0) {
|
||||
print_level = CLOUDLOG_DEBUG;
|
||||
} else if (strcmp(print_lvl, "info") == 0) {
|
||||
print_level = CLOUDLOG_INFO;
|
||||
} else if (strcmp(print_lvl, "warning") == 0) {
|
||||
print_level = CLOUDLOG_WARNING;
|
||||
}
|
||||
}
|
||||
|
||||
ctx_j = json11::Json::object{};
|
||||
if (char* dongle_id = getenv("DONGLE_ID")) {
|
||||
ctx_j["dongle_id"] = dongle_id;
|
||||
}
|
||||
if (char* git_origin = getenv("GIT_ORIGIN")) {
|
||||
ctx_j["origin"] = git_origin;
|
||||
}
|
||||
if (char* git_branch = getenv("GIT_BRANCH")) {
|
||||
ctx_j["branch"] = git_branch;
|
||||
}
|
||||
if (char* git_commit = getenv("GIT_COMMIT")) {
|
||||
ctx_j["commit"] = git_commit;
|
||||
}
|
||||
if (char* daemon_name = getenv("MANAGER_DAEMON")) {
|
||||
ctx_j["daemon"] = daemon_name;
|
||||
}
|
||||
ctx_j["version"] = IQPILOT_VERSION;
|
||||
ctx_j["dirty"] = !getenv("CLEAN");
|
||||
ctx_j["device"] = Hardware::get_name();
|
||||
}
|
||||
|
||||
~SwaglogState() {
|
||||
zmq_close(sock);
|
||||
zmq_ctx_destroy(zctx);
|
||||
}
|
||||
|
||||
void log(int levelnum, const char* filename, int lineno, const char* func, const char* msg, const std::string& log_s) {
|
||||
std::lock_guard lk(lock);
|
||||
if (levelnum >= print_level) {
|
||||
printf("%s: %s\n", filename, msg);
|
||||
}
|
||||
zmq_send(sock, log_s.data(), log_s.length(), ZMQ_NOBLOCK);
|
||||
}
|
||||
|
||||
std::mutex lock;
|
||||
void* zctx = nullptr;
|
||||
void* sock = nullptr;
|
||||
int print_level;
|
||||
json11::Json::object ctx_j;
|
||||
};
|
||||
|
||||
bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
|
||||
uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();
|
||||
|
||||
static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
|
||||
char* msg_buf, const json11::Json::object &msg_j={}) {
|
||||
static SwaglogState s;
|
||||
|
||||
json11::Json::object log_j = json11::Json::object {
|
||||
{"ctx", s.ctx_j},
|
||||
{"levelnum", levelnum},
|
||||
{"filename", filename},
|
||||
{"lineno", lineno},
|
||||
{"funcname", func},
|
||||
{"created", seconds_since_epoch()}
|
||||
};
|
||||
if (msg_j.empty()) {
|
||||
log_j["msg"] = msg_buf;
|
||||
} else {
|
||||
log_j["msg"] = msg_j;
|
||||
}
|
||||
|
||||
std::string log_s;
|
||||
log_s += (char)levelnum;
|
||||
((json11::Json)log_j).dump(log_s);
|
||||
s.log(levelnum, filename, lineno, func, msg_buf, log_s);
|
||||
|
||||
free(msg_buf);
|
||||
}
|
||||
|
||||
void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
|
||||
const char* fmt, ...) {
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
char* msg_buf = nullptr;
|
||||
int ret = vasprintf(&msg_buf, fmt, args);
|
||||
va_end(args);
|
||||
if (ret <= 0 || !msg_buf) return;
|
||||
cloudlog_common(levelnum, filename, lineno, func, msg_buf);
|
||||
}
|
||||
|
||||
void cloudlog_t_common(int levelnum, const char* filename, int lineno, const char* func,
|
||||
uint32_t frame_id, const char* fmt, va_list args) {
|
||||
if (!LOG_TIMESTAMPS) return;
|
||||
char* msg_buf = nullptr;
|
||||
int ret = vasprintf(&msg_buf, fmt, args);
|
||||
if (ret <= 0 || !msg_buf) return;
|
||||
json11::Json::object tspt_j = json11::Json::object{
|
||||
{"event", msg_buf},
|
||||
{"time", std::to_string(nanos_since_boot())}
|
||||
};
|
||||
if (frame_id < NO_FRAME_ID) {
|
||||
tspt_j["frame_id"] = std::to_string(frame_id);
|
||||
}
|
||||
tspt_j = json11::Json::object{{"timestamp", tspt_j}};
|
||||
cloudlog_common(levelnum, filename, lineno, func, msg_buf, tspt_j);
|
||||
}
|
||||
|
||||
|
||||
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
|
||||
const char* fmt, ...) {
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
cloudlog_t_common(levelnum, filename, lineno, func, NO_FRAME_ID, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
|
||||
uint32_t frame_id, const char* fmt, ...) {
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
cloudlog_t_common(levelnum, filename, lineno, func, frame_id, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
76
common/swaglog.h
Normal file
76
common/swaglog.h
Normal file
@@ -0,0 +1,76 @@
|
||||
#pragma once
|
||||
|
||||
#include "common/timing.h"
|
||||
|
||||
#define CLOUDLOG_DEBUG 10
|
||||
#define CLOUDLOG_INFO 20
|
||||
#define CLOUDLOG_WARNING 30
|
||||
#define CLOUDLOG_ERROR 40
|
||||
#define CLOUDLOG_CRITICAL 50
|
||||
|
||||
|
||||
#ifdef __GNUC__
|
||||
#define SWAG_LOG_CHECK_FMT(a, b) __attribute__ ((format (printf, a, b)))
|
||||
#else
|
||||
#define SWAG_LOG_CHECK_FMT(a, b)
|
||||
#endif
|
||||
|
||||
void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
|
||||
const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
|
||||
|
||||
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
|
||||
const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
|
||||
|
||||
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
|
||||
uint32_t frame_id, const char* fmt, ...) SWAG_LOG_CHECK_FMT(6, 7);
|
||||
|
||||
|
||||
#define cloudlog(lvl, fmt, ...) cloudlog_e(lvl, __FILE__, __LINE__, \
|
||||
__func__, \
|
||||
fmt, ## __VA_ARGS__)
|
||||
|
||||
#define cloudlog_t(lvl, ...) cloudlog_te(lvl, __FILE__, __LINE__, \
|
||||
__func__, \
|
||||
__VA_ARGS__)
|
||||
|
||||
|
||||
#define cloudlog_rl(burst, millis, lvl, fmt, ...) \
|
||||
{ \
|
||||
static uint64_t __begin = 0; \
|
||||
static int __printed = 0; \
|
||||
static int __missed = 0; \
|
||||
\
|
||||
int __burst = (burst); \
|
||||
int __millis = (millis); \
|
||||
uint64_t __ts = nanos_since_boot(); \
|
||||
\
|
||||
if (!__begin) { __begin = __ts; } \
|
||||
\
|
||||
if (__begin + __millis*1000000ULL < __ts) { \
|
||||
if (__missed) { \
|
||||
cloudlog(CLOUDLOG_WARNING, "cloudlog: %d messages suppressed", __missed); \
|
||||
} \
|
||||
__begin = 0; \
|
||||
__printed = 0; \
|
||||
__missed = 0; \
|
||||
} \
|
||||
\
|
||||
if (__printed < __burst) { \
|
||||
cloudlog(lvl, fmt, ## __VA_ARGS__); \
|
||||
__printed++; \
|
||||
} else { \
|
||||
__missed++; \
|
||||
} \
|
||||
}
|
||||
|
||||
|
||||
#define LOGT(...) cloudlog_t(CLOUDLOG_DEBUG, __VA_ARGS__)
|
||||
#define LOGD(fmt, ...) cloudlog(CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
|
||||
#define LOG(fmt, ...) cloudlog(CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
|
||||
#define LOGW(fmt, ...) cloudlog(CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
|
||||
#define LOGE(fmt, ...) cloudlog(CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
|
||||
|
||||
#define LOGD_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
|
||||
#define LOG_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
|
||||
#define LOGW_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
|
||||
#define LOGE_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
|
||||
144
common/swaglog.py
Normal file
144
common/swaglog.py
Normal file
@@ -0,0 +1,144 @@
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from logging.handlers import BaseRotatingHandler
|
||||
|
||||
import zmq
|
||||
|
||||
from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
|
||||
|
||||
def get_file_handler():
|
||||
Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True)
|
||||
base_filename = os.path.join(Paths.swaglog_root(), "swaglog")
|
||||
handler = SwaglogRotatingFileHandler(base_filename)
|
||||
return handler
|
||||
|
||||
class SwaglogRotatingFileHandler(BaseRotatingHandler):
|
||||
def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
|
||||
super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
|
||||
self.base_filename = base_filename
|
||||
self.interval = interval # seconds
|
||||
self.max_bytes = max_bytes
|
||||
self.backup_count = backup_count
|
||||
self.log_files = self.get_existing_logfiles()
|
||||
log_indexes = [f.split(".")[-1] for f in self.log_files]
|
||||
self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
|
||||
self.last_rollover = None
|
||||
self.doRollover()
|
||||
|
||||
def _open(self):
|
||||
self.last_rollover = time.monotonic()
|
||||
self.last_file_idx += 1
|
||||
next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
|
||||
stream = open(next_filename, self.mode, encoding=self.encoding)
|
||||
self.log_files.insert(0, next_filename)
|
||||
return stream
|
||||
|
||||
def get_existing_logfiles(self):
|
||||
log_files = list()
|
||||
base_dir = os.path.dirname(self.base_filename)
|
||||
for fn in os.listdir(base_dir):
|
||||
fp = os.path.join(base_dir, fn)
|
||||
if fp.startswith(self.base_filename) and os.path.isfile(fp):
|
||||
log_files.append(fp)
|
||||
return sorted(log_files)
|
||||
|
||||
def shouldRollover(self, record):
|
||||
size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
|
||||
time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= time.monotonic()
|
||||
return size_exceeded or time_exceeded
|
||||
|
||||
def doRollover(self):
|
||||
if self.stream:
|
||||
self.stream.close()
|
||||
self.stream = self._open()
|
||||
|
||||
if self.backup_count > 0:
|
||||
while len(self.log_files) > self.backup_count:
|
||||
to_delete = self.log_files.pop()
|
||||
if os.path.exists(to_delete): # just being safe, should always exist
|
||||
os.remove(to_delete)
|
||||
|
||||
class UnixDomainSocketHandler(logging.Handler):
|
||||
def __init__(self, formatter):
|
||||
logging.Handler.__init__(self)
|
||||
self.setFormatter(formatter)
|
||||
self.pid = None
|
||||
|
||||
self.zctx = None
|
||||
self.sock = None
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
if self.sock is not None:
|
||||
self.sock.close()
|
||||
if self.zctx is not None:
|
||||
self.zctx.term()
|
||||
|
||||
def connect(self):
|
||||
self.zctx = zmq.Context()
|
||||
self.sock = self.zctx.socket(zmq.PUSH)
|
||||
self.sock.setsockopt(zmq.LINGER, 10)
|
||||
self.sock.connect(Paths.swaglog_ipc())
|
||||
self.pid = os.getpid()
|
||||
|
||||
def emit(self, record):
|
||||
if os.getpid() != self.pid:
|
||||
# TODO suppresses warning about forking proc with zmq socket, fix root cause
|
||||
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<zmq.*>")
|
||||
self.connect()
|
||||
|
||||
msg = self.format(record).rstrip('\n')
|
||||
# print("SEND".format(repr(msg)))
|
||||
try:
|
||||
s = chr(record.levelno)+msg
|
||||
self.sock.send(s.encode('utf8'), zmq.NOBLOCK)
|
||||
except zmq.error.Again:
|
||||
# drop :/
|
||||
pass
|
||||
|
||||
|
||||
class ForwardingHandler(logging.Handler):
|
||||
def __init__(self, target_logger):
|
||||
super().__init__()
|
||||
self.target_logger = target_logger
|
||||
|
||||
def emit(self, record):
|
||||
self.target_logger.handle(record)
|
||||
|
||||
|
||||
def add_file_handler(log):
|
||||
"""
|
||||
Function to add the file log handler to swaglog.
|
||||
This can be used to store logs when logmessaged is not running.
|
||||
"""
|
||||
handler = get_file_handler()
|
||||
handler.setFormatter(SwagLogFileFormatter(log))
|
||||
log.addHandler(handler)
|
||||
|
||||
|
||||
cloudlog = log = SwagLogger()
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
outhandler = logging.StreamHandler()
|
||||
|
||||
print_level = os.environ.get('LOGPRINT', 'warning')
|
||||
if print_level == 'debug':
|
||||
outhandler.setLevel(logging.DEBUG)
|
||||
elif print_level == 'info':
|
||||
outhandler.setLevel(logging.INFO)
|
||||
elif print_level == 'warning':
|
||||
outhandler.setLevel(logging.WARNING)
|
||||
|
||||
ipchandler = UnixDomainSocketHandler(SwagFormatter(log))
|
||||
|
||||
log.addHandler(outhandler)
|
||||
# logs are sent through IPC before writing to disk to prevent disk I/O blocking
|
||||
log.addHandler(ipchandler)
|
||||
1
common/tests/.gitignore
vendored
Normal file
1
common/tests/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
test_common
|
||||
0
common/tests/__init__.py
Normal file
0
common/tests/__init__.py
Normal file
19
common/tests/test_file_helpers.py
Normal file
19
common/tests/test_file_helpers.py
Normal file
@@ -0,0 +1,19 @@
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from openpilot.common.utils import atomic_write
|
||||
|
||||
|
||||
class TestFileHelpers:
|
||||
def run_atomic_write_func(self, atomic_write_func):
|
||||
path = f"/tmp/tmp{uuid4()}"
|
||||
with atomic_write_func(path) as f:
|
||||
f.write("test")
|
||||
assert not os.path.exists(path)
|
||||
|
||||
with open(path) as f:
|
||||
assert f.read() == "test"
|
||||
os.remove(path)
|
||||
|
||||
def test_atomic_write(self):
|
||||
self.run_atomic_write_func(atomic_write)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user