mawei/sunnypilot
1
0
Fork 0
mirror of https://github.com/sunnypilot/sunnypilot.git synced 2026-08-21 02:43:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

sunnypilot v2026.003.000 release

Browse Source 
date: 2026-08-19T09:43:43
master commit: ba29a38507
This commit is contained in:
github-actions[bot]
2026-08-19 09:43:44 +00:00
commit e01ac7f80f
4046 changed files with 997234 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openpilot/__init__.py
View File
openpilot/cereal/README.md
+95
View File
@@ -0,0 +1,95 @@
# What is cereal?
cereal is the messaging system for openpilot. It uses [msgq](https://github.com/commaai/msgq) as a pub/sub backend, and [Cap'n proto](https://capnproto.org/capnp-tool.html) for serialization of the structs.
## Messaging Spec
You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`.
All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type.
### Best Practices
- **All fields must describe quantities in SI units**, unless otherwise specified in the field name.
- In the context of the message they are in, field names should be completely unambiguous.
- All values should be easy to plot and be human-readable with minimal parsing.
### Maintaining backwards-compatibility
When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can
be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other
things are not. Read more details [here](https://capnproto.org/language.html).
### Custom forks
Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
spec, however this could conflict with future changes made in mainline openpilot's cereal spec. Rebasing against mainline openpilot
then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
[custom.capnp](custom.capnp) that we will leave empty in mainline openpilot's cereal spec. **If you only modify those, you can ensure your
fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
An example of compatible changes:
```diff
diff --git a/openpilot/cereal/custom.capnp b/openpilot/cereal/custom.capnp
index 3348e859e..3365c7b98 100644
--- a/openpilot/cereal/custom.capnp
+++ b/openpilot/cereal/custom.capnp
@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
# DO rename the structs
# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
-struct CustomReserved0 @0x81c2f05a394cf4af {
+struct SteeringInfo @0x81c2f05a394cf4af {
+ active @0 :Bool;
+ steeringAngleDeg @1 :Float32;
+ steeringRateDeg @2 :Float32;
+ steeringAccelDeg @3 :Float32;
}
struct CustomReserved1 @0xaedffd8f31e7b55d {
diff --git a/openpilot/cereal/log.capnp b/openpilot/cereal/log.capnp
index 1209f3fd9..b189f58b6 100644
--- a/openpilot/cereal/log.capnp
+++ b/openpilot/cereal/log.capnp
@@ -2558,14 +2558,14 @@ struct Event {
# DO change the name of the field
# DON'T change anything after the "@"
- customReservedRawData0 @124 :Data;
+ rawCanData @124 :Data;
customReservedRawData1 @125 :Data;
customReservedRawData2 @126 :Data;
# DO change the name of the field and struct
# DON'T change the ID (e.g. @107)
# DON'T change which struct it points to
- customReserved0 @107 :Custom.CustomReserved0;
+ steeringInfo @107 :Custom.SteeringInfo;
customReserved1 @108 :Custom.CustomReserved1;
customReserved2 @109 :Custom.CustomReserved2;
customReserved3 @110 :Custom.CustomReserved3;
```
---
Example
---
```python
import openpilot.cereal.messaging as messaging
# in subscriber
sm = messaging.SubMaster(['sensorEvents'])
while 1:
sm.update()
print(sm['sensorEvents'])
```
```python
# in publisher
pm = messaging.PubMaster(['sensorEvents'])
dat = messaging.new_message('sensorEvents', size=1)
dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
pm.send('sensorEvents', dat)
```
openpilot/cereal/SConscript
+22
View File
@@ -0,0 +1,22 @@
Import('env', 'common', 'msgq')
cereal_dir = Dir('.')
gen_dir = Dir('gen')
# Build cereal
schema_files = ['log.capnp', 'deprecated.capnp', 'custom.capnp']
car_capnp = '#opendbc_repo/opendbc/car/car.capnp'
all_output = schema_files + ['car.capnp']
env.Command([f'gen/cpp/{s}.c++' for s in all_output] + [f'gen/cpp/{s}.h' for s in all_output],
schema_files + [car_capnp],
f"capnpc --src-prefix={cereal_dir.path} --src-prefix=opendbc_repo/opendbc/car --import-path=opendbc_repo/opendbc/car $SOURCES -o c++:{gen_dir.path}/cpp/")
cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in all_output])
# Build messaging
services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
Export('cereal', 'socketmaster')
openpilot/cereal/__init__.py
+11
View File
@@ -0,0 +1,11 @@
import os
import capnp
from importlib.resources import as_file, files
capnp.remove_import_hook()
with as_file(files("openpilot.cereal")) as fspath, as_file(files("opendbc")) as opendbc_path:
CEREAL_PATH = fspath.as_posix()
opendbc_import_path = os.path.join(os.path.realpath(opendbc_path.as_posix()), 'car')
log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"), imports=[opendbc_import_path])
custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"), imports=[opendbc_import_path])
openpilot/cereal/custom.capnp
+499
View File
@@ -0,0 +1,499 @@
using Cxx = import "/include/c++.capnp";
$Cxx.namespace("cereal");
@0xb526ba661d550a59;
# custom.capnp: a home for empty structs reserved for custom forks
# These structs are guaranteed to remain reserved and empty in mainline
# cereal, so use these if you want custom events in your fork.
# DO rename the structs
# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
struct ModularAssistiveDrivingSystem {
state @0 :ModularAssistiveDrivingSystemState;
enabled @1 :Bool;
active @2 :Bool;
available @3 :Bool;
enum ModularAssistiveDrivingSystemState {
disabled @0;
paused @1;
enabled @2;
softDisabling @3;
overriding @4;
}
}
struct IntelligentCruiseButtonManagement {
state @0 :IntelligentCruiseButtonManagementState;
sendButton @1 :SendButtonState;
vTarget @2 :Float32;
enum IntelligentCruiseButtonManagementState {
inactive @0; # No button press or default state
preActive @1; # Pre-active state before transitioning to increasing or decreasing
increasing @2; # Increasing speed
decreasing @3; # Decreasing speed
holding @4; # Holding steady speed
}
enum SendButtonState {
none @0;
increase @1;
decrease @2;
}
}
# Same struct as Log.RadarState.LeadData
struct LeadData {
dRel @0 :Float32;
yRel @1 :Float32;
vRel @2 :Float32;
aRel @3 :Float32;
vLead @4 :Float32;
dPath @6 :Float32;
vLat @7 :Float32;
vLeadK @8 :Float32;
aLeadK @9 :Float32;
fcw @10 :Bool;
status @11 :Bool;
aLeadTau @12 :Float32;
modelProb @13 :Float32;
radar @14 :Bool;
radarTrackId @15 :Int32 = -1;
aLeadDEPRECATED @5 :Float32;
}
struct SelfdriveStateSP @0x81c2f05a394cf4af {
mads @0 :ModularAssistiveDrivingSystem;
intelligentCruiseButtonManagement @1 :IntelligentCruiseButtonManagement;
buttonsPressed @2 :UInt16;
buttonsReleaseToggle @3 :UInt16;
enum AudibleAlert {
none @0;
engage @1;
disengage @2;
refuse @3;
warningSoft @4;
warningImmediate @5;
prompt @6;
promptRepeat @7;
promptDistracted @8;
# unused, these are reserved for upstream events so we don't collide
reserved9 @9;
reserved10 @10;
reserved11 @11;
reserved12 @12;
reserved13 @13;
reserved14 @14;
reserved15 @15;
reserved16 @16;
reserved17 @17;
reserved18 @18;
reserved19 @19;
reserved20 @20;
reserved21 @21;
reserved22 @22;
reserved23 @23;
reserved24 @24;
reserved25 @25;
reserved26 @26;
reserved27 @27;
reserved28 @28;
reserved29 @29;
reserved30 @30;
promptSingleLow @31;
promptSingleHigh @32;
}
}
struct ModelManagerSP @0xaedffd8f31e7b55d {
activeBundle @0 :ModelBundle;
selectedBundle @1 :ModelBundle;
availableBundles @2 :List(ModelBundle);
struct DownloadUri {
uri @0 :Text;
sha256 @1 :Text;
}
enum DownloadStatus {
notDownloading @0;
downloading @1;
downloaded @2;
cached @3;
failed @4;
}
struct DownloadProgress {
status @0 :DownloadStatus;
progress @1 :Float32;
eta @2 :UInt32;
}
struct Chunk {
fileName @0 :Text;
sha256 @1 :Text;
}
struct Artifact {
fileName @0 :Text;
downloadUri @1 :DownloadUri;
downloadProgress @2 :DownloadProgress;
chunks @3 :List(Chunk);
}
struct Model {
type @0 :Type;
artifact @1 :Artifact; # Main artifact
metadata @2 :Artifact; # Metadata artifact
enum Type {
supercombo @0;
navigation @1;
vision @2;
policy @3;
offPolicy @4;
onPolicy @5;
chunked @6;
}
}
enum Runner {
snpe @0;
tinygrad @1;
stock @2;
}
struct Override {
key @0 :Text;
value @1 :Text;
}
struct ModelBundle {
index @0 :UInt32;
internalName @1 :Text;
displayName @2 :Text;
models @3 :List(Model);
status @4 :DownloadStatus;
generation @5 :UInt32;
environment @6 :Text;
runner @7 :Runner;
is20hz @8 :Bool;
ref @9 :Text;
minimumSelectorVersion @10 :UInt32;
overrides @11 :List(Override);
}
}
struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 {
dec @0 :DynamicExperimentalControl;
longitudinalPlanSource @1 :LongitudinalPlanSource;
smartCruiseControl @2 :SmartCruiseControl;
speedLimit @3 :SpeedLimit;
vTarget @4 :Float32;
aTarget @5 :Float32;
events @6 :List(OnroadEventSP.Event);
e2eAlerts @7 :E2eAlerts;
struct DynamicExperimentalControl {
state @0 :DynamicExperimentalControlState;
enabled @1 :Bool;
active @2 :Bool;
enum DynamicExperimentalControlState {
acc @0;
blended @1;
}
}
struct SmartCruiseControl {
vision @0 :Vision;
map @1 :Map;
struct Vision {
state @0 :VisionState;
vTarget @1 :Float32;
aTarget @2 :Float32;
currentLateralAccel @3 :Float32;
maxPredictedLateralAccel @4 :Float32;
enabled @5 :Bool;
active @6 :Bool;
}
struct Map {
state @0 :MapState;
vTarget @1 :Float32;
aTarget @2 :Float32;
enabled @3 :Bool;
active @4 :Bool;
}
enum VisionState {
disabled @0; # System disabled or inactive.
enabled @1; # No predicted substantial turn on vision range.
entering @2; # A substantial turn is predicted ahead, adapting speed to turn comfort levels.
turning @3; # Actively turning. Managing acceleration to provide a roll on turn feeling.
leaving @4; # Road ahead straightens. Start to allow positive acceleration.
overriding @5; # System overriding with manual control.
}
enum MapState {
disabled @0; # System disabled or inactive.
enabled @1; # No predicted substantial turn on map range.
turning @2; # Actively turning. Managing acceleration to provide a roll on turn feeling.
overriding @3; # System overriding with manual control.
}
}
struct SpeedLimit {
resolver @0 :Resolver;
assist @1 :Assist;
struct Resolver {
speedLimit @0 :Float32;
distToSpeedLimit @1 :Float32;
source @2 :Source;
speedLimitOffset @3 :Float32;
speedLimitLast @4 :Float32;
speedLimitFinal @5 :Float32;
speedLimitFinalLast @6 :Float32;
speedLimitValid @7 :Bool;
speedLimitLastValid @8 :Bool;
}
struct Assist {
state @0 :AssistState;
enabled @1 :Bool;
active @2 :Bool;
vTarget @3 :Float32;
aTarget @4 :Float32;
}
enum Source {
none @0;
car @1;
map @2;
}
enum AssistState {
disabled @0;
inactive @1; # No speed limit set or not enabled by parameter.
preActive @2;
pending @3; # Awaiting new speed limit.
adapting @4; # Reducing speed to match new speed limit.
active @5; # Cruising at speed limit.
}
}
enum LongitudinalPlanSource {
cruise @0;
sccVision @1;
sccMap @2;
speedLimitAssist @3;
}
struct E2eAlerts {
greenLightAlert @0 :Bool;
leadDepartAlert @1 :Bool;
}
}
struct OnroadEventSP @0xda96579883444c35 {
events @0 :List(Event);
struct Event {
name @0 :EventName;
# event types
enable @1 :Bool;
noEntry @2 :Bool;
warning @3 :Bool; # alerts presented only when enabled or soft disabling
userDisable @4 :Bool;
softDisable @5 :Bool;
immediateDisable @6 :Bool;
preEnable @7 :Bool;
permanent @8 :Bool; # alerts presented regardless of openpilot state
overrideLateral @10 :Bool;
overrideLongitudinal @9 :Bool;
}
enum EventName {
lkasEnable @0;
lkasDisable @1;
manualSteeringRequired @2;
manualLongitudinalRequired @3;
silentLkasEnable @4;
silentLkasDisable @5;
silentBrakeHold @6;
silentWrongGear @7;
silentReverseGear @8;
silentDoorOpen @9;
silentSeatbeltNotLatched @10;
silentParkBrake @11;
controlsMismatchLateral @12;
hyundaiRadarTracksConfirmed @13;
experimentalModeSwitched @14;
wrongCarModeAlertOnly @15;
pedalPressedAlertOnly @16;
laneTurnLeft @17;
laneTurnRight @18;
speedLimitPreActive @19;
speedLimitActive @20;
speedLimitChanged @21;
speedLimitPending @22;
e2eChime @23;
laneChangeRoadEdge @24;
}
}
struct CarParamsSP @0x80ae746ee2596b11 {
flags @0 :UInt32; # flags for car specific quirks in sunnypilot
safetyParam @1 : Int16; # flags for sunnypilot's custom safety flags
pcmCruiseSpeed @3 :Bool;
intelligentCruiseButtonManagementAvailable @4 :Bool;
enableGasInterceptor @5 :Bool;
neuralNetworkLateralControl @2 :NeuralNetworkLateralControl;
struct NeuralNetworkLateralControl {
model @0 :Model;
fuzzyFingerprint @1 :Bool;
struct Model {
path @0 :Text;
name @1 :Text;
}
}
}
struct CarControlSP @0xa5cd762cd951a455 {
mads @0 :ModularAssistiveDrivingSystem;
params @1 :List(Param);
leadOne @2 :LeadData;
leadTwo @3 :LeadData;
intelligentCruiseButtonManagement @4 :IntelligentCruiseButtonManagement;
struct Param {
key @0 :Text;
type @2 :ParamType;
value @3 :Data;
valueDEPRECATED @1 :Text; # The data type change may cause issues with backwards compatibility.
}
enum ParamType {
string @0;
bool @1;
int @2;
float @3;
time @4;
json @5;
bytes @6;
}
}
struct BackupManagerSP @0xf98d843bfd7004a3 {
backupStatus @0 :Status;
restoreStatus @1 :Status;
backupProgress @2 :Float32;
restoreProgress @3 :Float32;
lastError @4 :Text;
currentBackup @5 :BackupInfo;
backupHistory @6 :List(BackupInfo);
enum Status {
idle @0;
inProgress @1;
completed @2;
failed @3;
}
struct Version {
major @0 :UInt16;
minor @1 :UInt16;
patch @2 :UInt16;
build @3 :UInt16;
branch @4 :Text;
}
struct MetadataEntry {
key @0 :Text;
value @1 :Text;
tags @2 :List(Text);
}
struct BackupInfo {
deviceId @0 :Text;
version @1 :UInt32;
config @2 :Text;
isEncrypted @3 :Bool;
createdAt @4 :Text; # ISO timestamp
updatedAt @5 :Text; # ISO timestamp
sunnypilotVersion @6 :Version;
backupMetadata @7 :List(MetadataEntry);
}
}
struct CarStateSP @0xb86e6369214c01c8 {
speedLimit @0 :Float32;
}
struct LiveMapDataSP @0xf416ec09499d9d19 {
speedLimitValid @0 :Bool;
speedLimit @1 :Float32;
speedLimitAheadValid @2 :Bool;
speedLimitAhead @3 :Float32;
speedLimitAheadDistance @4 :Float32;
roadName @5 :Text;
}
struct ModelDataV2SP @0xa1680744031fdb2d {
laneTurnDirection @0 :TurnDirection;
leftLaneChangeEdgeBlock @1 :Bool;
rightLaneChangeEdgeBlock @2 :Bool;
enum TurnDirection {
none @0;
turnLeft @1;
turnRight @2;
}
}
struct CustomReserved10 @0xcb9fd56c7057593a {
}
struct CustomReserved11 @0xc2243c65e0340384 {
}
struct CustomReserved12 @0x9ccdc8676701b412 {
}
struct CustomReserved13 @0xcd96dafb67a082d0 {
}
struct CustomReserved14 @0xb057204d7deadf3f {
}
struct CustomReserved15 @0xbd443b539493bc68 {
}
struct CustomReserved16 @0xfc6241ed8877b611 {
}
struct CustomReserved17 @0xa30662f84033036c {
}
struct CustomReserved18 @0xc86a3d38d13eb3ef {
}
struct CustomReserved19 @0xa4f1eb3323f5f582 {
}
openpilot/cereal/deprecated.capnp
+787
View File
@@ -0,0 +1,787 @@
using Cxx = import "/include/c++.capnp";
$Cxx.namespace("cereal");
@0x80ef1ec4889c2a63;
# deprecated.capnp: a home for deprecated structs
struct LogRotate @0x9811e1f38f62f2d1 {
segmentNum @0 :Int32;
path @1 :Text;
}
struct LiveUI @0xc08240f996aefced {
rearViewCam @0 :Bool;
alertText1 @1 :Text;
alertText2 @2 :Text;
awarenessStatus @3 :Float32;
}
struct UiLayoutState @0x88dcce08ad29dda0 {
activeApp @0 :App;
sidebarCollapsed @1 :Bool;
mapEnabled @2 :Bool;
mockEngaged @3 :Bool;
enum App @0x9917470acf94d285 {
home @0;
music @1;
nav @2;
settings @3;
none @4;
}
}
struct OrbslamCorrection @0x8afd33dc9b35e1aa {
correctionMonoTime @0 :UInt64;
prePositionECEF @1 :List(Float64);
postPositionECEF @2 :List(Float64);
prePoseQuatECEF @3 :List(Float32);
postPoseQuatECEF @4 :List(Float32);
numInliers @5 :UInt32;
}
struct EthernetPacket @0xa99a9d5b33cf5859 {
pkt @0 :Data;
ts @1 :Float32;
}
struct CellInfo @0xcff7566681c277ce {
timestamp @0 :UInt64;
repr @1 :Text; # android toString() for now
}
struct WifiScan @0xd4df5a192382ba0b {
bssid @0 :Text;
ssid @1 :Text;
capabilities @2 :Text;
frequency @3 :Int32;
level @4 :Int32;
timestamp @5 :Int64;
centerFreq0 @6 :Int32;
centerFreq1 @7 :Int32;
channelWidth @8 :ChannelWidth;
operatorFriendlyName @9 :Text;
venueName @10 :Text;
is80211mcResponder @11 :Bool;
passpoint @12 :Bool;
distanceCm @13 :Int32;
distanceSdCm @14 :Int32;
enum ChannelWidth @0xcb6a279f015f6b51 {
w20Mhz @0;
w40Mhz @1;
w80Mhz @2;
w160Mhz @3;
w80Plus80Mhz @4;
}
}
struct LiveEventData @0x94b7baa90c5c321e {
name @0 :Text;
value @1 :Int32;
}
struct ModelData @0xb8aad62cffef28a9 {
frameId @0 :UInt32;
frameAge @12 :UInt32;
frameDropPerc @13 :Float32;
timestampEof @9 :UInt64;
modelExecutionTime @14 :Float32;
gpuExecutionTime @16 :Float32;
rawPred @15 :Data;
path @1 :PathData;
leftLane @2 :PathData;
rightLane @3 :PathData;
lead @4 :LeadData;
freePath @6 :List(Float32);
settings @5 :ModelSettings;
leadFuture @7 :LeadData;
speed @8 :List(Float32);
meta @10 :MetaData;
longitudinal @11 :LongitudinalData;
struct PathData @0x8817eeea389e9f08 {
points @0 :List(Float32);
prob @1 :Float32;
std @2 :Float32;
stds @3 :List(Float32);
poly @4 :List(Float32);
validLen @5 :Float32;
}
struct LeadData @0xd1c9bef96d26fa91 {
dist @0 :Float32;
prob @1 :Float32;
std @2 :Float32;
relVel @3 :Float32;
relVelStd @4 :Float32;
relY @5 :Float32;
relYStd @6 :Float32;
relA @7 :Float32;
relAStd @8 :Float32;
}
struct ModelSettings @0xa26e3710efd3e914 {
bigBoxX @0 :UInt16;
bigBoxY @1 :UInt16;
bigBoxWidth @2 :UInt16;
bigBoxHeight @3 :UInt16;
boxProjection @4 :List(Float32);
yuvCorrection @5 :List(Float32);
inputTransform @6 :List(Float32);
}
struct MetaData @0x9744f25fb60f2bf8 {
engagedProb @0 :Float32;
desirePrediction @1 :List(Float32);
brakeDisengageProb @2 :Float32;
gasDisengageProb @3 :Float32;
steerOverrideProb @4 :Float32;
desireState @5 :List(Float32);
}
struct LongitudinalData @0xf98f999c6a071122 {
distances @2 :List(Float32);
speeds @0 :List(Float32);
accelerations @1 :List(Float32);
}
}
struct ECEFPoint @0xc25bbbd524983447 {
x @0 :Float64;
y @1 :Float64;
z @2 :Float64;
}
struct ECEFPointDEPRECATED @0xe10e21168db0c7f7 {
x @0 :Float32;
y @1 :Float32;
z @2 :Float32;
}
struct GPSPlannerPoints @0xab54c59699f8f9f3 {
curPosDEPRECATED @0 :ECEFPointDEPRECATED;
pointsDEPRECATED @1 :List(ECEFPointDEPRECATED);
curPos @6 :ECEFPoint;
points @7 :List(ECEFPoint);
valid @2 :Bool;
trackName @3 :Text;
speedLimit @4 :Float32;
accelTarget @5 :Float32;
}
struct GPSPlannerPlan @0xf5ad1d90cdc1dd6b {
valid @0 :Bool;
poly @1 :List(Float32);
trackName @2 :Text;
speed @3 :Float32;
acceleration @4 :Float32;
pointsDEPRECATED @5 :List(ECEFPointDEPRECATED);
points @6 :List(ECEFPoint);
xLookahead @7 :Float32;
}
struct UiNavigationEvent @0x90c8426c3eaddd3b {
type @0: Type;
status @1: Status;
distanceTo @2: Float32;
endRoadPointDEPRECATED @3: ECEFPointDEPRECATED;
endRoadPoint @4: ECEFPoint;
enum Type @0xe8db07dcf8fcea05 {
none @0;
laneChangeLeft @1;
laneChangeRight @2;
mergeLeft @3;
mergeRight @4;
turnLeft @5;
turnRight @6;
}
enum Status @0xb9aa88c75ef99a1f {
none @0;
passive @1;
approaching @2;
active @3;
}
}
struct LiveLocationData @0xb99b2bc7a57e8128 {
status @0 :UInt8;
# 3D fix
lat @1 :Float64;
lon @2 :Float64;
alt @3 :Float32; # m
# speed
speed @4 :Float32; # m/s
# NED velocity components
vNED @5 :List(Float32);
# roll, pitch, heading (x,y,z)
roll @6 :Float32; # WRT to center of earth?
pitch @7 :Float32; # WRT to center of earth?
heading @8 :Float32; # WRT to north?
# what are these?
wanderAngle @9 :Float32;
trackAngle @10 :Float32;
# car frame -- https://upload.wikimedia.org/wikipedia/commons/f/f5/RPY_angles_of_cars.png
# gyro, in car frame, deg/s
gyro @11 :List(Float32);
# accel, in car frame, m/s^2
accel @12 :List(Float32);
accuracy @13 :Accuracy;
source @14 :SensorSource;
# if we are fixing a location in the past
fixMonoTime @15 :UInt64;
gpsWeek @16 :Int32;
timeOfWeek @17 :Float64;
positionECEF @18 :List(Float64);
poseQuatECEF @19 :List(Float32);
pitchCalibration @20 :Float32;
yawCalibration @21 :Float32;
imuFrame @22 :List(Float32);
struct Accuracy @0x943dc4625473b03f {
pNEDError @0 :List(Float32);
vNEDError @1 :List(Float32);
rollError @2 :Float32;
pitchError @3 :Float32;
headingError @4 :Float32;
ellipsoidSemiMajorError @5 :Float32;
ellipsoidSemiMinorError @6 :Float32;
ellipsoidOrientationError @7 :Float32;
}
enum SensorSource @0xc871d3cc252af657 {
applanix @0;
kalman @1;
orbslam @2;
timing @3;
dummy @4;
}
}
struct OrbOdometry @0xd7700859ed1f5b76 {
# timing first
startMonoTime @0 :UInt64;
endMonoTime @1 :UInt64;
# fundamental matrix and error
f @2: List(Float64);
err @3: Float64;
# number of inlier points
inliers @4: Int32;
# for debug only
# indexed by endMonoTime features
# value is startMonoTime feature match
# -1 if no match
matches @5: List(Int16);
}
struct OrbFeatures @0xcd60164a8a0159ef {
timestampEof @0 :UInt64;
# transposed arrays of normalized image coordinates
# len(xs) == len(ys) == len(descriptors) * 32
xs @1 :List(Float32);
ys @2 :List(Float32);
descriptors @3 :Data;
octaves @4 :List(Int8);
# match index to last OrbFeatures
# -1 if no match
timestampLastEof @5 :UInt64;
matches @6: List(Int16);
}
struct OrbFeaturesSummary @0xd500d30c5803fa4f {
timestampEof @0 :UInt64;
timestampLastEof @1 :UInt64;
featureCount @2 :UInt16;
matchCount @3 :UInt16;
computeNs @4 :UInt64;
}
struct OrbKeyFrame @0xc8233c0345e27e24 {
# this is a globally unique id for the KeyFrame
id @0: UInt64;
# this is the location of the KeyFrame
pos @1: ECEFPoint;
# these are the features in the world
# len(dpos) == len(descriptors) * 32
dpos @2 :List(ECEFPoint);
descriptors @3 :Data;
}
struct KalmanOdometry @0x92e21bb7ea38793a {
trans @0 :List(Float32); # m/s in device frame
rot @1 :List(Float32); # rad/s in device frame
transStd @2 :List(Float32); # std m/s in device frame
rotStd @3 :List(Float32); # std rad/s in device frame
}
struct OrbObservation @0x9b326d4e436afec7 {
observationMonoTime @0 :UInt64;
normalizedCoordinates @1 :List(Float32);
locationECEF @2 :List(Float64);
matchDistance @3: UInt32;
}
struct CalibrationFeatures @0x8fdfadb254ea867a {
frameId @0 :UInt32;
p0 @1 :List(Float32);
p1 @2 :List(Float32);
status @3 :List(Int8);
}
struct NavStatus @0xbd8822120928120c {
isNavigating @0 :Bool;
currentAddress @1 :Address;
struct Address @0xce7cd672cacc7814 {
title @0 :Text;
lat @1 :Float64;
lng @2 :Float64;
house @3 :Text;
address @4 :Text;
street @5 :Text;
city @6 :Text;
state @7 :Text;
country @8 :Text;
}
}
struct NavUpdate @0xdb98be6565516acb {
isNavigating @0 :Bool;
curSegment @1 :Int32;
segments @2 :List(Segment);
struct LatLng @0x9eaef9187cadbb9b {
lat @0 :Float64;
lng @1 :Float64;
}
struct Segment @0xa5b39b4fc4d7da3f {
from @0 :LatLng;
to @1 :LatLng;
updateTime @2 :Int32;
distance @3 :Int32;
crossTime @4 :Int32;
exitNo @5 :Int32;
instruction @6 :Instruction;
parts @7 :List(LatLng);
enum Instruction @0xc5417a637451246f {
turnLeft @0;
turnRight @1;
keepLeft @2;
keepRight @3;
straight @4;
roundaboutExitNumber @5;
roundaboutExit @6;
roundaboutTurnLeft @7;
unkn8 @8;
roundaboutStraight @9;
unkn10 @10;
roundaboutTurnRight @11;
unkn12 @12;
roundaboutUturn @13;
unkn14 @14;
arrive @15;
exitLeft @16;
exitRight @17;
unkn18 @18;
uturn @19;
# ...
}
}
}
struct TrafficEvent @0xacfa74a094e62626 {
type @0 :Type;
distance @1 :Float32;
action @2 :Action;
resuming @3 :Bool;
enum Type @0xd85d75253435bf4b {
stopSign @0;
lightRed @1;
lightYellow @2;
lightGreen @3;
stopLight @4;
}
enum Action @0xa6f6ce72165ccb49 {
none @0;
yield @1;
stop @2;
resumeReady @3;
}
}
struct AndroidGnss @0xdfdf30d03fc485bd {
union {
measurements @0 :Measurements;
navigationMessage @1 :NavigationMessage;
}
struct Measurements @0xa20710d4f428d6cd {
clock @0 :Clock;
measurements @1 :List(Measurement);
struct Clock @0xa0e27b453a38f450 {
timeNanos @0 :Int64;
hardwareClockDiscontinuityCount @1 :Int32;
hasTimeUncertaintyNanos @2 :Bool;
timeUncertaintyNanos @3 :Float64;
hasLeapSecond @4 :Bool;
leapSecond @5 :Int32;
hasFullBiasNanos @6 :Bool;
fullBiasNanos @7 :Int64;
hasBiasNanos @8 :Bool;
biasNanos @9 :Float64;
hasBiasUncertaintyNanos @10 :Bool;
biasUncertaintyNanos @11 :Float64;
hasDriftNanosPerSecond @12 :Bool;
driftNanosPerSecond @13 :Float64;
hasDriftUncertaintyNanosPerSecond @14 :Bool;
driftUncertaintyNanosPerSecond @15 :Float64;
}
struct Measurement @0xd949bf717d77614d {
svId @0 :Int32;
constellation @1 :Constellation;
timeOffsetNanos @2 :Float64;
state @3 :Int32;
receivedSvTimeNanos @4 :Int64;
receivedSvTimeUncertaintyNanos @5 :Int64;
cn0DbHz @6 :Float64;
pseudorangeRateMetersPerSecond @7 :Float64;
pseudorangeRateUncertaintyMetersPerSecond @8 :Float64;
accumulatedDeltaRangeState @9 :Int32;
accumulatedDeltaRangeMeters @10 :Float64;
accumulatedDeltaRangeUncertaintyMeters @11 :Float64;
hasCarrierFrequencyHz @12 :Bool;
carrierFrequencyHz @13 :Float32;
hasCarrierCycles @14 :Bool;
carrierCycles @15 :Int64;
hasCarrierPhase @16 :Bool;
carrierPhase @17 :Float64;
hasCarrierPhaseUncertainty @18 :Bool;
carrierPhaseUncertainty @19 :Float64;
hasSnrInDb @20 :Bool;
snrInDb @21 :Float64;
multipathIndicator @22 :MultipathIndicator;
enum Constellation @0x9ef1f3ff0deb5ffb {
unknown @0;
gps @1;
sbas @2;
glonass @3;
qzss @4;
beidou @5;
galileo @6;
}
enum State @0xcbb9490adce12d72 {
unknown @0;
codeLock @1;
bitSync @2;
subframeSync @3;
towDecoded @4;
msecAmbiguous @5;
symbolSync @6;
gloStringSync @7;
gloTodDecoded @8;
bdsD2BitSync @9;
bdsD2SubframeSync @10;
galE1bcCodeLock @11;
galE1c2ndCodeLock @12;
galE1bPageSync @13;
sbasSync @14;
}
enum MultipathIndicator @0xc04e7b6231d4caa8 {
unknown @0;
detected @1;
notDetected @2;
}
}
}
struct NavigationMessage @0xe2517b083095fd4e {
type @0 :Int32;
svId @1 :Int32;
messageId @2 :Int32;
submessageId @3 :Int32;
data @4 :Data;
status @5 :Status;
enum Status @0xec1ff7996b35366f {
unknown @0;
parityPassed @1;
parityRebuilt @2;
}
}
}
struct LidarPts @0xe3d6685d4e9d8f7a {
r @0 :List(UInt16); # uint16 m*500.0
theta @1 :List(UInt16); # uint16 deg*100.0
reflect @2 :List(UInt8); # uint8 0-255
# For storing out of file.
idx @3 :UInt64;
# For storing in file
pkt @4 :Data;
}
struct LiveTracksDEPRECATED @0xb16f60103159415a {
trackId @0 :Int32;
dRel @1 :Float32;
yRel @2 :Float32;
vRel @3 :Float32;
aRel @4 :Float32;
timeStamp @5 :Float32;
status @6 :Float32;
currentTime @7 :Float32;
stationary @8 :Bool;
oncoming @9 :Bool;
}
struct LiveMpcData @0x92a5e332a85f32a0 {
x @0 :List(Float32);
y @1 :List(Float32);
psi @2 :List(Float32);
curvature @3 :List(Float32);
qpIterations @4 :UInt32;
calculationTime @5 :UInt64;
cost @6 :Float64;
}
struct LiveLongitudinalMpcData @0xe7e17c434f865ae2 {
xEgo @0 :List(Float32);
vEgo @1 :List(Float32);
aEgo @2 :List(Float32);
xLead @3 :List(Float32);
vLead @4 :List(Float32);
aLead @5 :List(Float32);
aLeadTau @6 :Float32; # lead accel time constant
qpIterations @7 :UInt32;
mpcId @8 :UInt32;
calculationTime @9 :UInt64;
cost @10 :Float64;
}
struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
frameId @0 :UInt32;
modelExecutionTime @14 :Float32;
dspExecutionTime @16 :Float32;
rawPredictions @15 :Data;
faceOrientation @3 :List(Float32);
facePosition @4 :List(Float32);
faceProb @5 :Float32;
leftEyeProb @6 :Float32;
rightEyeProb @7 :Float32;
leftBlinkProb @8 :Float32;
rightBlinkProb @9 :Float32;
faceOrientationStd @11 :List(Float32);
facePositionStd @12 :List(Float32);
sunglassesProb @13 :Float32;
poorVision @17 :Float32;
partialFace @18 :Float32;
distractedPose @19 :Float32;
distractedEyes @20 :Float32;
eyesOnRoad @21 :Float32;
phoneUse @22 :Float32;
occludedProb @23 :Float32;
readyProb @24 :List(Float32);
notReadyProb @25 :List(Float32);
irPwrDEPRECATED @10 :Float32;
descriptorDEPRECATED @1 :List(Float32);
stdDEPRECATED @2 :Float32;
}
struct NavModelData @0xac3de5c437be057a {
frameId @0 :UInt32;
locationMonoTime @6 :UInt64;
modelExecutionTime @1 :Float32;
dspExecutionTime @2 :Float32;
features @3 :List(Float32);
# predicted future position
position @4 :XYData;
desirePrediction @5 :List(Float32);
# All SI units and in device frame
struct XYData @0xbe09e615b2507e26 {
x @0 :List(Float32);
y @1 :List(Float32);
xStd @2 :List(Float32);
yStd @3 :List(Float32);
}
}
struct AndroidBuildInfo @0xfe2919d5c21f426c {
board @0 :Text;
bootloader @1 :Text;
brand @2 :Text;
device @3 :Text;
display @4 :Text;
fingerprint @5 :Text;
hardware @6 :Text;
host @7 :Text;
id @8 :Text;
manufacturer @9 :Text;
model @10 :Text;
product @11 :Text;
radioVersion @12 :Text;
serial @13 :Text;
supportedAbis @14 :List(Text);
tags @15 :Text;
time @16 :Int64;
type @17 :Text;
user @18 :Text;
versionCodename @19 :Text;
versionRelease @20 :Text;
versionSdk @21 :Int32;
versionSecurityPatch @22 :Text;
}
struct AndroidSensor @0x9b513b93a887dbcd {
id @0 :Int32;
name @1 :Text;
vendor @2 :Text;
version @3 :Int32;
handle @4 :Int32;
type @5 :Int32;
maxRange @6 :Float32;
resolution @7 :Float32;
power @8 :Float32;
minDelay @9 :Int32;
fifoReservedEventCount @10 :UInt32;
fifoMaxEventCount @11 :UInt32;
stringType @12 :Text;
maxDelay @13 :Int32;
}
struct IosBuildInfo @0xd97e3b28239f5580 {
appVersion @0 :Text;
appBuild @1 :UInt32;
osVersion @2 :Text;
deviceModel @3 :Text;
}
enum FrameTypeDEPRECATED @0xa37f0d8558e193fd {
unknown @0;
neo @1;
chffrAndroid @2;
front @3;
}
struct AndroidCaptureResult @0xbcc3efbac41d2048 {
sensitivity @0 :Int32;
frameDuration @1 :Int64;
exposureTime @2 :Int64;
rollingShutterSkew @3 :UInt64;
colorCorrectionTransform @4 :List(Int32);
colorCorrectionGains @5 :List(Float32);
displayRotation @6 :Int8;
}
enum UsbPowerModeDEPRECATED @0xa8883583b32c9877 {
none @0;
client @1;
cdp @2;
dcp @3;
}
struct LateralINDIState @0x939463348632375e {
active @0 :Bool;
steeringAngleDeg @1 :Float32;
steeringRateDeg @2 :Float32;
steeringAccelDeg @3 :Float32;
rateSetPoint @4 :Float32;
accelSetPoint @5 :Float32;
accelError @6 :Float32;
delayedOutput @7 :Float32;
delta @8 :Float32;
output @9 :Float32;
saturated @10 :Bool;
steeringAngleDesiredDeg @11 :Float32;
steeringRateDesiredDeg @12 :Float32;
}
struct LateralLQRState @0x9024e2d790c82ade {
active @0 :Bool;
steeringAngleDeg @1 :Float32;
i @2 :Float32;
output @3 :Float32;
lqrOutput @4 :Float32;
saturated @5 :Bool;
steeringAngleDesiredDeg @6 :Float32;
}
struct LateralPlannerSolution @0x84caeca5a6b4acfe {
x @0 :List(Float32);
y @1 :List(Float32);
yaw @2 :List(Float32);
yawRate @3 :List(Float32);
xStd @4 :List(Float32);
yStd @5 :List(Float32);
yawStd @6 :List(Float32);
yawRateStd @7 :List(Float32);
}
struct GpsTrajectory @0x8cfeb072f5301000 {
x @0 :List(Float32);
y @1 :List(Float32);
}
struct AudioFeedbackDEPRECATED @0xed47e3c075be372a {
audio @0 :AudioData;
blockNum @1 :UInt16;
struct AudioData {
data @0 :Data;
sampleRate @1 :UInt32;
}
}
openpilot/cereal/include/c++.capnp
+26
View File
@@ -0,0 +1,26 @@
# Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
# Licensed under the MIT License:
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
@0xbdf87d7bb8304e81;
$namespace("capnp::annotations");
annotation namespace(file): Text;
annotation name(field, enumerant, struct, enum, interface, method, param, group, union): Text;
openpilot/cereal/log.capnp
+2711
View File
File diff suppressed because it is too large Load Diff
openpilot/cereal/messaging/__init__.py
+300
View File
@@ -0,0 +1,300 @@
# must be built with scons
from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
import msgq
import os
import capnp
import time
from typing import Union
from openpilot.cereal import log
from openpilot.cereal.services import SERVICE_LIST
from openpilot.common.utils import MovingAverage
__all__ = (
"NO_TRAVERSAL_LIMIT",
"Context",
"FrequencyTracker",
"IpcError",
"MultiplePublishersError",
"Poller",
"PubMaster",
"PubSocket",
"SocketEventHandle",
"SubMaster",
"SubSocket",
"delete_fake_prefix",
"drain_sock",
"drain_sock_raw",
"fake_event_handle",
"get_fake_prefix",
"log_from_bytes",
"new_message",
"pub_sock",
"recv_one",
"recv_one_or_none",
"recv_one_retry",
"recv_sock",
"reset_context",
"set_fake_prefix",
"sub_sock",
"toggle_fake_events",
"wait_for_one_event",
)
NO_TRAVERSAL_LIMIT = 2**64-1
def pub_sock(endpoint: str) -> PubSocket:
service = SERVICE_LIST.get(endpoint)
segment_size = service.queue_size if service else 0
return msgq.pub_sock(endpoint, segment_size)
def sub_sock(endpoint: str, poller: Poller | None = None, addr: str = "127.0.0.1",
conflate: bool = False, timeout: int | None = None) -> SubSocket:
service = SERVICE_LIST.get(endpoint)
segment_size = service.queue_size if service else 0
return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
timeout=timeout, segment_size=segment_size)
def reset_context():
msgq.context = Context()
def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
return msg
def new_message(service: str | None, size: int | None = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
args = {
'valid': False,
'logMonoTime': int(time.monotonic() * 1e9),
**kwargs
}
dat = log.Event.new_message(**args)
if service is not None:
if size is None:
dat.init(service)
else:
dat.init(service, size)
return dat
def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> list[capnp.lib.capnp._DynamicStructReader]:
"""Receive all message currently available on the queue"""
msgs = drain_sock_raw(sock, wait_for_one=wait_for_one)
return [log_from_bytes(m) for m in msgs]
# TODO: print when we drop packets?
def recv_sock(sock: SubSocket, wait: bool = False) -> capnp.lib.capnp._DynamicStructReader | None:
"""Same as drain sock, but only returns latest message. Consider using conflate instead."""
dat = None
while 1:
if wait and dat is None:
recv = sock.receive()
else:
recv = sock.receive(non_blocking=True)
if recv is None: # Timeout hit
break
dat = recv
if dat is not None:
dat = log_from_bytes(dat)
return dat
def recv_one(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader | None:
dat = sock.receive()
if dat is not None:
dat = log_from_bytes(dat)
return dat
def recv_one_or_none(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader | None:
dat = sock.receive(non_blocking=True)
if dat is not None:
dat = log_from_bytes(dat)
return dat
def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
"""Keep receiving until we get a message"""
while True:
dat = sock.receive()
if dat is not None:
return log_from_bytes(dat)
class FrequencyTracker:
def __init__(self, service_freq: float, update_freq: float, is_poll: bool):
freq = max(min(service_freq, update_freq), 1.)
if is_poll:
min_freq = max_freq = freq
else:
max_freq = min(freq, update_freq)
if service_freq >= 2 * update_freq:
min_freq = update_freq
elif update_freq >= 2* service_freq:
min_freq = freq
else:
min_freq = min(freq, freq / 2.)
self.min_freq = min_freq * 0.8
self.max_freq = max_freq * 1.2
self.avg_dt = MovingAverage(int(10 * freq))
self.recent_avg_dt = MovingAverage(int(freq))
self.prev_time = 0.0
def record_recv_time(self, cur_time: float) -> None:
# TODO: Handle case where cur_time is less than prev_time
if self.prev_time > 1e-5:
dt = cur_time - self.prev_time
self.avg_dt.add_value(dt)
self.recent_avg_dt.add_value(dt)
self.prev_time = cur_time
@property
def valid(self) -> bool:
if self.avg_dt.count == 0:
return False
avg_freq = 1.0 / self.avg_dt.get_average()
if self.min_freq <= avg_freq <= self.max_freq:
return True
avg_freq_recent = 1.0 / self.recent_avg_dt.get_average()
return self.min_freq <= avg_freq_recent <= self.max_freq
class SubMaster:
def __init__(self, services: list[str], poll: str | None = None,
ignore_alive: list[str] | None = None, ignore_avg_freq: list[str] | None = None,
ignore_valid: list[str] | None = None, addr: str = "127.0.0.1", frequency: float | None = None):
self.frame = -1
self.services = services
self.seen = dict.fromkeys(services, False)
self.updated = dict.fromkeys(services, False)
self.recv_time = dict.fromkeys(services, 0.0)
self.recv_frame = dict.fromkeys(services, 0)
self.sock = {}
self.data = {}
self.logMonoTime = dict.fromkeys(services, 0)
# zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
self.static_freq_services = {s for s in services if not on_demand[s]}
self.alive = {s: on_demand[s] for s in services}
self.freq_ok = {s: on_demand[s] for s in services}
self.valid = {s: on_demand[s] for s in services}
self.freq_tracker: dict[str, FrequencyTracker] = {}
self.poller = Poller()
polled_services = set([poll, ] if poll is not None else services)
self.non_polled_services = set(services) - polled_services
self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq
self.ignore_alive = [] if ignore_alive is None else ignore_alive
self.ignore_valid = [] if ignore_valid is None else ignore_valid
self.simulation = bool(int(os.getenv("SIMULATION", "0")))
# if freq and poll aren't specified, assume the max to be conservative
assert frequency is None or poll is None, "Do not specify 'frequency' - frequency of the polled service will be used."
self.update_freq = frequency or max([SERVICE_LIST[s].frequency for s in polled_services])
for s in services:
p = self.poller if s not in self.non_polled_services else None
self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True)
try:
data = new_message(s)
except capnp.lib.capnp.KjException:
data = new_message(s, 0) # lists
self.data[s] = getattr(data.as_reader(), s)
self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll)
def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
return self.data[s]
def _check_avg_freq(self, s: str) -> bool:
return SERVICE_LIST[s].frequency > 0.99 and (s not in self.ignore_average_freq) and (s not in self.ignore_alive)
def update(self, timeout: int = 100) -> None:
msgs = []
for sock in self.poller.poll(timeout):
msgs.append(recv_one_or_none(sock))
# non-blocking receive for non-polled sockets
for s in self.non_polled_services:
msgs.append(recv_one_or_none(self.sock[s]))
self.update_msgs(time.monotonic(), msgs)
def update_msgs(self, cur_time: float, msgs: list[capnp.lib.capnp._DynamicStructReader]) -> None:
self.frame += 1
self.updated = dict.fromkeys(self.services, False)
for msg in msgs:
if msg is None:
continue
s = msg.which()
self.seen[s] = True
self.updated[s] = True
self.freq_tracker[s].record_recv_time(cur_time)
self.recv_time[s] = cur_time
self.recv_frame[s] = self.frame
self.data[s] = getattr(msg, s)
self.logMonoTime[s] = msg.logMonoTime
self.valid[s] = msg.valid
for s in self.static_freq_services:
# alive if delay is within 10x the expected frequency; checks relaxed in simulator
self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
def all_alive(self, service_list: list[str] | None = None) -> bool:
return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
def all_freq_ok(self, service_list: list[str] | None = None) -> bool:
return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
def all_valid(self, service_list: list[str] | None = None) -> bool:
return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
def all_checks(self, service_list: list[str] | None = None) -> bool:
return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
class PubMaster:
def __init__(self, services: list[str]):
self.sock = {}
for s in services:
self.sock[s] = pub_sock(s)
def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder]) -> None:
if not isinstance(dat, bytes):
dat = dat.to_bytes()
self.sock[s].send(dat)
def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
try:
self.sock[s].wait_for_readers(timeout=timeout, interval=dt)
return True
except TimeoutError:
return False
def all_readers_updated(self, s: str) -> bool:
return self.sock[s].all_readers_updated()
openpilot/cereal/messaging/bridge.cc
+72
View File
@@ -0,0 +1,72 @@
#include <cassert>
#include "openpilot/cereal/messaging/msgq_to_zmq.h"
#include "openpilot/cereal/services.h"
#include "common/util.h"
ExitHandler do_exit;
static std::vector<std::string> get_services(const std::string &whitelist_str, bool zmq_to_msgq) {
std::vector<std::string> service_list;
for (const auto& it : services) {
std::string name = it.second.name;
bool in_whitelist = whitelist_str.find(name) != std::string::npos;
if (zmq_to_msgq && !in_whitelist) {
continue;
}
service_list.push_back(name);
}
return service_list;
}
void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &ip) {
MsgqToZmq bridge;
bridge.run(endpoints, ip);
}
void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
auto poller = std::make_unique<BridgeZmqPoller>();
auto pub_context = std::make_unique<Context>();
auto sub_context = std::make_unique<BridgeZmqContext>();
std::map<BridgeZmqSubSocket *, PubSocket *> sub2pub;
for (auto endpoint : endpoints) {
auto pub_sock = new PubSocket();
auto sub_sock = new BridgeZmqSubSocket();
size_t queue_size = services.at(endpoint).queue_size;
pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
sub_sock->connect(sub_context.get(), endpoint, ip, false);
poller->registerSocket(sub_sock);
sub2pub[sub_sock] = pub_sock;
}
while (!do_exit) {
for (auto sub_sock : poller->poll(100)) {
std::unique_ptr<Message> msg(sub_sock->receive(true));
if (msg) {
sub2pub[sub_sock]->sendMessage(msg.get());
}
}
}
// Clean up allocated sockets
for (auto &[sub_sock, pub_sock] : sub2pub) {
delete sub_sock;
delete pub_sock;
}
}
int main(int argc, char **argv) {
bool is_zmq_to_msgq = argc > 2;
std::string ip = is_zmq_to_msgq ? argv[1] : "127.0.0.1";
std::string whitelist_str = is_zmq_to_msgq ? std::string(argv[2]) : "";
std::vector<std::string> endpoints = get_services(whitelist_str, is_zmq_to_msgq);
if (is_zmq_to_msgq) {
zmq_to_msgq(endpoints, ip);
} else {
msgq_to_zmq(endpoints, ip);
}
return 0;
}
openpilot/cereal/messaging/bridge_zmq.cc
+170
View File
@@ -0,0 +1,170 @@
#include "openpilot/cereal/messaging/bridge_zmq.h"
#include <cassert>
#include <cstring>
#include <unistd.h>
static size_t fnv1a_hash(const std::string &str) {
const size_t fnv_prime = 0x100000001b3;
size_t hash_value = 0xcbf29ce484222325;
for (char c : str) {
hash_value ^= (unsigned char)c;
hash_value *= fnv_prime;
}
return hash_value;
}
// FIXME: This is a hack to get the port number from the socket name, might have collisions.
static int get_port(std::string endpoint) {
size_t hash_value = fnv1a_hash(endpoint);
int start_port = 8023;
int max_port = 65535;
return start_port + (hash_value % (max_port - start_port));
}
BridgeZmqContext::BridgeZmqContext() {
context = zmq_ctx_new();
}
BridgeZmqContext::~BridgeZmqContext() {
if (context != nullptr) {
zmq_ctx_term(context);
}
}
void BridgeZmqMessage::init(size_t sz) {
size = sz;
data = new char[size];
}
void BridgeZmqMessage::init(char *d, size_t sz) {
size = sz;
data = new char[size];
memcpy(data, d, size);
}
void BridgeZmqMessage::close() {
if (size > 0) {
delete[] data;
}
data = nullptr;
size = 0;
}
BridgeZmqMessage::~BridgeZmqMessage() {
close();
}
int BridgeZmqSubSocket::connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint) {
sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
if (sock == nullptr) {
return -1;
}
zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
if (conflate) {
int arg = 1;
zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
}
int reconnect_ivl = 500;
zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
full_endpoint = "tcp://" + address + ":";
if (check_endpoint) {
full_endpoint += std::to_string(get_port(endpoint));
} else {
full_endpoint += endpoint;
}
return zmq_connect(sock, full_endpoint.c_str());
}
void BridgeZmqSubSocket::setTimeout(int timeout) {
zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
}
Message *BridgeZmqSubSocket::receive(bool non_blocking) {
zmq_msg_t msg;
assert(zmq_msg_init(&msg) == 0);
int flags = non_blocking ? ZMQ_DONTWAIT : 0;
int rc = zmq_msg_recv(&msg, sock, flags);
Message *ret = nullptr;
if (rc >= 0) {
ret = new BridgeZmqMessage;
ret->init((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
}
zmq_msg_close(&msg);
return ret;
}
BridgeZmqSubSocket::~BridgeZmqSubSocket() {
if (sock != nullptr) {
zmq_close(sock);
}
}
int BridgeZmqPubSocket::connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint) {
sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
if (sock == nullptr) {
return -1;
}
full_endpoint = "tcp://*:";
if (check_endpoint) {
full_endpoint += std::to_string(get_port(endpoint));
} else {
full_endpoint += endpoint;
}
// ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same.
pid = getpid();
return zmq_bind(sock, full_endpoint.c_str());
}
int BridgeZmqPubSocket::sendMessage(Message *message) {
assert(pid == getpid());
return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
}
int BridgeZmqPubSocket::send(char *data, size_t size) {
assert(pid == getpid());
return zmq_send(sock, data, size, ZMQ_DONTWAIT);
}
BridgeZmqPubSocket::~BridgeZmqPubSocket() {
if (sock != nullptr) {
zmq_close(sock);
}
}
void BridgeZmqPoller::registerSocket(BridgeZmqSubSocket *socket) {
assert(num_polls + 1 < (sizeof(polls) / sizeof(polls[0])));
polls[num_polls].socket = socket->getRawSocket();
polls[num_polls].events = ZMQ_POLLIN;
sockets.push_back(socket);
num_polls++;
}
std::vector<BridgeZmqSubSocket *> BridgeZmqPoller::poll(int timeout) {
std::vector<BridgeZmqSubSocket *> ret;
int rc = zmq_poll(polls, num_polls, timeout);
if (rc < 0) {
return ret;
}
for (size_t i = 0; i < num_polls; i++) {
if (polls[i].revents) {
ret.push_back(sockets[i]);
}
}
return ret;
}
openpilot/cereal/messaging/bridge_zmq.h
+72
View File
@@ -0,0 +1,72 @@
#pragma once
#include <cstddef>
#include <string>
#include <vector>
#include <zmq.h>
#include "msgq/ipc.h"
class BridgeZmqContext {
public:
BridgeZmqContext();
void *getRawContext() { return context; }
~BridgeZmqContext();
private:
void *context = nullptr;
};
class BridgeZmqMessage : public Message {
public:
void init(size_t size);
void init(char *data, size_t size);
void close();
size_t getSize() { return size; }
char *getData() { return data; }
~BridgeZmqMessage();
private:
char *data = nullptr;
size_t size = 0;
};
class BridgeZmqSubSocket {
public:
int connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate = false, bool check_endpoint = true);
void setTimeout(int timeout);
Message *receive(bool non_blocking = false);
void *getRawSocket() { return sock; }
~BridgeZmqSubSocket();
private:
void *sock = nullptr;
std::string full_endpoint;
};
class BridgeZmqPubSocket {
public:
int connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint = true);
int sendMessage(Message *message);
int send(char *data, size_t size);
void *getRawSocket() { return sock; }
~BridgeZmqPubSocket();
private:
void *sock = nullptr;
std::string full_endpoint;
int pid = -1;
};
class BridgeZmqPoller {
public:
void registerSocket(BridgeZmqSubSocket *socket);
std::vector<BridgeZmqSubSocket *> poll(int timeout);
private:
static constexpr size_t MAX_BRIDGE_ZMQ_POLLERS = 128;
std::vector<BridgeZmqSubSocket *> sockets;
zmq_pollitem_t polls[MAX_BRIDGE_ZMQ_POLLERS] = {};
size_t num_polls = 0;
};
openpilot/cereal/messaging/messaging.h
+102
View File
@@ -0,0 +1,102 @@
#pragma once
#include <cstddef>
#include <map>
#include <string>
#include <vector>
#include <utility>
#include <capnp/serialize.h>
#include "openpilot/cereal/gen/cpp/log.capnp.h"
#include "common/timing.h"
#include "msgq/ipc.h"
class SubMaster {
public:
SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll = {},
const char *address = nullptr, const std::vector<const char *> &ignore_alive = {});
void update(int timeout = 1000);
void update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages);
inline bool allAlive(const std::vector<const char *> &service_list = {}) { return all_(service_list, false, true); }
inline bool allValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, false); }
inline bool allAliveAndValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, true); }
void drain();
~SubMaster();
uint64_t frame = 0;
bool updated(const char *name) const;
bool alive(const char *name) const;
bool valid(const char *name) const;
uint64_t rcv_frame(const char *name) const;
uint64_t rcv_time(const char *name) const;
cereal::Event::Reader &operator[](const char *name) const;
private:
bool all_(const std::vector<const char *> &service_list, bool valid, bool alive);
Poller *poller_ = nullptr;
struct SubMessage;
std::map<SubSocket *, SubMessage *> messages_;
std::map<std::string, SubMessage *> services_;
};
class MessageBuilder : public capnp::MallocMessageBuilder {
public:
MessageBuilder() = default;
cereal::Event::Builder initEvent(bool valid = true) {
cereal::Event::Builder event = initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
event.setValid(valid);
return event;
}
kj::ArrayPtr<capnp::byte> toBytes() {
heapArray_ = capnp::messageToFlatArray(*this);
return heapArray_.asBytes();
}
size_t getSerializedSize() {
return capnp::computeSerializedSizeInWords(*this) * sizeof(capnp::word);
}
int serializeToBuffer(unsigned char *buffer, size_t buffer_size) {
size_t serialized_size = getSerializedSize();
if (serialized_size > buffer_size) { return -1; }
kj::ArrayOutputStream out(kj::ArrayPtr<capnp::byte>(buffer, buffer_size));
capnp::writeMessage(out, *this);
return serialized_size;
}
private:
kj::Array<capnp::word> heapArray_;
};
class PubMaster {
public:
PubMaster(const std::vector<const char *> &service_list);
inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); }
int send(const char *name, MessageBuilder &msg);
~PubMaster();
private:
std::map<std::string, PubSocket *> sockets_;
};
class AlignedBuffer {
public:
kj::ArrayPtr<const capnp::word> align(const char *data, const size_t size) {
words_size = size / sizeof(capnp::word) + 1;
if (aligned_buf.size() < words_size) {
aligned_buf = kj::heapArray<capnp::word>(words_size < 512 ? 512 : words_size);
}
memcpy(aligned_buf.begin(), data, size);
return aligned_buf.slice(0, words_size);
}
inline kj::ArrayPtr<const capnp::word> align(Message *m) {
return align(m->getData(), m->getSize());
}
private:
kj::Array<capnp::word> aligned_buf;
size_t words_size;
};
openpilot/cereal/messaging/msgq_to_zmq.cc
+146
View File
@@ -0,0 +1,146 @@
#include "openpilot/cereal/messaging/msgq_to_zmq.h"
#include <cassert>
#include "openpilot/cereal/services.h"
#include "common/util.h"
extern ExitHandler do_exit;
// Max messages to process per socket per poll
constexpr int MAX_MESSAGES_PER_SOCKET = 50;
static std::string recv_zmq_msg(void *sock) {
zmq_msg_t msg;
zmq_msg_init(&msg);
std::string ret;
if (zmq_msg_recv(&msg, sock, 0) > 0) {
ret.assign((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
}
zmq_msg_close(&msg);
return ret;
}
void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
zmq_context = std::make_unique<BridgeZmqContext>();
msgq_context = std::make_unique<Context>();
// Create ZMQPubSockets for each endpoint
for (const auto &endpoint : endpoints) {
auto &socket_pair = socket_pairs.emplace_back();
socket_pair.endpoint = endpoint;
socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
if (ret != 0) {
printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
return;
}
}
// Start ZMQ monitoring thread to monitor socket events
std::thread thread(&MsgqToZmq::zmqMonitorThread, this);
// Main loop for processing messages
while (!do_exit) {
{
std::unique_lock lk(mutex);
cv.wait(lk, [this]() { return do_exit || !sub2pub.empty(); });
if (do_exit) break;
for (auto sub_sock : msgq_poller->poll(100)) {
// Process messages for each socket
BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
if (!msg) break;
while (pub_sock->sendMessage(msg.get()) == -1) {
if (errno != EINTR) break;
}
}
}
}
util::sleep_for(1); // Give zmqMonitorThread a chance to acquire the mutex
}
thread.join();
}
void MsgqToZmq::zmqMonitorThread() {
std::vector<zmq_pollitem_t> pollitems;
// Set up ZMQ monitor for each pub socket
for (int i = 0; i < socket_pairs.size(); ++i) {
std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
zmq_connect(monitor_socket, addr.c_str());
pollitems.emplace_back(zmq_pollitem_t{.socket = monitor_socket, .events = ZMQ_POLLIN});
}
while (!do_exit) {
int ret = zmq_poll(pollitems.data(), pollitems.size(), 1000);
if (ret < 0) {
if (errno == EINTR) {
// Due to frequent EINTR signals from msgq, introduce a brief delay (200 ms)
// to reduce CPU usage during retry attempts.
util::sleep_for(200);
}
continue;
}
for (int i = 0; i < pollitems.size(); ++i) {
if (pollitems[i].revents & ZMQ_POLLIN) {
// First frame in message contains event number and value
std::string frame = recv_zmq_msg(pollitems[i].socket);
if (frame.empty()) continue;
uint16_t event_type = *(uint16_t *)(frame.data());
// Second frame in message contains event address
frame = recv_zmq_msg(pollitems[i].socket);
if (frame.empty()) continue;
std::unique_lock lk(mutex);
auto &pair = socket_pairs[i];
if (event_type & ZMQ_EVENT_ACCEPTED) {
printf("socket [%s] connected\n", pair.endpoint.c_str());
if (++pair.connected_clients == 1) {
// Create new MSGQ subscriber socket and map to ZMQ publisher
pair.sub_sock = std::make_unique<MSGQSubSocket>();
size_t queue_size = services.at(pair.endpoint).queue_size;
pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
registerSockets();
}
} else if (event_type & ZMQ_EVENT_DISCONNECTED) {
printf("socket [%s] disconnected\n", pair.endpoint.c_str());
if (pair.connected_clients == 0 || --pair.connected_clients == 0) {
// Remove MSGQ subscriber socket from mapping and reset it
sub2pub.erase(pair.sub_sock.get());
pair.sub_sock.reset(nullptr);
registerSockets();
}
}
cv.notify_one();
}
}
}
// Clean up monitor sockets
for (int i = 0; i < pollitems.size(); ++i) {
zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
zmq_close(pollitems[i].socket);
}
cv.notify_one();
}
void MsgqToZmq::registerSockets() {
msgq_poller = std::make_unique<MSGQPoller>();
for (const auto &socket_pair : socket_pairs) {
if (socket_pair.sub_sock) {
msgq_poller->registerSocket(socket_pair.sub_sock.get());
}
}
}
openpilot/cereal/messaging/msgq_to_zmq.h
+36
View File
@@ -0,0 +1,36 @@
#pragma once
#include <condition_variable>
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
#include "msgq/impl_msgq.h"
#include "openpilot/cereal/messaging/bridge_zmq.h"
class MsgqToZmq {
public:
MsgqToZmq() {}
void run(const std::vector<std::string> &endpoints, const std::string &ip);
protected:
void registerSockets();
void zmqMonitorThread();
struct SocketPair {
std::string endpoint;
std::unique_ptr<BridgeZmqPubSocket> pub_sock;
std::unique_ptr<MSGQSubSocket> sub_sock;
int connected_clients = 0;
};
std::unique_ptr<Context> msgq_context;
std::unique_ptr<BridgeZmqContext> zmq_context;
std::mutex mutex;
std::condition_variable cv;
std::unique_ptr<MSGQPoller> msgq_poller;
std::map<SubSocket *, BridgeZmqPubSocket *> sub2pub;
std::vector<SocketPair> socket_pairs;
};
openpilot/cereal/messaging/socketmaster.cc
+204
View File
@@ -0,0 +1,204 @@
#include <assert.h>
#include <stdlib.h>
#include <string>
#include <mutex>
#include "openpilot/cereal/services.h"
#include "openpilot/cereal/messaging/messaging.h"
const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
static inline bool inList(const std::vector<const char *> &list, const char *value) {
for (auto &v : list) {
if (strcmp(value, v) == 0) return true;
}
return false;
}
class MessageContext {
public:
MessageContext() : ctx_(nullptr) {}
~MessageContext() { delete ctx_; }
inline Context *context() {
std::call_once(init_flag, [=]() { ctx_ = Context::create(); });
return ctx_;
}
private:
Context *ctx_;
std::once_flag init_flag;
};
MessageContext message_context;
struct SubMaster::SubMessage {
std::string name;
SubSocket *socket = nullptr;
float freq = 0.0f;
bool updated = false, alive = false, valid = false, ignore_alive;
uint64_t rcv_time = 0, rcv_frame = 0;
void *allocated_msg_reader = nullptr;
bool is_polled = false;
capnp::FlatArrayMessageReader *msg_reader = nullptr;
AlignedBuffer aligned_buf;
cereal::Event::Reader event;
};
SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll,
const char *address, const std::vector<const char *> &ignore_alive) {
poller_ = Poller::create();
for (auto name : service_list) {
assert(services.count(std::string(name)) > 0);
service serv = services.at(std::string(name));
SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
assert(socket != 0);
bool is_polled = inList(poll, name) || poll.empty();
if (is_polled) poller_->registerSocket(socket);
SubMessage *m = new SubMessage{
.name = name,
.socket = socket,
.freq = serv.frequency,
.ignore_alive = inList(ignore_alive, name),
.allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
.is_polled = is_polled};
m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader({});
messages_[socket] = m;
services_[name] = m;
}
}
void SubMaster::update(int timeout) {
for (auto &kv : messages_) kv.second->updated = false;
auto sockets = poller_->poll(timeout);
// add non-polled sockets for non-blocking receive
for (auto &kv : messages_) {
SubMessage *m = kv.second;
SubSocket *s = kv.first;
if (!m->is_polled) sockets.push_back(s);
}
uint64_t current_time = nanos_since_boot();
std::vector<std::pair<std::string, cereal::Event::Reader>> messages;
for (auto s : sockets) {
Message *msg = s->receive(true);
if (msg == nullptr) continue;
SubMessage *m = messages_.at(s);
m->msg_reader->~FlatArrayMessageReader();
capnp::ReaderOptions options;
options.traversalLimitInWords = kj::maxValue; // Don't limit
m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(m->aligned_buf.align(msg), options);
delete msg;
messages.push_back({m->name, m->msg_reader->getRoot<cereal::Event>()});
}
update_msgs(current_time, messages);
}
void SubMaster::update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages){
if (++frame == UINT64_MAX) frame = 1;
for (auto &kv : messages) {
auto m_find = services_.find(kv.first);
if (m_find == services_.end()){
continue;
}
SubMessage *m = m_find->second;
m->event = kv.second;
m->updated = true;
m->rcv_time = current_time;
m->rcv_frame = frame;
m->valid = m->event.getValid();
if (SIMULATION) m->alive = true;
}
if (!SIMULATION) {
for (auto &kv : messages_) {
SubMessage *m = kv.second;
m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq));
}
}
}
bool SubMaster::all_(const std::vector<const char *> &service_list, bool valid, bool alive) {
int found = 0;
for (auto &kv : messages_) {
SubMessage *m = kv.second;
if (service_list.size() == 0 || inList(service_list, m->name.c_str())) {
found += (!valid || m->valid) && (!alive || (m->alive || m->ignore_alive));
}
}
return service_list.size() == 0 ? found == messages_.size() : found == service_list.size();
}
void SubMaster::drain() {
while (true) {
auto polls = poller_->poll(0);
if (polls.size() == 0)
break;
for (auto sock : polls) {
Message *msg = sock->receive(true);
delete msg;
}
}
}
bool SubMaster::updated(const char *name) const {
return services_.at(name)->updated;
}
bool SubMaster::alive(const char *name) const {
return services_.at(name)->alive;
}
bool SubMaster::valid(const char *name) const {
return services_.at(name)->valid;
}
uint64_t SubMaster::rcv_frame(const char *name) const {
return services_.at(name)->rcv_frame;
}
uint64_t SubMaster::rcv_time(const char *name) const {
return services_.at(name)->rcv_time;
}
cereal::Event::Reader &SubMaster::operator[](const char *name) const {
return services_.at(name)->event;
}
SubMaster::~SubMaster() {
delete poller_;
for (auto &kv : messages_) {
SubMessage *m = kv.second;
m->msg_reader->~FlatArrayMessageReader();
free(m->allocated_msg_reader);
delete m->socket;
delete m;
}
}
PubMaster::PubMaster(const std::vector<const char *> &service_list) {
for (auto name : service_list) {
assert(services.count(name) > 0);
service serv = services.at(std::string(name));
PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
assert(socket);
sockets_[name] = socket;
}
}
int PubMaster::send(const char *name, MessageBuilder &msg) {
auto bytes = msg.toBytes();
return send(name, bytes.begin(), bytes.size());
}
PubMaster::~PubMaster() {
for (auto s : sockets_) delete s.second;
}
openpilot/cereal/messaging/tests/__init__.py
View File
openpilot/cereal/messaging/tests/test_messaging.py
+164
View File
@@ -0,0 +1,164 @@
import capnp
import multiprocessing
import numbers
import random
import threading
import time
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.parameterized import parameterized
from openpilot.cereal import log
from opendbc.car.structs import car
import openpilot.cereal.messaging as messaging
from openpilot.cereal.services import SERVICE_LIST
events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
def random_sock():
return random.choice(events)
def random_socks(num_socks=10):
return list({random_sock() for _ in range(num_socks)})
def random_bytes(length=1000):
return bytes([random.randrange(0xFF) for _ in range(length)])
# TODO: this should take any capnp struct and returrn a msg with random populated data
def random_carstate():
fields = ["vEgo", "aEgo", "steeringTorque", "steeringAngleDeg"]
msg = messaging.new_message("carState")
cs = msg.carState
for f in fields:
setattr(cs, f, random.random() * 10)
return msg
# TODO: this should compare any capnp structs
def assert_carstate(cs1, cs2):
for f in car.CarState.schema.non_union_fields:
# TODO: check all types
val1, val2 = getattr(cs1, f), getattr(cs2, f)
if isinstance(val1, numbers.Number):
assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'"
def delayed_send(delay, sock, dat):
def send_func():
sock.send(dat)
threading.Timer(delay, send_func).start()
class TestMessaging(OpenpilotTestCase):
@parameterized.expand(events)
def test_new_message(self, evt):
try:
msg = messaging.new_message(evt)
except capnp.lib.capnp.KjException:
msg = messaging.new_message(evt, random.randrange(200))
assert (time.monotonic() - msg.logMonoTime) < 0.1
assert not msg.valid
assert evt == msg.which()
@parameterized.expand(events)
def test_pub_sock(self, evt):
messaging.pub_sock(evt)
@parameterized.expand(events)
def test_sub_sock(self, evt):
messaging.sub_sock(evt)
@parameterized.expand([
(messaging.drain_sock, capnp._DynamicStructReader),
(messaging.drain_sock_raw, bytes),
])
def test_drain_sock(self, func, expected_type):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=1000)
# no wait and no msgs in queue
msgs = func(sub_sock)
assert isinstance(msgs, list)
assert len(msgs) == 0
# no wait but msgs are queued up
pub_sock.send(messaging.new_message(sock).to_bytes())
assert sub_sock.receive() is not None # synchronize the PUB/SUB connection
num_msgs = random.randrange(3, 10)
for _ in range(num_msgs):
pub_sock.send(messaging.new_message(sock).to_bytes())
msgs = func(sub_sock)
assert isinstance(msgs, list)
assert all(isinstance(msg, expected_type) for msg in msgs)
assert len(msgs) == num_msgs
def test_recv_sock(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=100)
# no wait and no msg in queue, socket should timeout
recvd = messaging.recv_sock(sub_sock)
assert recvd is None
# no wait and one msg in queue
msg = random_carstate()
pub_sock.send(msg.to_bytes())
time.sleep(0.01)
recvd = messaging.recv_sock(sub_sock)
assert isinstance(recvd, capnp._DynamicStructReader)
# https://github.com/python/mypy/issues/13038
assert_carstate(msg.carState, recvd.carState)
def test_recv_one(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=10)
# no msg in queue, socket should timeout
recvd = messaging.recv_one(sub_sock)
assert recvd is None
# one msg in queue
msg = random_carstate()
pub_sock.send(msg.to_bytes())
recvd = messaging.recv_one(sub_sock)
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)
def test_recv_one_or_none(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock)
# no msg in queue, socket shouldn't block
recvd = messaging.recv_one_or_none(sub_sock)
assert recvd is None
# one msg in queue
msg = random_carstate()
pub_sock.send(msg.to_bytes())
recvd = messaging.recv_one_or_none(sub_sock)
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)
def test_recv_one_retry(self):
sock = "carState"
sock_timeout = 0.005
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
# wait 5 socket timeouts and make sure it's still retrying
p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
p.start()
time.sleep(sock_timeout*5)
assert p.is_alive()
p.terminate()
# wait 5 socket timeouts before sending
msg = random_carstate()
start_time = time.monotonic()
delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
recvd = messaging.recv_one_retry(sub_sock)
assert (time.monotonic() - start_time) >= sock_timeout*5
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)
openpilot/cereal/messaging/tests/test_pub_sub_master.py
+142
View File
@@ -0,0 +1,142 @@
import random
import time
from typing import cast
from collections.abc import Sized
from openpilot.common.test import OpenpilotTestCase
import openpilot.cereal.messaging as messaging
from openpilot.cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
random_bytes, random_carstate, assert_carstate
from openpilot.cereal.services import SERVICE_LIST
class TestSubMaster(OpenpilotTestCase):
def test_init(self):
sm = messaging.SubMaster(events)
for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
sm.sock, sm.data, sm.logMonoTime, sm.valid]:
assert len(cast(Sized, p)) == len(events)
def test_init_state(self):
socks = random_socks()
sm = messaging.SubMaster(socks)
assert sm.frame == -1
assert not any(sm.updated.values())
assert not any(sm.seen.values())
on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in sm.services}
assert all(sm.alive[s] == sm.valid[s] == sm.freq_ok[s] == on_demand[s] for s in sm.services)
assert all(t == 0. for t in sm.recv_time.values())
assert all(f == 0 for f in sm.recv_frame.values())
assert all(t == 0 for t in sm.logMonoTime.values())
for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
sm.sock, sm.data, sm.logMonoTime, sm.valid]:
assert len(cast(Sized, p)) == len(socks)
def test_getitem(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
msg = random_carstate()
pub_sock.send(msg.to_bytes())
sm.update(1000)
assert_carstate(msg.carState, sm[sock])
# TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs
def test_update(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
for i in range(10):
msg = messaging.new_message(sock)
pub_sock.send(msg.to_bytes())
sm.update(1000)
assert sm.frame == i
assert all(sm.updated.values())
def test_update_timeout(self):
sock = random_sock()
sm = messaging.SubMaster([sock,])
timeout = random.randrange(10, 30)
start_time = time.monotonic()
sm.update(timeout)
t = time.monotonic() - start_time
assert t >= timeout/1000.
assert t < 0.1
assert not any(sm.updated.values())
def test_avg_frequency_checks(self):
for poll in (True, False):
sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "extrinsicsCalibration"],
poll=("modelV2" if poll else None),
frequency=(20. if not poll else None))
checks = {
"carState": (20, 20),
"modelV2": (20, 20 if poll else 10),
"cameraOdometry": (20, 10),
"extrinsicsCalibration": (4, 4),
"carParams": (None, None),
"userBookmark": (None, None),
}
for service, (max_freq, min_freq) in checks.items():
if max_freq is not None:
assert min_freq is not None
assert sm._check_avg_freq(service)
assert sm.freq_tracker[service].max_freq == max_freq*1.2
assert sm.freq_tracker[service].min_freq == min_freq*0.8
else:
assert not sm._check_avg_freq(service)
# SubMaster should always conflate
def test_conflate(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
pub_sock.send(messaging.new_message(sock).to_bytes())
sm.update(1000) # synchronize the PUB/SUB connection
n = 10
for i in range(n+1):
msg = messaging.new_message(sock)
msg.carState.vEgo = i
pub_sock.send(msg.to_bytes())
sm.update(1000)
assert sm[sock].vEgo == n
class TestPubMaster(OpenpilotTestCase):
def test_init(self):
messaging.PubMaster(events)
def test_send(self):
socks = random_socks()
pm = messaging.PubMaster(socks)
sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
# PubMaster accepts either a capnp msg builder or bytes
for capnp in [True, False]:
for i in range(100):
sock = socks[i % len(socks)]
if capnp:
try:
msg = messaging.new_message(sock)
except Exception:
msg = messaging.new_message(sock, random.randrange(50))
else:
msg = random_bytes()
pm.send(sock, msg)
recvd = sub_socks[sock].receive()
if capnp:
msg.clear_write_flag()
msg = msg.to_bytes()
assert msg == recvd, i
openpilot/cereal/messaging/tests/test_services.py
+22
View File
@@ -0,0 +1,22 @@
import subprocess
import tempfile
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.parameterized import parameterized
import openpilot.cereal.services as services
from openpilot.cereal.services import SERVICE_LIST
class TestServices(OpenpilotTestCase):
@parameterized.expand(SERVICE_LIST.keys())
def test_services(self, s):
service = SERVICE_LIST[s]
assert service.frequency <= 104
assert service.decimation != 0
def test_generated_header(self):
with tempfile.NamedTemporaryFile(suffix=".h") as f:
ret = subprocess.run(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11", shell=True).returncode
assert ret == 0, "generated services header is not valid C"
openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py
Executable
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""Validate sunnypilot routes are parseable by stock commaai/openpilot.
Cap'n Proto is wire-compatible across renames, type relocations, and
additive fields. The only breaking change is a union variant that
upstream doesn't recognize — an unknown discriminant makes the entire
union unreadable.
This script checks: for every struct with a union that exists in both
schemas, does sunnypilot introduce union variants upstream doesn't have?
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from typing import Any
NO_DISCRIMINANT = 0xFFFF
def hex_id(value: int) -> str:
return f"0x{value:016x}"
def encode_struct(schema: Any) -> dict:
node = schema.node
fields = []
for name, field in schema.fields.items():
proto = field.proto
ordinal = proto.ordinal.explicit if proto.ordinal.which() == "explicit" else None
discriminant = proto.discriminantValue if proto.discriminantValue != NO_DISCRIMINANT else None
fields.append({"name": name, "ordinal": ordinal, "discriminant": discriminant})
return {
"typeId": hex_id(node.id),
"displayName": node.displayName,
"hasUnion": node.struct.discriminantCount > 0,
"fields": fields,
}
def _child_struct_schema(field: Any) -> Any:
proto = field.proto
if proto.which() == "group":
return field.schema
type_node = proto.slot.type
which = type_node.which()
if which == "struct":
return field.schema
if which == "list":
container = field.schema
element_type = type_node.list.elementType
while element_type.which() == "list":
container = container.elementType
element_type = element_type.list.elementType
if element_type.which() == "struct":
return container.elementType
return None
def collect_schema(root: Any) -> dict[str, dict]:
structs: dict[str, dict] = {}
stack = [root]
while stack:
schema = stack.pop()
type_id = hex_id(schema.node.id)
if type_id in structs:
continue
structs[type_id] = encode_struct(schema)
for _name, field in schema.fields.items():
try:
child = _child_struct_schema(field)
except Exception:
child = None
if child is not None:
stack.append(child)
return structs
def load_log(cereal_dir: str, extra_imports: list[str] | None = None) -> Any:
import capnp
cereal_dir = os.path.abspath(cereal_dir)
capnp.remove_import_hook()
imports = [cereal_dir] + [os.path.abspath(p) for p in (extra_imports or [])]
return capnp.load(os.path.join(cereal_dir, "log.capnp"), imports=imports)
def dump_schema(cereal_dir: str, path: str, extra_imports: list[str] | None = None) -> None:
log = load_log(cereal_dir, extra_imports)
payload = {
"root": hex_id(log.Event.schema.node.id),
"structs": collect_schema(log.Event.schema),
}
with open(path, "w", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2, sort_keys=True)
print(f"wrote schema dump with {len(payload['structs'])} structs to {path}")
def compare(sunnypilot_dump: dict, upstream_dump: dict) -> list[str]:
violations: list[str] = []
sunnypilot_structs = sunnypilot_dump["structs"]
upstream_structs = upstream_dump["structs"]
for type_id, sp_struct in sunnypilot_structs.items():
if not sp_struct["hasUnion"]:
continue
up_struct = upstream_structs.get(type_id)
if up_struct is None:
continue
up_ordinals = {f["ordinal"] for f in up_struct["fields"] if f.get("discriminant") is not None}
display = sp_struct["displayName"]
for field in sp_struct["fields"]:
if field.get("discriminant") is None:
continue
if field["ordinal"] not in up_ordinals:
violations.append(
f"{display} @{field['ordinal']} '{field['name']}': "
f"union variant not present upstream (discriminant={field['discriminant']})"
)
return violations
def run_read(cereal_dir: str, peer_path: str, extra_imports: list[str] | None = None) -> int:
log = load_log(cereal_dir, extra_imports)
with open(peer_path, "r", encoding="utf-8") as f:
peer_dump = json.load(f)
local_dump = {
"root": hex_id(log.Event.schema.node.id),
"structs": collect_schema(log.Event.schema),
}
violations = compare(sunnypilot_dump=peer_dump, upstream_dump=local_dump)
if not violations:
print("cereal compat OK: upstream can parse sunnypilot routes.")
return 0
print(f"cereal compat FAIL ({len(violations)} leaked union variant(s)):")
for v in violations:
print(f" {v}")
return 1
def main() -> int:
parser = argparse.ArgumentParser(description="sunnypilot cereal upstream compat check")
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("-g", "--generate", action="store_true", help="dump local schema to JSON")
mode.add_argument("-r", "--read", action="store_true", help="validate against peer schema")
parser.add_argument("-f", "--file", default="schema.json", help="JSON file path")
parser.add_argument("--cereal-dir", required=True, help="path to cereal directory")
parser.add_argument("-I", "--import-path", action="append", default=[], help="extra capnp import paths")
args = parser.parse_args()
if args.generate:
dump_schema(args.cereal_dir, args.file, args.import_path)
return 0
return run_read(args.cereal_dir, args.file, args.import_path)
if __name__ == "__main__":
sys.exit(main())
openpilot/cereal/services.py
Executable
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
from enum import IntEnum
# TODO: this should be automatically determined using the capnp schema
class QueueSize(IntEnum):
BIG = 10 * 1024 * 1024 # 10MB - video frames, large AI outputs
MEDIUM = 2 * 1024 * 1024 # 2MB - high freq (CAN), livestream
SMALL = 250 * 1024 # 250KB - most services
class Service:
def __init__(self, should_log: bool, frequency: float, decimation: int | None = None,
queue_size: QueueSize = QueueSize.SMALL):
self.should_log = should_log
self.frequency = frequency
self.decimation = decimation
self.queue_size = queue_size
_services: dict[str, tuple] = {
# service: (should_log, frequency, qlog decimation (optional))
# note: the "EncodeIdx" packets will still be in the log
"gyroscope": (True, 104., 104),
"accelerometer": (True, 104., 104),
"temperatureSensor": (True, 2., 200),
"deviceState": (True, 2., 1),
"chestnutState": (True, 10., 10),
"touch": (True, 20., 1),
"can": (True, 100., 2053, QueueSize.BIG), # decimation gives ~3 msgs in a full segment
"controlsState": (True, 100., 10, QueueSize.MEDIUM),
"selfdriveState": (True, 100., 10),
"pandaStates": (True, 10., 1),
"peripheralState": (True, 2., 1),
"radarState": (True, 20., 5),
"narrowRoadEncodeIdx": (False, 20., 1),
"radarTracks": (True, 20.),
"sendcan": (True, 100., 139, QueueSize.MEDIUM),
"logMessage": (True, 0., None, QueueSize.BIG),
"errorLogMessage": (True, 0., 1, QueueSize.BIG),
"extrinsicsCalibration": (True, 4., 4),
"lateralTorqueParameters": (True, 4., 1),
"lateralDelay": (True, 4., 1),
"operatingSystemLog": (True, 0.),
"carState": (True, 100., 10),
"carControl": (True, 100., 10),
"carOutput": (True, 100., 10),
"longitudinalPlan": (True, 20., 10),
"lateralManeuverPlan": (True, 20.),
"driverAssistance": (True, 20., 20),
"procLog": (True, 0.5, 15, QueueSize.BIG),
"gpsLocationExternal": (True, 10., 10),
"gpsLocation": (True, 1., 1),
"ubloxGnss": (True, 10.),
"qcomGnss": (True, 2.),
"clocks": (True, 0.1, 1),
"ubloxRaw": (True, 20.),
"deviceMotion": (True, 20., 4),
"vehicleParameters": (True, 20., 5),
"cameraOdometry": (True, 20., 10),
"thumbnail": (True, 1 / 60., 1),
"onroadEvents": (True, 1., 1),
"carParams": (True, 0.02, 1),
"narrowRoadCameraState": (True, 20., 20),
"cabinCameraState": (True, 20., 20),
"cabinEncodeIdx": (False, 20., 1),
"driverStateV2": (True, 20., 10),
"driverMonitoringState": (True, 20., 10),
"wideRoadEncodeIdx": (False, 20., 1),
"wideRoadCameraState": (True, 20., 20),
"drivingModelData": (True, 20., 10),
"modelV2": (True, 20., None, QueueSize.BIG),
"managerState": (True, 2., 1),
"qNarrowRoadEncodeIdx": (False, 20.),
"userBookmark": (True, 0., 1),
"soundPressure": (True, 10., 10),
"rawAudioData": (False, 20.),
"bookmarkButton": (True, 0., 1),
"narrowRoadEncodeData": (False, 20., None, QueueSize.BIG),
"cabinEncodeData": (False, 20., None, QueueSize.BIG),
"wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
"qNarrowRoadEncodeData": (False, 20., None, QueueSize.BIG),
# sunnypilot
"modelManagerSP": (False, 1., 1, QueueSize.BIG),
"backupManagerSP": (False, 1., 1, QueueSize.BIG),
"selfdriveStateSP": (True, 100., 10),
"longitudinalPlanSP": (True, 20., 10),
"onroadEventsSP": (True, 1., 1),
"carParamsSP": (True, 0.02, 1),
"carControlSP": (True, 100., 10),
"carStateSP": (True, 100., 10),
"liveMapDataSP": (True, 1., 1),
"modelDataV2SP": (True, 20., None, QueueSize.BIG),
"liveLocationKalman": (True, 20.),
# debug
"uiDebug": (True, 0., 1),
"testJoystick": (True, 0.),
"alertDebug": (True, 20., 5),
"livestreamWideRoadEncodeIdx": (False, 20.),
"livestreamNarrowRoadEncodeIdx": (False, 20.),
"livestreamCabinEncodeIdx": (False, 20.),
"livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
"livestreamNarrowRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
"livestreamCabinEncodeData": (False, 20., None, QueueSize.MEDIUM),
"customReservedRawData0": (True, 0.),
}
SERVICE_LIST = {name: Service(*vals) for
idx, (name, vals) in enumerate(_services.items())}
def build_header():
h = ""
h += "/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */\n"
h += "#ifndef __SERVICES_H\n"
h += "#define __SERVICES_H\n"
h += "#include <map>\n"
h += "#include <string>\n"
h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
h += "static std::map<std::string, service> services = {\n"
for k, v in SERVICE_LIST.items():
should_log = "true" if v.should_log else "false"
decimation = -1 if v.decimation is None else v.decimation
h += f' {{ "{k}", {{"{k}", {should_log}, {v.frequency:f}, {decimation:d}, {v.queue_size:d}}}}},\n'
h += "};\n"
h += "#endif\n"
return h
if __name__ == "__main__":
print(build_header())
openpilot/cereal/visionipc.py
+8
View File
@@ -0,0 +1,8 @@
from enum import IntEnum
class VisionStreamType(IntEnum):
VISION_STREAM_NARROW_ROAD = 0
VISION_STREAM_CABIN = 1
VISION_STREAM_WIDE_ROAD = 2
VISION_STREAM_MAP = 3
openpilot/cereal/visionstream.h
+10
View File
@@ -0,0 +1,10 @@
#pragma once
#include "msgq/visionipc/visionbuf.h"
enum VisionStreamValues : VisionStreamType {
VISION_STREAM_NARROW_ROAD = 0,
VISION_STREAM_CABIN = 1,
VISION_STREAM_WIDE_ROAD = 2,
VISION_STREAM_MAP = 3,
};
openpilot/common/SConscript
+19
View File
@@ -0,0 +1,19 @@
Import('env')
common_libs = [
'params.cc',
'swaglog.cc',
'util.cc',
'ratekeeper.cc',
'yuv.cc',
]
_common = env.Library('common', common_libs, LIBS="json11")
Export('_common')
params_python = env.SharedLibrary('params_c', 'params_c.cc', LIBS=[_common, 'zmq', 'json11'])
common_python = [params_python]
Export('common_python')
if GetOption('extras'):
env.Program('tests/test_swaglog', 'tests/test_swaglog.cc', LIBS=[_common, 'json11', 'zmq', 'pthread'])
openpilot/common/__init__.py
View File
openpilot/common/api/__init__.py
+26
View File
@@ -0,0 +1,26 @@
from openpilot.common.api.comma_connect import CommaConnectApi
class Api:
def __init__(self, dongle_id):
self.service = CommaConnectApi(dongle_id)
def request(self, method, endpoint, **params):
return self.service.request(method, endpoint, **params)
def get(self, *args, **kwargs):
return self.service.get(*args, **kwargs)
def post(self, *args, **kwargs):
return self.service.post(*args, **kwargs)
def get_token(self, payload_extra=None, expiry_hours=1):
return self.service.get_token(payload_extra, expiry_hours)
def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
return CommaConnectApi(None).get_key_pair()
openpilot/common/api/base.py
+73
View File
@@ -0,0 +1,73 @@
import jwt
import os
import requests
import unicodedata
from datetime import datetime, timedelta, UTC
from openpilot.common.hardware.hw import Paths
from openpilot.common.version import get_version
# name: jwt signature algorithm
KEYS = {"id_rsa": "RS256",
"id_ecdsa": "ES256"}
class BaseApi:
def __init__(self, dongle_id, api_host, user_agent="openpilot-"):
self.dongle_id = dongle_id
self.api_host = api_host
self.user_agent = user_agent
self.jwt_algorithm, self.private_key, _ = self.get_key_pair()
def get(self, *args, **kwargs):
return self.request('GET', *args, **kwargs)
def post(self, *args, **kwargs):
return self.request('POST', *args, **kwargs)
def request(self, method, endpoint, timeout=None, access_token=None, **params):
return self.api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
def _get_token(self, payload_extra=None, expiry_hours=1, **extra_payload):
now = datetime.now(UTC).replace(tzinfo=None)
payload = {
'identity': self.dongle_id,
'nbf': now,
'iat': now,
'exp': now + timedelta(hours=expiry_hours),
**extra_payload
}
if payload_extra is not None:
payload.update(payload_extra)
assert self.private_key is not None
token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
if isinstance(token, bytes):
token = token.decode('utf8')
return token
def get_token(self, payload_extra=None, expiry_hours=1):
return self._get_token(payload_extra, expiry_hours)
def remove_non_ascii_chars(self, text):
normalized_text = unicodedata.normalize('NFD', text)
ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
return ascii_encoded_text.decode()
def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
headers = {}
if access_token is not None:
headers['Authorization'] = "JWT " + access_token
version = self.remove_non_ascii_chars(get_version())
headers['User-Agent'] = self.user_agent + version
# TODO: add session to Api
req = requests if session is None else session
return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
@staticmethod
def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
for key in KEYS:
if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
return KEYS[key], private.read(), public.read()
return None, None, None
openpilot/common/api/comma_connect.py
+11
View File
@@ -0,0 +1,11 @@
import os
from openpilot.common.api.base import BaseApi
API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
class CommaConnectApi(BaseApi):
def __init__(self, dongle_id):
super().__init__(dongle_id, API_HOST)
self.user_agent = "openpilot-"
openpilot/common/basedir.py
+4
View File
@@ -0,0 +1,4 @@
import os
BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../.."))
openpilot/common/constants.py
+23
View File
@@ -0,0 +1,23 @@
import numpy as np
# conversions
class CV:
# Speed
MPH_TO_KPH = 1.609344
KPH_TO_MPH = 1. / MPH_TO_KPH
MS_TO_KPH = 3.6
KPH_TO_MS = 1. / MS_TO_KPH
MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
MS_TO_KNOTS = 1.9438
KNOTS_TO_MS = 1. / MS_TO_KNOTS
# Angle
DEG_TO_RAD = np.pi / 180.
RAD_TO_DEG = 1. / DEG_TO_RAD
# Mass
LB_TO_KG = 0.453592
ACCELERATION_DUE_TO_GRAVITY = 9.81 # m/s^2
openpilot/common/esim/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from openpilot.common.esim.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
__all__ = ["LPABase", "LPAError", "LPAProfileNotFoundError", "Profile"]
openpilot/common/esim/base.py
+55
View File
@@ -0,0 +1,55 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass
class LPAError(RuntimeError):
pass
class LPAProfileNotFoundError(LPAError):
pass
@dataclass
class Profile:
iccid: str
nickname: str
enabled: bool
provider: str
@property
def is_comma(self) -> bool:
return self.provider == 'Webbing' and self.iccid.startswith('8985235')
class LPABase(ABC):
@abstractmethod
def list_profiles(self) -> list[Profile]:
pass
@abstractmethod
def get_active_profile(self) -> Profile | None:
pass
@abstractmethod
def delete_profile(self, iccid: str) -> None:
pass
@abstractmethod
def download_profile(self, qr: str, nickname: str | None = None) -> None:
pass
@abstractmethod
def nickname_profile(self, iccid: str, nickname: str) -> None:
pass
@abstractmethod
def switch_profile(self, iccid: str) -> None:
pass
def process_notifications(self) -> None:
pass
@abstractmethod
def is_euicc(self) -> bool:
pass
openpilot/common/esim/esim.py
Executable
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
import argparse
import sys
import time
from openpilot.common.hardware import HARDWARE
from openpilot.common.esim.base import LPABase, Profile
def sorted_profiles(lpa: LPABase) -> list[Profile]:
return sorted(lpa.list_profiles(), key=lambda p: p.iccid)
def resolve_iccid(lpa: LPABase, ref: str) -> str:
# ref is either a 1-based index into the sorted list, or a literal iccid
if ref.isdigit():
profiles = sorted_profiles(lpa)
idx = int(ref) - 1
if not 0 <= idx < len(profiles):
raise SystemExit(f'no profile at index {ref} (have {len(profiles)})')
return profiles[idx].iccid
return ref
def print_profiles(lpa: LPABase) -> None:
profiles = sorted_profiles(lpa)
print(f'\n{len(profiles)} profile{"s" if len(profiles) != 1 else ""}:')
for i, p in enumerate(profiles, start=1):
print(f'{i}. {p.iccid} (nickname: {p.nickname or "<none provided>"}) (provider: {p.provider}) - {"enabled" if p.enabled else "disabled"}')
def execute_and_process_notifications(lpa: LPABase, operation) -> None:
try:
operation()
finally:
time.sleep(1) # Need to wait for 1s after the operation is finished so the eUICC/modem can settle down.
try:
lpa.process_notifications()
except Exception as e:
print(f'failed to process eSIM notifications: {e}', file=sys.stderr)
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='esim.py', description='manage eSIM profiles on your comma device', epilog='comma.ai')
sub = parser.add_subparsers(dest='cmd')
sub.add_parser('list', help='list profiles')
p_switch = sub.add_parser('switch', help='switch to profile')
p_switch.add_argument('profile', help='iccid or 1-based index from `list`')
p_delete = sub.add_parser('delete', help='delete profile (warning: this cannot be undone)')
p_delete.add_argument('profile', help='iccid or 1-based index from `list`')
p_download = sub.add_parser('download', help='download a profile using QR code (format: LPA:1$rsp.truphone.com$QRF-SPEEDTEST)')
p_download.add_argument('qr')
p_download.add_argument('name')
p_nickname = sub.add_parser('nickname', help='update the nickname for a profile')
p_nickname.add_argument('profile', help='iccid or 1-based index from `list`')
p_nickname.add_argument('name')
args = parser.parse_args()
lpa = HARDWARE.get_sim_lpa()
if not lpa.is_euicc():
raise SystemExit("no eUICC detected")
if args.cmd == 'switch':
iccid = resolve_iccid(lpa, args.profile)
execute_and_process_notifications(lpa, lambda: lpa.switch_profile(iccid))
elif args.cmd == 'delete':
iccid = resolve_iccid(lpa, args.profile)
confirm = input(f'are you sure you want to delete profile {iccid}? (y/N) ')
if confirm == 'y':
execute_and_process_notifications(lpa, lambda: lpa.delete_profile(iccid))
else:
print('cancelled')
exit(0)
elif args.cmd == 'download':
execute_and_process_notifications(lpa, lambda: lpa.download_profile(args.qr, args.name))
elif args.cmd == 'nickname':
lpa.nickname_profile(resolve_iccid(lpa, args.profile), args.name)
else:
if args.cmd is None:
parser.print_help()
print_profiles(lpa)
openpilot/common/esim/gsma_ci_bundle.pem
+133
View File
@@ -0,0 +1,133 @@
# GSMA Certificate Issuer (CI) bundle for eSIM RSP
# Source: https://euicc-manual.osmocom.org/docs/pki/ci/bundle.pem
issuer=
countryName = CH
organizationName = OISTE Foundation
commonName = OISTE GSMA CI G1
notBefore=2024-01-16 23:17:39Z
notAfter=2059-01-07 23:17:38Z
-----BEGIN CERTIFICATE-----
MIIB9zCCAZ2gAwIBAgIUSpBSCCDYPOEG/IFHUCKpZ2pIAQMwCgYIKoZIzj0EAwIw
QzELMAkGA1UEBhMCQ0gxGTAXBgNVBAoMEE9JU1RFIEZvdW5kYXRpb24xGTAXBgNV
BAMMEE9JU1RFIEdTTUEgQ0kgRzEwIBcNMjQwMTE2MjMxNzM5WhgPMjA1OTAxMDcy
MzE3MzhaMEMxCzAJBgNVBAYTAkNIMRkwFwYDVQQKDBBPSVNURSBGb3VuZGF0aW9u
MRkwFwYDVQQDDBBPSVNURSBHU01BIENJIEcxMFkwEwYHKoZIzj0CAQYIKoZIzj0D
AQcDQgAEvZ3s3PFC4NgrCcCMmHJ6DJ66uzAHuLcvjJnOn+TtBNThS7YHLDyHCa2v
7D+zTP+XTtgqgcLoB56Gha9EQQQ4xKNtMGswDwYDVR0TAQH/BAUwAwEB/zAQBgNV
HREECTAHiAVghXQFDjAXBgNVHSABAf8EDTALMAkGB2eBEgECAQAwHQYDVR0OBBYE
FEwnlnrSDBSzkelgHkHmBK1XwCIvMA4GA1UdDwEB/wQEAwIBBjAKBggqhkjOPQQD
AgNIADBFAiBVcywTj017jKpAQ+gwy4MqK2hQvzve6lkvQkgSP6ykHwIhAI0KFwCD
jnPbmcJsG41hUrWNlf+IcrMvFuYii0DasBNi
-----END CERTIFICATE-----
issuer=
organizationName = GSM Association
commonName = GSM Association - RSP2 Root CI1
notBefore=2017-02-22 00:00:00Z
notAfter=2052-02-21 23:59:59Z
-----BEGIN CERTIFICATE-----
MIICSTCCAe+gAwIBAgIQbmhWeneg7nyF7hg5Y9+qejAKBggqhkjOPQQDAjBEMRgw
FgYDVQQKEw9HU00gQXNzb2NpYXRpb24xKDAmBgNVBAMTH0dTTSBBc3NvY2lhdGlv
biAtIFJTUDIgUm9vdCBDSTEwIBcNMTcwMjIyMDAwMDAwWhgPMjA1MjAyMjEyMzU5
NTlaMEQxGDAWBgNVBAoTD0dTTSBBc3NvY2lhdGlvbjEoMCYGA1UEAxMfR1NNIEFz
c29jaWF0aW9uIC0gUlNQMiBSb290IENJMTBZMBMGByqGSM49AgEGCCqGSM49AwEH
A0IABJ1qutL0HCMX52GJ6/jeibsAqZfULWj/X10p/Min6seZN+hf5llovbCNuB2n
unLz+O8UD0SUCBUVo8e6n9X1TuajgcAwgb0wDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
EwEB/wQFMAMBAf8wEwYDVR0RBAwwCogIKwYBBAGC6WAwFwYDVR0gAQH/BA0wCzAJ
BgdngRIBAgEAME0GA1UdHwRGMEQwQqBAoD6GPGh0dHA6Ly9nc21hLWNybC5zeW1h
dXRoLmNvbS9vZmZsaW5lY2EvZ3NtYS1yc3AyLXJvb3QtY2kxLmNybDAdBgNVHQ4E
FgQUgTcPUSXQsdQI1MOyMubSXnlb6/swCgYIKoZIzj0EAwIDSAAwRQIgIJdYsOMF
WziPK7l8nh5mu0qiRiVf25oa9ullG/OIASwCIQDqCmDrYf+GziHXBOiwJwnBaeBO
aFsiLzIEOaUuZwdNUw==
-----END CERTIFICATE-----
issuer=
countryName = US
organizationName = Entrust, Inc.
organizationalUnitName = See www.entrust.net/legal-terms
organizationalUnitName = (c) 2016 Entrust, Inc. - for authorized use only
commonName = Entrust eSIM Certification Authority
notBefore=2016-11-16 16:04:02Z
notAfter=2051-10-16 16:34:02Z
-----BEGIN CERTIFICATE-----
MIIC6DCCAo2gAwIBAgIRAIy4GT7M5nHsAAAAAFgsinowCgYIKoZIzj0EAwIwgbkx
CzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9T
ZWUgd3d3LmVudHJ1c3QubmV0L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAx
NiBFbnRydXN0LCBJbmMuIC0gZm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxLTArBgNV
BAMTJEVudHJ1c3QgZVNJTSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAgFw0xNjEx
MTYxNjA0MDJaGA8yMDUxMTAxNjE2MzQwMlowgbkxCzAJBgNVBAYTAlVTMRYwFAYD
VQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1c3QubmV0
L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxNiBFbnRydXN0LCBJbmMuIC0g
Zm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxLTArBgNVBAMTJEVudHJ1c3QgZVNJTSBD
ZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA
BAdzwGHeQ1Wb2f4DmHTByR5/IWL3JugQ1U3908a++bHdlt+TTA7K4c5cYZ+51Yz/
hg/bacxguPDh9uQUK6Wg3a6jcjBwMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/
BAQDAgEGMBcGA1UdIAEB/wQNMAswCQYHZ4ESAQIBADAVBgNVHREEDjAMiApghkgB
hvpsFAoAMB0GA1UdDgQWBBQWcEt/NR42B/GMS3AAXDoAPf1BSjAKBggqhkjOPQQD
AgNJADBGAiEAspjXMvaBZyAg86Z0AAtT0yBRAi1EyaAfNz9kDJeAE04CIQC3efj8
ATL7/tDBOhANy3cK8PS/1NIlu9vqMLCZsZvJ0Q==
-----END CERTIFICATE-----
issuer=
countryName = FR
organizationName = OBERTHUR TECHNOLOGIES
organizationalUnitName = TELECOM
commonName = MC4 OT ROOT CI v1
notBefore=2016-11-15 00:00:01Z
notAfter=2046-11-08 23:59:59Z
-----BEGIN CERTIFICATE-----
MIICOjCCAeGgAwIBAgIBATAKBggqhkjOPQQDAjBbMQswCQYDVQQGEwJGUjEeMBwG
A1UEChMVT0JFUlRIVVIgVEVDSE5PTE9HSUVTMRAwDgYDVQQLEwdURUxFQ09NMRow
GAYDVQQDExFNQzQgT1QgUk9PVCBDSSB2MTAeFw0xNjExMTUwMDAwMDFaFw00NjEx
MDgyMzU5NTlaMFsxCzAJBgNVBAYTAkZSMR4wHAYDVQQKExVPQkVSVEhVUiBURUNI
Tk9MT0dJRVMxEDAOBgNVBAsTB1RFTEVDT00xGjAYBgNVBAMTEU1DNCBPVCBST09U
IENJIHYxMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEHb/Gajt3OZxuaDSklBQE
D4lOd6PGPLSvtfkM952ubdyy45tJwAeA0eEii0CLrFT6tcfXkW+H/5mQyMRXaAUk
T6OBlTCBkjAfBgNVHSMEGDAWgBTNbmC3LXoGPLyEYluR6A/jBAbhPjAdBgNVHQ4E
FgQUzW5gty16Bjy8hGJbkegP4wQG4T4wDgYDVR0PAQH/BAQDAgAGMBcGA1UdIAEB
/wQNMAswCQYHZ4ESAQIBADAWBgNVHREEDzANiAsrBgEEAYHvb7OITTAPBgNVHRMB
Af8EBTADAQH/MAoGCCqGSM49BAMCA0cAMEQCIEw4Nc7f2fDtoH+6ON/bknfDQxmT
ikThXjhpLtSrSKN2AiAxHxgC87L0FDnH8dJNlkdGX9c0JIx6oLheIplfS6k+jg==
-----END CERTIFICATE-----
issuer=
commonName = SubMan V4.2 CI Google Pixel
organizationName = Giesecke and Devrient GmbH
organizationalUnitName = Mobile Security
countryName = DE
notBefore=2017-05-10 00:00:00Z
notAfter=2027-05-10 00:00:00Z
-----BEGIN CERTIFICATE-----
MIICaTCCAg6gAwIBAgICASwwCgYIKoZIzj0EAwIwczElMCMGA1UEAxMcIFN1Yk1h
biBWNC4yIENJIEdvb2dsZSBQaXhlbDEjMCEGA1UEChMaR2llc2Vja2UgYW5kIERl
dnJpZW50IEdtYkgxGDAWBgNVBAsTD01vYmlsZSBTZWN1cml0eTELMAkGA1UEBhMC
REUwHhcNMTcwNTEwMDAwMDAwWhcNMjcwNTEwMDAwMDAwWjBzMSUwIwYDVQQDExwg
U3ViTWFuIFY0LjIgQ0kgR29vZ2xlIFBpeGVsMSMwIQYDVQQKExpHaWVzZWNrZSBh
bmQgRGV2cmllbnQgR21iSDEYMBYGA1UECxMPTW9iaWxlIFNlY3VyaXR5MQswCQYD
VQQGEwJERTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABHNorfaJsGzqWNawyAhl
IAv9QL2/+b9RsUoso06t/dKX1MRr5CUJ51acvv5TAFhQKIml+dwLbFnV5aO+8W6Z
wxajgZEwgY4wHwYDVR0jBBgwFoAUtg8LiX/WMLiM/tYWH46oCMU4KsMwHQYDVR0O
BBYEFLYPC4l/1jC4jP7WFh+OqAjFOCrDMA4GA1UdDwEB/wQEAwIBBjAXBgNVHSAB
Af8EDTALMAkGB2eBEgECAQAwDwYDVR0TAQH/BAUwAwEB/zASBgNVHREECzAJiAcr
BgEEAdwPMAoGCCqGSM49BAMCA0kAMEYCIQDpoZcuAQrjATW8U+AWqMUJ0dY6nWW1
R1QmFzVZ1yMXSwIhALCvRqkCtgiavdeFeSgsSNbY5Fhd+QoCltuSh1U4TE7A
-----END CERTIFICATE-----
issuer=
countryName = DE
commonName = SubMan V4.2 CI
organizationName = Giesecke and Devrient
organizationalUnitName = Mobile Security
notBefore=2016-08-12 13:51:48Z
notAfter=2026-08-12 13:51:48Z
-----BEGIN CERTIFICATE-----
MIICUjCCAfigAwIBAgIDQgAAMAoGCCqGSM49BAMCMGAxCzAJBgNVBAYTAkRFMRcw
FQYDVQQDEw5TdWJNYW4gVjQuMiBDSTEeMBwGA1UEChMVR2llc2Vja2UgYW5kIERl
dnJpZW50MRgwFgYDVQQLEw9Nb2JpbGUgU2VjdXJpdHkwHhcNMTYwODEyMTM1MTQ4
WhcNMjYwODEyMTM1MTQ4WjBgMQswCQYDVQQGEwJERTEXMBUGA1UEAxMOU3ViTWFu
IFY0LjIgQ0kxHjAcBgNVBAoTFUdpZXNlY2tlIGFuZCBEZXZyaWVudDEYMBYGA1UE
CxMPTW9iaWxlIFNlY3VyaXR5MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYIgl
VQr9wbXOlwPp8qMg5Df08Cli9Mc+lpr3Lwa9PlVA3QWlLeX4GfD4H3phLBqVIa17
yHttmtheTxi0KoEqhKOBoDCBnTAdBgNVHQ4EFgQU6lOt7zMpuVCa/XVf1Ei4LcG8
7P8wDgYDVR0PAQH/BAQDAgEGMBcGA1UdIAEB/wQNMAswCQYHZ4ESAQIBADAPBgNV
HRMBAf8EBTADAQH/MBIGA1UdEQQLMAmIBysGAQQB3A8wLgYDVR0fBCcwJTAjoCGg
H4YdaHR0cDovL2dpLWRlLmNvbS90ZXN0LmNybC5wZW0wCgYIKoZIzj0EAwIDSAAw
RQIhAMMx2L/VHDiOW+Fl/OuFmhCdizYM17Yn9zAVieKO2T0iAiANWtCMmY+DzkqK
yHxBFX0U2tBd682zP4DpgRt8j3Ylew==
-----END CERTIFICATE-----
openpilot/common/esim/lpa.py
+791
View File
@@ -0,0 +1,791 @@
# SGP.22 v2.3: https://www.gsma.com/solutions-and-impact/technologies/esim/wp-content/uploads/2021/07/SGP.22-v2.3.pdf
import atexit
import base64
import fcntl
import hashlib
import os
import requests
import subprocess
import sys
import termios
import time
from collections.abc import Callable, Generator
from contextlib import contextmanager
from typing import Any
from pathlib import Path
from openpilot.common.time_helpers import system_time_valid
from openpilot.common.esim.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
from openpilot.common.serial import Serial, SerialException
GSMA_CI_BUNDLE = str(Path(__file__).parent / "gsma_ci_bundle.pem")
DEFAULT_DEVICE = "/dev/modem_at0"
DEFAULT_BAUD = 9600
DEFAULT_TIMEOUT = 5.0
# https://euicc-manual.osmocom.org/docs/lpa/applet-id/
ISDR_AID = "A0000005591010FFFFFFFF8900000100"
ES10X_MSS = 120
HTTP_TIMEOUT = 30
OPEN_ISDR_RETRIES = 10
OPEN_ISDR_RETRY_DELAY_S = 0.25
OPEN_ISDR_RESET_ATTEMPT = 5
SEND_APDU_RETRIES = 3
LOCK_FILE = '/dev/shm/modem.lock'
DEBUG = os.environ.get("DEBUG") == "1"
# TLV Tags
TAG_ICCID = 0x5A
TAG_STATUS = 0x80
TAG_EUICC_INFO = 0xBF20
TAG_PREPARE_DOWNLOAD = 0xBF21
TAG_BPP_COMMAND = 0xBF23
TAG_PROFILE_METADATA = 0xBF25
TAG_INSTALL_RESULT_DATA = 0xBF27
TAG_LIST_NOTIFICATION = 0xBF28
TAG_SET_NICKNAME = 0xBF29
TAG_RETRIEVE_NOTIFICATION = 0xBF2B
TAG_PROFILE_INFO_LIST = 0xBF2D
TAG_EUICC_CHALLENGE = 0xBF2E
TAG_NOTIFICATION_METADATA = 0xBF2F
TAG_NOTIFICATION_SENT = 0xBF30
TAG_ENABLE_PROFILE = 0xBF31
TAG_DELETE_PROFILE = 0xBF33
TAG_BPP = 0xBF36
TAG_PROFILE_INSTALL_RESULT = 0xBF37
TAG_AUTH_SERVER = 0xBF38
TAG_CANCEL_SESSION = 0xBF41
TAG_OK = 0xA0
PROFILE_OK = 0x00
PROFILE_NOT_IN_DISABLED_STATE = 0x02
PROFILE_CAT_BUSY = 0x05
PROFILE_ERROR_CODES = {
0x01: "iccidOrAidNotFound", PROFILE_NOT_IN_DISABLED_STATE: "profileNotInDisabledState",
0x03: "disallowedByPolicy", 0x04: "wrongProfileReenabling",
PROFILE_CAT_BUSY: "catBusy", 0x06: "undefinedError",
}
AUTH_SERVER_ERROR_CODES = {
0x01: "eUICCVerificationFailed", 0x02: "eUICCCertificateExpired",
0x03: "eUICCCertificateRevoked", 0x05: "invalidServerSignature",
0x06: "euiccCiPKUnknown", 0x0A: "matchingIdRefused",
0x10: "insufficientMemory",
}
BPP_COMMAND_NAMES = {
0: "initialiseSecureChannel", 1: "configureISDP", 2: "storeMetadata",
3: "storeMetadata2", 4: "replaceSessionKeys", 5: "loadProfileElements",
}
BPP_ERROR_REASONS = {
1: "incorrectInputValues", 2: "invalidSignature", 3: "invalidTransactionId",
4: "unsupportedCrtValues", 5: "unsupportedRemoteOperationType",
6: "unsupportedProfileClass", 7: "scp03tStructureError", 8: "scp03tSecurityError",
9: "iccidAlreadyExistsOnEuicc", 10: "insufficientMemoryForProfile",
11: "installInterrupted", 12: "peProcessingError", 13: "dataMismatch",
14: "invalidNAA",
}
BPP_ERROR_MESSAGES = {
9: "This eSIM profile is already installed on this device.",
10: "Not enough memory on the eUICC to install this profile.",
12: "Profile installation failed. The QR code may have already been used.",
}
# SGP.22 §5.2.6 SM-DP+ reason/subject codes mapped to user-friendly messages
ES9P_ERROR_MESSAGES: dict[tuple[str, str], str] = {
('3.8', '8.2.6'): "This eSIM profile is already installed on another device. Please use a new QR code.",
('3.8', '8.2.1'): "This eSIM profile has expired. Please request a new QR code.",
('3.8', '8.1'): "The SM-DP+ server refused this request.",
('3.1', '8.2.6'): "This eSIM profile has been revoked by the carrier.",
('3.9', '8.2.6'): "This eSIM profile download has already been completed.",
('2.1', '8.8'): "The device is not compatible with this eSIM profile.",
('1.2', '8.1'): "The SM-DP+ server is temporarily unavailable. Try again later.",
}
NOTIFICATION_OPERATIONS = {0x80: "install", 0x40: "enable", 0x20: "disable", 0x10: "delete"}
STATE_LABELS = {0: "disabled", 1: "enabled", 255: "unknown"}
ICON_LABELS = {0: "jpeg", 1: "png", 255: "unknown"}
CLASS_LABELS = {0: "test", 1: "provisioning", 2: "operational", 255: "unknown"}
# TLV tag -> (field_name, decoder)
FieldMap = dict[int, tuple[str, Callable[[bytes], Any]]]
def b64e(data: bytes) -> str:
return base64.b64encode(data).decode("ascii")
def base64_trim(s: str) -> str:
return "".join(c for c in s if c not in "\n\r \t")
def b64d(s: str) -> bytes:
return base64.b64decode(base64_trim(s))
class AtClient:
def __init__(self, device: str, baud: int, timeout: float) -> None:
self.channel: str | None = None
self._device = device
self._baud = baud
self._timeout = timeout
self._serial: Serial | None = None
def send_raw(self, data: bytes) -> None:
self._ensure_serial()
self._serial.reset_input_buffer()
self._serial.write(data)
self._serial.flush()
def close(self) -> None:
try:
if self.channel:
try:
self.query(f"AT+CCHC={self.channel}")
except (RuntimeError, TimeoutError):
pass
self.channel = None
finally:
if self._serial:
self._serial.close()
def _send(self, cmd: str) -> None:
if DEBUG:
print(f"SER >> {cmd}", file=sys.stderr)
self._serial.write((cmd + "\r").encode("ascii"))
def _expect(self) -> list[str]:
lines: list[str] = []
while True:
raw = self._serial.readline()
if not raw:
raise TimeoutError("AT command timed out")
line = raw.decode(errors="ignore").strip()
if not line:
continue
if DEBUG:
print(f"SER << {line}", file=sys.stderr)
if line == "OK":
return lines
if line == "ERROR" or line.startswith("+CME ERROR"):
raise RuntimeError(f"AT command failed: {line}")
lines.append(line)
def _ensure_serial(self, reconnect: bool = False) -> None:
if reconnect:
self.channel = None
try:
if self._serial:
self._serial.close()
except Exception:
pass
self._serial = None
if self._serial is None:
self._serial = Serial(self._device, baudrate=self._baud, timeout=self._timeout)
def query(self, cmd: str) -> list[str]:
self._ensure_serial()
try:
self._send(cmd)
return self._expect()
except SerialException:
self._ensure_serial(reconnect=True)
self._send(cmd)
return self._expect()
def _open_isdr_once(self) -> None:
if self.channel:
try:
self.query(f"AT+CCHC={self.channel}")
except RuntimeError:
pass
self.channel = None
# drain any unsolicited responses before opening
if self._serial:
try:
self._serial.reset_input_buffer()
except (OSError, SerialException, termios.error):
self._ensure_serial(reconnect=True)
for line in self.query(f'AT+CCHO="{ISDR_AID}"'):
if line.startswith("+CCHO:") and (ch := line.split(":", 1)[1].strip()):
self.channel = ch
return
raise RuntimeError("Failed to open ISD-R application")
def _reset_modem(self) -> None:
if self._serial:
try:
self._serial.close()
except Exception:
pass
self._serial = None
subprocess.run(['/usr/comma/lte/lte.sh', 'start'], capture_output=True)
def open_isdr(self) -> None:
for attempt in range(OPEN_ISDR_RETRIES):
try:
self._open_isdr_once()
return
except (RuntimeError, TimeoutError, termios.error, SerialException):
time.sleep(OPEN_ISDR_RETRY_DELAY_S)
if attempt == OPEN_ISDR_RESET_ATTEMPT:
self._reset_modem()
raise RuntimeError("Failed to open ISD-R after retries")
def send_apdu(self, apdu: bytes) -> tuple[bytes, int, int]:
for attempt in range(SEND_APDU_RETRIES):
try:
if not self.channel:
self.open_isdr()
hex_payload = apdu.hex().upper()
for line in self.query(f'AT+CGLA={self.channel},{len(hex_payload)},"{hex_payload}"'):
if line.startswith("+CGLA:"):
parts = line.split(":", 1)[1].split(",", 1)
if len(parts) == 2:
data = bytes.fromhex(parts[1].strip().strip('"'))
if len(data) >= 2:
return data[:-2], data[-2], data[-1]
raise RuntimeError("Missing +CGLA response")
except (RuntimeError, ValueError):
self.channel = None
if attempt == SEND_APDU_RETRIES - 1:
raise
raise RuntimeError("send_apdu failed")
# --- TLV utilities ---
def iter_tlv(data: bytes, with_positions: bool = False) -> Generator:
idx, length = 0, len(data)
while idx < length:
start_pos = idx
tag = data[idx]
idx += 1
if tag & 0x1F == 0x1F: # Multi-byte tag
tag_value = tag
while idx < length:
next_byte = data[idx]
idx += 1
tag_value = (tag_value << 8) | next_byte
if not (next_byte & 0x80):
break
else:
tag_value = tag
if idx >= length:
break
size = data[idx]
idx += 1
if size & 0x80: # Multi-byte length
num_bytes = size & 0x7F
if idx + num_bytes > length:
break
size = int.from_bytes(data[idx : idx + num_bytes], "big")
idx += num_bytes
if idx + size > length:
break
value = data[idx : idx + size]
idx += size
yield (tag_value, value, start_pos, idx) if with_positions else (tag_value, value)
def find_tag(data: bytes, target: int) -> bytes | None:
return next((v for t, v in iter_tlv(data) if t == target), None)
def require_tag(data: bytes, target: int, label: str = "") -> bytes:
v = find_tag(data, target)
if v is None:
raise RuntimeError(f"Missing {label or f'tag 0x{target:X}'}")
return v
def tbcd_to_string(raw: bytes) -> str:
return "".join(str(n) for b in raw for n in (b & 0x0F, b >> 4) if n <= 9)
def string_to_tbcd(s: str) -> bytes:
digits = [int(c) for c in s if c.isdigit()]
return bytes(digits[i] | ((digits[i + 1] if i + 1 < len(digits) else 0xF) << 4) for i in range(0, len(digits), 2))
def encode_tlv(tag: int, value: bytes) -> bytes:
tag_bytes = bytes([(tag >> 8) & 0xFF, tag & 0xFF]) if tag > 255 else bytes([tag])
vlen = len(value)
if vlen <= 127:
return tag_bytes + bytes([vlen]) + value
length_bytes = vlen.to_bytes((vlen.bit_length() + 7) // 8, "big")
return tag_bytes + bytes([0x80 | len(length_bytes)]) + length_bytes + value
def int_bytes(n: int) -> bytes:
"""Encode a positive integer as minimal big-endian bytes (at least 1 byte)."""
return n.to_bytes((n.bit_length() + 7) // 8 or 1, "big")
PROFILE: FieldMap = {
TAG_ICCID: ("iccid", tbcd_to_string),
0x4F: ("isdpAid", lambda v: v.hex().upper()),
0x9F70: ("profileState", lambda v: STATE_LABELS.get(v[0], "unknown")),
0x90: ("profileNickname", lambda v: v.decode("utf-8", errors="ignore") or None),
0x91: ("serviceProviderName", lambda v: v.decode("utf-8", errors="ignore") or None),
0x92: ("profileName", lambda v: v.decode("utf-8", errors="ignore") or None),
0x93: ("iconType", lambda v: ICON_LABELS.get(v[0], "unknown")),
0x94: ("icon", b64e),
0x95: ("profileClass", lambda v: CLASS_LABELS.get(v[0], "unknown")),
}
def decode_struct(data: bytes, field_map: FieldMap) -> dict[str, Any]:
"""Parse TLV data using a {tag: (field_name, decoder)} map into a dict."""
result: dict[str, Any] = {name: None for name, _ in field_map.values()}
for tag, value in iter_tlv(data):
if (field := field_map.get(tag)):
result[field[0]] = field[1](value)
return result
# --- ES10x command transport ---
def es10x_command(client: AtClient, data: bytes) -> bytes:
response = bytearray()
sequence = 0
offset = 0
while offset < len(data):
chunk = data[offset : offset + ES10X_MSS]
offset += len(chunk)
is_last = offset == len(data)
apdu = bytes([0x80, 0xE2, 0x91 if is_last else 0x11, sequence & 0xFF, len(chunk)]) + chunk
segment, sw1, sw2 = client.send_apdu(apdu)
response.extend(segment)
while True:
if sw1 == 0x61: # More data available
segment, sw1, sw2 = client.send_apdu(bytes([0x80, 0xC0, 0x00, 0x00, sw2 or 0]))
response.extend(segment)
continue
if (sw1 & 0xF0) == 0x90:
break
raise RuntimeError(f"APDU failed with SW={sw1:02X}{sw2:02X}")
sequence += 1
return bytes(response)
# --- Profile operations ---
NOTIFICATION: FieldMap = {
TAG_STATUS: ("seqNumber", lambda v: int.from_bytes(v, "big")),
0x81: ("profileManagementOperation",
lambda v: NOTIFICATION_OPERATIONS.get(next((m for m in NOTIFICATION_OPERATIONS if len(v) >= 2 and v[1] & m), 0), "unknown")),
0x0C: ("notificationAddress", lambda v: v.decode("utf-8", errors="ignore")),
TAG_ICCID: ("iccid", tbcd_to_string),
}
def decode_profiles(blob: bytes) -> list[dict]:
root = require_tag(blob, TAG_PROFILE_INFO_LIST, "ProfileInfoList")
list_ok = find_tag(root, TAG_OK)
if list_ok is None:
return []
return [decode_struct(value, PROFILE) for tag, value in iter_tlv(list_ok) if tag == 0xE3]
def list_profiles(client: AtClient) -> list[dict]:
return decode_profiles(es10x_command(client, TAG_PROFILE_INFO_LIST.to_bytes(2, "big") + b"\x00"))
def set_profile_nickname(client: AtClient, iccid: str, nickname: str) -> None:
nickname_bytes = nickname.encode("utf-8")
if len(nickname_bytes) > 64:
raise ValueError("Profile nickname must be 64 bytes or less")
content = encode_tlv(TAG_ICCID, string_to_tbcd(iccid)) + encode_tlv(0x90, nickname_bytes)
response = es10x_command(client, encode_tlv(TAG_SET_NICKNAME, content))
code = require_tag(require_tag(response, TAG_SET_NICKNAME, "SetNicknameResponse"), TAG_STATUS, "SetNickname status")[0]
if code == 0x01:
raise LPAError(f"profile {iccid} not found")
if code != 0x00:
raise RuntimeError(f"SetNickname failed with status 0x{code:02X}")
# --- ES9P HTTP ---
def es9p_request(smdp_address: str, endpoint: str, payload: dict, error_prefix: str = "Request", session: requests.Session | None = None) -> dict:
url = f"https://{smdp_address}/gsma/rsp2/es9plus/{endpoint}"
headers = {"User-Agent": "gsma-rsp-lpad", "X-Admin-Protocol": "gsma/rsp/v2.3.0", "Content-Type": "application/json"}
http = session or requests
resp = http.post(url, json=payload, headers=headers, timeout=HTTP_TIMEOUT, verify=GSMA_CI_BUNDLE)
resp.raise_for_status()
if not resp.content:
return {}
data = resp.json()
if "header" in data and "functionExecutionStatus" in data["header"]:
status = data["header"]["functionExecutionStatus"]
if status.get("status") == "Failed":
sd = status.get("statusCodeData", {})
reason = sd.get("reasonCode", "unknown")
subject = sd.get("subjectCode", "unknown")
msg = ES9P_ERROR_MESSAGES.get((reason, subject),
f"{error_prefix} failed: {reason}/{subject} - {sd.get('message', 'unknown')}")
raise RuntimeError(msg)
return data
# --- Notifications ---
def list_notifications(client: AtClient) -> list[dict]:
response = es10x_command(client, encode_tlv(TAG_LIST_NOTIFICATION, b""))
root = require_tag(response, TAG_LIST_NOTIFICATION, "ListNotificationResponse")
metadata_list = find_tag(root, TAG_OK)
if metadata_list is None:
return []
return [decode_struct(value, NOTIFICATION) for tag, value in iter_tlv(metadata_list) if tag == TAG_NOTIFICATION_METADATA]
def process_notifications(client: AtClient) -> None:
for notification in list_notifications(client):
seq_number, smdp_address = notification["seqNumber"], notification["notificationAddress"]
try:
request = encode_tlv(TAG_RETRIEVE_NOTIFICATION, encode_tlv(TAG_OK, encode_tlv(TAG_STATUS, int_bytes(seq_number))))
response = es10x_command(client, request)
content = require_tag(require_tag(response, TAG_RETRIEVE_NOTIFICATION, "RetrieveNotificationsListResponse"),
TAG_OK, "RetrieveNotificationsListResponse")
pending_notif = next((content[start:end] for t, _, start, end in iter_tlv(content, with_positions=True) if t in (TAG_PROFILE_INSTALL_RESULT, 0x30)), None)
if pending_notif is None:
raise RuntimeError("Missing PendingNotification")
es9p_request(smdp_address, "handleNotification", {"pendingNotification": b64e(pending_notif)}, "HandleNotification")
response = es10x_command(client, encode_tlv(TAG_NOTIFICATION_SENT, encode_tlv(TAG_STATUS, int_bytes(seq_number))))
root = require_tag(response, TAG_NOTIFICATION_SENT, "NotificationSentResponse")
if int.from_bytes(require_tag(root, TAG_STATUS, "RemoveNotificationFromList status"), "big") != 0:
raise RuntimeError("RemoveNotificationFromList failed")
except Exception as e:
print(f"notification {seq_number} failed: {e}", file=sys.stderr)
# --- Authentication & Download ---
def get_challenge_and_info(client: AtClient) -> tuple[bytes, bytes]:
challenge_resp = es10x_command(client, encode_tlv(TAG_EUICC_CHALLENGE, b""))
challenge = require_tag(require_tag(challenge_resp, TAG_EUICC_CHALLENGE, "GetEuiccDataResponse"),
TAG_STATUS, "challenge in response")
info_resp = es10x_command(client, encode_tlv(TAG_EUICC_INFO, b""))
require_tag(info_resp, TAG_EUICC_INFO, "GetEuiccInfo1Response")
return challenge, info_resp
def authenticate_server(client: AtClient, b64_signed1: str, b64_sig1: str, b64_pk_id: str, b64_cert: str, matching_id: str) -> str:
tac = bytes([0x35, 0x29, 0x06, 0x11])
device_info = encode_tlv(TAG_STATUS, tac) + encode_tlv(0xA1, b"")
ctx_inner = encode_tlv(TAG_STATUS, matching_id.encode("utf-8")) + encode_tlv(0xA1, device_info)
content = b64d(b64_signed1) + b64d(b64_sig1) + b64d(b64_pk_id) + b64d(b64_cert) + encode_tlv(0xA0, ctx_inner)
response = es10x_command(client, encode_tlv(TAG_AUTH_SERVER, content))
root = require_tag(response, TAG_AUTH_SERVER, "AuthenticateServerResponse")
error_tag = find_tag(root, 0xA1)
if error_tag is not None:
code = int.from_bytes(error_tag, "big") if error_tag else 0
raise RuntimeError(f"AuthenticateServer rejected by eUICC: {AUTH_SERVER_ERROR_CODES.get(code, 'unknown')} (0x{code:02X})")
return b64e(response)
def prepare_download(client: AtClient, b64_signed2: str, b64_sig2: str, b64_cert: str, cc: str | None = None) -> str:
smdp_signed2 = b64d(b64_signed2)
smdp_signature2 = b64d(b64_sig2)
smdp_certificate = b64d(b64_cert)
smdp_signed2_root = find_tag(smdp_signed2, 0x30)
if smdp_signed2_root is None:
raise RuntimeError("Invalid smdpSigned2")
transaction_id = find_tag(smdp_signed2_root, TAG_STATUS)
cc_required_flag = find_tag(smdp_signed2_root, 0x01)
if transaction_id is None or cc_required_flag is None:
raise RuntimeError("Invalid smdpSigned2")
content = smdp_signed2 + smdp_signature2
if int.from_bytes(cc_required_flag, "big") != 0:
if not cc:
raise RuntimeError("Confirmation code required but not provided")
content += encode_tlv(0x04, hashlib.sha256(hashlib.sha256(cc.encode("utf-8")).digest() + transaction_id).digest())
content += smdp_certificate
response = es10x_command(client, encode_tlv(TAG_PREPARE_DOWNLOAD, content))
require_tag(response, TAG_PREPARE_DOWNLOAD, "PrepareDownloadResponse")
return b64e(response)
def _parse_tlv_header_len(data: bytes) -> int:
tag_len = 2 if data[0] & 0x1F == 0x1F else 1
length_byte = data[tag_len]
return tag_len + (1 + (length_byte & 0x7F) if length_byte & 0x80 else 1)
def _split_bpp(bpp: bytes) -> list[bytes]:
"""Split a BoundProfilePackage into APDU chunks per SGP.22 §5.7.6."""
root_value = None
for tag, value, start, end in iter_tlv(bpp, with_positions=True):
if tag == TAG_BPP:
root_value = value
val_start = start + _parse_tlv_header_len(bpp[start:end])
break
if root_value is None:
raise RuntimeError("Invalid BoundProfilePackage")
chunks: list[bytes] = []
for tag, value, start, end in iter_tlv(root_value, with_positions=True):
if tag == TAG_BPP_COMMAND:
chunks.append(bpp[0 : val_start + end])
elif tag in (0xA0, 0xA2):
chunks.append(bpp[val_start + start : val_start + end])
elif tag in (0xA1, 0xA3):
hdr_len = _parse_tlv_header_len(root_value[start:end])
chunks.append(bpp[val_start + start : val_start + start + hdr_len])
for _, _, cs, ce in iter_tlv(value, with_positions=True):
chunks.append(value[cs:ce])
return chunks
def _parse_install_result(response: bytes) -> dict[str, Any] | None:
"""Parse a ProfileInstallResult from an APDU response, or None if not present."""
root = find_tag(response, TAG_PROFILE_INSTALL_RESULT)
if not root:
return None
result_data = find_tag(root, TAG_INSTALL_RESULT_DATA)
if not result_data:
return None
result: dict[str, Any] = {"seqNumber": 0, "success": False, "bppCommandId": None, "errorReason": None}
notif_meta = find_tag(result_data, TAG_NOTIFICATION_METADATA)
if notif_meta:
seq_num = find_tag(notif_meta, TAG_STATUS)
if seq_num:
result["seqNumber"] = int.from_bytes(seq_num, "big")
final_result = find_tag(result_data, 0xA2)
if final_result:
for tag, value in iter_tlv(final_result):
if tag == 0xA0:
result["success"] = True
elif tag == 0xA1:
bpp_cmd = find_tag(value, TAG_STATUS)
if bpp_cmd:
result["bppCommandId"] = int.from_bytes(bpp_cmd, "big")
err = find_tag(value, 0x81)
if err:
result["errorReason"] = int.from_bytes(err, "big")
return result
def load_bpp(client: AtClient, b64_bpp: str) -> dict:
bpp = b64d(b64_bpp)
result = None
for chunk in _split_bpp(bpp):
response = es10x_command(client, chunk)
if response and (parsed := _parse_install_result(response)):
result = parsed
break
if result is None:
raise RuntimeError("Profile installation failed: no result from eUICC")
if not result["success"] and result["errorReason"] is not None:
msg = BPP_ERROR_MESSAGES.get(result["errorReason"])
if not msg:
cmd_name = BPP_COMMAND_NAMES.get(result["bppCommandId"], f"unknown({result['bppCommandId']})")
err_name = BPP_ERROR_REASONS.get(result["errorReason"], f"unknown({result['errorReason']})")
msg = f"Profile installation failed at {cmd_name}: {err_name}"
raise RuntimeError(msg)
if not result["success"]:
raise RuntimeError("Profile installation failed: no result from eUICC")
return result
def parse_metadata(b64_metadata: str) -> dict:
root = find_tag(b64d(b64_metadata), TAG_PROFILE_METADATA)
if root is None:
raise RuntimeError("Invalid profileMetadata")
return decode_struct(root, PROFILE)
def cancel_session(client: AtClient, transaction_id: bytes, reason: int = 127) -> str:
content = encode_tlv(0x80, transaction_id) + encode_tlv(0x81, bytes([reason]))
response = es10x_command(client, encode_tlv(TAG_CANCEL_SESSION, content))
return b64e(response)
def parse_lpa_activation_code(activation_code: str) -> tuple[str, str]:
"""Parse 'LPA:1$smdp.example.com$MATCHING-ID' into (smdp_address, matching_id)."""
if not activation_code.startswith("LPA:"):
raise ValueError("Invalid activation code format")
parts = activation_code[4:].split("$")
if len(parts) != 3:
raise ValueError("Invalid activation code format")
return parts[1], parts[2]
def _b64_field(data: dict, key: str) -> str:
return base64_trim(data[key])
def _cancel_session_safe(client: AtClient, smdp: str, tx_id: str, session: requests.Session) -> None:
b64_cancel = ""
try:
b64_cancel = cancel_session(client, b64d(tx_id))
except Exception:
pass
try:
es9p_request(smdp, "cancelSession", {"transactionId": tx_id, "cancelSessionResponse": b64_cancel}, "CancelSession", session=session)
except Exception:
pass
def download_profile(client: AtClient, activation_code: str) -> str:
"""Download and install an eSIM profile. Returns the ICCID of the installed profile."""
if not system_time_valid():
raise RuntimeError("System time is not set; TLS certificate validation requires a valid clock")
smdp, matching_id = parse_lpa_activation_code(activation_code)
challenge, euicc_info = get_challenge_and_info(client)
session = requests.Session()
tx_id = None
try:
# step 1: initiate authentication
auth = es9p_request(smdp, "initiateAuthentication", {
"smdpAddress": smdp, "euiccChallenge": b64e(challenge),
"euiccInfo1": b64e(euicc_info), "matchingId": matching_id,
}, "Authentication", session=session)
tx_id = _b64_field(auth, "transactionId")
# step 2: authenticate server
b64_auth = authenticate_server(client,
_b64_field(auth, "serverSigned1"), _b64_field(auth, "serverSignature1"),
_b64_field(auth, "euiccCiPKIdToBeUsed"), _b64_field(auth, "serverCertificate"),
matching_id)
# step 3: authenticate client + get metadata
cli = es9p_request(smdp, "authenticateClient", {
"transactionId": tx_id, "authenticateServerResponse": b64_auth,
}, "Authentication", session=session)
iccid = parse_metadata(_b64_field(cli, "profileMetadata"))["iccid"]
# step 4: prepare download
b64_prep = prepare_download(client,
_b64_field(cli, "smdpSigned2"), _b64_field(cli, "smdpSignature2"),
_b64_field(cli, "smdpCertificate"))
# step 5: get and install bound profile package
bpp = es9p_request(smdp, "getBoundProfilePackage", {
"transactionId": tx_id, "prepareDownloadResponse": b64_prep,
}, "GetBoundProfilePackage", session=session)
load_bpp(client, _b64_field(bpp, "boundProfilePackage"))
return iccid
except Exception:
if tx_id:
_cancel_session_safe(client, smdp, tx_id, session)
raise
finally:
session.close()
class LPA(LPABase):
def __init__(self):
if hasattr(self, '_client'):
return
self._client = AtClient(DEFAULT_DEVICE, DEFAULT_BAUD, DEFAULT_TIMEOUT)
atexit.register(self._client.close)
@contextmanager
def _acquire_lock(self):
fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR)
try:
fcntl.flock(fd, fcntl.LOCK_EX)
yield
finally:
fcntl.flock(fd, fcntl.LOCK_UN)
os.close(fd)
@contextmanager
def _acquire_channel(self):
with self._acquire_lock():
try:
self._client.open_isdr()
yield
finally:
if self._client.channel:
try:
self._client.query(f"AT+CCHC={self._client.channel}")
except (RuntimeError, TimeoutError):
pass
self._client.channel = None
def list_profiles(self) -> list[Profile]:
with self._acquire_channel():
return [
Profile(
iccid=p.get("iccid", ""),
nickname=p.get("profileNickname") or "",
enabled=p.get("profileState") == "enabled",
provider=p.get("serviceProviderName") or "",
)
for p in list_profiles(self._client)
]
def get_active_profile(self) -> Profile | None:
return None
def process_notifications(self) -> None:
if not system_time_valid():
raise RuntimeError("System time is not set; TLS certificate validation requires a valid clock")
with self._acquire_channel():
process_notifications(self._client)
def delete_profile(self, iccid: str) -> None:
profile = next((p for p in self.list_profiles() if p.iccid == iccid), None)
if profile is None:
raise LPAProfileNotFoundError(f"profile not found: {iccid}")
if profile.is_comma:
raise LPAError("refusing to delete a comma profile")
with self._acquire_channel():
request = encode_tlv(TAG_DELETE_PROFILE, encode_tlv(TAG_ICCID, string_to_tbcd(iccid)))
response = es10x_command(self._client, request)
code = require_tag(require_tag(response, TAG_DELETE_PROFILE, "DeleteProfileResponse"), TAG_STATUS, "DeleteProfile status")[0]
if code != PROFILE_OK:
raise LPAError(f"DeleteProfile failed: {PROFILE_ERROR_CODES.get(code, 'unknown')} (0x{code:02X})")
def download_profile(self, qr: str, nickname: str | None = None) -> None:
with self._acquire_channel():
iccid = download_profile(self._client, qr)
if nickname and iccid:
set_profile_nickname(self._client, iccid, nickname)
def nickname_profile(self, iccid: str, nickname: str) -> None:
with self._acquire_channel():
set_profile_nickname(self._client, iccid, nickname)
def _enable_profile(self, iccid: str) -> int:
inner = encode_tlv(TAG_OK, encode_tlv(TAG_ICCID, string_to_tbcd(iccid)))
inner += b'\x01\x01\x01' # refreshFlag=1
response = es10x_command(self._client, encode_tlv(TAG_ENABLE_PROFILE, inner))
return require_tag(require_tag(response, TAG_ENABLE_PROFILE, "EnableProfileResponse"), TAG_STATUS, "EnableProfile status")[0]
def switch_profile(self, iccid: str) -> None:
with self._acquire_channel():
code = self._enable_profile(iccid)
if code == PROFILE_CAT_BUSY: # stale eUICC transaction, reset and retry
self._client._reset_modem()
self._client.open_isdr()
code = self._enable_profile(iccid)
if code not in (PROFILE_OK, PROFILE_NOT_IN_DISABLED_STATE):
raise LPAError(f"EnableProfile failed: {PROFILE_ERROR_CODES.get(code, 'unknown')} (0x{code:02X})")
def is_euicc(self) -> bool:
# +CCHO:<n> -> ISD-R applet present, eUICC. Any error -> non-eUICC.
with self._acquire_lock():
try:
lines = self._client.query(f'AT+CCHO="{ISDR_AID}"')
except RuntimeError:
return False
for line in lines:
if line.startswith("+CCHO:") and (ch := line.split(":", 1)[1].strip()):
try:
self._client.query(f"AT+CCHC={ch}")
except (RuntimeError, TimeoutError):
pass
self._client.channel = None
return True
return False
openpilot/common/file_chunker.py
Executable
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env python3
import io
import sys
import math
import os
from pathlib import Path
CHUNK_SIZE = 45 * 1024 * 1024 # 45MB, under GitHub's 50MB limit
def get_chunk_name(name, idx, num_chunks):
return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
def get_manifest_path(name):
return f"{name}.chunkmanifest"
def _chunk_paths(path, num_chunks):
return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
def get_chunk_targets(path, file_size):
num_chunks = math.ceil(file_size / CHUNK_SIZE)
return _chunk_paths(path, num_chunks)
def chunk_file(path, targets):
manifest_path, *chunk_paths = targets
actual_num_chunks = max(1, math.ceil(os.path.getsize(path) / CHUNK_SIZE))
assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
with open(path, 'rb') as f:
for chunk_path in chunk_paths:
with open(chunk_path, 'wb') as out:
out.write(f.read(CHUNK_SIZE))
Path(manifest_path).write_text(str(len(chunk_paths)))
os.remove(path)
def get_existing_chunks(path):
if os.path.isfile(path):
return [path]
if os.path.isfile(manifest := get_manifest_path(path)):
num_chunks = int(Path(manifest).read_text().strip())
return _chunk_paths(path, num_chunks)
raise FileNotFoundError(path)
class ChunkStream(io.RawIOBase):
def __init__(self, paths):
self._paths = iter(paths)
self._f = None
def readable(self):
return True
def readinto(self, b):
n = 0
view = memoryview(b)
while n < len(b):
if self._f is None:
p = next(self._paths, None)
if p is None:
break
self._f = open(p, 'rb')
count = self._f.readinto(view[n:])
if not count:
self._f.close()
self._f = None
continue
n += count
return n
def open_file_chunked(path):
manifest_path = get_manifest_path(path)
if os.path.isfile(manifest_path):
num_chunks = int(Path(manifest_path).read_text().strip())
paths = [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
elif os.path.isfile(path):
paths = [path]
else:
raise FileNotFoundError(path)
return io.BufferedReader(ChunkStream(paths))
if __name__ == "__main__":
path = sys.argv[1]
chunk_paths = get_chunk_targets(path, os.path.getsize(path))
chunk_file(path, chunk_paths)
openpilot/common/filter_simple.py
+34
View File
@@ -0,0 +1,34 @@
class FirstOrderFilter:
def __init__(self, x0, rc, dt, initialized=True):
self.x = x0
self.dt = dt
self.update_alpha(rc)
self.initialized = initialized
def update_alpha(self, rc):
self.alpha = self.dt / (rc + self.dt)
def update(self, x):
if self.initialized:
self.x = (1. - self.alpha) * self.x + self.alpha * x
else:
self.initialized = True
self.x = x
return self.x
class BounceFilter(FirstOrderFilter):
def __init__(self, x0, rc, dt, initialized=True, bounce=2):
self.velocity = FirstOrderFilter(0.0, 0.15, dt)
self.bounce = bounce
super().__init__(x0, rc, dt, initialized)
def update(self, x):
super().update(x)
scale = self.dt / (1.0 / 60.0) # tuned at 60 fps
self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
self.velocity.update(0.0)
if abs(self.velocity.x) < 1e-3:
self.velocity.x = 0.0
self.x += self.velocity.x
return self.x
openpilot/common/fuzzy.py
+240
View File
@@ -0,0 +1,240 @@
import math
import os
import random
import secrets
import struct
from collections.abc import Callable, Sequence
from functools import wraps
from typing import Any, TypeVar
import capnp
T = TypeVar("T")
_EDGE_SLOTS = 16
_MINIMAL_EXAMPLES = 10
_INTEGER_RANGES = {
"int8": (-2**7, 2**7 - 1),
"int16": (-2**15, 2**15 - 1),
"int32": (-2**31, 2**31 - 1),
"int64": (-2**63, 2**63 - 1),
"uint8": (0, 2**8 - 1),
"uint16": (0, 2**16 - 1),
"uint32": (0, 2**32 - 1),
"uint64": (0, 2**64 - 1),
}
# One seed is shared by the whole test process. Individual tests derive their seed
# from their unittest ID, so FUZZ_SEED is reproducible under the parallel runner too.
FUZZ_SEED = int(os.environ.get("FUZZ_SEED", secrets.randbits(64)))
class Fuzzy:
"""Fast, deterministic data generator with systematic boundary coverage."""
def __init__(self, seed: int | str, example_index: int):
self.example_index = example_index
self._random = random.Random(seed)
self._draw_index = 0
def _draw(self, edges: Sequence[T], random_value: Callable[[], T]) -> T:
draw_index = self._draw_index
self._draw_index += 1
# Preserve the cheap minimal prefix Hypothesis produced, then interleave
# systematic boundaries and random values at every draw site.
if self.example_index < _MINIMAL_EXAMPLES:
return edges[0]
search_example = self.example_index - _MINIMAL_EXAMPLES
if search_example < _EDGE_SLOTS * 2 and search_example % 2 == 0:
return edges[(search_example // 2 + draw_index) % len(edges)]
if self._random.randrange(4) == 0:
return self._random.choice(edges)
return random_value()
def boolean(self) -> bool:
return self._draw((False, True), lambda: bool(self._random.getrandbits(1)))
def choice(self, values: Sequence[T]) -> T:
if not values:
raise ValueError("cannot choose from an empty sequence")
return self._draw(values, lambda: self._random.choice(values))
def integer(self, min_value: int, max_value: int) -> int:
if min_value > max_value:
raise ValueError(f"{min_value=} must not exceed {max_value=}")
edges = [
0, 1, -1, min_value, max_value,
min_value + 1, max_value - 1,
]
edges.extend(1 << bit for bit in range(max_value.bit_length()))
edges.extend(-(1 << bit) for bit in range((-min_value).bit_length()))
valid_edges = tuple(dict.fromkeys(v for v in edges if min_value <= v <= max_value))
return self._draw(valid_edges, lambda: self._random.randint(min_value, max_value))
def floating(self, width: int = 64, *, allow_nan: bool = True, allow_infinity: bool = True) -> float:
if width not in (32, 64):
raise ValueError("float width must be 32 or 64")
if width == 32:
unpack_format = "!f"
finite_edges = (
0.0, -0.0, 1.0, -1.0,
struct.unpack(unpack_format, b"\x00\x00\x00\x01")[0],
struct.unpack(unpack_format, b"\x80\x00\x00\x01")[0],
struct.unpack(unpack_format, b"\x7f\x7f\xff\xff")[0],
struct.unpack(unpack_format, b"\xff\x7f\xff\xff")[0],
struct.unpack(unpack_format, b"\x00\x80\x00\x00")[0],
struct.unpack(unpack_format, b"\x80\x80\x00\x00")[0],
)
else:
unpack_format = "!d"
finite_edges = (
0.0, -0.0, 1.0, -1.0,
math.ulp(0.0), -math.ulp(0.0),
float.fromhex("0x1.fffffffffffffp+1023"), -float.fromhex("0x1.fffffffffffffp+1023"),
float.fromhex("0x1p-1022"), -float.fromhex("0x1p-1022"),
)
edges = list(finite_edges)
if allow_infinity:
edges.extend((math.inf, -math.inf))
if allow_nan:
edges.append(math.nan)
def random_float() -> float:
while True:
value = struct.unpack(unpack_format, self._random.randbytes(width // 8))[0]
if (allow_nan or not math.isnan(value)) and (allow_infinity or not math.isinf(value)):
return value
return self._draw(tuple(edges), random_float)
def _length(self, min_length: int, max_length: int | None) -> int:
if min_length < 0:
raise ValueError("minimum length must be non-negative")
if max_length is not None and min_length > max_length:
raise ValueError(f"{min_length=} must not exceed {max_length=}")
if max_length == min_length:
return min_length
offsets = (0, 1, 2, 4, 8, 16, 32)
edges = tuple(min_length + offset for offset in offsets if max_length is None or min_length + offset <= max_length)
def random_length() -> int:
# A geometric tail keeps ordinary examples small without placing an
# artificial ceiling on an unbounded list.
length = min_length
while max_length is None or length < max_length:
if self._random.randrange(8) == 0:
break
length += 1
return length
return self._draw(edges, random_length)
def binary(self, min_size: int = 0, max_size: int | None = None) -> bytes:
size = self._length(min_size, max_size)
patterns = (
bytes(size),
b"\xff" * size,
(b"\xaa\x55" * ((size + 1) // 2))[:size],
bytes(i & 0xff for i in range(size)),
)
return self._draw(patterns, lambda: self._random.randbytes(size))
def text(self, min_size: int = 0, max_size: int | None = None) -> str:
size = self._length(min_size, max_size)
def scalar() -> str:
value = self._random.randrange(0x110000 - 0x800)
if value >= 0xd800:
value += 0x800
return chr(value)
patterns = (
"",
"a" * size,
"\0" * size,
"\U0010ffff" * size,
)
valid_patterns = tuple(value for value in patterns if len(value) == size)
return self._draw(valid_patterns, lambda: "".join(scalar() for _ in range(size)))
def list(self, generate: Callable[[], T], min_size: int = 0, max_size: int | None = None) -> list[T]:
return [generate() for _ in range(self._length(min_size, max_size))]
def fuzzy_test(max_examples: int) -> Callable[[Callable[..., None]], Callable[..., None]]:
"""Run a unittest method repeatedly with independent, reproducible fuzzy data."""
max_examples = int(os.environ.get("MAX_EXAMPLES", max_examples))
assert max_examples >= 1
def decorator(fn: Callable[..., None]) -> Callable[..., None]:
@wraps(fn)
def wrapper(*args: Any, **kwargs: Any) -> None:
test_seed = f"{FUZZ_SEED}:{args[0].id()}"
selected_example = os.environ.get("FUZZ_EXAMPLE")
examples = [int(selected_example, 0)] if selected_example is not None else range(max_examples)
for example_index in examples:
if not 0 <= example_index < max_examples:
raise ValueError(f"FUZZ_EXAMPLE={example_index} is outside [0, {max_examples})")
try:
fn(*args, **kwargs, fuzzy=Fuzzy(f"{test_seed}:{example_index}", example_index))
except Exception as exc:
exc.add_note(f"reproduce with FUZZ_SEED={FUZZ_SEED} FUZZ_EXAMPLE={example_index}")
raise
return wrapper
return decorator
def capnp_random_dict(fuzzy: Fuzzy, schema: Any, event: str | None = None, *, real_floats: bool = False) -> dict[str, Any]:
"""Generate a dictionary accepted by a pycapnp struct constructor."""
def native(type_name: str) -> bool | int | float | str | bytes:
if type_name == "bool":
return fuzzy.boolean()
if type_name in _INTEGER_RANGES:
return fuzzy.integer(*_INTEGER_RANGES[type_name])
if type_name in ("float32", "float64"):
return fuzzy.floating(width=int(type_name[-2:]), allow_nan=not real_floats, allow_infinity=not real_floats)
if type_name == "text":
return fuzzy.text(max_size=1000)
if type_name == "anyPointer":
return fuzzy.text()
if type_name == "data":
return fuzzy.binary(max_size=1000)
raise NotImplementedError(f"invalid Cap'n Proto type: {type_name}")
def generate_field(field: Any) -> Any:
def rec(field_type: Any, base_type: str) -> Any:
type_name = field_type.which()
if type_name == "struct":
struct_schema = field.schema.elementType if base_type == "list" else field.schema
return capnp_random_dict(fuzzy, struct_schema, real_floats=real_floats)
if type_name == "list":
return fuzzy.list(lambda: rec(field_type.list.elementType, "list"))
if type_name == "enum":
enum_schema = field.schema.elementType if base_type == "list" else field.schema
return fuzzy.choice(tuple(enum_schema.enumerants))
return native(type_name)
try:
if hasattr(field.proto, "slot"):
slot_type = field.proto.slot.type
return rec(slot_type, slot_type.which())
return capnp_random_dict(fuzzy, field.schema, real_floats=real_floats)
except capnp.lib.capnp.KjException:
return capnp_random_dict(fuzzy, field.schema, real_floats=real_floats)
union_field = event or (fuzzy.choice(tuple(schema.union_fields)) if schema.union_fields else None)
fields = schema.non_union_fields + ((union_field,) if union_field else ())
return {
field_name: generate_field(schema.fields[field_name])
for field_name in fields
if not field_name.endswith("DEPRECATED") and field_name != "deprecated"
}
openpilot/common/git.py
+33
View File
@@ -0,0 +1,33 @@
from functools import cache
import subprocess
from openpilot.common.utils import run_cmd, run_cmd_default
@cache
def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
@cache
def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
@cache
def get_short_branch(cwd: str | None = None) -> str:
return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
@cache
def get_branch(cwd: str | None = None) -> str:
return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
@cache
def get_origin(cwd: str | None = None) -> str:
try:
local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd)
except subprocess.CalledProcessError: # Not on a branch, fallback
return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd)
openpilot/common/gpio.py
+89
View File
@@ -0,0 +1,89 @@
import os
import fcntl
import ctypes
from functools import cache
def gpio_init(pin: int, output: bool) -> None:
try:
with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
f.write(b"out" if output else b"in")
except Exception as e:
print(f"Failed to set gpio {pin} direction: {e}")
def gpio_set(pin: int, high: bool) -> None:
try:
with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
f.write(b"1" if high else b"0")
except Exception as e:
print(f"Failed to set gpio {pin} value: {e}")
def gpio_read(pin: int) -> bool | None:
val = None
try:
with open(f"/sys/class/gpio/gpio{pin}/value", 'rb') as f:
val = bool(int(f.read().strip()))
except Exception as e:
print(f"Failed to set gpio {pin} value: {e}")
return val
def gpio_export(pin: int) -> None:
if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
return
try:
with open("/sys/class/gpio/export", 'w') as f:
f.write(str(pin))
except Exception:
print(f"Failed to export gpio {pin}")
@cache
def get_irq_action(irq: int) -> list[str]:
try:
with open(f"/sys/kernel/irq/{irq}/actions") as f:
actions = f.read().strip().split(',')
return actions
except FileNotFoundError:
return []
def get_irqs_for_action(action: str) -> list[str]:
ret = []
with open("/proc/interrupts") as f:
for l in f.readlines():
irq = l.split(':')[0].strip()
if irq.isdigit() and action in get_irq_action(irq):
ret.append(irq)
return ret
# *** gpiochip ***
class gpioevent_data(ctypes.Structure):
_fields_ = [
("timestamp", ctypes.c_uint64),
("id", ctypes.c_uint32),
]
class gpioevent_request(ctypes.Structure):
_fields_ = [
("lineoffset", ctypes.c_uint32),
("handleflags", ctypes.c_uint32),
("eventflags", ctypes.c_uint32),
("label", ctypes.c_char * 32),
("fd", ctypes.c_int)
]
def gpiochip_get_ro_value_fd(label: str, gpiochip_id: int, pin: int) -> int:
GPIOEVENT_REQUEST_BOTH_EDGES = 0x3
GPIOHANDLE_REQUEST_INPUT = 0x1
GPIO_GET_LINEEVENT_IOCTL = 0xc030b404
rq = gpioevent_request()
rq.lineoffset = pin
rq.handleflags = GPIOHANDLE_REQUEST_INPUT
rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES
rq.label = label.encode('utf-8')[:31] + b'\0'
fd = os.open(f"/dev/gpiochip{gpiochip_id}", os.O_RDONLY)
fcntl.ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, rq)
os.close(fd)
return int(rq.fd)
openpilot/common/gps.py
+8
View File
@@ -0,0 +1,8 @@
from openpilot.common.params import Params
def get_gps_location_service(params: Params) -> str:
if params.get_bool("UbloxAvailable"):
return "gpsLocationExternal"
else:
return "gpsLocation"
openpilot/common/hardware/__init__.py
+16
View File
@@ -0,0 +1,16 @@
import os
from typing import cast
from openpilot.common.hardware.base import HardwareBase
from openpilot.common.hardware.comma.hardware import HardwareComma
from openpilot.common.hardware.pc.hardware import HardwarePc
AGNOS = os.path.isfile('/AGNOS')
COMMA_HARDWARE = AGNOS
PC = not COMMA_HARDWARE
if COMMA_HARDWARE:
HARDWARE = cast(HardwareBase, HardwareComma())
else:
HARDWARE = cast(HardwareBase, HardwarePc())
openpilot/common/hardware/base.h
+25
View File
@@ -0,0 +1,25 @@
#pragma once
#include <cstdlib>
#include <fstream>
#include <map>
#include <string>
#include "openpilot/cereal/gen/cpp/log.capnp.h"
// no-op base hw class
class HardwareNone {
public:
static std::string get_name() { return ""; }
static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::UNKNOWN; }
static std::string get_serial() { return "cccccc"; }
static std::map<std::string, std::string> get_init_logs(bool route_log = false) {
return {};
}
static void set_ir_power(int percentage) {}
static bool PC() { return false; }
};
openpilot/common/hardware/base.py
+161
View File
@@ -0,0 +1,161 @@
import os
from abc import abstractmethod, ABC
from dataclasses import dataclass, fields
from openpilot.cereal import log
from openpilot.common.esim.base import LPABase
NetworkType = log.DeviceState.NetworkType
NetworkStrength = log.DeviceState.NetworkStrength
@dataclass
class ThermalZone:
# a zone from /sys/class/thermal/thermal_zone*
name: str # a.k.a type
scale: float = 1000. # scale to get degrees in C
zone_number = -1
def read(self) -> float:
if self.zone_number < 0:
for n in os.listdir("/sys/devices/virtual/thermal"):
if not n.startswith("thermal_zone"):
continue
with open(os.path.join("/sys/devices/virtual/thermal", n, "type")) as f:
if f.read().strip() == self.name:
self.zone_number = int(n.removeprefix("thermal_zone"))
break
try:
with open(f"/sys/devices/virtual/thermal/thermal_zone{self.zone_number}/temp") as f:
return int(f.read()) / self.scale
except FileNotFoundError:
return 0
@dataclass
class ThermalConfig:
cpu: list[ThermalZone] | None = None
gpu: list[ThermalZone] | None = None
dsp: ThermalZone | None = None
pmic: list[ThermalZone] | None = None
memory: ThermalZone | None = None
intake: ThermalZone | None = None
exhaust: ThermalZone | None = None
gnss: ThermalZone | None = None
bottomSoc: ThermalZone | None = None
def get_msg(self):
ret = {}
for f in fields(ThermalConfig):
v = getattr(self, f.name)
if v is not None:
if isinstance(v, list):
ret[f.name + "TempC"] = [x.read() for x in v]
else:
ret[f.name + "TempC"] = v.read()
return ret
class HardwareBase(ABC):
@staticmethod
def get_cmdline() -> dict[str, str]:
with open('/proc/cmdline') as f:
cmdline = f.read()
return {kv[0]: kv[1] for kv in [s.split('=') for s in cmdline.split(' ')] if len(kv) == 2}
@staticmethod
def read_param_file(path, parser, default=0):
try:
with open(path) as f:
return parser(f.read())
except Exception:
return default
def booted(self) -> bool:
return True
def reboot(self, reason=None):
print("REBOOT!")
def uninstall(self):
print("uninstall")
def get_os_version(self):
return None
@abstractmethod
def get_device_type(self):
pass
def get_imei(self) -> str:
return ""
def get_serial(self):
return ""
def get_network_info(self):
return None
def get_network_type(self):
return NetworkType.none
def get_sim_info(self):
return {
'sim_id': '',
'mcc_mnc': None,
'network_type': ["Unknown"],
'sim_state': ["ABSENT"],
'data_connected': False
}
def get_sim_lpa(self) -> LPABase:
raise NotImplementedError("SIM LPA not available")
def get_network_strength(self, network_type):
return NetworkStrength.unknown
def get_network_metered(self, network_type) -> bool:
return network_type not in (NetworkType.none, NetworkType.wifi, NetworkType.ethernet)
def get_current_power_draw(self):
return 0
def get_som_power_draw(self):
return 0
def shutdown(self):
print("SHUTDOWN!")
def get_thermal_config(self):
return ThermalConfig()
def set_display_power(self, on: bool):
pass
def set_screen_brightness(self, percentage):
pass
def get_screen_brightness(self):
return 0
def set_power_save(self, powersave_enabled):
pass
def get_gpu_usage_percent(self):
return 0
def get_modem_temperatures(self):
return []
def initialize_hardware(self):
pass
def reset_internal_panda(self):
pass
def recover_internal_panda(self):
pass
def get_modem_data_usage(self):
return -1, -1
def set_ir_power(self, percent: int):
pass
openpilot/common/hardware/comma/__init__.py
+1
View File
@@ -0,0 +1 @@
openpilot/common/hardware/comma/agnos.json
+84
View File
@@ -0,0 +1,84 @@
[
{
"name": "xbl",
"url": "https://commadist.azureedge.net/agnosupdate/xbl-e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb.img.xz",
"hash": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
"hash_raw": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
"size": 3282256,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "bea7f1a24428c3ededf672fa4fc78baf180cfbd8aafb77c974655b38517283e3"
},
{
"name": "xbl_config",
"url": "https://commadist.azureedge.net/agnosupdate/xbl_config-758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85.img.xz",
"hash": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
"hash_raw": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
"size": 98124,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "fb18cde08a98a168961ecd357e92474823046752b94e112f59fe51a6acd7197d"
},
{
"name": "abl",
"url": "https://commadist.azureedge.net/agnosupdate/abl-29fd7ed1c012e599420764840f9f11286d34dbff4adaf102a447f06d8c5e0b35.img.xz",
"hash": "29fd7ed1c012e599420764840f9f11286d34dbff4adaf102a447f06d8c5e0b35",
"hash_raw": "29fd7ed1c012e599420764840f9f11286d34dbff4adaf102a447f06d8c5e0b35",
"size": 274432,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "29fd7ed1c012e599420764840f9f11286d34dbff4adaf102a447f06d8c5e0b35"
},
{
"name": "aop",
"url": "https://commadist.azureedge.net/agnosupdate/aop-78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1.img.xz",
"hash": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
"hash_raw": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
"size": 184364,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "6c9135446bd3fc075fcee59b887a12e49029ab1f98ed8d6d1e32c73569d47de3"
},
{
"name": "devcfg",
"url": "https://commadist.azureedge.net/agnosupdate/devcfg-f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc.img.xz",
"hash": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
"hash_raw": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
"size": 40336,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "2a67971602012c1b43544964709da13c322786b456a8e78568b117e8b1540ce3"
},
{
"name": "boot",
"url": "https://commadist.azureedge.net/agnosupdate/boot-b30f5eef65ec3878f3aa3dcaf2cc95c09e2c1e661cd3a38e94da37dee76f68bd.img.xz",
"hash": "b30f5eef65ec3878f3aa3dcaf2cc95c09e2c1e661cd3a38e94da37dee76f68bd",
"hash_raw": "b30f5eef65ec3878f3aa3dcaf2cc95c09e2c1e661cd3a38e94da37dee76f68bd",
"size": 46897152,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "6650e4c46df99ae6dfd6ee895a34b8a2a3cc490a8ce18e16cc3c451c3f822b6e"
},
{
"name": "system",
"url": "https://commadist.azureedge.net/agnosupdate/system-5b6ce7965904a157fd3a134ccfcb854f9ca5c1cc2a26b7cb80a4fa4e1cc4aaa3.img.xz",
"hash": "b134fd04e9da27fa1d359ea0f2742c216fa21a08b5c47e9be22ab3b0563d9b9b",
"hash_raw": "5b6ce7965904a157fd3a134ccfcb854f9ca5c1cc2a26b7cb80a4fa4e1cc4aaa3",
"size": 4718592000,
"sparse": true,
"full_check": false,
"has_ab": true,
"ondevice_hash": "91242772af771ae96fe2eebc105f2b80a7e1dbaaf6003c2574b62d51b806f468",
"alt": {
"hash": "5b6ce7965904a157fd3a134ccfcb854f9ca5c1cc2a26b7cb80a4fa4e1cc4aaa3",
"url": "https://commadist.azureedge.net/agnosupdate/system-5b6ce7965904a157fd3a134ccfcb854f9ca5c1cc2a26b7cb80a4fa4e1cc4aaa3.img",
"size": 4718592000
}
}
]
openpilot/common/hardware/comma/agnos.py
Executable
+287
View File
@@ -0,0 +1,287 @@
#!/usr/bin/env python3
import hashlib
import json
import lzma
import os
import struct
import subprocess
import time
from collections.abc import Generator
import requests
SPARSE_CHUNK_FMT = struct.Struct('H2xI4x')
AGNOS_MANIFEST_FILE = "openpilot/system/hardware/comma/agnos.json"
class StreamingDecompressor:
def __init__(self, url: str) -> None:
self.buf = b""
self.req = requests.get(url, stream=True, headers={'Accept-Encoding': 'identity'}, timeout=60)
self.it = self.req.iter_content(chunk_size=1024 * 1024)
self.decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_AUTO)
self.eof = False
self.sha256 = hashlib.sha256()
def read(self, length: int) -> bytes:
while len(self.buf) < length and not self.eof:
if self.decompressor.needs_input:
self.req.raise_for_status()
try:
compressed = next(self.it)
except StopIteration:
self.eof = True
break
else:
compressed = b''
self.buf += self.decompressor.decompress(compressed, max_length=length)
if self.decompressor.eof:
self.eof = True
break
result = self.buf[:length]
self.buf = self.buf[length:]
self.sha256.update(result)
return result
def unsparsify(f: StreamingDecompressor) -> Generator[bytes, None, None]:
# https://source.android.com/devices/bootloader/images#sparse-format
magic = struct.unpack("I", f.read(4))[0]
assert(magic == 0xed26ff3a)
# Version
major = struct.unpack("H", f.read(2))[0]
minor = struct.unpack("H", f.read(2))[0]
assert(major == 1 and minor == 0)
f.read(2) # file header size
f.read(2) # chunk header size
block_sz = struct.unpack("I", f.read(4))[0]
f.read(4) # total blocks
num_chunks = struct.unpack("I", f.read(4))[0]
f.read(4) # crc checksum
for _ in range(num_chunks):
chunk_type, out_blocks = SPARSE_CHUNK_FMT.unpack(f.read(12))
if chunk_type == 0xcac1: # Raw
# TODO: yield in smaller chunks. Yielding only block_sz is too slow. Largest observed data chunk is 252 MB.
yield f.read(out_blocks * block_sz)
elif chunk_type == 0xcac2: # Fill
filler = f.read(4) * (block_sz // 4)
for _ in range(out_blocks):
yield filler
elif chunk_type == 0xcac3: # Don't care
yield b""
else:
raise Exception("Unhandled sparse chunk type")
# noop wrapper with same API as unsparsify() for non sparse images
def noop(f: StreamingDecompressor) -> Generator[bytes, None, None]:
while len(chunk := f.read(1024 * 1024)) > 0:
yield chunk
def get_target_slot_number() -> int:
current_slot = subprocess.check_output(["abctl", "--boot_slot"], encoding='utf-8').strip()
return 1 if current_slot == "_a" else 0
def slot_number_to_suffix(slot_number: int) -> str:
assert slot_number in (0, 1)
return '_a' if slot_number == 0 else '_b'
def get_partition_path(target_slot_number: int, partition: dict) -> str:
path = f"/dev/disk/by-partlabel/{partition['name']}"
if partition.get('has_ab', True):
path += slot_number_to_suffix(target_slot_number)
return path
def get_raw_hash(path: str, partition_size: int) -> str:
raw_hash = hashlib.sha256()
pos, chunk_size = 0, 1024 * 1024
with open(path, 'rb+') as out:
while pos < partition_size:
n = min(chunk_size, partition_size - pos)
raw_hash.update(out.read(n))
pos += n
return raw_hash.hexdigest().lower()
def verify_partition(target_slot_number: int, partition: dict[str, str | int], force_full_check: bool = False) -> bool:
full_check = partition['full_check'] or force_full_check
path = get_partition_path(target_slot_number, partition)
if not isinstance(partition['size'], int):
return False
partition_size: int = partition['size']
if not isinstance(partition['hash_raw'], str):
return False
partition_hash: str = partition['hash_raw']
if full_check:
return get_raw_hash(path, partition_size) == partition_hash.lower()
else:
with open(path, 'rb+') as out:
out.seek(partition_size)
return out.read(64) == partition_hash.lower().encode()
def clear_partition_hash(target_slot_number: int, partition: dict) -> None:
path = get_partition_path(target_slot_number, partition)
with open(path, 'wb+') as out:
partition_size = partition['size']
out.seek(partition_size)
out.write(b"\x00" * 64)
os.sync()
def extract_compressed_image(target_slot_number: int, partition: dict, cloudlog):
path = get_partition_path(target_slot_number, partition)
downloader = StreamingDecompressor(partition['url'])
with open(path, 'wb+') as out:
# Flash partition
last_p = 0
raw_hash = hashlib.sha256()
f = unsparsify if partition['sparse'] else noop
for chunk in f(downloader):
raw_hash.update(chunk)
out.write(chunk)
p = int(out.tell() / partition['size'] * 100)
if p != last_p:
last_p = p
print(f"Installing {partition['name']}: {p}", flush=True)
if raw_hash.hexdigest().lower() != partition['hash_raw'].lower():
raise Exception(f"Raw hash mismatch '{raw_hash.hexdigest().lower()}'")
if downloader.sha256.hexdigest().lower() != partition['hash'].lower():
raise Exception("Uncompressed hash mismatch")
if out.tell() != partition['size']:
raise Exception("Uncompressed size mismatch")
os.sync()
def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalone=False):
cloudlog.info(f"Downloading and writing {partition['name']}")
if verify_partition(target_slot_number, partition):
cloudlog.info(f"Already flashed {partition['name']}")
return
# Clear hash before flashing in case we get interrupted
full_check = partition['full_check']
if not full_check:
clear_partition_hash(target_slot_number, partition)
path = get_partition_path(target_slot_number, partition)
extract_compressed_image(target_slot_number, partition, cloudlog)
# Write hash after successful flash
if not full_check:
with open(path, 'wb+') as out:
out.seek(partition['size'])
out.write(partition['hash_raw'].lower().encode())
def swap(manifest_path: str, target_slot_number: int, cloudlog) -> None:
update = json.load(open(manifest_path))
for partition in update:
if not partition.get('full_check', False):
clear_partition_hash(target_slot_number, partition)
while True:
out = subprocess.check_output(f"abctl --set_active {target_slot_number}", shell=True, stderr=subprocess.STDOUT, encoding='utf8')
if ("No such file or directory" not in out) and ("lun as boot lun" in out):
cloudlog.info(f"Swap successful {out}")
break
else:
cloudlog.error(f"Swap failed {out}")
def flash_agnos_update(manifest_path: str, target_slot_number: int, cloudlog, standalone=False) -> None:
update = json.load(open(manifest_path))
cloudlog.info(f"Target slot {target_slot_number}")
# set target slot as unbootable
subprocess.run(f"abctl --set_unbootable {target_slot_number}", shell=True)
for partition in update:
success = False
for retries in range(10):
try:
flash_partition(target_slot_number, partition, cloudlog, standalone)
success = True
break
except requests.exceptions.RequestException:
cloudlog.exception("Failed")
cloudlog.info(f"Failed to download {partition['name']}, retrying ({retries})")
time.sleep(10)
if not success:
cloudlog.info(f"Failed to flash {partition['name']}, aborting")
raise Exception("Maximum retries exceeded")
cloudlog.info(f"AGNOS ready on slot {target_slot_number}")
def verify_agnos_update(manifest_path: str, target_slot_number: int) -> bool:
update = json.load(open(manifest_path))
return all(verify_partition(target_slot_number, partition) for partition in update)
if __name__ == "__main__":
import argparse
import logging
parser = argparse.ArgumentParser(description="Flash and verify AGNOS update",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--verify", action="store_true", help="Verify and perform swap if update ready")
parser.add_argument("--swap", action="store_true", help="Verify and perform swap, downloads if necessary")
parser.add_argument("manifest", help="Manifest json")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
target_slot_number = get_target_slot_number()
if args.verify:
if verify_agnos_update(args.manifest, target_slot_number):
swap(args.manifest, target_slot_number, logging)
exit(0)
exit(1)
elif args.swap:
while not verify_agnos_update(args.manifest, target_slot_number):
logging.error("Verification failed. Flashing AGNOS")
flash_agnos_update(args.manifest, target_slot_number, logging, standalone=True)
logging.warning(f"Verification succeeded. Swapping to slot {target_slot_number}")
swap(args.manifest, target_slot_number, logging)
else:
flash_agnos_update(args.manifest, target_slot_number, logging, standalone=True)
openpilot/common/hardware/comma/all-partitions.json
+389
View File
@@ -0,0 +1,389 @@
[
{
"name": "gpt_main_0",
"url": "https://commadist.azureedge.net/agnosupdate/gpt_main_0-8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd.img.xz",
"hash": "8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd",
"hash_raw": "8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd",
"size": 24576,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd",
"gpt": {
"lun": 0,
"start_sector": 0,
"num_sectors": 6
}
},
{
"name": "gpt_main_1",
"url": "https://commadist.azureedge.net/agnosupdate/gpt_main_1-fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6.img.xz",
"hash": "fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6",
"hash_raw": "fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6",
"size": 24576,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6",
"gpt": {
"lun": 1,
"start_sector": 0,
"num_sectors": 6
}
},
{
"name": "gpt_main_2",
"url": "https://commadist.azureedge.net/agnosupdate/gpt_main_2-5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21.img.xz",
"hash": "5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21",
"hash_raw": "5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21",
"size": 24576,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21",
"gpt": {
"lun": 2,
"start_sector": 0,
"num_sectors": 6
}
},
{
"name": "gpt_main_3",
"url": "https://commadist.azureedge.net/agnosupdate/gpt_main_3-c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159.img.xz",
"hash": "c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159",
"hash_raw": "c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159",
"size": 24576,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159",
"gpt": {
"lun": 3,
"start_sector": 0,
"num_sectors": 6
}
},
{
"name": "gpt_main_4",
"url": "https://commadist.azureedge.net/agnosupdate/gpt_main_4-e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e.img.xz",
"hash": "e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e",
"hash_raw": "e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e",
"size": 24576,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e",
"gpt": {
"lun": 4,
"start_sector": 0,
"num_sectors": 6
}
},
{
"name": "gpt_main_5",
"url": "https://commadist.azureedge.net/agnosupdate/gpt_main_5-21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3.img.xz",
"hash": "21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3",
"hash_raw": "21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3",
"size": 24576,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3",
"gpt": {
"lun": 5,
"start_sector": 0,
"num_sectors": 6
}
},
{
"name": "persist",
"url": "https://commadist.azureedge.net/agnosupdate/persist-d6af4ec18df180c7417353b52a9e05e43a6480b29425f087874136436cefe786.img.xz",
"hash": "d6af4ec18df180c7417353b52a9e05e43a6480b29425f087874136436cefe786",
"hash_raw": "d6af4ec18df180c7417353b52a9e05e43a6480b29425f087874136436cefe786",
"size": 4096,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "d6af4ec18df180c7417353b52a9e05e43a6480b29425f087874136436cefe786"
},
{
"name": "systemrw",
"url": "https://commadist.azureedge.net/agnosupdate/systemrw-8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e.img.xz",
"hash": "8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e",
"hash_raw": "8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e",
"size": 16777216,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e"
},
{
"name": "cache",
"url": "https://commadist.azureedge.net/agnosupdate/cache-ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4.img.xz",
"hash": "ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4",
"hash_raw": "ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4",
"size": 134217728,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4"
},
{
"name": "xbl",
"url": "https://commadist.azureedge.net/agnosupdate/xbl-e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb.img.xz",
"hash": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
"hash_raw": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
"size": 3282256,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "bea7f1a24428c3ededf672fa4fc78baf180cfbd8aafb77c974655b38517283e3"
},
{
"name": "xbl_config",
"url": "https://commadist.azureedge.net/agnosupdate/xbl_config-758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85.img.xz",
"hash": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
"hash_raw": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
"size": 98124,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "fb18cde08a98a168961ecd357e92474823046752b94e112f59fe51a6acd7197d"
},
{
"name": "abl",
"url": "https://commadist.azureedge.net/agnosupdate/abl-b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c.img.xz",
"hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
"hash_raw": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
"size": 274432,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c"
},
{
"name": "aop",
"url": "https://commadist.azureedge.net/agnosupdate/aop-78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1.img.xz",
"hash": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
"hash_raw": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
"size": 184364,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "6c9135446bd3fc075fcee59b887a12e49029ab1f98ed8d6d1e32c73569d47de3"
},
{
"name": "bluetooth",
"url": "https://commadist.azureedge.net/agnosupdate/bluetooth-9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533.img.xz",
"hash": "9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533",
"hash_raw": "9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533",
"size": 1048576,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533"
},
{
"name": "cmnlib64",
"url": "https://commadist.azureedge.net/agnosupdate/cmnlib64-1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3.img.xz",
"hash": "1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3",
"hash_raw": "1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3",
"size": 524288,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3"
},
{
"name": "cmnlib",
"url": "https://commadist.azureedge.net/agnosupdate/cmnlib-63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82.img.xz",
"hash": "63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82",
"hash_raw": "63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82",
"size": 524288,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82"
},
{
"name": "devcfg",
"url": "https://commadist.azureedge.net/agnosupdate/devcfg-f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc.img.xz",
"hash": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
"hash_raw": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
"size": 40336,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "2a67971602012c1b43544964709da13c322786b456a8e78568b117e8b1540ce3"
},
{
"name": "devinfo",
"url": "https://commadist.azureedge.net/agnosupdate/devinfo-143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3.img.xz",
"hash": "143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3",
"hash_raw": "143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3",
"size": 4096,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3"
},
{
"name": "dsp",
"url": "https://commadist.azureedge.net/agnosupdate/dsp-4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248.img.xz",
"hash": "4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248",
"hash_raw": "4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248",
"size": 33554432,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248"
},
{
"name": "hyp",
"url": "https://commadist.azureedge.net/agnosupdate/hyp-ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927.img.xz",
"hash": "ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927",
"hash_raw": "ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927",
"size": 524288,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927"
},
{
"name": "keymaster",
"url": "https://commadist.azureedge.net/agnosupdate/keymaster-5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04.img.xz",
"hash": "5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04",
"hash_raw": "5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04",
"size": 524288,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04"
},
{
"name": "limits",
"url": "https://commadist.azureedge.net/agnosupdate/limits-94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1.img.xz",
"hash": "94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1",
"hash_raw": "94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1",
"size": 4096,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1"
},
{
"name": "logfs",
"url": "https://commadist.azureedge.net/agnosupdate/logfs-b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220.img.xz",
"hash": "b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220",
"hash_raw": "b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220",
"size": 8388608,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220"
},
{
"name": "modem",
"url": "https://commadist.azureedge.net/agnosupdate/modem-a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994.img.xz",
"hash": "a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994",
"hash_raw": "a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994",
"size": 125829120,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994"
},
{
"name": "qupfw",
"url": "https://commadist.azureedge.net/agnosupdate/qupfw-64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a.img.xz",
"hash": "64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a",
"hash_raw": "64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a",
"size": 65536,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a"
},
{
"name": "splash",
"url": "https://commadist.azureedge.net/agnosupdate/splash-5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08.img.xz",
"hash": "5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08",
"hash_raw": "5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08",
"size": 34226176,
"sparse": false,
"full_check": true,
"has_ab": false,
"ondevice_hash": "5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08"
},
{
"name": "storsec",
"url": "https://commadist.azureedge.net/agnosupdate/storsec-4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce.img.xz",
"hash": "4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce",
"hash_raw": "4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce",
"size": 131072,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce"
},
{
"name": "tz",
"url": "https://commadist.azureedge.net/agnosupdate/tz-e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16.img.xz",
"hash": "e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16",
"hash_raw": "e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16",
"size": 2097152,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16"
},
{
"name": "boot",
"url": "https://commadist.azureedge.net/agnosupdate/boot-5463317ca7a231f2ddd55a667a8c2b0d6692982359773ca875a99a0e6fd67fb4.img.xz",
"hash": "5463317ca7a231f2ddd55a667a8c2b0d6692982359773ca875a99a0e6fd67fb4",
"hash_raw": "5463317ca7a231f2ddd55a667a8c2b0d6692982359773ca875a99a0e6fd67fb4",
"size": 46897152,
"sparse": false,
"full_check": true,
"has_ab": true,
"ondevice_hash": "daefa6f897bdcc277e50dbf562fd63e57d50d9fc7a599f2d171208a490e725d9"
},
{
"name": "system",
"url": "https://commadist.azureedge.net/agnosupdate/system-2e1ca22762e66898aae384fcc21f488cb015c020b6b588862a64fc8128f0ad1d.img.xz",
"hash": "14ab660bb955604ab6c482b3f59e5239e2c40c54e798201afcaed43d5cb0dcfd",
"hash_raw": "2e1ca22762e66898aae384fcc21f488cb015c020b6b588862a64fc8128f0ad1d",
"size": 4718592000,
"sparse": true,
"full_check": false,
"has_ab": true,
"ondevice_hash": "18666ca3b529304f5e7a8fc2e3c9dce715b55941e3965a7af43c613f8e18cfce",
"alt": {
"hash": "2e1ca22762e66898aae384fcc21f488cb015c020b6b588862a64fc8128f0ad1d",
"url": "https://commadist.azureedge.net/agnosupdate/system-2e1ca22762e66898aae384fcc21f488cb015c020b6b588862a64fc8128f0ad1d.img",
"size": 4718592000
}
},
{
"name": "userdata_90",
"url": "https://commadist.azureedge.net/agnosupdate/userdata_90-871adca25cb254ad1704ce427f3faaae1027f74aff43a3e150188c8a1c3d3cb8.img.xz",
"hash": "e4f59e5dcf4758759da9284479163e420febf1c118bf33c83b402e80e241608d",
"hash_raw": "871adca25cb254ad1704ce427f3faaae1027f74aff43a3e150188c8a1c3d3cb8",
"size": 96636764160,
"sparse": true,
"full_check": true,
"has_ab": false,
"ondevice_hash": "17b4ec7449e62c52be5df551542684b953e5c7ffc95103008043602f79ae7112"
},
{
"name": "userdata_89",
"url": "https://commadist.azureedge.net/agnosupdate/userdata_89-9d84bd64ef237a32675f936594c16932f8acbb52cf2dd82373a0e718cd32dfcf.img.xz",
"hash": "652e61ecb5b12250dbfeb9f12fd74af16b867b5ea9bf8a08b5731ffd63a76317",
"hash_raw": "9d84bd64ef237a32675f936594c16932f8acbb52cf2dd82373a0e718cd32dfcf",
"size": 95563022336,
"sparse": true,
"full_check": true,
"has_ab": false,
"ondevice_hash": "acad63821bb48823c871910b1a363ab924d2635a7574cc5045e877c438d3012f"
}
]
openpilot/common/hardware/comma/amplifier.py
Executable
+123
View File
@@ -0,0 +1,123 @@
#!/usr/bin/env python3
import time
from collections import namedtuple
from openpilot.common.i2c import SMBus
# https://datasheets.maximintegrated.com/en/ds/MAX98089.pdf
AmpConfig = namedtuple('AmpConfig', ['name', 'value', 'register', 'offset', 'mask'])
CONFIG = [
AmpConfig("MCLK prescaler", 0b01, 0x10, 4, 0b00110000),
AmpConfig("PM: enable speakers", 0b11, 0x4D, 4, 0b00110000),
AmpConfig("PM: enable DACs", 0b11, 0x4D, 0, 0b00000011),
AmpConfig("Enable PLL1", 0b1, 0x12, 7, 0b10000000),
AmpConfig("Enable PLL2", 0b1, 0x1A, 7, 0b10000000),
AmpConfig("DAI1: I2S mode", 0b00100, 0x14, 2, 0b01111100),
AmpConfig("DAI2: I2S mode", 0b00100, 0x1C, 2, 0b01111100),
AmpConfig("DAI1 Passband filtering: music mode", 0b1, 0x18, 7, 0b10000000),
AmpConfig("DAI1 voice mode gain (DV1G)", 0b00, 0x2F, 4, 0b00110000),
AmpConfig("DAI1 attenuation (DV1)", 0x0, 0x2F, 0, 0b00001111),
AmpConfig("DAI2 attenuation (DV2)", 0x0, 0x31, 0, 0b00001111),
AmpConfig("DAI2: DC blocking", 0b1, 0x20, 0, 0b00000001),
AmpConfig("DAI2: High sample rate", 0b0, 0x20, 3, 0b00001000),
AmpConfig("ALC enable", 0b1, 0x43, 7, 0b10000000),
AmpConfig("ALC/excursion limiter release time", 0b101, 0x43, 4, 0b01110000),
AmpConfig("ALC multiband enable", 0b1, 0x43, 3, 0b00001000),
AmpConfig("DAI1 EQ enable", 0b0, 0x49, 0, 0b00000001),
AmpConfig("DAI2 EQ clip detection disabled", 0b1, 0x32, 4, 0b00010000),
AmpConfig("DAI2 EQ attenuation", 0x5, 0x32, 0, 0b00001111),
AmpConfig("Excursion limiter upper corner freq", 0b100, 0x41, 4, 0b01110000),
AmpConfig("Excursion limiter lower corner freq", 0b00, 0x41, 0, 0b00000011),
AmpConfig("Excursion limiter threshold", 0b000, 0x42, 0, 0b00001111),
AmpConfig("Distortion limit (THDCLP)", 0x6, 0x46, 4, 0b11110000),
AmpConfig("Distortion limiter release time constant", 0b0, 0x46, 0, 0b00000001),
AmpConfig("Right DAC input mixer: DAI1 left", 0b0, 0x22, 3, 0b00001000),
AmpConfig("Right DAC input mixer: DAI1 right", 0b0, 0x22, 2, 0b00000100),
AmpConfig("Right DAC input mixer: DAI2 left", 0b1, 0x22, 1, 0b00000010),
AmpConfig("Right DAC input mixer: DAI2 right", 0b0, 0x22, 0, 0b00000001),
AmpConfig("DAI1 audio port selector", 0b10, 0x16, 6, 0b11000000),
AmpConfig("DAI2 audio port selector", 0b01, 0x1E, 6, 0b11000000),
AmpConfig("Enable left digital microphone", 0b1, 0x48, 5, 0b00100000),
AmpConfig("Enable right digital microphone", 0b1, 0x48, 4, 0b00010000),
AmpConfig("Enhanced volume smoothing disabled", 0b0, 0x49, 7, 0b10000000),
AmpConfig("Volume adjustment smoothing disabled", 0b0, 0x49, 6, 0b01000000),
AmpConfig("Zero-crossing detection disabled", 0b0, 0x49, 5, 0b00100000),
AmpConfig("Left speaker output from left DAC", 0b1, 0x2B, 0, 0b11111111),
AmpConfig("Right speaker output from right DAC", 0b1, 0x2C, 0, 0b11111111),
AmpConfig("Left Speaker Mixer Gain", 0b00, 0x2D, 0, 0b00000011),
AmpConfig("Right Speaker Mixer Gain", 0b00, 0x2D, 2, 0b00001100),
AmpConfig("Left speaker output volume", 0x17, 0x3D, 0, 0b00011111),
AmpConfig("Right speaker output volume", 0x17, 0x3E, 0, 0b00011111),
AmpConfig("DAI2 EQ enable", 0b0, 0x49, 1, 0b00000010),
AmpConfig("DAI2: DC blocking", 0b0, 0x20, 0, 0b00000001),
AmpConfig("ALC enable", 0b0, 0x43, 7, 0b10000000),
AmpConfig("DAI2 EQ attenuation", 0x2, 0x32, 0, 0b00001111),
AmpConfig("Excursion limiter upper corner freq", 0b001, 0x41, 4, 0b01110000),
AmpConfig("Excursion limiter threshold", 0b100, 0x42, 0, 0b00001111),
AmpConfig("Distortion limit (THDCLP)", 0x0, 0x46, 4, 0b11110000),
AmpConfig("Distortion limiter release time constant", 0b1, 0x46, 0, 0b00000001),
AmpConfig("Left DAC input mixer: DAI1 left", 0b0, 0x22, 7, 0b10000000),
AmpConfig("Left DAC input mixer: DAI1 right", 0b0, 0x22, 6, 0b01000000),
AmpConfig("Left DAC input mixer: DAI2 left", 0b1, 0x22, 5, 0b00100000),
AmpConfig("Left DAC input mixer: DAI2 right", 0b0, 0x22, 4, 0b00010000),
AmpConfig("Right DAC input mixer: DAI2 left", 0b0, 0x22, 1, 0b00000010),
AmpConfig("Right DAC input mixer: DAI2 right", 0b1, 0x22, 0, 0b00000001),
AmpConfig("Volume adjustment smoothing disabled", 0b1, 0x49, 6, 0b01000000),
]
class Amplifier:
AMP_I2C_BUS = 0
AMP_ADDRESS = 0x10
def __init__(self, debug=False):
self.debug = debug
def _get_shutdown_config(self, amp_disabled: bool) -> AmpConfig:
return AmpConfig("Global shutdown", 0b0 if amp_disabled else 0b1, 0x51, 7, 0b10000000)
def _set_configs(self, configs: list[AmpConfig]) -> None:
with SMBus(self.AMP_I2C_BUS) as bus:
for config in configs:
if self.debug:
print(f"Setting \"{config.name}\" to {config.value}:")
old_value = bus.read_byte_data(self.AMP_ADDRESS, config.register, force=True)
new_value = (old_value & (~config.mask)) | ((config.value << config.offset) & config.mask)
bus.write_byte_data(self.AMP_ADDRESS, config.register, new_value, force=True)
if self.debug:
print(f" Changed {hex(config.register)}: {hex(old_value)} -> {hex(new_value)}")
def set_configs(self, configs: list[AmpConfig]) -> bool:
# retry in case panda is using the amp
tries = 15
backoff = 0.
for i in range(tries):
try:
self._set_configs(configs)
return True
except OSError:
backoff += 0.1
time.sleep(backoff)
print(f"Failed to set amp config, {tries - i - 1} retries left")
return False
def set_global_shutdown(self, amp_disabled: bool) -> bool:
return self.set_configs([self._get_shutdown_config(amp_disabled), ])
def initialize_configuration(self) -> bool:
cfgs = [
self._get_shutdown_config(True),
*CONFIG,
self._get_shutdown_config(False),
]
return self.set_configs(cfgs)
if __name__ == "__main__":
amp = Amplifier()
amp.initialize_configuration()
openpilot/common/hardware/comma/hardware.h
+89
View File
@@ -0,0 +1,89 @@
#pragma once
#include <cassert>
#include <fstream>
#include <map>
#include <string>
#include <algorithm> // for std::clamp
#include "common/util.h"
#include "common/hardware/base.h"
class HardwareComma : public HardwareNone {
public:
static std::string get_name() {
static const std::string name = []() {
std::string model = util::read_file("/sys/firmware/devicetree/base/model");
return util::strip(model.substr(std::string("comma ").size()));
}();
return name;
}
static cereal::InitData::DeviceType get_device_type() {
static const std::map<std::string, cereal::InitData::DeviceType> device_map = {
{"tizi", cereal::InitData::DeviceType::TIZI},
{"mici", cereal::InitData::DeviceType::MICI}
};
static const auto it = device_map.find(get_name());
assert(it != device_map.end());
return it->second;
}
static std::string get_serial() {
static std::string serial("");
if (serial.empty()) {
std::ifstream stream("/proc/cmdline");
std::string cmdline;
std::getline(stream, cmdline);
auto start = cmdline.find("serialno=");
if (start == std::string::npos) {
serial = "cccccc";
} else {
auto end = cmdline.find(" ", start + 9);
serial = cmdline.substr(start + 9, end - start - 9);
}
}
return serial;
}
static void set_ir_power(int percent) {
auto device = get_device_type();
if (device == cereal::InitData::DeviceType::TIZI) {
return;
}
int value = util::map_val(std::clamp(percent, 0, 100), 0, 100, 0, 300);
std::ofstream("/sys/class/leds/led:switch_2/brightness") << 0 << "\n";
std::ofstream("/sys/class/leds/led:torch_2/brightness") << value << "\n";
std::ofstream("/sys/class/leds/led:switch_2/brightness") << value << "\n";
}
static std::map<std::string, std::string> get_init_logs(bool route_log = false) {
std::map<std::string, std::string> ret = {
{"/BUILD", util::read_file("/BUILD")},
{"lsblk", util::check_output("lsblk -o NAME,SIZE,STATE,VENDOR,MODEL,REV,SERIAL")},
{"SOM ID", util::read_file("/sys/devices/platform/vendor/vendor:gpio-som-id/som_id")},
};
std::string bs = util::check_output("abctl --boot_slot");
ret["boot slot"] = bs.substr(0, bs.find_first_of("\n"));
std::string temp = util::read_file("/dev/disk/by-partlabel/ssd");
temp.erase(temp.find_last_not_of(std::string("\0\r\n", 3))+1);
ret["boot temp"] = temp;
// TODO: these are too slow to do on route log inits. need to do it async?
if (!route_log) {
for (std::string part : {"xbl", "abl", "aop", "devcfg", "xbl_config"}) {
for (std::string slot : {"a", "b"}) {
std::string partition = part + "_" + slot;
std::string hash = util::check_output("sha256sum /dev/disk/by-partlabel/" + partition);
ret[partition] = hash.substr(0, hash.find_first_of(" "));
}
}
}
return ret;
}
};
openpilot/common/hardware/comma/hardware.py
+429
View File
@@ -0,0 +1,429 @@
import configparser
import json
import os
import socket
import subprocess
import time
from functools import cached_property, lru_cache
from pathlib import Path
from openpilot.cereal import log
from openpilot.common.utils import sudo_read, sudo_write
from openpilot.common.gpio import gpio_set, gpio_init, get_irqs_for_action
from openpilot.common.esim.base import LPABase
from openpilot.common.hardware.base import HardwareBase, ThermalConfig, ThermalZone
from openpilot.common.hardware.comma.pins import GPIO
from openpilot.common.hardware.comma.amplifier import Amplifier
MODEM_STATE_PATH = "/dev/shm/modem"
NetworkType = log.DeviceState.NetworkType
NetworkStrength = log.DeviceState.NetworkStrength
def affine_irq(val, action):
irqs = get_irqs_for_action(action)
if len(irqs) == 0:
print(f"No IRQs found for '{action}'")
return
for i in irqs:
sudo_write(str(val), f"/proc/irq/{i}/smp_affinity_list")
@lru_cache
def get_device_type():
# lru_cache and cache can cause memory leaks when used in classes
with open("/sys/firmware/devicetree/base/model") as f:
model = f.read().strip('\x00')
return model.split('comma ')[-1]
def wpa_supplicant_cmd(cmd: str, timeout: float = 0.2) -> dict[str, str]:
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
sock.settimeout(timeout)
sock.bind(f"\0openpilot-wpa-{os.getpid()}-{time.monotonic_ns()}")
sock.connect("/run/wpa_supplicant/wlan0")
sock.send(cmd.encode())
while True:
out = sock.recv(8192).decode("utf-8", "replace")
if out.startswith("<"):
continue
if out.startswith("FAIL"):
return {}
return dict(l.split("=", 1) for l in out.splitlines() if "=" in l)
def get_default_route_iface():
with open("/proc/net/route") as f:
routes = [(int(route[6]), route[0]) for line in f.readlines()[1:] if (route := line.split())[1] == "00000000" and int(route[3], 16) & 0x1]
return min(routes)[1] if routes else None
class HardwareComma(HardwareBase):
"""
This platform covers the Snapdragon 845-based comma devices:
- tici = comma three
- tizi = comma 3X
- mici = comma four
We strictly use only the device codenames in this codebase for
consistency, though all user-facing UI should use the product names.
"""
@cached_property
def amplifier(self):
if self.get_device_type() == "mici":
return None
return Amplifier()
def get_modem_state(self) -> dict:
try:
with open(MODEM_STATE_PATH) as f:
return json.load(f)
except (FileNotFoundError, json.JSONDecodeError):
return {}
def get_os_version(self):
with open("/VERSION") as f:
return f.read().strip()
def get_device_type(self):
return get_device_type()
def reboot(self, reason=None):
subprocess.check_output(["sudo", "reboot"])
def uninstall(self):
Path("/data/__system_reset__").touch()
os.sync()
self.reboot()
def get_serial(self):
return self.get_cmdline()['androidboot.serialno']
def get_voltage(self):
with open("/sys/class/hwmon/hwmon1/in1_input") as f:
return int(f.read())
def get_current(self):
with open("/sys/class/hwmon/hwmon1/curr1_input") as f:
return int(f.read())
def set_ir_power(self, percent: int):
if self.get_device_type() == "tizi":
return
value = int((percent / 100) * 300)
with open("/sys/class/leds/led:switch_2/brightness", "w") as f:
f.write("0\n")
with open("/sys/class/leds/led:torch_2/brightness", "w") as f:
f.write(f"{value}\n")
with open("/sys/class/leds/led:switch_2/brightness", "w") as f:
f.write(f"{value}\n")
def get_network_type(self):
try:
if (iface := get_default_route_iface()):
if iface.startswith('wlan'):
return NetworkType.wifi
if iface.startswith('eth'):
return NetworkType.ethernet
except Exception:
pass
ms = self.get_modem_state()
if ms.get('connected'):
nt = ms.get('network_type', '')
if nt == 'nr':
return NetworkType.cell5G
elif nt == 'lte':
return NetworkType.cell4G
elif nt in ('utran', 'umts'):
return NetworkType.cell3G
elif nt == 'gsm':
return NetworkType.cell2G
return NetworkType.none
def get_sim_info(self):
ms = self.get_modem_state()
sim_id = ms.get('iccid', '')
return {
'sim_id': sim_id,
'mcc_mnc': ms.get('mcc_mnc') or None,
'network_type': ["Unknown"],
'sim_state': ["ABSENT"] if not sim_id else ["READY"],
'data_connected': ms.get('connected', False),
}
def get_sim_lpa(self) -> LPABase:
from openpilot.common.esim.lpa import LPA
return LPA()
def get_imei(self):
return self.get_modem_state().get('imei', '')
def get_network_info(self):
if self.get_device_type() == "mici":
return None
ms = self.get_modem_state()
return {
'technology': ms.get('network_type', '').upper() if ms.get('network_type') else '',
'operator': ms.get('operator', ''),
'band': ms.get('band', ''),
'channel': ms.get('channel', 0),
'extra': ms.get('extra', ''),
'state': ms.get('state', 'UNKNOWN'),
}
def parse_strength(self, percentage):
if percentage < 25:
return NetworkStrength.poor
elif percentage < 50:
return NetworkStrength.moderate
elif percentage < 75:
return NetworkStrength.good
else:
return NetworkStrength.great
def get_network_strength(self, network_type):
network_strength = NetworkStrength.unknown
try:
if network_type == NetworkType.none:
pass
elif network_type == NetworkType.ethernet:
network_strength = NetworkStrength.great
elif network_type == NetworkType.wifi:
rssi = wpa_supplicant_cmd("SIGNAL_POLL").get("RSSI")
if rssi is not None:
dbm = int(rssi)
if -100 < dbm <= 0:
network_strength = self.parse_strength(120 + max(-100, min(-20, dbm)))
else: # Cellular
network_strength = self.parse_strength(self.get_modem_state().get('signal_quality', 0))
except Exception:
pass
return network_strength
def get_network_metered(self, network_type) -> bool:
if network_type in (NetworkType.cell2G, NetworkType.cell3G, NetworkType.cell4G, NetworkType.cell5G):
from openpilot.common.params import Params
return Params().get_bool("GsmMetered")
try:
if network_type == NetworkType.wifi:
ssid = wpa_supplicant_cmd("STATUS").get("ssid", "")
if ssid:
# wpa_supplicant escapes non-printable bytes as \xNN; NM keyfile stores ASCII SSIDs as a literal and others as a byte;byte; list
ssid_bytes = ssid.encode().decode('unicode_escape').encode('latin-1')
ssid_keyfile_list = ';'.join(str(b) for b in ssid_bytes) + ';'
nm_dirs = ("/run/NetworkManager/system-connections", "/data/etc/NetworkManager/system-connections")
for fpath in (p for d in nm_dirs for p in Path(d).glob("*.nmconnection")):
raw = sudo_read(str(fpath))
if not raw:
continue
cp = configparser.ConfigParser(interpolation=None)
try:
cp.read_string(raw)
keyfile_ssid = cp.get("wifi", "ssid", fallback="")
if keyfile_ssid != ssid and keyfile_ssid != ssid_keyfile_list:
continue
metered = cp.getint("connection", "metered", fallback=0)
except (configparser.Error, ValueError):
continue
if metered == 1: # NM_METERED_YES
return True
if metered == 2: # NM_METERED_NO
return False
break
except Exception:
pass
return super().get_network_metered(network_type)
def get_modem_temperatures(self):
return self.get_modem_state().get('temperatures', [])
def get_current_power_draw(self):
return (self.read_param_file("/sys/class/hwmon/hwmon1/power1_input", int) / 1e6)
def get_som_power_draw(self):
return (self.read_param_file("/sys/class/power_supply/bms/voltage_now", int) * self.read_param_file("/sys/class/power_supply/bms/current_now", int) / 1e12)
def shutdown(self):
subprocess.run("sudo poweroff", shell=True)
def get_thermal_config(self):
intake, exhaust, gnss, bottomSoc = None, None, None, None
if self.get_device_type() == "mici":
gnss = ThermalZone("gnss")
intake = ThermalZone("intake")
exhaust = ThermalZone("exhaust")
bottomSoc = ThermalZone("bottom_soc")
return ThermalConfig(cpu=[ThermalZone(f"cpu{i}-silver-usr") for i in range(4)] +
[ThermalZone(f"cpu{i}-gold-usr") for i in range(4)],
gpu=[ThermalZone("gpu0-usr"), ThermalZone("gpu1-usr")],
dsp=ThermalZone("compute-hvx-usr"),
memory=ThermalZone("ddr-usr"),
pmic=[ThermalZone("pm8998_tz"), ThermalZone("pm8005_tz")],
intake=intake,
exhaust=exhaust,
gnss=gnss,
bottomSoc=bottomSoc)
def set_display_power(self, on):
try:
with open("/sys/class/backlight/panel0-backlight/bl_power", "w") as f:
f.write("0" if on else "4")
except Exception:
pass
def set_screen_brightness(self, percentage):
try:
with open("/sys/class/backlight/panel0-backlight/max_brightness") as f:
max_brightness = float(f.read().strip())
val = int(percentage * (max_brightness / 100.))
with open("/sys/class/backlight/panel0-backlight/brightness", "w") as f:
f.write(str(val))
except Exception:
pass
def get_screen_brightness(self):
try:
with open("/sys/class/backlight/panel0-backlight/max_brightness") as f:
max_brightness = float(f.read().strip())
with open("/sys/class/backlight/panel0-backlight/brightness") as f:
return int(float(f.read()) / (max_brightness / 100.))
except Exception:
return 0
def set_power_save(self, powersave_enabled):
# amplifier, 100mW at idle
if self.amplifier is not None:
self.amplifier.set_global_shutdown(amp_disabled=powersave_enabled)
if not powersave_enabled:
self.amplifier.initialize_configuration()
# *** CPU config ***
# offline big cluster
for i in range(4, 8):
val = '0' if powersave_enabled else '1'
sudo_write(val, f'/sys/devices/system/cpu/cpu{i}/online')
for n in ('0', '4'):
if powersave_enabled and n == '4':
continue
gov = 'ondemand' if powersave_enabled else 'performance'
sudo_write(gov, f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_governor')
if not powersave_enabled:
# cap max core freq to 1689 Mhz
sudo_write('1689600', f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_max_freq')
# *** IRQ config ***
# GPU, modeld core
affine_irq(7, "kgsl-3d0")
# camerad core
camera_irqs = ("a5", "cci", "cpas_camnoc", "cpas-cdm", "csid", "ife", "csid-lite", "ife-lite")
for n in camera_irqs:
affine_irq(6, n)
def get_gpu_usage_percent(self):
try:
with open('/sys/class/kgsl/kgsl-3d0/gpubusy') as f:
used, total = f.read().strip().split()
return 100.0 * int(used) / int(total)
except Exception:
return 0
def initialize_hardware(self):
if self.amplifier is not None:
self.amplifier.initialize_configuration()
# Allow hardwared to write engagement status to kmsg
subprocess.run("sudo chmod a+w /dev/kmsg", shell=True)
# Ensure fan gpio is enabled so fan runs until shutdown, also turned on at boot by the ABL
gpio_init(GPIO.SOM_ST_IO, True)
gpio_set(GPIO.SOM_ST_IO, True)
# *** IRQ config ***
# mask off big cluster from default affinity
sudo_write("f", "/proc/irq/default_smp_affinity")
# move these off the default core
affine_irq(1, "msm_vidc") # encoders
affine_irq(1, "i2c_geni") # sensors
# *** GPU config ***
# https://github.com/commaai/agnos-kernel-sdm845/blob/master/arch/arm64/boot/dts/qcom/sdm845-gpu.dtsi#L216
affine_irq(5, "fts_ts") # touch
affine_irq(5, "msm_drm") # display
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/min_pwrlevel")
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/max_pwrlevel")
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_bus_on")
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_clk_on")
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_rail_on")
sudo_write("1000", "/sys/class/kgsl/kgsl-3d0/idle_timer")
sudo_write("performance", "/sys/class/kgsl/kgsl-3d0/devfreq/governor")
sudo_write("710", "/sys/class/kgsl/kgsl-3d0/max_clock_mhz")
# setup governors
sudo_write("performance", "/sys/class/devfreq/soc:qcom,cpubw/governor")
sudo_write("performance", "/sys/class/devfreq/soc:qcom,memlat-cpu0/governor")
sudo_write("performance", "/sys/class/devfreq/soc:qcom,memlat-cpu4/governor")
# *** VIDC (encoder) config ***
sudo_write("N", "/sys/kernel/debug/msm_vidc/clock_scaling")
sudo_write("Y", "/sys/kernel/debug/msm_vidc/disable_thermal_mitigation")
# pandad core
affine_irq(3, "spi_geni") # SPI
try:
pid = subprocess.check_output(["pgrep", "-f", "spi0"], encoding='utf8').strip()
subprocess.call(["sudo", "chrt", "-f", "-p", "1", pid])
subprocess.call(["sudo", "taskset", "-pc", "3", pid])
except subprocess.CalledProcessError as e:
print(str(e))
def get_modem_data_usage(self):
ms = self.get_modem_state()
return ms.get('tx_bytes', -1), ms.get('rx_bytes', -1)
def reset_internal_panda(self):
gpio_init(GPIO.STM_RST_N, True)
gpio_init(GPIO.STM_BOOT0, True)
gpio_set(GPIO.STM_RST_N, True)
gpio_set(GPIO.STM_BOOT0, False)
time.sleep(0.01)
gpio_set(GPIO.STM_RST_N, False)
def recover_internal_panda(self):
gpio_init(GPIO.STM_RST_N, True)
gpio_init(GPIO.STM_BOOT0, True)
gpio_set(GPIO.STM_RST_N, True)
gpio_set(GPIO.STM_BOOT0, True)
time.sleep(0.01)
gpio_set(GPIO.STM_RST_N, False)
time.sleep(0.01)
gpio_set(GPIO.STM_BOOT0, False)
def booted(self):
# this normally boots within 8s, but on rare occasions takes 30+s
encoder_state = sudo_read("/sys/kernel/debug/msm_vidc/core0/info")
if "Core state: 0" in encoder_state and (time.monotonic() < 60*2):
return False
return True
if __name__ == "__main__":
t = HardwareComma()
t.initialize_hardware()
t.set_power_save(False)
print(t.get_sim_info())
openpilot/common/hardware/comma/id_rsa
+28
View File
@@ -0,0 +1,28 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC+iXXq30Tq+J5N
Kat3KWHCzcmwZ55nGh6WggAqECa5CasBlM9VeROpVu3beA+5h0MibRgbD4DMtVXB
t6gEvZ8nd04E7eLA9LTZyFDZ7SkSOVj4oXOQsT0GnJmKrASW5KslTWqVzTfo2XCt
Z+004ikLxmyFeBO8NOcErW1pa8gFdQDToH9FrA7kgysic/XVESTOoe7XlzRoe/eZ
acEQ+jtnmFd21A4aEADkk00Ahjr0uKaJiLUAPatxs2icIXWpgYtfqqtaKF23wSt6
1OTu6cAwXbOWr3m+IUSRUO0IRzEIQS3z1jfd1svgzSgSSwZ1Lhj4AoKxIEAIc8qJ
rO4uymCJAgMBAAECggEBAISFevxHGdoL3Z5xkw6oO5SQKO2GxEeVhRzNgmu/HA+q
x8OryqD6O1CWY4037kft6iWxlwiLOdwna2P25ueVM3LxqdQH2KS4DmlCx+kq6FwC
gv063fQPMhC9LpWimvaQSPEC7VUPjQlo4tPY6sTTYBUOh0A1ihRm/x7juKuQCWix
Cq8C/DVnB1X4mGj+W3nJc5TwVJtgJbbiBrq6PWrhvB/3qmkxHRL7dU2SBb2iNRF1
LLY30dJx/cD73UDKNHrlrsjk3UJc29Mp4/MladKvUkRqNwlYxSuAtJV0nZ3+iFkL
s3adSTHdJpClQer45R51rFDlVsDz2ZBpb/hRNRoGDuECgYEA6A1EixLq7QYOh3cb
Xhyh3W4kpVvA/FPfKH1OMy3ONOD/Y9Oa+M/wthW1wSoRL2n+uuIW5OAhTIvIEivj
6bAZsTT3twrvOrvYu9rx9aln4p8BhyvdjeW4kS7T8FP5ol6LoOt2sTP3T1LOuJPO
uQvOjlKPKIMh3c3RFNWTnGzMPa0CgYEA0jNiPLxP3A2nrX0keKDI+VHuvOY88gdh
0W5BuLMLovOIDk9aQFIbBbMuW1OTjHKv9NK+Lrw+YbCFqOGf1dU/UN5gSyE8lX/Q
FsUGUqUZx574nJZnOIcy3ONOnQLcvHAQToLFAGUd7PWgP3CtHkt9hEv2koUwL4vo
ikTP1u9Gkc0CgYEA2apoWxPZrY963XLKBxNQecYxNbLFaWq67t3rFnKm9E8BAICi
4zUaE5J1tMVi7Vi9iks9Ml9SnNyZRQJKfQ+kaebHXbkyAaPmfv+26rqHKboA0uxA
nDOZVwXX45zBkp6g1sdHxJx8JLoGEnkC9eyvSi0C//tRLx86OhLErXwYcNkCf1it
VMRKrWYoXJTUNo6tRhvodM88UnnIo3u3CALjhgU4uC1RTMHV4ZCGBwiAOb8GozSl
s5YD1E1iKwEULloHnK6BIh6P5v8q7J6uf/xdqoKMjlWBHgq6/roxKvkSPA1DOZ3l
jTadcgKFnRUmc+JT9p/ZbCxkA/ALFg8++G+0ghECgYA8vG3M/utweLvq4RI7l7U7
b+i2BajfK2OmzNi/xugfeLjY6k2tfQGRuv6ppTjehtji2uvgDWkgjJUgPfZpir3I
RsVMUiFgloWGHETOy0Qvc5AwtqTJFLTD1Wza2uBilSVIEsg6Y83Gickh+ejOmEsY
6co17RFaAZHwGfCFFjO76Q==
-----END RSA PRIVATE KEY-----
openpilot/common/hardware/comma/modem.py
Executable
+630
View File
@@ -0,0 +1,630 @@
#!/usr/bin/env python3
import fcntl
import json
import logging
import os
import select
import signal
import struct
import subprocess
import tempfile
import termios
import time
from contextlib import contextmanager
from ipaddress import IPv4Address, AddressValueError
from enum import Enum
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s.%(msecs)03d %(levelname)-7s modem: %(message)s",
datefmt="%H:%M:%S",
)
AT_PORT = "/dev/modem_at0"
PPP_PORT = "/dev/modem_at1"
STATE_PATH = "/dev/shm/modem"
AT_LOCK = "/dev/shm/modem.lock" # shared with LPA
AT_INIT = [
"ATE0", # disable command echo
"ATV1", # verbose result codes (CONNECT/BUSY/NO CARRIER, not numeric)
"AT+CMEE=1", # numeric +CME ERROR codes on failures (per 3GPP 27.007)
"ATX4", # extended result codes: busy + dial tone detection, line speed in CONNECT
"AT&C1", # DCD pin follows carrier state (V.250 default)
"AT+CREG=2", # registration URCs include location info
"AT+CGREG=2", # GPRS registration URCs include location info
]
CREG = {0: "not_registered", 1: "home", 2: "searching", 3: "denied", 4: "unknown", 5: "roaming"}
# 3GPP TS 27.007 +COPS <AcT> -> network type
NETWORK_TYPE = {0: "gsm", 1: "gsm", 3: "gsm", 8: "gsm",
2: "utran", 4: "utran", 5: "utran", 6: "utran",
7: "lte", 9: "lte", 10: "lte",
11: "nr", 12: "nr", 13: "nr"}
DIAL_CID = 1
WEBBING_ICCID_PREFIX = "8985235"
PPPD_CMD = [
"sudo", "pppd", PPP_PORT, "460800", "noauth", "nodetach", "noipdefault", "usepeerdns",
"nodefaultroute", "connect",
"/usr/sbin/chat -v ABORT 'NO CARRIER' ABORT 'NO DIALTONE' ABORT 'BUSY' " +
f"ABORT 'NO ANSWER' ABORT 'ERROR' TIMEOUT 5 '' AT OK ATD*99***{DIAL_CID}# CONNECT ''",
"lcp-echo-interval", "30", "lcp-echo-failure", "4", "mtu", "1500", "mru", "1500",
"novj", "novjccomp", "ipcp-accept-local", "ipcp-accept-remote", "nomagic",
"user", '""', "password", '""',
]
INITIAL_STATE: dict[str, object] = {
"seconds_since_boot": 0,
"state": "INITIALIZING",
"connected": False, "ip_address": "",
"iccid": "", "mcc_mnc": "", "imei": "", "modem_version": "",
"signal_strength": 0, "signal_quality": 0,
"network_type": "unknown", "operator": "", "band": "", "channel": 0,
"registration": "unknown", "temperatures": [], "extra": "",
"tx_bytes": 0, "rx_bytes": 0,
}
@contextmanager
def _serial_port(port: str, baudrate: int):
fd = os.open(port, os.O_RDWR | os.O_NOCTTY)
try:
attrs = termios.tcgetattr(fd)
attrs[0] = 0
attrs[1] = 0
attrs[2] = termios.CLOCAL | termios.CREAD | termios.CS8
attrs[3] = 0
attrs[4] = attrs[5] = getattr(termios, f"B{baudrate}")
attrs[6][termios.VMIN] = 0
attrs[6][termios.VTIME] = 0
termios.tcsetattr(fd, termios.TCSANOW, attrs)
yield fd
finally:
os.close(fd)
def _read_line(fd: int, timeout: float) -> bytes:
data = bytearray()
deadline = time.monotonic() + timeout
while True:
readable, _, _ = select.select([fd], [], [], max(0.0, deadline - time.monotonic()))
if not readable:
return bytes(data)
byte = os.read(fd, 1)
if not byte:
return bytes(data)
data.extend(byte)
if byte == b"\n":
return bytes(data)
class State(Enum):
INITIALIZING = "INITIALIZING"
SEARCHING = "SEARCHING"
CONNECTING = "CONNECTING"
CONNECTED = "CONNECTED"
DISCONNECTING = "DISCONNECTING"
STATE_WAIT = 1.0 # seconds to wait after each state handler returns
class PPPSession:
"""Owns pppd lifecycle, fail tracking, and PPP routing."""
MAX_FAILS = 3
def __init__(self):
self._proc: subprocess.Popen | None = None
self._fails = 0
self._peer = ""
def start(self):
self._proc = subprocess.Popen(PPPD_CMD, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self._peer = ""
logging.info(f"PPP dialing CID {DIAL_CID}")
def kill(self):
subprocess.run(["sudo", "killall", "-9", "pppd"], capture_output=True)
self._peer = ""
@staticmethod
def reset_data_port():
"""Drop DTR on PPP_PORT so the modem terminates any stuck PPP session."""
try:
with _serial_port(PPP_PORT, 460800) as fd:
dtr = struct.pack("I", termios.TIOCM_DTR)
fcntl.ioctl(fd, termios.TIOCMBIC, dtr)
time.sleep(0.2)
fcntl.ioctl(fd, termios.TIOCMBIS, dtr)
except Exception as e:
logging.warning(f"data port reset failed: {e}")
def has_exited(self) -> bool:
return self._proc is not None and self._proc.poll() is not None
def reset_fail_counter(self):
self._fails = 0
def record_fail(self) -> bool:
"""Bump fail counter; return True if at the give-up limit."""
self._fails += 1
return self._fails >= self.MAX_FAILS
@property
def fails(self) -> int:
return self._fails
def maybe_install_routes(self, ip: str, peer: str) -> bool:
"""Install routes if peer changed; kill the session on failure so the state machine reconnects."""
if not peer or peer == self._peer:
return False
try:
IPv4Address(ip)
IPv4Address(peer)
except AddressValueError:
logging.warning(f"refusing route install with non-IPv4 ip={ip!r} peer={peer!r}")
self.kill()
return False
self.cleanup_routes()
cmds = [
["sudo", "ip", "route", "add", "default", "via", peer, "dev", "ppp0", "metric", "1000"],
["sudo", "ip", "route", "add", "default", "via", peer, "dev", "ppp0", "table", "1000"],
["sudo", "ip", "rule", "add", "from", ip, "table", "1000"],
]
for cmd in cmds:
r = subprocess.run(cmd, capture_output=True, text=True)
if r.returncode != 0:
logging.warning(f"route install failed ({' '.join(cmd[1:])}): {r.stderr.strip()}")
self.cleanup_routes()
self.kill()
return False
logging.info(f"route set up for {ip} via {peer}")
self._peer = peer
return True
def maybe_install_dns(self, dns_servers: list[str]) -> bool:
"""Register DNS servers with systemd-resolved; kill the session on failure to force a retry."""
if not dns_servers:
return False
for cmd in (["sudo", "resolvectl", "dns", "ppp0", *dns_servers],
["sudo", "resolvectl", "default-route", "ppp0", "yes"]):
r = subprocess.run(cmd, capture_output=True, text=True)
if r.returncode != 0:
logging.warning(f"resolvectl failed ({' '.join(cmd[1:])}): {r.stderr.strip()}")
self.kill()
return False
logging.info(f"resolvectl: ppp0 DNS = {dns_servers}")
return True
@staticmethod
def cleanup_routes():
subprocess.run(["sudo", "ip", "route", "del", "default", "dev", "ppp0"], capture_output=True)
subprocess.run(["sudo", "ip", "route", "flush", "table", "1000"], capture_output=True)
# rules don't have a flush; delete until none remain
while subprocess.run(["sudo", "ip", "rule", "del", "table", "1000"], capture_output=True).returncode == 0:
pass
subprocess.run(["sudo", "resolvectl", "revert", "ppp0"], capture_output=True)
class Modem:
def __init__(self):
self._ppp = PPPSession()
self._sim_change = False
self._apn = "" # blank = network-provided via PCO
self._roaming_allowed = True
self.running = True
self.S = INITIAL_STATE.copy()
@staticmethod
def _read_param(key):
try:
with open(f"/data/params/d/{key}") as f:
return f.read().strip()
except FileNotFoundError:
return ""
@staticmethod
def _parse_reg(v: str) -> str:
try:
return CREG.get(int(v.split(",")[1].strip('"')), "unknown")
except (ValueError, IndexError):
return "unknown"
@staticmethod
def _has_modem_manager() -> bool:
return os.path.isfile("/lib/systemd/system/ModemManager.service")
def _is_roaming_allowed(self) -> bool:
if self.S["iccid"].startswith(WEBBING_ICCID_PREFIX):
return True
return self._read_param("GsmRoaming") == "1"
def _publish_state(self, **kwargs):
self.S.update(kwargs)
self.S["seconds_since_boot"] = time.monotonic()
with tempfile.NamedTemporaryFile(mode="w", dir="/dev/shm", delete=False) as f:
json.dump(self.S, f, indent=2)
os.chmod(f.name, 0o644)
os.replace(f.name, STATE_PATH)
def _at(self, cmd):
"""Send AT command, return response lines. [] on error or if LPA holds port."""
fd = os.open(AT_LOCK, os.O_CREAT | os.O_RDWR, 0o666)
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
os.close(fd)
return []
try:
with _serial_port(AT_PORT, 9600) as serial_fd:
termios.tcflush(serial_fd, termios.TCIFLUSH)
command = (cmd + "\r").encode()
while command:
command = command[os.write(serial_fd, command):]
lines = []
while True:
raw = _read_line(serial_fd, 5)
if not raw:
raise TimeoutError("AT timeout")
line = raw.decode(errors="ignore").strip()
if not line:
continue
if line == "OK":
break
if line == "ERROR" or line.startswith("+CME ERROR"):
raise RuntimeError(line)
lines.append(line)
return lines
except (RuntimeError, TimeoutError, OSError) as e:
logging.info(f"AT {cmd} failed: {e}")
return []
finally:
fcntl.flock(fd, fcntl.LOCK_UN)
os.close(fd)
def _atv(self, cmd, pfx):
for line in self._at(cmd):
if pfx in line and ":" in line:
return line.split(":", 1)[1].strip()
return None
def _init_at_channel(self) -> bool:
"""Run AT_INIT and confirm ATE0 took effect. Returns False if echo is still on."""
for c in AT_INIT:
self._at(c)
r = self._at("AT+CGMI")
return bool(r) and not r[0].startswith("AT")
def _configure_modem(self, modem_version: str):
if not modem_version.startswith("EG25"):
return
cmds = [
# clear initial EPS bearer APN (some carriers reject the default)
'AT+CGDCONT=0,"IP",""',
# SIM hot swap
'AT+QSIMDET=1,0',
'AT+QSIMSTAT=1',
# configure modem as data-centric
'AT+QNVW=5280,0,"0102000000000000"',
'AT+QNVFW="/nv/item_files/ims/IMS_enable",00',
'AT+QNVFW="/nv/item_files/modem/mmode/ue_usage_setting",01',
]
for c in cmds:
self._at(c)
def _do_initializing(self):
if not os.path.exists(AT_PORT):
return State.INITIALIZING
logging.info("port found, initializing")
self._ppp.kill()
self._ppp.cleanup_routes()
if not self._init_at_channel():
logging.warning("AT echo still on, retrying")
return State.INITIALIZING
identity = self._read_identity()
if not identity["iccid"] or not identity["imei"]:
logging.warning(f"identity read incomplete: {identity}, retrying")
return State.INITIALIZING
self._configure_modem(identity["modem_version"])
self.S.update(identity)
self._apn = self._read_param("GsmApn")
self._roaming_allowed = self._is_roaming_allowed()
# blank APN lets the carrier supply one via PCO
self._at(f'AT+CGDCONT={DIAL_CID},"IP","{self._apn}"')
logging.info(f"APN '{self._apn or '(network-provided)'}' written to CID {DIAL_CID}, roaming={'on' if self._roaming_allowed else 'off'}")
self._sim_change = False # clear since we just re-read identity with the new SIM
self._publish_state(**identity)
return State.SEARCHING
def _read_identity(self):
def first_line(cmd):
r = self._at(cmd)
return r[0].strip() if r else ""
imei = first_line("AT+CGSN")
if not (imei.isdigit() and 14 <= len(imei) <= 17): # 3GPP TS 23.003
imei = ""
iccid = (self._atv("AT+QCCID", "+QCCID:") or "").rstrip("F")
if not iccid.isdigit():
iccid = ""
imsi = first_line("AT+CIMI")
mcc_mnc = imsi[:6] if imsi.isdigit() and len(imsi) >= 6 else ""
modem_version = first_line("AT+GMR")
logging.info(f"imei={imei} iccid={iccid} mcc_mnc={mcc_mnc} ver={modem_version}")
return {"imei": imei, "iccid": iccid, "mcc_mnc": mcc_mnc, "modem_version": modem_version}
def _do_searching(self):
new_roaming = self._is_roaming_allowed()
if new_roaming != self._roaming_allowed:
logging.info(f"roaming changed: {self._roaming_allowed} -> {new_roaming}")
self._roaming_allowed = new_roaming
v = self._atv("AT+CREG?", "+CREG:")
if not v:
return self._searching_idle()
reg = self._parse_reg(v)
greg = self._parse_reg(self._atv("AT+CGREG?", "+CGREG:") or "")
logging.debug(f"creg={reg} cgreg={greg} roaming_allowed={self._roaming_allowed}")
if reg == "roaming" and not self._roaming_allowed:
self._publish_state(registration=reg)
return State.SEARCHING
if reg in ("home", "roaming") and greg in ("home", "roaming"):
self._publish_state(registration=reg)
return State.CONNECTING
if reg != self.S.get("registration"):
self._publish_state(registration=reg)
return self._searching_idle()
def _searching_idle(self):
if self._sim_change or not os.path.exists(AT_PORT):
logging.info(f"-> reconnecting (sim_change={self._sim_change} port={os.path.exists(AT_PORT)})")
return State.DISCONNECTING
return State.SEARCHING
def _do_connecting(self):
logging.info("starting pppd")
self._ppp.reset_fail_counter()
self._sim_change = False
self._ppp.start()
return State.CONNECTED
def _handle_pppd_exit(self):
if self._sim_change or not os.path.exists(AT_PORT):
return State.DISCONNECTING
give_up = self._ppp.record_fail()
if give_up:
logging.warning(f"PPP fail {self._ppp.fails}/{self._ppp.MAX_FAILS}, reconnecting")
return State.DISCONNECTING
logging.warning(f"PPP fail {self._ppp.fails}/{self._ppp.MAX_FAILS}, retrying")
self._ppp.reset_data_port()
if not os.path.exists(AT_PORT):
return State.DISCONNECTING
self._ppp.start()
return State.CONNECTED
def _params_changed(self) -> bool:
new_apn = self._read_param("GsmApn")
if new_apn != self._apn:
logging.info(f"GsmApn changed: '{self._apn}' -> '{new_apn}'")
return True
new_roaming = self._is_roaming_allowed()
if new_roaming != self._roaming_allowed:
logging.info(f"roaming changed: {self._roaming_allowed} -> {new_roaming}")
return True
return False
def _check_iccid(self, state):
if state in (State.INITIALIZING, State.DISCONNECTING) or not self.S["iccid"]:
return
iccid = (self._atv("AT+QCCID", "+QCCID:") or "").rstrip("F")
if iccid and iccid != self.S["iccid"]:
logging.warning(f"iccid changed: {self.S['iccid']} -> {iccid}")
self._sim_change = True
def _do_connected(self):
if self._ppp.has_exited():
return self._handle_pppd_exit()
if self._sim_change or not os.path.exists(AT_PORT) or self._params_changed():
return State.DISCONNECTING
self._poll()
return State.CONNECTED
def _do_disconnecting(self):
logging.warning("reconnecting")
self._publish_state(**INITIAL_STATE)
self._ppp.kill()
self._ppp.cleanup_routes()
self._ppp.reset_data_port()
self._sim_change = False
return State.INITIALIZING
def _poll_signal(self) -> dict:
v = self._atv("AT+CSQ", "+CSQ:")
if not v:
return {}
try:
rssi = int(v.split(",")[0])
if rssi == 99:
return {}
return {"signal_strength": rssi, "signal_quality": min(100, int(rssi / 31 * 100))}
except (ValueError, IndexError):
return {}
def _poll_operator(self) -> dict:
v = self._atv("AT+COPS?", "+COPS:")
if not v:
return {}
p = v.split(",")
out: dict = {}
try:
if len(p) >= 3:
out["operator"] = p[2].strip('"')
if len(p) >= 4:
out["network_type"] = NETWORK_TYPE.get(int(p[3]), "unknown")
except (ValueError, IndexError):
pass
return out
def _poll_band(self) -> dict:
v = self._atv("AT+QNWINFO", "+QNWINFO:")
if not v:
return {}
info = v.replace('"', '').split(",")
try:
if len(info) >= 4:
return {"band": info[2], "channel": int(info[3])}
except ValueError:
pass
return {}
def _poll_extra(self) -> dict:
v = self._atv('AT+QENG="servingcell"', "+QENG:")
return {"extra": v.replace('"', '')} if v else {}
def _poll_temps(self) -> dict:
v = self._atv("AT+QTEMP", "+QTEMP:")
if not v:
return {}
try:
return {"temperatures": [t for t in (int(x) for x in v.split(",") if x.strip()) if t != 255]}
except (ValueError, IndexError):
return {}
def _poll_iface(self) -> dict:
try:
r = subprocess.run(["ip", "-4", "addr", "show", "ppp0"], capture_output=True, text=True, timeout=2)
ip, peer = "", ""
for line in r.stdout.splitlines():
# `inet 10.x.x.x peer 10.64.64.64/32 ...`
parts = line.strip().split()
if "inet" in parts:
i = parts.index("inet")
ip = parts[i + 1].split("/")[0]
if "peer" in parts:
peer = parts[parts.index("peer") + 1].split("/")[0]
break
if ip:
if self._ppp.maybe_install_routes(ip, peer):
self._ppp.maybe_install_dns(self._read_cellular_dns())
return {"ip_address": ip, "connected": True}
if self.S["connected"]:
return {"connected": False, "ip_address": ""}
except Exception:
pass
return {}
def _read_cellular_dns(self) -> list[str]:
v = self._atv(f"AT+CGCONTRDP={DIAL_CID}", "+CGCONTRDP:")
if not v:
return []
# +CGCONTRDP: <cid>,<bearer_id>,<apn>,<local_addr>,<gw_addr>,<dns_prim>,<dns_sec>,...
fields = [f.strip().strip('"') for f in v.split(",")]
dns_servers = []
for d in fields[5:7]:
try:
dns_servers.append(str(IPv4Address(d)))
except (AddressValueError, ValueError):
pass
if not dns_servers:
dns_servers = [
"8.8.8.8", # Google
"1.1.1.1", # Cloudflare
]
logging.warning(f"no cellular DNS servers reported by modem: {v!r}; using fallback {dns_servers}")
return dns_servers
def _poll_byte_counters(self) -> dict:
try:
with open("/sys/class/net/ppp0/statistics/tx_bytes") as f:
tx = int(f.read().strip())
with open("/sys/class/net/ppp0/statistics/rx_bytes") as f:
rx = int(f.read().strip())
except Exception:
return {}
return {"tx_bytes": tx, "rx_bytes": rx}
def _poll(self):
s: dict = {}
for fn in (self._poll_signal, self._poll_operator, self._poll_band,
self._poll_extra, self._poll_temps, self._poll_iface,
self._poll_byte_counters):
s.update(fn())
if s:
self._publish_state(**s)
def run(self):
logging.info("starting")
self._publish_state(state=State.INITIALIZING.value)
if self._has_modem_manager():
subprocess.run(["sudo", "systemctl", "mask", "--runtime", "ModemManager"], capture_output=True)
subprocess.run(["sudo", "systemctl", "stop", "ModemManager"], capture_output=True)
self._ppp.kill()
state = State.INITIALIZING
handlers = {
State.INITIALIZING: self._do_initializing,
State.SEARCHING: self._do_searching,
State.CONNECTING: self._do_connecting,
State.CONNECTED: self._do_connected,
State.DISCONNECTING: self._do_disconnecting,
}
while self.running:
try:
self._check_iccid(state)
prev = state
state = handlers[state]()
if state != prev:
self._publish_state(state=state.value)
logging.info(f"{prev.value} -> {state.value}")
except Exception:
logging.exception(f"error in {state.value}")
state = State.DISCONNECTING
time.sleep(STATE_WAIT)
def stop(self):
self.running = False
self._ppp.kill()
self._ppp.cleanup_routes()
try:
os.remove(STATE_PATH)
except FileNotFoundError:
pass
if self._has_modem_manager():
subprocess.run(["sudo", "systemctl", "unmask", "--runtime", "ModemManager"], capture_output=True)
subprocess.run(["sudo", "systemctl", "start", "ModemManager"], capture_output=True)
def main():
m = Modem()
def _sig(*_):
m.running = False
signal.signal(signal.SIGINT, _sig)
signal.signal(signal.SIGTERM, _sig)
m.run()
m.stop()
if __name__ == "__main__":
main()
openpilot/common/hardware/comma/pins.py
+27
View File
@@ -0,0 +1,27 @@
# GPIO pin definitions
class GPIO:
# both GPIO_STM_RST_N and GPIO_LTE_RST_N are misnamed, they are high to reset
HUB_RST_N = 30
UBLOX_RST_N = 32
UBLOX_SAFEBOOT_N = 33
GNSS_PWR_EN = 34 # SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN
STM_RST_N = 124
STM_BOOT0 = 134
STM_PWR_EN_N = 41 # because STM32H7 RST doesn't generate a full power-on-reset
SIREN = 42
SOM_ST_IO = 49
LTE_RST_N = 50
LTE_PWRKEY = 116
LTE_BOOT = 52
# GPIO_CAM0_DVDD_EN = /sys/kernel/debug/regulator/camera_rear_ldo
CAM0_AVDD_EN = 8
CAM0_RSTN = 9
CAM1_RSTN = 7
CAM2_RSTN = 12
# Sensor interrupts
LSM_INT = 84
openpilot/common/hardware/comma/power_monitor.py
Executable
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
import sys
import time
import datetime
import numpy as np
from collections import deque
from openpilot.common.realtime import Ratekeeper
from openpilot.common.filter_simple import FirstOrderFilter
def read_power():
with open("/sys/bus/i2c/devices/0-0040/hwmon/hwmon1/power1_input") as f:
return int(f.read()) / 1e6
def sample_power(seconds=5) -> list[float]:
rate = 123
rk = Ratekeeper(rate, print_delay_threshold=None)
pwrs = []
for _ in range(rate*seconds):
pwrs.append(read_power())
rk.keep_time()
return pwrs
def get_power(seconds=5):
pwrs = sample_power(seconds)
return np.mean(pwrs)
def wait_for_power(min_pwr, max_pwr, min_secs_in_range, timeout):
start_time = time.monotonic()
pwrs = deque([min_pwr - 1.]*min_secs_in_range, maxlen=min_secs_in_range)
while (time.monotonic() - start_time < timeout):
pwrs.append(get_power(1))
if all(min_pwr <= p <= max_pwr for p in pwrs):
break
return np.mean(pwrs)
if __name__ == "__main__":
duration = None
if len(sys.argv) > 1:
duration = int(sys.argv[1])
rate = 23
rk = Ratekeeper(rate, print_delay_threshold=None)
fltr = FirstOrderFilter(0, 5, 1. / rate, initialized=False)
measurements = []
start_time = time.monotonic()
try:
while duration is None or time.monotonic() - start_time < duration:
fltr.update(read_power())
if rk.frame % rate == 0:
measurements.append(fltr.x)
t = datetime.timedelta(seconds=time.monotonic() - start_time)
avg = sum(measurements) / len(measurements)
print(f"Now: {fltr.x:.2f} W, Avg: {avg:.2f} W over {t}")
rk.keep_time()
except KeyboardInterrupt:
pass
t = datetime.timedelta(seconds=time.monotonic() - start_time)
avg = sum(measurements) / len(measurements)
print(f"\nAverage power: {avg:.2f}W over {t}")
openpilot/common/hardware/comma/tests/__init__.py
+1
View File
@@ -0,0 +1 @@
openpilot/common/hardware/comma/tests/test_agnos_updater.py
+21
View File
@@ -0,0 +1,21 @@
import json
import os
import requests
TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)))
MANIFEST = os.path.join(TEST_DIR, "../agnos.json")
from openpilot.common.test import OpenpilotTestCase
class TestAgnosUpdater(OpenpilotTestCase):
def test_manifest(self):
with open(MANIFEST) as f:
m = json.load(f)
for img in m:
r = requests.head(img['url'], timeout=10)
r.raise_for_status()
assert r.headers['Content-Type'] == "application/x-xz"
if not img['sparse']:
assert img['hash'] == img['hash_raw']
openpilot/common/hardware/comma/tests/test_amplifier.py
Executable
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
import time
import subprocess
import unittest
from panda import Panda
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.hardware import HARDWARE
from openpilot.common.hardware.comma.amplifier import Amplifier
class TestAmplifier(OpenpilotTestCase):
COMMA_HARDWARE_TEST = True
def setup_method(self):
# clear dmesg
subprocess.check_call("sudo dmesg -C", shell=True)
HARDWARE.reset_internal_panda()
Panda.wait_for_panda(None, 30)
self.panda = Panda()
def teardown_method(self):
HARDWARE.reset_internal_panda()
def _check_for_i2c_errors(self, expected):
dmesg = subprocess.check_output("dmesg", shell=True, encoding='utf8')
i2c_lines = [l for l in dmesg.strip().splitlines() if 'i2c_geni a88000.i2c' in l]
i2c_str = '\n'.join(i2c_lines)
if not expected:
return len(i2c_lines) == 0
else:
return "i2c error :-107" in i2c_str or "Bus arbitration lost" in i2c_str
def test_init(self):
amp = Amplifier(debug=True)
r = amp.initialize_configuration()
assert r
assert self._check_for_i2c_errors(False)
def test_shutdown(self):
amp = Amplifier(debug=True)
for _ in range(10):
r = amp.set_global_shutdown(True)
r = amp.set_global_shutdown(False)
# amp config should be successful, with no i2c errors
assert r
assert self._check_for_i2c_errors(False)
def test_init_while_siren_play(self):
for _ in range(10):
self.panda.set_siren(False)
time.sleep(0.1)
self.panda.set_siren(True)
time.sleep(0.1)
amp = Amplifier(debug=True)
r = amp.initialize_configuration()
assert r
if self._check_for_i2c_errors(True):
break
else:
self.fail("didn't hit any i2c errors")
if __name__ == "__main__":
unittest.main()
openpilot/common/hardware/comma/updater
Executable
BIN
View File
Binary file not shown.
openpilot/common/hardware/hw.h
+62
View File
@@ -0,0 +1,62 @@
#pragma once
#include <string>
#include "common/hardware/base.h"
#include "common/util.h"
#if __COMMA_HARDWARE__
#include "common/hardware/comma/hardware.h"
#define Hardware HardwareComma
#else
#include "common/hardware/pc/hardware.h"
#define Hardware HardwarePC
#endif
namespace Path {
inline std::string openpilot_prefix() {
return util::getenv("OPENPILOT_PREFIX", "");
}
inline std::string comma_home() {
return util::getenv("HOME") + "/.comma" + Path::openpilot_prefix();
}
inline std::string log_root() {
if (const char *env = getenv("LOG_ROOT")) {
return env;
}
return Hardware::PC() ? Path::comma_home() + "/media/0/realdata" : "/data/media/0/realdata";
}
inline std::string params() {
return util::getenv("PARAMS_ROOT", Hardware::PC() ? (Path::comma_home() + "/params") : "/data/params");
}
inline std::string rsa_file() {
return Hardware::PC() ? Path::comma_home() + "/persist/comma/id_rsa" : "/persist/comma/id_rsa";
}
inline std::string swaglog_ipc() {
return "ipc:///tmp/logmessage" + Path::openpilot_prefix();
}
inline std::string download_cache_root() {
if (const char *env = getenv("COMMA_CACHE")) {
return env;
}
return "/tmp/comma_download_cache" + Path::openpilot_prefix() + "/";
}
inline std::string shm_path() {
#ifdef __APPLE__
return"/tmp";
#else
return "/dev/shm";
#endif
}
inline std::string model_root() {
return Hardware::PC() ? Path::comma_home() + "/media/0/models" : "/data/media/0/models";
}
} // namespace Path
openpilot/common/hardware/hw.py
+97
View File
@@ -0,0 +1,97 @@
import os
import platform
from pathlib import Path
from openpilot.common.hardware import PC
DEFAULT_DOWNLOAD_CACHE_ROOT = "/tmp/comma_download_cache"
class Paths:
@staticmethod
def comma_home() -> str:
return os.path.join(str(Path.home()), ".comma" + os.environ.get("OPENPILOT_PREFIX", ""))
@staticmethod
def log_root() -> str:
if os.environ.get('LOG_ROOT', False):
return os.environ['LOG_ROOT']
elif PC:
return str(Path(Paths.comma_home()) / "media" / "0" / "realdata")
else:
return '/data/media/0/realdata/'
@staticmethod
def log_root_external() -> str:
return '/mnt/external_realdata/'
@staticmethod
def swaglog_root() -> str:
if PC:
return os.path.join(Paths.comma_home(), "log")
else:
return "/data/log/"
@staticmethod
def swaglog_ipc() -> str:
return "ipc:///tmp/logmessage" + os.environ.get("OPENPILOT_PREFIX", "")
@staticmethod
def download_cache_root() -> str:
if os.environ.get('COMMA_CACHE', False):
return os.environ['COMMA_CACHE'] + "/"
return DEFAULT_DOWNLOAD_CACHE_ROOT + os.environ.get("OPENPILOT_PREFIX", "") + "/"
@staticmethod
def persist_root() -> str:
if PC:
return os.path.join(Paths.comma_home(), "persist")
else:
return "/persist/"
@staticmethod
def stats_root() -> str:
if PC:
return str(Path(Paths.comma_home()) / "stats")
else:
return "/data/stats/"
@staticmethod
def stats_sp_root() -> str:
if PC:
return str(Path(Paths.comma_home()) / "stats")
else:
return "/data/stats_sp/"
@staticmethod
def config_root() -> str:
if PC:
return Paths.comma_home()
else:
return "/tmp/.comma"
@staticmethod
def shm_path() -> str:
if PC and platform.system() == "Darwin":
return "/tmp" # This is not really shared memory on macOS, but it's the closest we can get
return "/dev/shm"
@staticmethod
def model_root() -> str:
if PC:
return str(Path(Paths.comma_home()) / "media" / "0" / "models")
else:
return "/data/media/0/models"
@staticmethod
def crash_log_root() -> str:
if PC:
return str(Path(Paths.comma_home()) / "community" / "crashes")
else:
return "/data/community/crashes"
@staticmethod
def mapd_root() -> str:
if PC:
return str(Path(Paths.comma_home()) / "media" / "0" / "osm")
else:
return "/data/media/0/osm"
openpilot/common/hardware/pc/__init__.py
View File
openpilot/common/hardware/pc/hardware.h
+12
View File
@@ -0,0 +1,12 @@
#pragma once
#include <string>
#include "common/hardware/base.h"
class HardwarePC : public HardwareNone {
public:
static std::string get_name() { return "pc"; }
static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::PC; }
static bool PC() { return true; }
};
openpilot/common/hardware/pc/hardware.py
+10
View File
@@ -0,0 +1,10 @@
from openpilot.cereal import log
from openpilot.common.hardware.base import HardwareBase
class HardwarePc(HardwareBase):
def get_device_type(self):
return "pc"
def get_network_type(self):
# some stuff is gated on wifi, so just assume for now
return log.DeviceState.NetworkType.wifi
openpilot/common/hardware/usb.py
+87
View File
@@ -0,0 +1,87 @@
import os
from pathlib import Path
CHESTNUT_FW_VERSION = "ed4e39b7"
CHESTNUT_USB_IDS = ((0xADD1, 0x0001), (0x3801, 0x0001))
CHESTNUT_ROM_USB_IDS = ((0x174C, 0x2464), (0x174C, 0x2463))
USB_DEVICES_PATH = Path("/sys/bus/usb/devices")
TYPEC_CC_ORIENTATION_PATH = Path("/sys/class/power_supply/usb/typec_cc_orientation")
PRIMARY_USB_CONTROLLER = "a600000.ssusb"
def get_usb_topology() -> set[str]:
try:
return set(os.listdir(USB_DEVICES_PATH))
except OSError:
return set()
def read(path: Path) -> str | None:
try:
return path.read_text().strip()
except OSError:
return None
def read_int(path: Path, base: int = 10) -> int:
try:
return int(path.read_text(), base)
except (OSError, ValueError, TypeError):
return 0
def usb_devices() -> list[Path]:
try:
devices = (d for d in USB_DEVICES_PATH.glob("*") if (d / "idVendor").exists())
return sorted(devices, key=lambda p: p.name)
except OSError:
return []
def controller(device: Path) -> Path | None:
try:
return next((parent for parent in device.resolve().parents if parent.name.endswith(".ssusb")), None)
except OSError:
return None
def get_usb_state() -> list[dict]:
devices = []
typec_orientation = read_int(TYPEC_CC_ORIENTATION_PATH)
for device in usb_devices():
vendor_id = read_int(device / "idVendor", 16)
product_id = read_int(device / "idProduct", 16)
ctrl = controller(device)
devices.append({
"busnum": read_int(device / "busnum"),
"devnum": read_int(device / "devnum"),
"vendorId": vendor_id,
"productId": product_id,
"speedMbps": read_int(device / "speed"),
"manufacturer": read(device / "manufacturer") or "",
"product": read(device / "product") or "",
"linkErrorCount": read_int(ctrl / "portli", 0) & 0xFFFF if ctrl is not None else 0,
"usb3Lane": {1: "a", 2: "b"}.get(typec_orientation, "unknown") if ctrl is not None and ctrl.name == PRIMARY_USB_CONTROLLER else "unknown",
})
return devices
def set_usb_state(device_state, devices: list[dict]) -> None:
entries = device_state.usbState.init('devices', len(devices))
chestnut_present = False
for entry, device in zip(entries, devices, strict=True):
entry.busnum = device["busnum"]
entry.devnum = device["devnum"]
entry.vendorId = device["vendorId"]
entry.productId = device["productId"]
entry.speedMbps = device["speedMbps"]
entry.manufacturer = device["manufacturer"]
entry.product = device["product"]
entry.linkErrorCount = device["linkErrorCount"]
entry.usb3Lane = device.get("usb3Lane", "unknown")
if (entry.vendorId, entry.productId) in CHESTNUT_USB_IDS:
chestnut_present = True
device_state.chestnutPresent = chestnut_present
openpilot/common/i2c.py
+81
View File
@@ -0,0 +1,81 @@
import os
import fcntl
import ctypes
# I2C constants from /usr/include/linux/i2c-dev.h
I2C_SLAVE = 0x0703
I2C_SLAVE_FORCE = 0x0706
I2C_SMBUS = 0x0720
# SMBus transfer types
I2C_SMBUS_READ = 1
I2C_SMBUS_WRITE = 0
I2C_SMBUS_BYTE_DATA = 2
I2C_SMBUS_I2C_BLOCK_DATA = 8
I2C_SMBUS_BLOCK_MAX = 32
class _I2cSmbusData(ctypes.Union):
_fields_ = [
("byte", ctypes.c_uint8),
("word", ctypes.c_uint16),
("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
]
class _I2cSmbusIoctlData(ctypes.Structure):
_fields_ = [
("read_write", ctypes.c_uint8),
("command", ctypes.c_uint8),
("size", ctypes.c_uint32),
("data", ctypes.POINTER(_I2cSmbusData)),
]
class SMBus:
def __init__(self, bus: int):
self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
def __enter__(self) -> 'SMBus':
return self
def __exit__(self, *args) -> None:
self.close()
def close(self) -> None:
if hasattr(self, '_fd') and self._fd >= 0:
os.close(self._fd)
self._fd = -1
def _set_address(self, addr: int, force: bool = False) -> None:
ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
fcntl.ioctl(self._fd, ioctl_arg, addr)
def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
self._set_address(addr, force)
data = _I2cSmbusData()
self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
return int(data.byte)
def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
self._set_address(addr, force)
data = _I2cSmbusData()
data.byte = value & 0xFF
self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
self._set_address(addr, force)
if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
data = _I2cSmbusData()
data.block[0] = length
self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
read_len = int(data.block[0]) or length
read_len = min(read_len, length)
return [int(b) for b in data.block[1 : read_len + 1]]
openpilot/common/linux.py
+50
View File
@@ -0,0 +1,50 @@
class LinuxSystemStats:
def __init__(self) -> None:
self._last_cpu_times = self._read_cpu_times()
@staticmethod
def _read_cpu_times() -> dict[int, tuple[int, int]]:
cpu_times = {}
with open('/proc/stat') as f:
for line in f:
name, *values = line.split()
if not name.startswith('cpu') or not name[3:].isdigit():
continue
times = [int(value) for value in values]
idle = sum(times[3:5])
total = sum(times[:8])
cpu_times[int(name[3:])] = (idle, total)
return cpu_times
def cpu_usage_percent(self) -> list[float]:
current_cpu_times = self._read_cpu_times()
usage = []
for cpu, (idle, total) in sorted(current_cpu_times.items()):
last_times = self._last_cpu_times.get(cpu)
if last_times is None:
usage.append(0.)
continue
last_idle, last_total = last_times
idle_delta = idle - last_idle
total_delta = total - last_total
if idle_delta < 0 or total_delta <= 0:
usage.append(0.)
else:
usage.append(max(0., min(100., 100. * (total_delta - idle_delta) / total_delta)))
self._last_cpu_times = current_cpu_times
return usage
@staticmethod
def memory_usage_percent() -> float:
memory = {}
with open('/proc/meminfo') as f:
for line in f:
key, value, *_ = line.split()
if key in ('MemTotal:', 'MemAvailable:'):
memory[key] = int(value)
total = memory['MemTotal:']
return max(0., min(100., 100. * (total - memory['MemAvailable:']) / total))
openpilot/common/logging_extra.py
+249
View File
@@ -0,0 +1,249 @@
import io
import os
import sys
import copy
import json
import time
import uuid
import socket
import logging
import traceback
import numpy as np
from threading import local
from collections import OrderedDict
from contextlib import contextmanager
LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
def json_handler(obj):
if isinstance(obj, np.bool_):
return bool(obj)
# if isinstance(obj, (datetime.date, datetime.time)):
# return obj.isoformat()
return repr(obj)
def json_robust_dumps(obj):
return json.dumps(obj, default=json_handler)
class NiceOrderedDict(OrderedDict):
def __str__(self):
return json_robust_dumps(self)
class SwagFormatter(logging.Formatter):
def __init__(self, swaglogger):
logging.Formatter.__init__(self, None, '%a %b %d %H:%M:%S %Z %Y')
self.swaglogger = swaglogger
self.host = socket.gethostname()
def format_dict(self, record):
record_dict = NiceOrderedDict()
if isinstance(record.msg, dict):
record_dict['msg'] = record.msg
else:
try:
record_dict['msg'] = record.getMessage()
except (ValueError, TypeError):
record_dict['msg'] = [record.msg]+record.args
record_dict['ctx'] = self.swaglogger.get_ctx()
if record.exc_info:
record_dict['exc_info'] = self.formatException(record.exc_info)
record_dict['level'] = record.levelname
record_dict['levelnum'] = record.levelno
record_dict['name'] = record.name
record_dict['filename'] = record.filename
record_dict['lineno'] = record.lineno
record_dict['pathname'] = record.pathname
record_dict['module'] = record.module
record_dict['funcName'] = record.funcName
record_dict['host'] = self.host
record_dict['process'] = record.process
record_dict['thread'] = record.thread
record_dict['threadName'] = record.threadName
record_dict['created'] = record.created
return record_dict
def format(self, record):
if self.swaglogger is None:
raise Exception("must set swaglogger before calling format()")
return json_robust_dumps(self.format_dict(record))
class SwagLogFileFormatter(SwagFormatter):
def fix_kv(self, k, v):
# append type to names to preserve legacy naming in logs
# avoids overlapping key namespaces with different types
# e.g. log.info() creates 'msg' -> 'msg$s'
# log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i'
# because overlapping namespace 'msg' caused problems
if isinstance(v, (str, bytes)):
k += "$s"
elif isinstance(v, float):
k += "$f"
elif isinstance(v, bool):
k += "$b"
elif isinstance(v, int):
k += "$i"
elif isinstance(v, dict):
nv = {}
for ik, iv in v.items():
ik, iv = self.fix_kv(ik, iv)
nv[ik] = iv
v = nv
elif isinstance(v, list):
k += "$a"
return k, v
def format(self, record):
if isinstance(record, str):
v = json.loads(record)
else:
v = self.format_dict(record)
mk, mv = self.fix_kv('msg', v['msg'])
del v['msg']
v[mk] = mv
v['id'] = uuid.uuid4().hex
return json_robust_dumps(v)
class SwagErrorFilter(logging.Filter):
def filter(self, record):
return record.levelno < logging.ERROR
def _tmpfunc():
return 0
def _srcfile():
return os.path.normcase(_tmpfunc.__code__.co_filename)
class SwagLogger(logging.Logger):
def __init__(self):
logging.Logger.__init__(self, "swaglog")
self.global_ctx = {}
self.log_local = local()
self.log_local.ctx = {}
def local_ctx(self):
try:
return self.log_local.ctx
except AttributeError:
self.log_local.ctx = {}
return self.log_local.ctx
def get_ctx(self):
return dict(self.local_ctx(), **self.global_ctx)
@contextmanager
def ctx(self, **kwargs):
old_ctx = self.local_ctx()
self.log_local.ctx = copy.copy(old_ctx) or {}
self.log_local.ctx.update(kwargs)
try:
yield
finally:
self.log_local.ctx = old_ctx
def bind(self, **kwargs):
self.local_ctx().update(kwargs)
def bind_global(self, **kwargs):
self.global_ctx.update(kwargs)
def event(self, event, *args, **kwargs):
evt = NiceOrderedDict()
evt['event'] = event
if args:
evt['args'] = args
evt.update(kwargs)
if 'error' in kwargs:
self.error(evt)
elif 'debug' in kwargs:
self.debug(evt)
else:
self.info(evt)
def timestamp(self, event_name):
if LOG_TIMESTAMPS:
t = time.monotonic()
tstp = NiceOrderedDict()
tstp['timestamp'] = NiceOrderedDict()
tstp['timestamp']["event"] = event_name
tstp['timestamp']["time"] = t*1e9
self.debug(tstp)
def findCaller(self, stack_info=False, stacklevel=1):
"""
Find the stack frame of the caller so that we can note the source
file name, line number and function name.
"""
f = sys._getframe(3)
#On some versions of IronPython, currentframe() returns None if
#IronPython isn't run with -X:Frames.
if f is not None:
f = f.f_back
orig_f = f
while f and stacklevel > 1:
f = f.f_back
stacklevel -= 1
if not f:
f = orig_f
rv = "(unknown file)", 0, "(unknown function)", None
while hasattr(f, "f_code"):
co = f.f_code
filename = os.path.normcase(co.co_filename)
if filename == _srcfile:
f = f.f_back
continue
sinfo = None
if stack_info:
sio = io.StringIO()
sio.write('Stack (most recent call last):\n')
traceback.print_stack(f, file=sio)
sinfo = sio.getvalue()
if sinfo[-1] == '\n':
sinfo = sinfo[:-1]
sio.close()
rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
break
return rv
if __name__ == "__main__":
log = SwagLogger()
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setLevel(logging.INFO)
stdout_handler.addFilter(SwagErrorFilter())
log.addHandler(stdout_handler)
stderr_handler = logging.StreamHandler(sys.stderr)
stderr_handler.setLevel(logging.ERROR)
log.addHandler(stderr_handler)
log.info("asdasd %s", "a")
log.info({'wut': 1})
log.warning("warning")
log.error("error")
log.critical("critical")
log.event("test", x="y")
with log.ctx():
stdout_handler.setFormatter(SwagFormatter(log))
stderr_handler.setFormatter(SwagFormatter(log))
log.bind(user="some user")
log.info("in req")
print("")
log.warning("warning")
print("")
log.error("error")
print("")
log.critical("critical")
print("")
log.event("do_req", a=1, b="c")
openpilot/common/markdown.py
+45
View File
@@ -0,0 +1,45 @@
HTML_REPLACEMENTS = [
(r'&', r'&amp;'),
(r'"', r'&quot;'),
]
def parse_markdown(text: str, tab_length: int = 2) -> str:
lines = text.split("\n")
output: list[str] = []
list_level = 0
def end_outstanding_lists(level: int, end_level: int) -> int:
while level > end_level:
level -= 1
output.append("</ul>")
if level > 0:
output.append("</li>")
return end_level
for i, line in enumerate(lines):
if i + 1 < len(lines) and lines[i + 1].startswith("==="): # heading
output.append(f"<h1>{line}</h1>")
elif line.startswith("==="):
pass
elif line.lstrip().startswith("* "): # list
line_level = 1 + line.count(" " * tab_length, 0, line.index("*"))
if list_level >= line_level:
list_level = end_outstanding_lists(list_level, line_level)
else:
list_level += 1
if list_level > 1:
output[-1] = output[-1].replace("</li>", "")
output.append("<ul>")
output.append(f"<li>{line.replace('*', '', 1).lstrip()}</li>")
else:
list_level = end_outstanding_lists(list_level, 0)
if len(line) > 0:
output.append(line)
end_outstanding_lists(list_level, 0)
output_str = "\n".join(output) + "\n"
for (fr, to) in HTML_REPLACEMENTS:
output_str = output_str.replace(fr, to)
return output_str
openpilot/common/mock/__init__.py
+50
View File
@@ -0,0 +1,50 @@
"""
Utilities for generating mock messages for testing.
example in openpilot/common/tests/test_mock.py
"""
import functools
import threading
from openpilot.cereal.messaging import PubMaster
from openpilot.cereal.services import SERVICE_LIST
from openpilot.common.mock.generators import generate_deviceMotion
from openpilot.common.realtime import Ratekeeper
MOCK_GENERATOR = {
"deviceMotion": generate_deviceMotion
}
def generate_messages_loop(services: list[str], done: threading.Event):
pm = PubMaster(services)
rk = Ratekeeper(100)
i = 0
while not done.is_set():
for s in services:
should_send = i % (100/SERVICE_LIST[s].frequency) == 0
if should_send:
message = MOCK_GENERATOR[s]()
pm.send(s, message)
i += 1
rk.keep_time()
def mock_messages(services: list[str] | str):
if isinstance(services, str):
services = [services]
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
done = threading.Event()
t = threading.Thread(target=generate_messages_loop, args=(services, done))
t.start()
try:
return func(*args, **kwargs)
finally:
done.set()
t.join()
return wrapper
return decorator
openpilot/common/mock/generators.py
+14
View File
@@ -0,0 +1,14 @@
from openpilot.cereal import messaging
def generate_deviceMotion():
msg = messaging.new_message('deviceMotion')
meas = {'x': 0.0, 'y': 0.0, 'z': 0.0, 'xStd': 0.0, 'yStd': 0.0, 'zStd': 0.0, 'valid': True}
msg.deviceMotion.orientationNED = meas
msg.deviceMotion.velocityDevice = meas
msg.deviceMotion.angularVelocityDevice = meas
msg.deviceMotion.accelerationDevice = meas
msg.deviceMotion.inputsOK = True
msg.deviceMotion.posenetOK = True
msg.deviceMotion.sensorsOK = True
return msg
openpilot/common/parameterized.py
+93
View File
@@ -0,0 +1,93 @@
import re
import sys
import inspect
import unittest
def _to_safe_name(s):
return re.sub(r"[^a-zA-Z0-9_]+", "_", str(s)).strip("_")
class parameterized:
@staticmethod
def expand(cases, names=None, ids=None, serial=False):
cases = list(cases)
if not cases:
return lambda func: unittest.skip("no parameterized cases")(func)
if serial:
def decorator(func):
normalized = [case if isinstance(case, tuple) else (case,) for case in cases]
def wrapper(self):
for case in normalized:
with self.subTest():
func(self, *case)
wrapper.__name__ = func.__name__
wrapper.__doc__ = func.__doc__
return wrapper
return decorator
return lambda func: _Expanded(func, cases, names, ids)
class _Expanded:
"""Descriptor that turns every parameter case into a real unittest method."""
def __init__(self, func, cases, names, ids):
self.func = func
self.cases = [c if isinstance(c, tuple) else (c,) for c in cases]
self.names = names
self.ids = ids
def __set_name__(self, owner, name):
params = [p for p in inspect.signature(self.func).parameters if p != "self"]
for index, case in enumerate(self.cases):
label = self.ids(*case) if self.ids is not None else None
method_name = f"{name}_{index}" + (f"_{_to_safe_name(label)}" if label is not None else "")
def test_method(test_case, current_case=case):
if self.names is None:
self.func(test_case, *current_case)
else:
values = dict(zip(self.names, current_case, strict=True))
values.update({param: test_case._fixture(param) for param in params if param not in values})
self.func(test_case, **values)
test_method.__name__ = method_name
test_method.__doc__ = self.func.__doc__
setattr(owner, method_name, test_method)
# The descriptor itself is only a method factory, not a test.
setattr(owner, name, None)
def parameterized_class(attrs, input_list=None):
if isinstance(attrs, list) and (not attrs or isinstance(attrs[0], dict)):
params_list = attrs
else:
assert input_list is not None
attr_names = (attrs,) if isinstance(attrs, str) else tuple(attrs)
params_list = [dict(zip(attr_names, v if isinstance(v, (tuple, list)) else (v,), strict=False)) for v in input_list]
def decorator(cls):
globs = sys._getframe(1).f_globals
for i, params in enumerate(params_list):
# Append sanitized values so unittest's -k can filter by them.
suffix = "_".join(filter(None, (_to_safe_name(v) for v in params.values() if isinstance(v, str))))
name = f"{cls.__name__}_{i}" + (f"_{suffix}" if suffix else "")
new_cls = type(name, (cls,), dict(params))
new_cls.__module__ = cls.__module__
new_cls.__unittest_skip__ = False
new_cls.__unittest_skip_why__ = "" # else inherited from the base and the collector drops it
globs[name] = new_cls
# Don't collect the un-parametrised base.
cls.__unittest_skip__ = True
cls.__unittest_skip_why__ = "parameterized base class"
return cls
return decorator
openpilot/common/params.cc
+244
View File
@@ -0,0 +1,244 @@
#include "common/params.h"
#include <dirent.h>
#include <sys/file.h>
#include <algorithm>
#include <cassert>
#include <csignal>
#include <unordered_map>
#include "common/params_keys.h"
#include "common/queue.h"
#include "common/swaglog.h"
#include "common/util.h"
#include "common/hardware/hw.h"
namespace {
volatile sig_atomic_t params_do_exit = 0;
void params_sig_handler(int signal) {
params_do_exit = 1;
}
int fsync_dir(const std::string &path) {
int result = -1;
int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
if (fd >= 0) {
result = HANDLE_EINTR(fsync(fd));
HANDLE_EINTR(close(fd));
}
return result;
}
bool create_params_path(const std::string &param_path, const std::string &key_path) {
// Make sure params path exists
if (!util::file_exists(param_path) && !util::create_directories(param_path, 0775)) {
return false;
}
// See if the symlink exists, otherwise create it
if (!util::file_exists(key_path)) {
// 1) Create temp folder
// 2) Symlink it to temp link
// 3) Move symlink to <params>/d
std::string tmp_path = param_path + "/.tmp_XXXXXX";
// this should be OK since mkdtemp just replaces characters in place
char *tmp_dir = mkdtemp((char *)tmp_path.c_str());
if (tmp_dir == NULL) {
return false;
}
std::string link_path = std::string(tmp_dir) + ".link";
if (symlink(tmp_dir, link_path.c_str()) != 0) {
return false;
}
// don't return false if it has been created by other
if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) {
return false;
}
}
return true;
}
std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
std::string params_path = path.empty() ? Path::params() : path;
if (!create_params_path(params_path, params_path + prefix)) {
throw std::runtime_error(util::string_format(
"Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
errno, params_path.c_str(), prefix.c_str()));
}
return params_path;
}
class FileLock {
public:
FileLock(const std::string &fn) {
fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
}
}
~FileLock() { close(fd_); }
private:
int fd_ = -1;
};
} // namespace
Params::Params(const std::string &path) {
params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
params_path = ensure_params_path(params_prefix, path);
}
Params::~Params() {
if (future.valid()) {
future.wait();
}
assert(queue.empty());
}
std::vector<std::string> Params::allKeys(ParamKeyFlag flag) const {
std::vector<std::string> ret;
for (auto &p : keys) {
if (flag == ALL || (p.second.flags & flag)) {
ret.push_back(p.first);
}
}
return ret;
}
bool Params::checkKey(const std::string &key) {
return keys.find(key) != keys.end();
}
ParamKeyFlag Params::getKeyFlag(const std::string &key) {
return static_cast<ParamKeyFlag>(keys[key].flags);
}
ParamKeyType Params::getKeyType(const std::string &key) {
return keys[key].type;
}
std::optional<std::string> Params::getKeyDefaultValue(const std::string &key) {
return keys[key].default_value;
}
int Params::put(const char* key, const char* value, size_t value_size) {
// Information about safely and atomically writing a file: https://lwn.net/Articles/457667/
// 1) Create temp file
// 2) Write data to temp file
// 3) fsync() the temp file
// 4) rename the temp file to the real name
// 5) fsync() the containing directory
std::string tmp_path = params_path + "/.tmp_value_XXXXXX";
int tmp_fd = mkstemp((char*)tmp_path.c_str());
if (tmp_fd < 0) return -1;
int result = -1;
do {
// Write value to temp.
ssize_t bytes_written = HANDLE_EINTR(write(tmp_fd, value, value_size));
if (bytes_written < 0 || (size_t)bytes_written != value_size) {
result = -20;
break;
}
// fsync to force persist the changes.
if ((result = HANDLE_EINTR(fsync(tmp_fd))) < 0) break;
FileLock file_lock(params_path + "/.lock");
// Move temp into place.
if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break;
// fsync parent directory
result = fsync_dir(getParamPath());
} while (false);
close(tmp_fd);
if (result != 0) {
::unlink(tmp_path.c_str());
}
return result;
}
int Params::remove(const std::string &key) {
FileLock file_lock(params_path + "/.lock");
int result = unlink(getParamPath(key).c_str());
if (result != 0) {
return result;
}
return fsync_dir(getParamPath());
}
std::string Params::get(const std::string &key, bool block) {
if (!block) {
return util::read_file(getParamPath(key));
} else {
// blocking read until successful
params_do_exit = 0;
void (*prev_handler_sigint)(int) = std::signal(SIGINT, params_sig_handler);
void (*prev_handler_sigterm)(int) = std::signal(SIGTERM, params_sig_handler);
std::string value;
while (!params_do_exit) {
if (value = util::read_file(getParamPath(key)); !value.empty()) {
break;
}
util::sleep_for(100); // 0.1 s
}
std::signal(SIGINT, prev_handler_sigint);
std::signal(SIGTERM, prev_handler_sigterm);
return value;
}
}
std::map<std::string, std::string> Params::readAll() {
FileLock file_lock(params_path + "/.lock");
return util::read_files_in_dir(getParamPath());
}
void Params::clearAll(ParamKeyFlag key_flag) {
FileLock file_lock(params_path + "/.lock");
// 1) delete params of key_flag
// 2) delete files that are not defined in the keys.
if (DIR *d = opendir(getParamPath().c_str())) {
struct dirent *de = NULL;
while ((de = readdir(d))) {
if (de->d_type != DT_DIR) {
auto it = keys.find(de->d_name);
if (it == keys.end() || (it->second.flags & key_flag)) {
unlink(getParamPath(de->d_name).c_str());
}
}
}
closedir(d);
}
fsync_dir(getParamPath());
}
void Params::putNonBlocking(const std::string &key, const std::string &val) {
queue.push(std::make_pair(key, val));
// start thread on demand
if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
future = std::async(std::launch::async, &Params::asyncWriteThread, this);
}
}
void Params::asyncWriteThread() {
// TODO: write the latest one if a key has multiple values in the queue.
std::pair<std::string, std::string> p;
while (queue.try_pop(p, 0)) {
// Params::put is Thread-Safe
put(p.first, p.second);
}
}
openpilot/common/params.h
+91
View File
@@ -0,0 +1,91 @@
#pragma once
#include <future>
#include <map>
#include <optional>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "common/queue.h"
enum ParamKeyFlag {
PERSISTENT = 0x02,
CLEAR_ON_MANAGER_START = 0x04,
CLEAR_ON_ONROAD_TRANSITION = 0x08,
CLEAR_ON_OFFROAD_TRANSITION = 0x10,
DONT_LOG = 0x20,
DEVELOPMENT_ONLY = 0x40,
CLEAR_ON_IGNITION_ON = 0x80,
BACKUP = 0x100,
ALL = 0xFFFFFFFF
};
enum ParamKeyType {
STRING = 0, // must be utf-8 decodable
BOOL = 1,
INT = 2,
FLOAT = 3,
TIME = 4, // ISO 8601
JSON = 5,
BYTES = 6
};
struct ParamKeyAttributes {
uint32_t flags;
ParamKeyType type;
std::optional<std::string> default_value = std::nullopt;
};
class Params {
public:
explicit Params(const std::string &path = {});
~Params();
// Not copyable.
Params(const Params&) = delete;
Params& operator=(const Params&) = delete;
std::vector<std::string> allKeys(ParamKeyFlag flag = ALL) const;
bool checkKey(const std::string &key);
ParamKeyFlag getKeyFlag(const std::string &key);
ParamKeyType getKeyType(const std::string &key);
std::optional<std::string> getKeyDefaultValue(const std::string &key);
inline std::string getParamPath(const std::string &key = {}) {
return params_path + params_prefix + (key.empty() ? "" : "/" + key);
}
// Delete a value
int remove(const std::string &key);
void clearAll(ParamKeyFlag flag);
// helpers for reading values
std::string get(const std::string &key, bool block = false);
inline bool getBool(const std::string &key, bool block = false) {
return get(key, block) == "1";
}
std::map<std::string, std::string> readAll();
// helpers for writing values
int put(const char *key, const char *val, size_t value_size);
inline int put(const std::string &key, const std::string &val) {
return put(key.c_str(), val.data(), val.size());
}
inline int putBool(const std::string &key, bool val) {
return put(key.c_str(), val ? "1" : "0", 1);
}
void putNonBlocking(const std::string &key, const std::string &val);
inline void putBoolNonBlocking(const std::string &key, bool val) {
putNonBlocking(key, val ? "1" : "0");
}
private:
void asyncWriteThread();
std::string params_path;
std::string params_prefix;
// for nonblocking write
std::future<void> future;
SafeQueue<std::pair<std::string, std::string>> queue;
};
openpilot/common/params.py
+221
View File
@@ -0,0 +1,221 @@
import sys
import json
import ctypes
import weakref
import builtins
import datetime
from pathlib import Path
from enum import IntEnum, IntFlag
from openpilot.common.swaglog import cloudlog
class ParamKeyFlag(IntFlag):
PERSISTENT = 0x02
CLEAR_ON_MANAGER_START = 0x04
CLEAR_ON_ONROAD_TRANSITION = 0x08
CLEAR_ON_OFFROAD_TRANSITION = 0x10
DEVELOPMENT_ONLY = 0x40
CLEAR_ON_IGNITION_ON = 0x80
BACKUP = 0x100
ALL = 0xFFFFFFFF
class ParamKeyType(IntEnum):
STRING = 0
BOOL = 1
INT = 2
FLOAT = 3
TIME = 4
JSON = 5
BYTES = 6
_suffix = ".dylib" if sys.platform == "darwin" else ".so"
lib = ctypes.CDLL(Path(__file__).with_name(f"libparams_c{_suffix}"))
ParamsHandle = ctypes.c_void_p
class ParamsBuffer(ctypes.Structure):
_fields_ = [("data", ctypes.c_void_p), ("size", ctypes.c_size_t)]
def _bind_raw(name, args, result=None):
function = getattr(lib, name)
function.argtypes = args
function.restype = result
return function
params_last_error = _bind_raw("params_last_error", [], ctypes.c_char_p)
def _bind(name, args, result=None):
function = _bind_raw(name, args, result)
def checked(*call_args):
value = function(*call_args)
if error := params_last_error():
raise RuntimeError(error.decode())
return value
return checked
params_create = _bind("params_create", [ctypes.c_char_p, ctypes.c_size_t], ParamsHandle)
params_destroy = _bind("params_destroy", [ParamsHandle])
params_clear_all = _bind("params_clear_all", [ParamsHandle, ctypes.c_uint])
params_check_key = _bind("params_check_key", [ParamsHandle, ctypes.c_char_p], ctypes.c_bool)
params_get_key_type = _bind("params_get_key_type", [ParamsHandle, ctypes.c_char_p], ctypes.c_int)
params_get_default = _bind("params_get_default", [ParamsHandle, ctypes.c_char_p], ParamsBuffer)
params_get = _bind("params_get", [ParamsHandle, ctypes.c_char_p, ctypes.c_bool], ParamsBuffer)
params_get_bool = _bind("params_get_bool", [ParamsHandle, ctypes.c_char_p, ctypes.c_bool], ctypes.c_bool)
params_put = _bind("params_put", [ParamsHandle, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.c_bool], ctypes.c_int)
params_put_bool = _bind("params_put_bool", [ParamsHandle, ctypes.c_char_p, ctypes.c_bool, ctypes.c_bool], ctypes.c_int)
params_remove = _bind("params_remove", [ParamsHandle, ctypes.c_char_p], ctypes.c_int)
params_get_path = _bind("params_get_path", [ParamsHandle, ctypes.c_char_p, ctypes.c_size_t], ParamsBuffer)
params_keys_size = _bind("params_keys_size", [ParamsHandle], ctypes.c_size_t)
params_key_at = _bind("params_key_at", [ParamsHandle, ctypes.c_size_t], ParamsBuffer)
params_keys_by_flag = _bind("params_keys_by_flag", [ParamsHandle, ctypes.c_uint, ctypes.POINTER(ParamsBuffer), ctypes.c_size_t], ctypes.c_size_t)
PYTHON_2_CPP = {
(str, ParamKeyType.STRING): lambda v: v,
(builtins.bool, ParamKeyType.BOOL): lambda v: "1" if v else "0",
(int, ParamKeyType.INT): str,
(float, ParamKeyType.FLOAT): str,
(datetime.datetime, ParamKeyType.TIME): lambda v: v.isoformat(),
(dict, ParamKeyType.JSON): json.dumps,
(list, ParamKeyType.JSON): json.dumps,
(bytes, ParamKeyType.BYTES): lambda v: v,
}
CPP_2_PYTHON = {
ParamKeyType.STRING: lambda v: v.decode("utf-8"),
ParamKeyType.BOOL: lambda v: v == b"1",
ParamKeyType.INT: int,
ParamKeyType.FLOAT: float,
ParamKeyType.TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
ParamKeyType.JSON: json.loads,
ParamKeyType.BYTES: lambda v: v,
}
def ensure_bytes(v):
return v.encode() if isinstance(v, str) else v
def _copy_string(value):
if value.data is None:
return None
return ctypes.string_at(value.data, value.size)
class UnknownKeyName(Exception):
pass
class Params:
def __init__(self, d=""):
path = ensure_bytes(d)
self.p = params_create(path, len(path))
self._finalizer = weakref.finalize(self, params_destroy, self.p)
self._finalizer.atexit = False # daemon threads can still be using live Params handles during interpreter shutdown
self.d = d
def __reduce__(self):
return (type(self), (self.d,))
def clear_all(self, tx_flag=ParamKeyFlag.ALL):
params_clear_all(self.p, int(tx_flag))
def check_key(self, key):
key = ensure_bytes(key)
if b"\0" in key or not params_check_key(self.p, key):
raise UnknownKeyName(key)
return key
def python2cpp(self, proposed_type, expected_type, value, key):
cast = PYTHON_2_CPP.get((proposed_type, expected_type))
if cast:
return cast(value)
raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
def _cpp2python(self, t, value, default, key):
if value is None:
return None
try:
return CPP_2_PYTHON[t](value)
except (KeyError, TypeError, ValueError):
cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
return self._cpp2python(t, default, None, key)
def _default(self, key):
return _copy_string(params_get_default(self.p, key))
def get(self, key, block=False, return_default=False):
k = self.check_key(key)
t = self.get_type(k)
default = self._default(k) if return_default else None
value = _copy_string(params_get(self.p, k, block))
if value == b"":
if block:
raise KeyboardInterrupt
return self._cpp2python(t, default, None, key)
return self._cpp2python(t, value, default, key)
def get_bool(self, key, block=False):
return bool(params_get_bool(self.p, self.check_key(key), block))
def _put_cast(self, key, dat):
return ensure_bytes(self.python2cpp(type(dat), self.get_type(key), dat, key))
def put(self, key, dat, block=False):
"""Write a parameter. block=True waits until it is persisted to disk."""
k = self.check_key(key)
value = self._put_cast(k, dat)
params_put(self.p, k, value, len(value), block)
def put_bool(self, key, val, block=False):
params_put_bool(self.p, self.check_key(key), val, block)
def remove(self, key):
params_remove(self.p, self.check_key(key))
def get_param_path(self, key=""):
key = ensure_bytes(key)
return _copy_string(params_get_path(self.p, key, len(key))).decode()
def get_type(self, key):
return ParamKeyType(params_get_key_type(self.p, self.check_key(key)))
def all_keys(self, flag=ParamKeyFlag.ALL):
if flag == ParamKeyFlag.ALL:
keys = []
for i in range(params_keys_size(self.p)):
keys.append(_copy_string(params_key_at(self.p, i)))
return keys
max_keys = 1024
buf = (ParamsBuffer * max_keys)()
count = params_keys_by_flag(self.p, int(flag), buf, max_keys)
return [_copy_string(buf[i]) for i in range(min(count, max_keys))]
def get_default_value(self, key):
k = self.check_key(key)
return self._cpp2python(self.get_type(k), self._default(k), None, key)
def cpp2python(self, key, value):
return self._cpp2python(self.get_type(key), value, None, key)
if __name__ == "__main__":
import sys
params = Params()
key = sys.argv[1]
params.check_key(key)
if len(sys.argv) == 3:
val = sys.argv[2]
print(f"SET: {key} = {val}")
params.put(key, val, block=True)
elif len(sys.argv) == 2:
print(f"GET: {key} = {params.get(key)}")
openpilot/common/params_c.cc
+174
View File
@@ -0,0 +1,174 @@
#include <cstddef>
#include <cstdio>
#include <exception>
#include <string>
#include <utility>
#include <vector>
#include "common/params.h"
typedef struct {
const char *data;
size_t size;
} ParamsBuffer;
struct ParamsHandle {
ParamsHandle(const char *path, size_t path_size) : params(std::string(path, path_size)), keys(params.allKeys()) {
}
Params params;
const std::vector<std::string> keys;
};
namespace {
thread_local char last_error[512] = {};
thread_local std::string result;
void set_error(const char *error) {
snprintf(last_error, sizeof(last_error), "%s", error);
}
ParamsBuffer return_string(std::string value) {
result = std::move(value);
return {result.data(), result.size()};
}
template <typename Result, typename Callable>
Result translate_exceptions(Result failure, Callable &&callable) noexcept {
last_error[0] = '\0';
try {
return callable();
} catch (const std::exception &e) {
set_error(e.what());
} catch (...) {
set_error("unknown C++ exception");
}
return failure;
}
template <typename Callable>
void translate_exceptions(Callable &&callable) noexcept {
translate_exceptions(false, [&]() {
callable();
return true;
});
}
} // namespace
extern "C" {
ParamsHandle *params_create(const char *path, size_t path_size) noexcept {
return translate_exceptions(static_cast<ParamsHandle *>(nullptr), [&]() {
return new ParamsHandle(path, path_size);
});
}
void params_destroy(ParamsHandle *handle) noexcept {
translate_exceptions([&]() {
delete handle;
});
}
const char *params_last_error() noexcept {
return last_error;
}
void params_clear_all(ParamsHandle *handle, unsigned int flag) noexcept {
translate_exceptions([&]() {
handle->params.clearAll(static_cast<ParamKeyFlag>(flag));
});
}
bool params_check_key(ParamsHandle *handle, const char *key) noexcept {
return translate_exceptions(false, [&]() {
return handle->params.checkKey(key);
});
}
int params_get_key_type(ParamsHandle *handle, const char *key) noexcept {
return translate_exceptions(-1, [&]() {
return static_cast<int>(handle->params.getKeyType(key));
});
}
ParamsBuffer params_get_default(ParamsHandle *handle, const char *key) noexcept {
return translate_exceptions(ParamsBuffer{nullptr, 0}, [&]() {
auto value = handle->params.getKeyDefaultValue(key);
if (!value.has_value()) {
return ParamsBuffer{nullptr, 0};
}
return return_string(*value);
});
}
ParamsBuffer params_get(ParamsHandle *handle, const char *key, bool block) noexcept {
return translate_exceptions(ParamsBuffer{nullptr, 0}, [&]() {
return return_string(handle->params.get(key, block));
});
}
bool params_get_bool(ParamsHandle *handle, const char *key, bool block) noexcept {
return translate_exceptions(false, [&]() {
return handle->params.getBool(key, block);
});
}
int params_put(ParamsHandle *handle, const char *key, const char *value, size_t size, bool block) noexcept {
return translate_exceptions(-1, [&]() {
if (block) {
return handle->params.put(key, value, size);
}
handle->params.putNonBlocking(key, std::string(value, size));
return 0;
});
}
int params_put_bool(ParamsHandle *handle, const char *key, bool value, bool block) noexcept {
return translate_exceptions(-1, [&]() {
if (block) {
return handle->params.putBool(key, value);
}
handle->params.putBoolNonBlocking(key, value);
return 0;
});
}
int params_remove(ParamsHandle *handle, const char *key) noexcept {
return translate_exceptions(-1, [&]() {
return handle->params.remove(key);
});
}
ParamsBuffer params_get_path(ParamsHandle *handle, const char *key, size_t key_size) noexcept {
return translate_exceptions(ParamsBuffer{nullptr, 0}, [&]() {
return return_string(handle->params.getParamPath(std::string(key, key_size)));
});
}
size_t params_keys_size(ParamsHandle *handle) noexcept {
return translate_exceptions(size_t{0}, [&]() {
return handle->keys.size();
});
}
ParamsBuffer params_key_at(ParamsHandle *handle, size_t index) noexcept {
return translate_exceptions(ParamsBuffer{nullptr, 0}, [&]() {
if (index >= handle->keys.size()) {
return ParamsBuffer{nullptr, 0};
}
return return_string(handle->keys[index]);
});
}
size_t params_keys_by_flag(ParamsHandle *handle, uint32_t flag, ParamsBuffer *out, size_t out_size) noexcept {
return translate_exceptions(size_t{0}, [&]() {
auto filtered = handle->params.allKeys(static_cast<ParamKeyFlag>(flag));
size_t count = std::min(filtered.size(), out_size);
for (size_t i = 0; i < count; i++) {
out[i] = return_string(filtered[i]);
}
return filtered.size();
});
}
} // extern "C"
openpilot/common/params_keys.h
+288
View File
@@ -0,0 +1,288 @@
#pragma once
#include <string>
#include <unordered_map>
#include "openpilot/cereal/gen/cpp/log.capnp.h"
inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
{"AdbEnabled", {PERSISTENT | BACKUP, BOOL}},
{"AlwaysOnDM", {PERSISTENT | BACKUP, BOOL}},
{"ApiCache_Device", {PERSISTENT, STRING}},
{"ApiCache_FirehoseStats", {PERSISTENT, JSON}},
{"AssistNowToken", {PERSISTENT, STRING}},
{"AthenadPid", {PERSISTENT, INT}},
{"AthenadUploadQueue", {PERSISTENT, JSON}},
{"AthenadRecentlyViewedRoutes", {PERSISTENT, STRING}},
{"BootCount", {PERSISTENT, INT}},
{"CalibrationParams", {PERSISTENT, BYTES}},
{"CameraDebugExpGain", {CLEAR_ON_MANAGER_START, STRING}},
{"CameraDebugExpTime", {CLEAR_ON_MANAGER_START, STRING}},
{"CarBatteryCapacity", {PERSISTENT, INT}},
{"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
{"CarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
{"CarParamsPersistent", {PERSISTENT, BYTES}},
{"CarParamsPrevRoute", {PERSISTENT, BYTES}},
{"CompletedTrainingVersion", {PERSISTENT, STRING, "0"}},
{"ControlsReady", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
{"CurrentBootlog", {PERSISTENT, STRING}},
{"CurrentRoute", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
{"DisableLogging", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
{"DisablePowerDown", {PERSISTENT | BACKUP, BOOL}},
{"DisableUpdates", {PERSISTENT | BACKUP, BOOL, "0"}},
{"DisengageOnAccelerator", {PERSISTENT | BACKUP, BOOL, "0"}},
{"DongleId", {PERSISTENT, STRING}},
{"DoReboot", {CLEAR_ON_MANAGER_START, BOOL}},
{"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
{"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
{"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
{"DriverLockoutCount", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, INT, "0"}},
{"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY | BACKUP, BOOL}},
{"ExperimentalMode", {PERSISTENT | BACKUP, BOOL}},
{"ExperimentalModeConfirmed", {PERSISTENT | BACKUP, BOOL}},
{"FirmwareQueryDone", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
{"ForcePowerDown", {PERSISTENT, BOOL}},
{"GitBranch", {PERSISTENT, STRING}},
{"GitCommit", {PERSISTENT, STRING}},
{"GitCommitDate", {PERSISTENT, STRING}},
{"GitDiff", {PERSISTENT, STRING}},
{"GithubSshKeys", {PERSISTENT | BACKUP, STRING}},
{"GithubUsername", {PERSISTENT | BACKUP, STRING}},
{"GitRemote", {PERSISTENT, STRING}},
{"GsmApn", {PERSISTENT | BACKUP, STRING}},
{"GsmMetered", {PERSISTENT | BACKUP, BOOL, "1"}},
{"GsmRoaming", {PERSISTENT | BACKUP, BOOL}},
{"HardwareSerial", {PERSISTENT, STRING}},
{"HasAcceptedTerms", {PERSISTENT, STRING, "0"}},
{"InstallDate", {PERSISTENT, TIME}},
{"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsEngaged", {PERSISTENT, BOOL}},
{"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}},
{"IsLiveStreaming", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsMetric", {PERSISTENT | BACKUP, BOOL}},
{"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsRhdDetected", {PERSISTENT, BOOL}},
{"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
{"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"LanguageSetting", {PERSISTENT | BACKUP, STRING, "en"}},
{"LastAthenaPingTime", {CLEAR_ON_MANAGER_START, INT}},
{"LastGPSPosition", {PERSISTENT, STRING}},
{"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
{"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
{"LastAgnosPowerMonitorShutdown", {CLEAR_ON_MANAGER_START, STRING}},
{"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
{"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
{"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
{"LastUpdateTime", {PERSISTENT, TIME}},
{"LastUpdateUptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
{"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
{"LiveParameters", {PERSISTENT, JSON}},
{"LiveParametersV2", {PERSISTENT, BYTES}},
{"LivestreamEncoderBitrate", {CLEAR_ON_MANAGER_START | DONT_LOG, INT}},
{"LivestreamRequestKeyframe", {CLEAR_ON_MANAGER_START | DONT_LOG, BOOL}},
{"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
{"LocationFilterInitialState", {PERSISTENT, BYTES}},
{"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"LongitudinalPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
{"NetworkMetered", {PERSISTENT | BACKUP, BOOL}},
{"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
{"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
{"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
{"Offroad_ChestnutBranch", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
{"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
{"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
{"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_DriverMonitoringUncertain", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
{"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
{"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
{"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"PrimeType", {PERSISTENT, INT}},
{"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
{"RecordFront", {PERSISTENT | BACKUP, BOOL}},
{"RecordFrontLock", {PERSISTENT, BOOL}}, // for the internal fleet
{"SecOCKey", {PERSISTENT | DONT_LOG | BACKUP, STRING}},
{"ShowDebugInfo", {PERSISTENT, BOOL}},
{"RouteCount", {PERSISTENT, INT, "0"}},
{"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"SshEnabled", {PERSISTENT | BACKUP, BOOL}},
{"TermsVersion", {PERSISTENT, STRING}},
{"TorqueBar", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TrainingVersion", {PERSISTENT, STRING}},
{"UbloxAvailable", {PERSISTENT, BOOL}},
{"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
{"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}},
{"UpdaterAvailableBranches", {PERSISTENT, STRING}},
{"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}},
{"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
{"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}},
{"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}},
{"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
{"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}},
{"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}},
{"UpdaterLastFetchTime", {PERSISTENT, TIME}},
{"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
{"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
{"UsbGpuActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
{"UsbGpuLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
{"Version", {PERSISTENT, STRING}},
// --- sunnypilot params --- //
{"ApiCache_DriveStats", {PERSISTENT, JSON}},
{"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
{"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
{"BlinkerLateralReengageDelay", {PERSISTENT | BACKUP, INT, "0"}}, // seconds
{"BlinkerMinLateralControlSpeed", {PERSISTENT | BACKUP, INT, "20"}}, // MPH or km/h
{"BlinkerPauseLateralControl", {PERSISTENT | BACKUP, INT, "0"}},
{"Brightness", {PERSISTENT | BACKUP, INT, "0"}},
{"CarList", {PERSISTENT, JSON}},
{"CarParamsSP", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
{"CarParamsSPCache", {CLEAR_ON_MANAGER_START, BYTES}},
{"CarParamsSPPersistent", {PERSISTENT, BYTES}},
{"CarPlatformBundle", {PERSISTENT | BACKUP, JSON}},
{"ChevronInfo", {PERSISTENT | BACKUP, INT, "4"}},
{"CompletedSunnylinkConsentVersion", {PERSISTENT, STRING, "0"}},
{"CustomAccIncrementsEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
{"CustomAccLongPressIncrement", {PERSISTENT | BACKUP, INT, "5"}},
{"CustomAccShortPressIncrement", {PERSISTENT | BACKUP, INT, "1"}},
{"DeviceBootMode", {PERSISTENT | BACKUP, INT, "0"}},
{"DevUIInfo", {PERSISTENT | BACKUP, INT, "0"}},
{"EnableCopyparty", {PERSISTENT | BACKUP, BOOL}},
{"EnableGithubRunner", {PERSISTENT | BACKUP, BOOL}},
{"GreenLightAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
{"GithubRunnerSufficientVoltage", {CLEAR_ON_MANAGER_START , BOOL}},
{"HasAcceptedTermsSP", {PERSISTENT, STRING, "0"}},
{"HideVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
{"IntelligentCruiseButtonManagement", {PERSISTENT | BACKUP , BOOL}},
{"InteractivityTimeout", {PERSISTENT | BACKUP, INT, "0"}},
{"IsDevelopmentBranch", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsReleaseSpBranch", {CLEAR_ON_MANAGER_START, BOOL}},
{"LastGPSPositionLLK", {PERSISTENT, STRING}},
{"LeadDepartAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
{"MaxTimeOffroad", {PERSISTENT | BACKUP, INT, "1800"}},
{"ModelRunnerTypeCache", {CLEAR_ON_ONROAD_TRANSITION, INT}},
{"OffroadMode", {CLEAR_ON_MANAGER_START, BOOL}},
{"Offroad_TiciSupport", {CLEAR_ON_MANAGER_START, JSON}},
{"OnroadScreenOffBrightness", {PERSISTENT | BACKUP, INT, "0"}},
{"OnroadScreenOffBrightnessMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
{"OnroadScreenOffTimer", {PERSISTENT | BACKUP, INT, "15"}},
{"OnroadScreenOffTimerMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
{"OnroadUploads", {PERSISTENT | BACKUP, BOOL, "1"}},
{"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
{"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
{"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
{"RoadEdgeLaneChangeEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
{"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ScreenSaverEnabled", {PERSISTENT | BACKUP, BOOL, "1"}},
{"ScreenSaverTimeout", {PERSISTENT | BACKUP, INT, "300"}},
{"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
{"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TrueVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
// MADS params
{"Mads", {PERSISTENT | BACKUP, BOOL, "1"}},
{"MadsMainCruiseAllowed", {PERSISTENT | BACKUP, BOOL, "1"}},
{"MadsSteeringMode", {PERSISTENT | BACKUP, INT, "0"}},
{"MadsUnifiedEngagementMode", {PERSISTENT | BACKUP, BOOL, "1"}},
// Model Manager params
{"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
{"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, STRING}},
{"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
{"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}},
{"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
{"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
{"ModelManager_LastSyncTime_USBGPU", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
{"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}},
{"ModelManager_ModelsCache_USBGPU", {PERSISTENT | BACKUP, JSON}},
// Neural Network Lateral Control
{"NeuralNetworkLateralControl", {PERSISTENT | BACKUP, BOOL, "0"}},
// sunnylink params
{"EnableSunnylinkUploader", {PERSISTENT | BACKUP, BOOL}},
{"LastSunnylinkPingTime", {CLEAR_ON_MANAGER_START, INT}},
{"ParamsVersion", {PERSISTENT, INT}},
{"SunnylinkCache_Roles", {PERSISTENT, STRING}},
{"SunnylinkCache_Users", {PERSISTENT, STRING}},
{"SunnylinkDongleId", {PERSISTENT, STRING}},
{"SunnylinkdPid", {PERSISTENT, INT}},
{"SunnylinkEnabled", {PERSISTENT, BOOL, "1"}},
{"SunnylinkTempFault", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL, "0"}},
// Backup Manager params
{"BackupManager_CreateBackup", {PERSISTENT, BOOL}},
{"BackupManager_RestoreVersion", {PERSISTENT, STRING}},
// sunnypilot car specific params
{"HyundaiLongitudinalTuning", {PERSISTENT | BACKUP, INT, "0"}},
{"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
{"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TeslaMadsScreenButton", {PERSISTENT | BACKUP, INT, "0"}},
{"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
{"DynamicExperimentalControl", {PERSISTENT | BACKUP, BOOL, "0"}},
{"BlindSpot", {PERSISTENT | BACKUP, BOOL, "0"}},
// sunnypilot model params
{"CameraOffset", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
{"LagdToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
{"LagdToggleDelay", {PERSISTENT | BACKUP, FLOAT, "0.2"}},
{"LagdValueCache", {PERSISTENT, FLOAT, "0.2"}},
{"LaneTurnDesire", {PERSISTENT | BACKUP, BOOL, "0"}},
{"LaneTurnValue", {PERSISTENT | BACKUP, FLOAT, "19.0"}},
{"PlanplusControl", {PERSISTENT | BACKUP, FLOAT, "1.0"}},
// mapd
{"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}},
{"MapdVersion", {PERSISTENT, STRING}},
{"MapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT, "0.0"}},
{"NextMapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, JSON}},
{"Offroad_OSMUpdateRequired", {CLEAR_ON_MANAGER_START, JSON}},
{"OsmDbUpdatesCheck", {CLEAR_ON_MANAGER_START, BOOL}}, // mapd database update happens with device ON, reset on boot
{"OSMDownloadBounds", {PERSISTENT, STRING}},
{"OsmDownloadedDate", {PERSISTENT, STRING, "0.0"}},
{"OSMDownloadLocations", {PERSISTENT, JSON}},
{"OSMDownloadProgress", {CLEAR_ON_MANAGER_START, JSON}},
{"OsmLocal", {PERSISTENT, BOOL}},
{"OsmLocationName", {PERSISTENT, STRING}},
{"OsmLocationTitle", {PERSISTENT, STRING}},
{"OsmLocationUrl", {PERSISTENT, STRING}},
{"OsmStateName", {PERSISTENT, STRING, "All"}},
{"OsmStateTitle", {PERSISTENT, STRING}},
{"OsmWayTest", {PERSISTENT, STRING}},
{"RoadName", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
{"RoadNameToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
// Speed Limit
{"SpeedLimitMode", {PERSISTENT | BACKUP, INT, "1"}},
{"SpeedLimitOffsetType", {PERSISTENT | BACKUP, INT, "0"}},
{"SpeedLimitPolicy", {PERSISTENT | BACKUP, INT, "3"}},
{"SpeedLimitValueOffset", {PERSISTENT | BACKUP, INT, "0"}},
// Smart Cruise Control
{"MapTargetVelocities", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
{"SmartCruiseControlMap", {PERSISTENT | BACKUP, BOOL, "0"}},
{"SmartCruiseControlVision", {PERSISTENT | BACKUP, BOOL, "0"}},
// Torque lateral control custom params
{"CustomTorqueParams", {PERSISTENT | BACKUP , BOOL}},
{"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
{"LateralJerkTorqueController", {PERSISTENT | BACKUP, BOOL, "0"}},
{"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
{"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
{"TorqueControlTune", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
{"TorqueParamsOverrideEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TorqueParamsOverrideFriction", {PERSISTENT | BACKUP, FLOAT, "0.1"}},
{"TorqueParamsOverrideLatAccelFactor", {PERSISTENT | BACKUP, FLOAT, "2.5"}},
};
openpilot/common/pid.py
+59
View File
@@ -0,0 +1,59 @@
import numpy as np
from collections.abc import Sequence
Gain = int | float | tuple[Sequence[float], Sequence[float]] | list[list[float]]
class PIDController:
def __init__(self, k_p: Gain, k_i: Gain, k_d: Gain = 0., pos_limit=1e308, neg_limit=-1e308, rate=100):
self._k_p = ([0], [k_p]) if isinstance(k_p, (int, float)) else k_p
self._k_i = ([0], [k_i]) if isinstance(k_i, (int, float)) else k_i
self._k_d = ([0], [k_d]) if isinstance(k_d, (int, float)) else k_d
self.set_limits(pos_limit, neg_limit)
self.i_dt = 1.0 / rate
self.speed = 0.0
self.reset()
@property
def k_p(self):
return np.interp(self.speed, self._k_p[0], self._k_p[1])
@property
def k_i(self):
return np.interp(self.speed, self._k_i[0], self._k_i[1])
@property
def k_d(self):
return np.interp(self.speed, self._k_d[0], self._k_d[1])
def reset(self):
self.p = 0.0
self.i = 0.0
self.d = 0.0
self.f = 0.0
self.control = 0
def set_limits(self, pos_limit, neg_limit):
self.pos_limit = pos_limit
self.neg_limit = neg_limit
def update(self, error, error_rate=0.0, speed=0.0, feedforward=0., freeze_integrator=False):
self.speed = speed
self.p = self.k_p * float(error)
self.d = self.k_d * error_rate
self.f = feedforward
if not freeze_integrator:
i = self.i + self.k_i * self.i_dt * error
# Don't allow windup if already clipping
test_control = self.p + i + self.d + self.f
i_upperbound = self.i if test_control > self.pos_limit else self.pos_limit
i_lowerbound = self.i if test_control < self.neg_limit else self.neg_limit
self.i = np.clip(i, i_lowerbound, i_upperbound)
control = self.p + self.i + self.d + self.f
self.control = np.clip(control, self.neg_limit, self.pos_limit)
return self.control
openpilot/common/prefix.h
+43
View File
@@ -0,0 +1,43 @@
#pragma once
#include <cassert>
#include <string>
#include "common/params.h"
#include "common/util.h"
#include "common/hardware/hw.h"
class OpenpilotPrefix {
public:
OpenpilotPrefix(std::string prefix = {}) {
if (prefix.empty()) {
prefix = util::random_string(15);
}
#ifdef __APPLE__
msgq_path = "/tmp/msgq_" + prefix;
#else
msgq_path = "/dev/shm/msgq_" + prefix;
#endif
bool ret = util::create_directories(msgq_path, 0777);
assert(ret);
setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
}
~OpenpilotPrefix() {
auto param_path = Params().getParamPath();
if (util::file_exists(param_path)) {
std::string real_path = util::readlink(param_path);
util::check_system(util::string_format("rm %s -rf", real_path.c_str()));
unlink(param_path.c_str());
}
if (getenv("COMMA_CACHE") == nullptr) {
util::check_system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()));
}
util::check_system(util::string_format("rm %s -rf", Path::comma_home().c_str()));
util::check_system(util::string_format("rm %s -rf", msgq_path.c_str()));
unsetenv("OPENPILOT_PREFIX");
}
private:
std::string msgq_path;
};
openpilot/common/prefix.py
+61
View File
@@ -0,0 +1,61 @@
import os
import platform
import shutil
import uuid
from openpilot.common.params import Params
from openpilot.common.hardware import PC
from openpilot.common.hardware.hw import Paths
from openpilot.common.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
class OpenpilotPrefix:
def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm"
self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix)
self.create_dirs_on_enter = create_dirs_on_enter
self.clean_dirs_on_exit = clean_dirs_on_exit
self.shared_download_cache = shared_download_cache
def __enter__(self):
self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
os.environ['OPENPILOT_PREFIX'] = self.prefix
if self.create_dirs_on_enter:
self.create_dirs()
if self.shared_download_cache:
os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
return self
def __exit__(self, exc_type, exc_obj, exc_tb):
if self.clean_dirs_on_exit:
self.clean_dirs()
try:
del os.environ['OPENPILOT_PREFIX']
if self.original_prefix is not None:
os.environ['OPENPILOT_PREFIX'] = self.original_prefix
except KeyError:
pass
return False
def create_dirs(self):
try:
os.mkdir(self.msgq_path)
except FileExistsError:
pass
os.makedirs(Paths.log_root(), exist_ok=True)
def clean_dirs(self):
symlink_path = Params().get_param_path()
if os.path.exists(symlink_path):
shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True)
os.remove(symlink_path)
shutil.rmtree(self.msgq_path, ignore_errors=True)
if PC:
shutil.rmtree(Paths.log_root(), ignore_errors=True)
if not os.environ.get("COMMA_CACHE", False):
shutil.rmtree(Paths.download_cache_root(), ignore_errors=True)
shutil.rmtree(Paths.comma_home(), ignore_errors=True)
openpilot/common/qrcode.py
+218
View File
@@ -0,0 +1,218 @@
"""Small QR encoder for the UI's byte-mode, error-correction-level-L codes."""
import numpy as np
import pyray as rl
# Indexes are QR versions. These are the only two Reed-Solomon parameters needed
# for error-correction level L.
_ECC_LEN = (0, 7, 10, 15, 20, 26, 18, 20, 24, 30, 18, 20, 24, 26, 30, 22, 24, 28, 30, 28, 28)
_NUM_BLOCKS = (0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4, 4, 4, 4, 4, 6, 6, 6, 6, 7, 8)
# 15 format-info bits for level L (01) with mask 0: ((0x08 << 10) | bch_remainder) ^ 0x5412
_FORMAT_BITS = 0b111011111000100
def _raw_modules(version: int) -> int:
result = (16 * version + 128) * version + 64
if version >= 2:
align = version // 7 + 2
result -= (25 * align - 10) * align - 55
return result - (36 if version >= 7 else 0)
def _capacity(version: int) -> int:
return _raw_modules(version) // 8 - _ECC_LEN[version] * _NUM_BLOCKS[version]
def _append_bits(bits: list[int], value: int, length: int) -> None:
bits.extend((value >> i) & 1 for i in range(length - 1, -1, -1))
def _data_codewords(data: bytes, version: int) -> bytes:
"""Byte-mode-encode the payload, terminated and padded to the version's capacity."""
capacity = _capacity(version)
bits: list[int] = []
_append_bits(bits, 4, 4) # byte mode
_append_bits(bits, len(data), 8 if version <= 9 else 16)
for value in data:
_append_bits(bits, value, 8)
bits.extend([0] * min(4, capacity * 8 - len(bits))) # terminator
bits.extend([0] * (-len(bits) % 8)) # byte alignment
result = bytearray(sum(bits[i + j] << (7 - j) for j in range(8)) for i in range(0, len(bits), 8))
pad = (0xEC, 0x11)
while len(result) < capacity:
result.append(pad[(len(result) - (len(bits) // 8)) & 1])
return bytes(result)
def _codewords(data: bytes, version: int) -> bytes:
"""Split data codewords into Reed-Solomon blocks and interleave data + ECC."""
data = _data_codewords(data, version)
num_blocks = _NUM_BLOCKS[version]
ecc_len = _ECC_LEN[version]
raw_codewords = _raw_modules(version) // 8
short_len = raw_codewords // num_blocks
num_short = num_blocks - raw_codewords % num_blocks
divisor = _divisor(ecc_len)
blocks: list[tuple[bytes, bytes]] = []
offset = 0
for i in range(num_blocks):
length = short_len - ecc_len + (0 if i < num_short else 1)
block = data[offset:offset + length]
blocks.append((block, _remainder(block, divisor)))
offset += length
result = bytearray()
for i in range(short_len - ecc_len + 1):
for block, _ in blocks:
result.extend(block[i:i + 1])
for i in range(ecc_len):
for _, ecc in blocks:
result.append(ecc[i])
return bytes(result)
def _multiply(x: int, y: int) -> int:
result = 0
for _ in range(8):
result = (result << 1) ^ (0x11D if result & 0x80 else 0)
if y & 0x80:
result ^= x
y <<= 1
return result
def _divisor(degree: int) -> bytes:
result = bytearray([0] * (degree - 1) + [1])
root = 1
for _ in range(degree):
for j in range(degree):
result[j] = _multiply(result[j], root)
if j + 1 < degree:
result[j] ^= result[j + 1]
root = _multiply(root, 2)
return bytes(result)
def _remainder(data: bytes, divisor: bytes) -> bytes:
result = bytearray(len(divisor))
for value in data:
factor = value ^ result.pop(0)
result.append(0)
for i, coefficient in enumerate(divisor):
result[i] ^= _multiply(coefficient, factor)
return bytes(result)
def _alignment_positions(version: int) -> list[int]:
if version == 1:
return []
count = version // 7 + 2
step = ((version * 4 + count * 2 + 1) // (count * 2 - 2)) * 2
return [6] + [version * 4 + 10 - step * i for i in range(count - 1)][::-1]
class _Qr:
def __init__(self, version: int, data: bytes):
self.version = version
self.size = version * 4 + 17
self.modules = [[False] * self.size for _ in range(self.size)]
self.function = [[False] * self.size for _ in range(self.size)]
self._draw_functions()
self._draw_data(_codewords(data, version))
for y in range(self.size):
for x in range(self.size):
if not self.function[y][x]:
self.modules[y][x] ^= (x + y) % 2 == 0
self._format()
def _set_function(self, x: int, y: int, dark: bool) -> None:
if 0 <= x < self.size and 0 <= y < self.size:
self.modules[y][x] = dark
self.function[y][x] = True
def _finder(self, x: int, y: int) -> None:
for dy in range(-4, 5):
for dx in range(-4, 5):
distance = max(abs(dx), abs(dy))
self._set_function(x + dx, y + dy, distance != 2 and distance != 4)
def _alignment(self, x: int, y: int) -> None:
for dy in range(-2, 3):
for dx in range(-2, 3):
self._set_function(x + dx, y + dy, max(abs(dx), abs(dy)) != 1)
def _draw_functions(self) -> None:
for i in range(self.size):
self._set_function(6, i, i % 2 == 0)
self._set_function(i, 6, i % 2 == 0)
self._finder(3, 3)
self._finder(self.size - 4, 3)
self._finder(3, self.size - 4)
positions = _alignment_positions(self.version)
for y in positions:
for x in positions:
if not ((x == 6 and y in (6, self.size - 7)) or (x == self.size - 7 and y == 6)):
self._alignment(x, y)
# reserve the format-info modules before the data is placed; the real
# values are written by the second _format call after masking
self._format()
if self.version >= 7:
value = self.version
for _ in range(12):
value = (value << 1) ^ ((value >> 11) * 0x1F25)
value = self.version << 12 | value
for i in range(18):
bit = ((value >> i) & 1) != 0
a = self.size - 11 + i % 3
b = i // 3
self._set_function(a, b, bit)
self._set_function(b, a, bit)
def _format(self) -> None:
for i in range(15):
bit = ((_FORMAT_BITS >> i) & 1) != 0
y_pos = i if i < 6 else i + 1 if i < 8 else self.size - 15 + i
self._set_function(8, y_pos, bit)
x_pos = self.size - 1 - i if i < 8 else 15 - i if i < 9 else 14 - i
self._set_function(x_pos, 8, bit)
self._set_function(8, self.size - 8, True)
def _draw_data(self, data: bytes) -> None:
bits = ((byte >> s) & 1 for byte in data for s in reversed(range(8)))
upward = True
right = self.size - 1
while right >= 1:
if right == 6: # skip the vertical timing column
right = 5
for vert in range(self.size):
y = self.size - 1 - vert if upward else vert
for x in (right, right - 1):
if not self.function[y][x]:
self.modules[y][x] = bool(next(bits, 0))
upward = not upward
right -= 2
def make_texture(data: str, inverted: bool = False) -> rl.Texture:
"""Render a URL as the RGBA QR texture used by the UI. The texture upload
copies the pixels, so the intermediate image/array don't need to outlive it."""
raw = data.encode()
for version in range(1, 21):
count_bits = 8 if version <= 9 else 16
if 4 + count_bits + len(raw) * 8 <= _capacity(version) * 8:
break
else:
raise ValueError("QR URL is too long")
modules = np.pad(_Qr(version, raw).modules, 0 if inverted else 4)
modules = np.repeat(np.repeat(modules, 10, axis=0), 10, axis=1)
gray = ((modules == inverted) * 255).astype(np.uint8)
img_array = np.dstack((gray, gray, gray, np.full_like(gray, 255)))
rl_image = rl.Image()
rl_image.data = rl.ffi.cast("void *", img_array.ctypes.data)
rl_image.width = img_array.shape[1]
rl_image.height = img_array.shape[0]
rl_image.mipmaps = 1
rl_image.format = rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_R8G8B8A8
return rl.load_texture_from_image(rl_image)
openpilot/common/queue.h
+52
View File
@@ -0,0 +1,52 @@
#pragma once
#include <condition_variable>
#include <mutex>
#include <queue>
template <class T>
class SafeQueue {
public:
SafeQueue() = default;
void push(const T& v) {
{
std::unique_lock lk(m);
q.push(v);
}
cv.notify_one();
}
T pop() {
std::unique_lock lk(m);
cv.wait(lk, [this] { return !q.empty(); });
T v = q.front();
q.pop();
return v;
}
bool try_pop(T& v, int timeout_ms = 0) {
std::unique_lock lk(m);
if (!cv.wait_for(lk, std::chrono::milliseconds(timeout_ms), [this] { return !q.empty(); })) {
return false;
}
v = q.front();
q.pop();
return true;
}
bool empty() const {
std::scoped_lock lk(m);
return q.empty();
}
size_t size() const {
std::scoped_lock lk(m);
return q.size();
}
private:
mutable std::mutex m;
std::condition_variable cv;
std::queue<T> q;
};
openpilot/common/ratekeeper.cc
+40
View File
@@ -0,0 +1,40 @@
#include "common/ratekeeper.h"
#include <algorithm>
#include "common/swaglog.h"
#include "common/timing.h"
#include "common/util.h"
RateKeeper::RateKeeper(const std::string &name_, float rate, float print_delay_threshold_)
: name(name_),
print_delay_threshold(std::max(0.f, print_delay_threshold_)) {
interval = 1 / rate;
last_monitor_time = seconds_since_boot();
next_frame_time = last_monitor_time + interval;
}
bool RateKeeper::keepTime() {
bool lagged = monitorTime();
if (remaining_ > 0) {
util::sleep_for(remaining_ * 1000);
}
return lagged;
}
bool RateKeeper::monitorTime() {
++frame_;
last_monitor_time = seconds_since_boot();
remaining_ = next_frame_time - last_monitor_time;
bool lagged = remaining_ < 0;
if (lagged) {
if (print_delay_threshold > 0 && remaining_ < -print_delay_threshold) {
LOGW("%s lagging by %.2f ms", name.c_str(), -remaining_ * 1000);
}
next_frame_time = last_monitor_time + interval;
} else {
next_frame_time += interval;
}
return lagged;
}
openpilot/common/ratekeeper.h
+23
View File
@@ -0,0 +1,23 @@
#pragma once
#include <cstdint>
#include <string>
class RateKeeper {
public:
RateKeeper(const std::string &name, float rate, float print_delay_threshold = 0);
~RateKeeper() {}
bool keepTime();
bool monitorTime();
inline uint64_t frame() const { return frame_; }
inline double remaining() const { return remaining_; }
private:
double interval;
double next_frame_time;
double last_monitor_time;
double remaining_ = 0;
float print_delay_threshold = 0;
uint64_t frame_ = 0;
std::string name;
};
openpilot/common/realtime.py
+101
View File
@@ -0,0 +1,101 @@
"""Utilities for reading real time clocks and keeping soft real time constraints."""
import gc
import os
import sys
import time
from setproctitle import getproctitle
from openpilot.common.utils import MovingAverage
from openpilot.common.hardware import PC
# time step for each process
DT_CTRL = 0.01 # controlsd
DT_MDL = 0.05 # model
DT_HW = 0.5 # hardwared and manager
DT_DMON = 0.05 # driver monitoring
class Priority:
# CORE 2
# - modeld = 55
# - camerad = 54
CTRL_LOW = 51 # plannerd & radard
# CORE 3
# - pandad = 55
CTRL_HIGH = 53
def drop_realtime() -> None:
if sys.platform == 'linux' and not PC:
os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0))
def set_core_affinity(cores: list[int]) -> None:
if sys.platform == 'linux' and not PC:
os.sched_setaffinity(0, cores)
def config_realtime_process(cores: int | list[int], priority: int) -> None:
gc.disable()
if sys.platform == 'linux' and not PC:
os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(priority))
c = cores if isinstance(cores, list) else [cores, ]
set_core_affinity(c)
class Ratekeeper:
def __init__(self, rate: float, print_delay_threshold: float | None = 0.0) -> None:
"""Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
self._interval = 1. / rate
self._print_delay_threshold = print_delay_threshold
self._frame = 0
self._remaining = 0.0
self._process_name = getproctitle()
self._last_monitor_time = -1.
self._next_frame_time = -1.
self.avg_dt = MovingAverage(100)
self.avg_dt.add_value(self._interval)
@property
def frame(self) -> int:
return self._frame
@property
def remaining(self) -> float:
return self._remaining
@property
def lagging(self) -> bool:
expected_dt = self._interval * (1 / 0.9)
return self.avg_dt.get_average() > expected_dt
# Maintain loop rate by calling this at the end of each loop
def keep_time(self) -> bool:
lagged = self.monitor_time()
if self._remaining > 0:
time.sleep(self._remaining)
return lagged
# Monitors the cumulative lag, but does not enforce a rate
def monitor_time(self) -> bool:
if self._last_monitor_time < 0:
self._next_frame_time = time.monotonic() + self._interval
self._last_monitor_time = time.monotonic()
prev = self._last_monitor_time
self._last_monitor_time = time.monotonic()
self.avg_dt.add_value(self._last_monitor_time - prev)
lagged = False
remaining = self._next_frame_time - time.monotonic()
self._next_frame_time += self._interval
if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
print(f"{self._process_name} lagging by {-remaining * 1000:.2f} ms")
lagged = True
self._frame += 1
self._remaining = remaining
return lagged
openpilot/common/serial.py
+272
View File
@@ -0,0 +1,272 @@
import errno
import fcntl
import os
import select
import struct
import termios
import time
# Modem control lines (linux/termios.h); fall back to common x86_64 values.
TIOCMBIS = getattr(termios, "TIOCMBIS", 0x5416)
TIOCMBIC = getattr(termios, "TIOCMBIC", 0x5417)
TIOCM_DTR = getattr(termios, "TIOCM_DTR", 0x002)
TIOCM_RTS = getattr(termios, "TIOCM_RTS", 0x004)
_TIOCM_DTR = struct.pack("I", TIOCM_DTR)
_TIOCM_RTS = struct.pack("I", TIOCM_RTS)
class SerialException(OSError):
pass
class Serial:
def __init__(self, port: str, baudrate: int = 9600, timeout: float | None = None, *,
rtscts: bool = False, dsrdtr: bool = False, exclusive: bool = False):
self._port = port
self._baudrate = baudrate
self._timeout = timeout
self._rtscts = rtscts
self._dsrdtr = dsrdtr
self._exclusive = exclusive
self._dtr = True
self._fd = -1
self.open()
def __enter__(self):
return self
def __exit__(self, *args) -> None:
self.close()
@property
def fd(self) -> int:
self._ensure_open()
return self._fd
@property
def baudrate(self) -> int:
return self._baudrate
@baudrate.setter
def baudrate(self, value: int) -> None:
self._baudrate = int(value)
if self._fd >= 0:
self._configure()
@property
def dtr(self) -> bool:
return self._dtr
@dtr.setter
def dtr(self, value: bool) -> None:
self._dtr = bool(value)
if self._fd >= 0:
self._set_line(TIOCM_DTR, _TIOCM_DTR, self._dtr)
def open(self) -> None:
if self._fd >= 0:
return
try:
self._fd = os.open(self._port, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
except OSError as e:
self._fd = -1
raise SerialException(e.errno, f"could not open port {self._port}: {e}") from e
try:
if self._exclusive:
try:
fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as e:
raise SerialException(e.errno, f"could not exclusively lock port {self._port}: {e}") from e
self._configure()
# When not using hardware DSR/DTR handshaking, drive lines ourselves.
if not self._dsrdtr:
try:
self._set_line(TIOCM_DTR, _TIOCM_DTR, self._dtr)
if not self._rtscts:
self._set_line(TIOCM_RTS, _TIOCM_RTS, True)
except OSError as e:
if e.errno not in (errno.EINVAL, errno.ENOTTY):
raise
self.reset_input_buffer()
except BaseException:
self._close_fd()
raise
def close(self) -> None:
self._close_fd()
def read(self, size: int = 1) -> bytes:
self._ensure_open()
if size <= 0:
return b""
buf = bytearray()
deadline = self._deadline()
while len(buf) < size:
remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
if not self._wait_readable(remaining):
break
try:
chunk = os.read(self._fd, size - len(buf))
except InterruptedError:
continue
except OSError as e:
if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
if self._timeout == 0:
break
continue
raise SerialException(e.errno, f"read failed: {e}") from e
if not chunk:
break
buf.extend(chunk)
return bytes(buf)
def readline(self) -> bytes:
self._ensure_open()
buf = bytearray()
deadline = self._deadline()
while True:
remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
if deadline is not None and remaining == 0.0 and not buf:
# match pyserial: timed-out readline returns empty
if not self._wait_readable(0.0):
return b""
elif not self._wait_readable(remaining):
return bytes(buf)
try:
chunk = os.read(self._fd, 1)
except InterruptedError:
continue
except OSError as e:
if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
if self._timeout == 0:
return bytes(buf)
continue
raise SerialException(e.errno, f"read failed: {e}") from e
if not chunk:
return bytes(buf)
buf.extend(chunk)
if chunk == b"\n":
return bytes(buf)
def write(self, data: bytes) -> int:
self._ensure_open()
if not data:
return 0
view = memoryview(data)
total = 0
while total < len(data):
try:
n = os.write(self._fd, view[total:])
except InterruptedError:
continue
except OSError as e:
if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
select.select([], [self._fd], [], None)
continue
raise SerialException(e.errno, f"write failed: {e}") from e
if n == 0:
raise SerialException("write returned 0")
total += n
return total
def flush(self) -> None:
self._ensure_open()
termios.tcdrain(self._fd)
def reset_input_buffer(self) -> None:
self._ensure_open()
termios.tcflush(self._fd, termios.TCIFLUSH)
def reset_output_buffer(self) -> None:
self._ensure_open()
termios.tcflush(self._fd, termios.TCOFLUSH)
def _close_fd(self) -> None:
if self._fd >= 0:
try:
if self._exclusive:
fcntl.flock(self._fd, fcntl.LOCK_UN)
except OSError:
pass
try:
os.close(self._fd)
except OSError:
pass
self._fd = -1
def _ensure_open(self) -> None:
if self._fd < 0:
raise SerialException("port is not open")
def _deadline(self) -> float | None:
if self._timeout is None:
return None
if self._timeout == 0:
return time.monotonic()
return time.monotonic() + self._timeout
def _wait_readable(self, timeout: float | None) -> bool:
"""Return True if fd is readable. timeout None blocks; 0 polls."""
if timeout is not None and timeout < 0:
timeout = 0.0
try:
ready, _, _ = select.select([self._fd], [], [], timeout)
except InterruptedError:
return False
return bool(ready)
def _baud_constant(self, baudrate: int) -> int:
try:
return getattr(termios, f"B{baudrate}")
except AttributeError as e:
raise ValueError(f"unsupported baud rate: {baudrate}") from e
def _configure(self) -> None:
self._ensure_open()
try:
attrs = termios.tcgetattr(self._fd)
except termios.error as e:
raise SerialException(f"could not get port attributes: {e}") from e
iflag, oflag, cflag, lflag, _ispeed, _ospeed, cc = attrs
# raw binary 8N1
iflag = 0
oflag = 0
lflag = 0
cflag |= termios.CLOCAL | termios.CREAD
cflag &= ~termios.CSIZE
cflag |= termios.CS8
cflag &= ~(termios.PARENB | termios.PARODD | termios.CSTOPB)
if hasattr(termios, "CRTSCTS"):
if self._rtscts:
cflag |= termios.CRTSCTS
else:
cflag &= ~termios.CRTSCTS
speed = self._baud_constant(self._baudrate)
cc = list(cc)
# Non-blocking reads are handled via select + O_NONBLOCK; keep VMIN/VTIME at 0.
cc[termios.VMIN] = 0
cc[termios.VTIME] = 0
try:
termios.tcsetattr(self._fd, termios.TCSANOW, [iflag, oflag, cflag, lflag, speed, speed, cc])
except termios.error as e:
raise SerialException(f"could not configure port: {e}") from e
# Keep the fd non-blocking so timeout=0 and select work consistently.
flags = fcntl.fcntl(self._fd, fcntl.F_GETFL)
fcntl.fcntl(self._fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
def _set_line(self, _bit: int, packed: bytes, enabled: bool) -> None:
request = TIOCMBIS if enabled else TIOCMBIC
fcntl.ioctl(self._fd, request, packed)
openpilot/common/simple_kalman.py
+54
View File
@@ -0,0 +1,54 @@
import numpy as np
def get_kalman_gain(dt, A, C, Q, R, iterations=100):
P = np.zeros_like(Q)
for _ in range(iterations):
P = A.dot(P).dot(A.T) + dt * Q
S = C.dot(P).dot(C.T) + R
K = P.dot(C.T).dot(np.linalg.inv(S))
P = (np.eye(len(P)) - K.dot(C)).dot(P)
return K
class KF1D:
# this EKF assumes constant covariance matrix, so calculations are much simpler
# the Kalman gain also needs to be precomputed using the control module
def __init__(self, x0, A, C, K):
self.x0_0 = x0[0][0]
self.x1_0 = x0[1][0]
self.A0_0 = A[0][0]
self.A0_1 = A[0][1]
self.A1_0 = A[1][0]
self.A1_1 = A[1][1]
self.C0_0 = C[0]
self.C0_1 = C[1]
self.K0_0 = K[0][0]
self.K1_0 = K[1][0]
self.A_K_0 = self.A0_0 - self.K0_0 * self.C0_0
self.A_K_1 = self.A0_1 - self.K0_0 * self.C0_1
self.A_K_2 = self.A1_0 - self.K1_0 * self.C0_0
self.A_K_3 = self.A1_1 - self.K1_0 * self.C0_1
# K matrix needs to be pre-computed as follow:
# import control
# (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
# self.K = np.transpose(K)
def update(self, meas):
#self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
x0_0 = self.A_K_0 * self.x0_0 + self.A_K_1 * self.x1_0 + self.K0_0 * meas
x1_0 = self.A_K_2 * self.x0_0 + self.A_K_3 * self.x1_0 + self.K1_0 * meas
self.x0_0 = x0_0
self.x1_0 = x1_0
return [self.x0_0, self.x1_0]
@property
def x(self):
return [[self.x0_0], [self.x1_0]]
def set_x(self, x):
self.x0_0 = x[0][0]
self.x1_0 = x[1][0]
openpilot/common/spinner.py
Executable
+52
View File
@@ -0,0 +1,52 @@
import os
import subprocess
from openpilot.common.basedir import BASEDIR
class Spinner:
def __init__(self):
try:
self.spinner_proc = subprocess.Popen(["./spinner.py"],
stdin=subprocess.PIPE,
cwd=os.path.join(BASEDIR, "openpilot/system", "ui"),
close_fds=True)
except OSError:
self.spinner_proc = None
def __enter__(self):
return self
def update(self, spinner_text: str):
if self.spinner_proc is not None:
self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
try:
self.spinner_proc.stdin.flush()
except BrokenPipeError:
pass
def update_progress(self, cur: float, total: float):
self.update(str(round(100 * cur / total)))
def close(self):
if self.spinner_proc is not None:
self.spinner_proc.kill()
try:
self.spinner_proc.communicate(timeout=2.)
except subprocess.TimeoutExpired:
print("WARNING: failed to kill spinner")
self.spinner_proc = None
def __del__(self):
self.close()
def __exit__(self, exc_type, exc_value, traceback):
self.close()
if __name__ == "__main__":
import time
with Spinner() as s:
s.update("Spinner text")
time.sleep(5.0)
print("gone")
time.sleep(5.0)
openpilot/common/stat_live.py
+70
View File
@@ -0,0 +1,70 @@
import numpy as np
class RunningStat:
# tracks realtime mean and standard deviation without storing any data
def __init__(self, priors=None, max_trackable=-1):
self.max_trackable = max_trackable
if priors is not None:
# initialize from history
self.M = priors[0]
self.S = priors[1]
self.n = priors[2]
self.M_last = self.M
self.S_last = self.S
else:
self.reset()
def reset(self):
self.M = 0.
self.S = 0.
self.M_last = 0.
self.S_last = 0.
self.n = 0
def push_data(self, new_data):
# short term memory hack
if self.max_trackable < 0 or self.n < self.max_trackable:
self.n += 1
if self.n == 0:
self.M_last = new_data
self.M = self.M_last
self.S_last = 0.
else:
self.M = self.M_last + (new_data - self.M_last) / self.n
self.S = self.S_last + (new_data - self.M_last) * (new_data - self.M)
self.M_last = self.M
self.S_last = self.S
def mean(self):
return self.M
def variance(self):
if self.n >= 2:
return self.S / (self.n - 1.)
else:
return 0
def std(self):
return np.sqrt(self.variance())
class RunningStatFilter:
def __init__(self, raw_priors=None, filtered_priors=None, max_trackable=-1):
self.raw_stat = RunningStat(raw_priors, -1)
self.filtered_stat = RunningStat(filtered_priors, max_trackable)
def reset(self):
self.raw_stat.reset()
self.filtered_stat.reset()
def push_and_update(self, new_data):
_std_last = self.raw_stat.std()
self.raw_stat.push_data(new_data)
_delta_std = self.raw_stat.std() - _std_last
if _delta_std <= 0:
self.filtered_stat.push_data(new_data)
else:
pass
# self.filtered_stat.push_data(self.filtered_stat.mean())
# class SequentialBayesian():
openpilot/common/swaglog.cc
+157
View File
@@ -0,0 +1,157 @@
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include "common/swaglog.h"
#include <cassert>
#include <limits>
#include <mutex>
#include <string>
#include <zmq.h>
#include <stdarg.h>
#include "json11/json11.hpp"
#include "common/version.h"
#include "common/hardware/hw.h"
#include "sunnypilot/common/version.h"
class SwaglogState {
public:
SwaglogState() {
zctx = zmq_ctx_new();
sock = zmq_socket(zctx, ZMQ_PUSH);
// Timeout on shutdown for messages to be received by the logging process
int timeout = 100;
zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
zmq_connect(sock, Path::swaglog_ipc().c_str());
// workaround for https://github.com/dropbox/json11/issues/38
setlocale(LC_NUMERIC, "C");
print_level = CLOUDLOG_WARNING;
if (const char* print_lvl = getenv("LOGPRINT")) {
if (strcmp(print_lvl, "debug") == 0) {
print_level = CLOUDLOG_DEBUG;
} else if (strcmp(print_lvl, "info") == 0) {
print_level = CLOUDLOG_INFO;
} else if (strcmp(print_lvl, "warning") == 0) {
print_level = CLOUDLOG_WARNING;
}
}
ctx_j = json11::Json::object{};
if (char* dongle_id = getenv("DONGLE_ID")) {
ctx_j["dongle_id"] = dongle_id;
}
if (char* git_origin = getenv("GIT_ORIGIN")) {
ctx_j["origin"] = git_origin;
}
if (char* git_branch = getenv("GIT_BRANCH")) {
ctx_j["branch"] = git_branch;
}
if (char* git_commit = getenv("GIT_COMMIT")) {
ctx_j["commit"] = git_commit;
}
if (char* daemon_name = getenv("MANAGER_DAEMON")) {
ctx_j["daemon"] = daemon_name;
}
ctx_j["version"] = SUNNYPILOT_VERSION;
ctx_j["dirty"] = !getenv("CLEAN");
ctx_j["device"] = Hardware::get_name();
}
~SwaglogState() {
zmq_close(sock);
zmq_ctx_destroy(zctx);
}
void log(int levelnum, const char* filename, int lineno, const char* func, const char* msg, const std::string& log_s) {
std::lock_guard lk(lock);
if (levelnum >= print_level) {
printf("%s: %s\n", filename, msg);
}
zmq_send(sock, log_s.data(), log_s.length(), ZMQ_NOBLOCK);
}
std::mutex lock;
void* zctx = nullptr;
void* sock = nullptr;
int print_level;
json11::Json::object ctx_j;
};
bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();
static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
char* msg_buf, const json11::Json::object &msg_j={}) {
static SwaglogState s;
json11::Json::object log_j = json11::Json::object {
{"ctx", s.ctx_j},
{"levelnum", levelnum},
{"filename", filename},
{"lineno", lineno},
{"funcname", func},
{"created", seconds_since_epoch()}
};
if (msg_j.empty()) {
log_j["msg"] = msg_buf;
} else {
log_j["msg"] = msg_j;
}
std::string log_s;
log_s += (char)levelnum;
((json11::Json)log_j).dump(log_s);
s.log(levelnum, filename, lineno, func, msg_buf, log_s);
free(msg_buf);
}
void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) {
va_list args;
va_start(args, fmt);
char* msg_buf = nullptr;
int ret = vasprintf(&msg_buf, fmt, args);
va_end(args);
if (ret <= 0 || !msg_buf) return;
cloudlog_common(levelnum, filename, lineno, func, msg_buf);
}
void cloudlog_t_common(int levelnum, const char* filename, int lineno, const char* func,
uint32_t frame_id, const char* fmt, va_list args) {
if (!LOG_TIMESTAMPS) return;
char* msg_buf = nullptr;
int ret = vasprintf(&msg_buf, fmt, args);
if (ret <= 0 || !msg_buf) return;
json11::Json::object tspt_j = json11::Json::object{
{"event", msg_buf},
{"time", std::to_string(nanos_since_boot())}
};
if (frame_id < NO_FRAME_ID) {
tspt_j["frame_id"] = std::to_string(frame_id);
}
tspt_j = json11::Json::object{{"timestamp", tspt_j}};
cloudlog_common(levelnum, filename, lineno, func, msg_buf, tspt_j);
}
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) {
va_list args;
va_start(args, fmt);
cloudlog_t_common(levelnum, filename, lineno, func, NO_FRAME_ID, fmt, args);
va_end(args);
}
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
uint32_t frame_id, const char* fmt, ...) {
va_list args;
va_start(args, fmt);
cloudlog_t_common(levelnum, filename, lineno, func, frame_id, fmt, args);
va_end(args);
}
openpilot/common/swaglog.h
+76
View File
@@ -0,0 +1,76 @@
#pragma once
#include "common/timing.h"
#define CLOUDLOG_DEBUG 10
#define CLOUDLOG_INFO 20
#define CLOUDLOG_WARNING 30
#define CLOUDLOG_ERROR 40
#define CLOUDLOG_CRITICAL 50
#ifdef __GNUC__
#define SWAG_LOG_CHECK_FMT(a, b) __attribute__ ((format (printf, a, b)))
#else
#define SWAG_LOG_CHECK_FMT(a, b)
#endif
void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
uint32_t frame_id, const char* fmt, ...) SWAG_LOG_CHECK_FMT(6, 7);
#define cloudlog(lvl, fmt, ...) cloudlog_e(lvl, __FILE__, __LINE__, \
__func__, \
fmt, ## __VA_ARGS__)
#define cloudlog_t(lvl, ...) cloudlog_te(lvl, __FILE__, __LINE__, \
__func__, \
__VA_ARGS__)
#define cloudlog_rl(burst, millis, lvl, fmt, ...) \
{ \
static uint64_t __begin = 0; \
static int __printed = 0; \
static int __missed = 0; \
\
int __burst = (burst); \
int __millis = (millis); \
uint64_t __ts = nanos_since_boot(); \
\
if (!__begin) { __begin = __ts; } \
\
if (__begin + __millis*1000000ULL < __ts) { \
if (__missed) { \
cloudlog(CLOUDLOG_WARNING, "cloudlog: %d messages suppressed", __missed); \
} \
__begin = 0; \
__printed = 0; \
__missed = 0; \
} \
\
if (__printed < __burst) { \
cloudlog(lvl, fmt, ## __VA_ARGS__); \
__printed++; \
} else { \
__missed++; \
} \
}
#define LOGT(...) cloudlog_t(CLOUDLOG_DEBUG, __VA_ARGS__)
#define LOGD(fmt, ...) cloudlog(CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
#define LOG(fmt, ...) cloudlog(CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
#define LOGW(fmt, ...) cloudlog(CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
#define LOGE(fmt, ...) cloudlog(CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
#define LOGD_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
#define LOG_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
#define LOGW_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
#define LOGE_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
openpilot/common/swaglog.py
+145
View File
@@ -0,0 +1,145 @@
import logging
import os
import time
import warnings
from pathlib import Path
from logging.handlers import BaseRotatingHandler
import zmq
from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
from openpilot.common.hardware.hw import Paths
def get_file_handler():
Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True)
base_filename = os.path.join(Paths.swaglog_root(), "swaglog")
handler = SwaglogRotatingFileHandler(base_filename)
return handler
class SwaglogRotatingFileHandler(BaseRotatingHandler):
def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
self.base_filename = base_filename
self.interval = interval # seconds
self.max_bytes = max_bytes
self.backup_count = backup_count
self.log_files = self.get_existing_logfiles()
log_indexes = [f.split(".")[-1] for f in self.log_files]
self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
self.last_rollover = 0.0
self.doRollover()
def _open(self):
self.last_rollover = time.monotonic()
self.last_file_idx += 1
next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
stream = open(next_filename, self.mode, encoding=self.encoding)
self.log_files.insert(0, next_filename)
return stream
def get_existing_logfiles(self):
log_files = []
base_dir = os.path.dirname(self.base_filename)
for fn in os.listdir(base_dir):
fp = os.path.join(base_dir, fn)
if fp.startswith(self.base_filename) and os.path.isfile(fp):
log_files.append(fp)
# newest first, matching _open()'s insert(0, ...) so doRollover()'s pop() deletes the oldest
return sorted(log_files, reverse=True)
def shouldRollover(self, record):
size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= time.monotonic()
return size_exceeded or time_exceeded
def doRollover(self):
if self.stream:
self.stream.close()
self.stream = self._open()
if self.backup_count > 0:
while len(self.log_files) > self.backup_count:
to_delete = self.log_files.pop()
if os.path.exists(to_delete): # just being safe, should always exist
os.remove(to_delete)
class UnixDomainSocketHandler(logging.Handler):
def __init__(self, formatter):
logging.Handler.__init__(self)
self.setFormatter(formatter)
self.pid = None
self.zctx = None
self.sock = None
def __del__(self):
self.close()
def close(self):
if self.sock is not None:
self.sock.close()
if self.zctx is not None:
self.zctx.term()
def connect(self):
self.zctx = zmq.Context()
self.sock = self.zctx.socket(zmq.PUSH)
self.sock.setsockopt(zmq.LINGER, 10)
self.sock.connect(Paths.swaglog_ipc())
self.pid = os.getpid()
def emit(self, record):
if os.getpid() != self.pid:
# TODO suppresses warning about forking proc with zmq socket, fix root cause
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<zmq.*>")
self.connect()
msg = self.format(record).rstrip('\n')
# print("SEND".format(repr(msg)))
try:
s = chr(record.levelno)+msg
self.sock.send(s.encode('utf8'), zmq.NOBLOCK)
except zmq.error.Again:
# drop :/
pass
class ForwardingHandler(logging.Handler):
def __init__(self, target_logger):
super().__init__()
self.target_logger = target_logger
def emit(self, record):
self.target_logger.handle(record)
def add_file_handler(log):
"""
Function to add the file log handler to swaglog.
This can be used to store logs when logmessaged is not running.
"""
handler = get_file_handler()
handler.setFormatter(SwagLogFileFormatter(log))
log.addHandler(handler)
cloudlog = log = SwagLogger()
log.setLevel(logging.DEBUG)
outhandler = logging.StreamHandler()
print_level = os.environ.get('LOGPRINT', 'warning')
if print_level == 'debug':
outhandler.setLevel(logging.DEBUG)
elif print_level == 'info':
outhandler.setLevel(logging.INFO)
elif print_level == 'warning':
outhandler.setLevel(logging.WARNING)
ipchandler = UnixDomainSocketHandler(SwagFormatter(log))
log.addHandler(outhandler)
# logs are sent through IPC before writing to disk to prevent disk I/O blocking
log.addHandler(ipchandler)
openpilot/common/test.py
+166
View File
@@ -0,0 +1,166 @@
import contextlib
import gc
import inspect
import os
import subprocess
import unittest
from unittest import mock
from openpilot.common.hardware import HARDWARE, COMMA_HARDWARE
from openpilot.common.prefix import OpenpilotPrefix
from openpilot.system.manager import manager
@contextlib.contextmanager
def clean_env():
starting_env = dict(os.environ)
try:
yield
finally:
os.environ.clear()
os.environ.update(starting_env)
class OpenpilotTestCase(unittest.TestCase):
"""TestCase with openpilot's per-test isolation."""
COMMA_HARDWARE_TEST = False
SHARED_DOWNLOAD_CACHE = False
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
# Preserve legacy xunit hooks and invoke them inside the prefix below.
for name in ("setup_method", "teardown_method"):
hook = cls.__dict__.get(name)
if hook is not None:
setattr(cls, f"openpilot_{name}", hook)
setattr(cls, name, None)
def _fixture(self, name):
if name == "mocker":
return Mocker(self.addCleanup)
if name == "monkeypatch":
return MonkeyPatch(self.addCleanup)
if name == "subtests":
return SubTests(self)
fixture = getattr(inspect.getmodule(type(self)), name)
kwargs = {p: self._fixture(p) for p in inspect.signature(fixture).parameters}
value = fixture(**kwargs)
if inspect.isgenerator(value):
generator = value
value = next(generator)
self.addCleanup(lambda: next(generator, None))
return value
def _callTestMethod(self, method):
params = [name for name, param in inspect.signature(method).parameters.items()
if param.default is inspect.Parameter.empty]
return method(**{name: self._fixture(name) for name in params})
def run(self, result=None):
# This boundary cannot live in setUp/tearDown: existing unittest classes
# are allowed to override those hooks without calling super().
if (self.COMMA_HARDWARE_TEST and not COMMA_HARDWARE) or getattr(type(self), "__unittest_skip__", False):
return super().run(result)
test_env = clean_env()
test_env.__enter__()
prefix = OpenpilotPrefix(shared_download_cache=self.SHARED_DOWNLOAD_CACHE)
prefix.__enter__()
try:
return super().run(result)
finally:
prefix.__exit__(None, None, None)
manager.manager_cleanup()
if not gc.isenabled():
gc.enable()
gc.collect()
test_env.__exit__(None, None, None)
@classmethod
def setUpClass(cls):
super().setUpClass()
if cls.COMMA_HARDWARE_TEST and not COMMA_HARDWARE:
raise unittest.SkipTest("Skipping comma hardware test on PC")
cls._class_env = clean_env()
cls._class_env.__enter__()
setup_class = getattr(cls, "setup_class", None)
if setup_class is not None:
setup_class()
@classmethod
def tearDownClass(cls):
try:
teardown_class = getattr(cls, "teardown_class", None)
if teardown_class is not None:
teardown_class()
finally:
cls._class_env.__exit__(None, None, None)
super().tearDownClass()
def setUp(self):
super().setUp()
if self.COMMA_HARDWARE_TEST and not COMMA_HARDWARE:
self.skipTest("Skipping comma hardware test on PC")
if self.COMMA_HARDWARE_TEST:
HARDWARE.initialize_hardware()
HARDWARE.set_power_save(False)
subprocess.run(["pkill", "-9", "-f", "athena"], check=False)
setup_method = getattr(self, "openpilot_setup_method", None)
if setup_method is not None:
setup_method()
def tearDown(self):
try:
teardown_method = getattr(self, "openpilot_teardown_method", None)
if teardown_method is not None:
teardown_method()
finally:
super().tearDown()
class Mocker:
Mock = mock.Mock
MagicMock = mock.MagicMock
call = mock.call
ANY = mock.ANY
def __init__(self, add_cleanup):
self._add_cleanup = add_cleanup
self.patch = Patch(self._start)
def _start(self, patcher):
value = patcher.start()
self._add_cleanup(patcher.stop)
return value
class Patch:
def __init__(self, start):
self._start = start
def __call__(self, *args, **kwargs):
return self._start(mock.patch(*args, **kwargs))
def object(self, *args, **kwargs):
return self._start(mock.patch.object(*args, **kwargs))
class MonkeyPatch:
def __init__(self, add_cleanup):
self._add_cleanup = add_cleanup
def setattr(self, target, name, value):
patcher = mock.patch.object(target, name, value)
patcher.start()
self._add_cleanup(patcher.stop)
class SubTests:
def __init__(self, test_case):
self._test_case = test_case
def test(self, label=None, **kwargs):
return self._test_case.subTest(**kwargs) if label is None else self._test_case.subTest(label, **kwargs)
openpilot/common/tests/.gitignore
+2
View File
@@ -0,0 +1,2 @@
test_common
test_swaglog
openpilot/common/tests/__init__.py
View File
openpilot/common/tests/native_test.h
+25
View File
@@ -0,0 +1,25 @@
#pragma once
#include <iostream>
#include <stdexcept>
#include <string>
inline void native_test_check(bool condition, const char *expression, const char *file, int line) {
if (!condition) {
throw std::runtime_error(std::string(file) + ":" + std::to_string(line) + ": check failed: " + expression);
}
}
#define CHECK(condition) native_test_check(static_cast<bool>(condition), #condition, __FILE__, __LINE__)
#define REQUIRE(...) CHECK((__VA_ARGS__))
template <typename Function>
int run_native_test(Function &&function) {
try {
function();
return 0;
} catch (const std::exception &error) {
std::cerr << error.what() << '\n';
return 1;
}
}
openpilot/common/tests/test_file_helpers.py
+20
View File
@@ -0,0 +1,20 @@
import os
from uuid import uuid4
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.utils import atomic_write
class TestFileHelpers(OpenpilotTestCase):
def run_atomic_write_func(self, atomic_write_func):
path = f"/tmp/tmp{uuid4()}"
with atomic_write_func(path) as f:
f.write("test")
assert not os.path.exists(path)
with open(path) as f:
assert f.read() == "test"
os.remove(path)
def test_atomic_write(self):
self.run_atomic_write_func(atomic_write)
openpilot/common/tests/test_markdown.py
+16
View File
@@ -0,0 +1,16 @@
import os
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.basedir import BASEDIR
from openpilot.common.markdown import parse_markdown
class TestMarkdown(OpenpilotTestCase):
def test_all_release_notes(self):
with open(os.path.join(BASEDIR, "CHANGELOG.md")) as f:
release_notes = f.read().split("\n\n")
assert len(release_notes) > 10
for rn in release_notes:
md = parse_markdown(rn)
assert len(md) > 0
openpilot/common/tests/test_params.py
+146
View File
@@ -0,0 +1,146 @@
import datetime
import os
import threading
import time
import uuid
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.params import Params, ParamKeyFlag, UnknownKeyName
class TestParams(OpenpilotTestCase):
def setup_method(self):
self.params = Params()
def test_params_put_and_get(self):
self.params.put("DongleId", "cb38263377b873ee", block=True)
assert self.params.get("DongleId") == "cb38263377b873ee"
def test_params_non_ascii(self):
st = b"\xe1\x90\xff"
self.params.put("CarParams", st, block=True)
assert self.params.get("CarParams") == st
def test_params_get_cleared_manager_start(self):
self.params.put("CarParams", b"test", block=True)
self.params.put("DongleId", "cb38263377b873ee", block=True)
assert self.params.get("CarParams") == b"test"
undefined_param = self.params.get_param_path(uuid.uuid4().hex)
with open(undefined_param, "w") as f:
f.write("test")
assert os.path.isfile(undefined_param)
self.params.clear_all(ParamKeyFlag.CLEAR_ON_MANAGER_START)
assert self.params.get("CarParams") is None
assert self.params.get("DongleId") is not None
assert not os.path.isfile(undefined_param)
def test_params_two_things(self):
self.params.put("DongleId", "bob", block=True)
self.params.put("AthenadPid", 123, block=True)
assert self.params.get("DongleId") == "bob"
assert self.params.get("AthenadPid") == 123
def test_params_get_block(self):
def _delayed_writer():
time.sleep(0.1)
self.params.put("CarParams", b"test", block=True)
threading.Thread(target=_delayed_writer).start()
assert self.params.get("CarParams") is None
assert self.params.get("CarParams", block=True) == b"test"
def test_params_unknown_key_fails(self):
with self.assertRaises(UnknownKeyName):
self.params.get("swag")
with self.assertRaises(UnknownKeyName):
self.params.get_bool("swag")
with self.assertRaises(UnknownKeyName):
self.params.put("swag", "abc", block=True)
with self.assertRaises(UnknownKeyName):
self.params.put_bool("swag", True, block=True)
with self.assertRaises(UnknownKeyName):
self.params.put(b"DongleId\0suffix", "abc", block=True)
assert self.params.get_param_path(b"key\0suffix").endswith("/key\0suffix")
def test_remove_not_there(self):
assert self.params.get("CarParams") is None
self.params.remove("CarParams")
assert self.params.get("CarParams") is None
def test_get_bool(self):
self.params.remove("IsMetric")
assert not self.params.get_bool("IsMetric")
self.params.put_bool("IsMetric", True, block=True)
assert self.params.get_bool("IsMetric")
self.params.put_bool("IsMetric", False, block=True)
assert not self.params.get_bool("IsMetric")
self.params.put("IsMetric", True, block=True)
assert self.params.get_bool("IsMetric")
self.params.put("IsMetric", False, block=True)
assert not self.params.get_bool("IsMetric")
def test_put_non_blocking_with_get_block(self):
q = Params()
def _delayed_writer():
time.sleep(0.1)
Params().put("CarParams", b"test")
threading.Thread(target=_delayed_writer).start()
assert q.get("CarParams") is None
assert q.get("CarParams", True) == b"test"
def test_put_bool_non_blocking_with_get_block(self):
q = Params()
def _delayed_writer():
time.sleep(0.1)
Params().put_bool("CarParams", True)
threading.Thread(target=_delayed_writer).start()
assert q.get("CarParams") is None
assert q.get("CarParams", True) == b"1"
def test_params_all_keys(self):
keys = Params().all_keys()
# sanity checks
assert len(keys) > 20
assert len(keys) == len(set(keys))
assert b"CarParams" in keys
def test_params_default_value(self):
self.params.remove("LanguageSetting")
self.params.remove("LongitudinalPersonality")
self.params.remove("LiveParametersV2")
assert self.params.get("LanguageSetting") is None
assert self.params.get("LanguageSetting", return_default=False) is None
assert isinstance(self.params.get("LanguageSetting", return_default=True), str)
assert isinstance(self.params.get("LongitudinalPersonality", return_default=True), int)
assert self.params.get("LiveParametersV2") is None
assert self.params.get("LiveParametersV2", return_default=True) is None
def test_params_get_type(self):
# json
self.params.put("ApiCache_FirehoseStats", {"a": 0}, block=True)
assert self.params.get("ApiCache_FirehoseStats") == {"a": 0}
# int
self.params.put("BootCount", 1441, block=True)
assert self.params.get("BootCount") == 1441
# bool
self.params.put("AdbEnabled", True, block=True)
assert self.params.get("AdbEnabled")
assert isinstance(self.params.get("AdbEnabled"), bool)
# time
now = datetime.datetime.now(datetime.UTC)
self.params.put("InstallDate", now, block=True)
assert self.params.get("InstallDate") == now

Some files were not shown because too many files have changed in this diff Show More