mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-10-29 01:23:58 +08:00
Contracts
Define persistent settings, typed schema fields and storage migrations before feature owners consume them. Keep calibration and parameter readers aligned with the shared data contracts. Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com> Co-authored-by: Danny <dannyirwin31@yahoo.com> Co-authored-by: Harald Schäfer <harald.the.engineer@gmail.com> Co-authored-by: Prabhaav Pillai <prabhaav@msu.edu> Co-authored-by: dirwin31 <83434411+dirwin31@users.noreply.github.com> Co-authored-by: firestarsdog <229254897+firestarsdog@users.noreply.github.com> Co-authored-by: haraschax <6804392+haraschax@users.noreply.github.com> Co-authored-by: Dom <168790843+firestar5683@users.noreply.github.com> Co-authored-by: prabhaavp <prabhaav@msu.edu>
This commit is contained in:
firestar5683
@@ -650,6 +650,7 @@ struct CarParams {
|
||||
volvo @36;
|
||||
bmw @37;
|
||||
mg @38;
|
||||
teslaPreap @39;
|
||||
}
|
||||
|
||||
enum SteerControlType {
|
||||
|
||||
@@ -109,6 +109,9 @@ class FwQueryConfig:
|
||||
# returns set of candidates. Only will match if one candidate is returned
|
||||
match_fw_to_car_fuzzy: Callable[[LiveFwVersions, str, OfflineFwVersions], set[str]] | None = None
|
||||
|
||||
# Reject contradictory firmware evidence after either exact or fuzzy matching.
|
||||
validate_fw_match: Callable[[str, LiveFwVersions, OfflineFwVersions], bool] | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
# Asserts that a request exists if extra ecus are used
|
||||
if len(self.extra_ecus):
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
def supported_pedal_detected(fingerprint, bus, *, supported):
|
||||
if not supported:
|
||||
return False
|
||||
try:
|
||||
return fingerprint[bus].get(0x201) == 6
|
||||
except (KeyError, IndexError, TypeError, AttributeError):
|
||||
return False
|
||||
@@ -79,3 +79,5 @@ legend = ["LAT_ACCEL_FACTOR", "MAX_LAT_ACCEL_MEASURED", "FRICTION"]
|
||||
"VOLKSWAGEN_JETTA_MK7" = [1.2271623034089392, 1.216955117387, 0.19437384688370712]
|
||||
"VOLKSWAGEN_PASSAT_MK8" = [1.3432120736752917, 1.7087275587362314, 0.19444383787326647]
|
||||
"VOLKSWAGEN_TIGUAN_MK2" = [0.9711965500094828, 1.0001565939459098, 0.1465626137072916]
|
||||
|
||||
"CHEVROLET_MALIBU_HYBRID_CC" = [1.5961527626411784, 1.8422651988094612, 0.1572393918005158]
|
||||
|
||||
+1296
-18
File diff suppressed because it is too large
Load Diff
+24
-21
@@ -2631,33 +2631,36 @@ struct Event {
|
||||
|
||||
# DO change the name of the field
|
||||
# DON'T change anything after the "@"
|
||||
customReservedRawData0 @124 :Data;
|
||||
customReservedRawData1 @125 :Data;
|
||||
customReservedRawData2 @126 :Data;
|
||||
aolSafetyWire @124 :Data;
|
||||
aolIntentWire @125 :Data;
|
||||
laneChangeAssistWire @126 :Data;
|
||||
|
||||
# DO change the name of the field and struct
|
||||
# DON'T change the ID (e.g. @107)
|
||||
# DON'T change which struct it points to
|
||||
customReserved0 @107 :Custom.CustomReserved0;
|
||||
customReserved1 @108 :Custom.CustomReserved1;
|
||||
customReserved2 @109 :Custom.CustomReserved2;
|
||||
customReserved3 @110 :Custom.CustomReserved3;
|
||||
customReserved4 @111 :Custom.CustomReserved4;
|
||||
starpilotNavigation @107 :Custom.StarPilotCarControl;
|
||||
autoSteeringLearning @108 :Custom.StarPilotCarParams;
|
||||
starpilotCarState @109 :Custom.StarPilotCarState;
|
||||
starpilotLongitudinalPlan @110 :Custom.StarPilotLongitudinalPlan;
|
||||
slcVisionObservation @111 :Custom.StarPilotModelDataV2;
|
||||
customReserved5 @112 :Custom.CustomReserved5;
|
||||
# Temporary historical starpilotPlan replay transport.
|
||||
# When old-route replay support is removed, retain reserved slot @113.
|
||||
# Adapter: openpilot/starpilot/ui/old_starpilot_route_shim.py.
|
||||
customReserved6 @113 :Custom.CustomReserved6;
|
||||
customReserved7 @114 :Custom.CustomReserved7;
|
||||
customReserved8 @115 :Custom.CustomReserved8;
|
||||
customReserved9 @116 :Custom.CustomReserved9;
|
||||
customReserved10 @136 :Custom.CustomReserved10;
|
||||
customReserved11 @137 :Custom.CustomReserved11;
|
||||
customReserved12 @138 :Custom.CustomReserved12;
|
||||
customReserved13 @139 :Custom.CustomReserved13;
|
||||
customReserved14 @140 :Custom.CustomReserved14;
|
||||
customReserved15 @141 :Custom.CustomReserved15;
|
||||
customReserved16 @142 :Custom.CustomReserved16;
|
||||
customReserved17 @143 :Custom.CustomReserved17;
|
||||
customReserved18 @144 :Custom.CustomReserved18;
|
||||
customReserved19 @145 :Custom.CustomReserved19;
|
||||
starpilotRadarState @114 :Custom.StarPilotRadarState;
|
||||
starpilotSelfdriveState @115 :Custom.StarPilotSelfdriveState;
|
||||
slcState @116 :Custom.SlcState;
|
||||
spotMonitorState @136 :Custom.SpotMonitorState;
|
||||
starpilotLateralState @137 :Custom.StarPilotLateralState;
|
||||
slcAction @138 :Custom.SlcAction;
|
||||
slcCruiseEvent @139 :Custom.SlcCruiseEvent;
|
||||
slcDashboardObservation @140 :Custom.SlcDashboardObservation;
|
||||
slcCruiseCommand @141 :Custom.SlcCruiseCommand;
|
||||
aolAxisState @142 :Custom.AolAxisState;
|
||||
mapdExtendedOut @143 :Custom.MapdExtendedOut;
|
||||
mapdIn @144 :Custom.MapdIn;
|
||||
mapdOut @145 :Custom.MapdOut;
|
||||
|
||||
# *********** legacy + deprecated ***********
|
||||
model @9 :Deprecated.ModelData; # TODO: rename modelV2 and mark this as deprecated
|
||||
|
||||
@@ -180,8 +180,8 @@ class FrequencyTracker:
|
||||
|
||||
class SubMaster:
|
||||
def __init__(self, services: list[str], poll: str | None = None,
|
||||
ignore_alive: list[str] | None = None, ignore_avg_freq: list[str] | None = None,
|
||||
ignore_valid: list[str] | None = None, addr: str = "127.0.0.1", frequency: float | None = None):
|
||||
ignore_alive: list[str] | set[str] | None = None, ignore_avg_freq: list[str] | set[str] | None = None,
|
||||
ignore_valid: list[str] | set[str] | None = None, addr: str = "127.0.0.1", frequency: float | None = None):
|
||||
self.frame = -1
|
||||
self.services = services
|
||||
self.seen = dict.fromkeys(services, False)
|
||||
@@ -234,7 +234,9 @@ class SubMaster:
|
||||
|
||||
def update(self, timeout: int = 100) -> None:
|
||||
msgs = []
|
||||
for sock in self.poller.poll(timeout):
|
||||
# Empty native polls can yield even with a zero timeout.
|
||||
socks = (self.sock[s] for s in self.services if s not in self.non_polled_services) if timeout == 0 else self.poller.poll(timeout)
|
||||
for sock in socks:
|
||||
msgs.append(recv_one_or_none(sock))
|
||||
|
||||
# non-blocking receive for non-polled sockets
|
||||
@@ -266,13 +268,43 @@ class SubMaster:
|
||||
self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
|
||||
|
||||
def all_alive(self, service_list: list[str] | None = None) -> bool:
|
||||
return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
|
||||
for s in (service_list or self.services):
|
||||
try:
|
||||
if s in self.ignore_alive:
|
||||
continue
|
||||
value = self.alive[s]
|
||||
except StopIteration as exc:
|
||||
raise RuntimeError('generator raised StopIteration') from exc
|
||||
if not value:
|
||||
return False
|
||||
del value
|
||||
return True
|
||||
|
||||
def all_freq_ok(self, service_list: list[str] | None = None) -> bool:
|
||||
return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
|
||||
for s in (service_list or self.services):
|
||||
try:
|
||||
if not self._check_avg_freq(s):
|
||||
continue
|
||||
value = self.freq_ok[s]
|
||||
except StopIteration as exc:
|
||||
raise RuntimeError('generator raised StopIteration') from exc
|
||||
if not value:
|
||||
return False
|
||||
del value
|
||||
return True
|
||||
|
||||
def all_valid(self, service_list: list[str] | None = None) -> bool:
|
||||
return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
|
||||
for s in (service_list or self.services):
|
||||
try:
|
||||
if s in self.ignore_valid:
|
||||
continue
|
||||
value = self.valid[s]
|
||||
except StopIteration as exc:
|
||||
raise RuntimeError('generator raised StopIteration') from exc
|
||||
if not value:
|
||||
return False
|
||||
del value
|
||||
return True
|
||||
|
||||
def all_checks(self, service_list: list[str] | None = None) -> bool:
|
||||
return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
|
||||
|
||||
@@ -2,6 +2,7 @@ import random
|
||||
import time
|
||||
from typing import cast
|
||||
from collections.abc import Sized
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from openpilot.common.test import OpenpilotTestCase
|
||||
import openpilot.cereal.messaging as messaging
|
||||
@@ -68,6 +69,62 @@ class TestSubMaster(OpenpilotTestCase):
|
||||
assert t < 0.1
|
||||
assert not any(sm.updated.values())
|
||||
|
||||
def test_nonblocking_update(self):
|
||||
services = ["modelV2", "carState", "carParams"]
|
||||
for poll in (None, "modelV2"):
|
||||
with self.subTest(poll=poll):
|
||||
pm = messaging.PubMaster(services)
|
||||
sm = messaging.SubMaster(services, poll=poll)
|
||||
sm.poller = Mock()
|
||||
sm.poller.poll.side_effect = AssertionError("nonblocking updates must not poll")
|
||||
|
||||
sm.update(0)
|
||||
assert sm.frame == 0
|
||||
assert not any(sm.updated.values())
|
||||
assert not any(sm.seen.values())
|
||||
assert not sm.alive["carState"] and not sm.alive["modelV2"]
|
||||
assert not any(sm.valid.values())
|
||||
|
||||
for i, service in enumerate(services):
|
||||
msg = messaging.new_message(service)
|
||||
msg.logMonoTime = 100 + i
|
||||
msg.valid = service != "carState"
|
||||
pm.send(service, msg)
|
||||
|
||||
with patch.object(messaging, "recv_one_or_none", wraps=messaging.recv_one_or_none) as recv:
|
||||
sm.update(0)
|
||||
order = [s for s in services if s not in sm.non_polled_services] + list(sm.non_polled_services)
|
||||
assert [call.args[0] for call in recv.call_args_list] == [sm.sock[s] for s in order]
|
||||
|
||||
assert all(sm.updated.values()) and all(sm.seen.values())
|
||||
assert sm.logMonoTime == dict(zip(services, (100, 101, 102), strict=True))
|
||||
assert sm.valid == {"modelV2": True, "carState": False, "carParams": True}
|
||||
assert not sm.all_valid() and sm.all_valid(["modelV2", "carParams"])
|
||||
assert len(set(sm.recv_time.values())) == 1
|
||||
assert all(f == sm.frame for f in sm.recv_frame.values())
|
||||
|
||||
received = (sm.recv_time.copy(), sm.recv_frame.copy(), sm.logMonoTime.copy())
|
||||
sm.update(0)
|
||||
assert not any(sm.updated.values())
|
||||
assert received == (sm.recv_time, sm.recv_frame, sm.logMonoTime)
|
||||
|
||||
for speed in range(3):
|
||||
msg = messaging.new_message("carState")
|
||||
msg.carState.vEgo = speed
|
||||
pm.send("carState", msg)
|
||||
sm.update(0)
|
||||
assert sm["carState"].vEgo == 2
|
||||
assert sm.updated == {"modelV2": False, "carState": True, "carParams": False}
|
||||
sm.poller.poll.assert_not_called()
|
||||
del sm, pm
|
||||
|
||||
def test_positive_timeout_uses_poller(self):
|
||||
sm = messaging.SubMaster(["carState"])
|
||||
sm.poller = Mock()
|
||||
sm.poller.poll.return_value = []
|
||||
sm.update(1)
|
||||
sm.poller.poll.assert_called_once_with(1)
|
||||
|
||||
def test_avg_frequency_checks(self):
|
||||
for poll in (True, False):
|
||||
sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "extrinsicsCalibration"],
|
||||
|
||||
@@ -19,6 +19,7 @@ class Service:
|
||||
|
||||
|
||||
_services: dict[str, tuple] = {
|
||||
"starpilotNavigation": (True, 1., 1),
|
||||
# service: (should_log, frequency, qlog decimation (optional))
|
||||
# note: the "EncodeIdx" packets will still be in the log
|
||||
"gyroscope": (True, 104., 104),
|
||||
@@ -30,27 +31,52 @@ _services: dict[str, tuple] = {
|
||||
"touch": (True, 20., 1),
|
||||
"can": (True, 100., 2053, QueueSize.BIG), # decimation gives ~3 msgs in a full segment
|
||||
"controlsState": (True, 100., 10, QueueSize.MEDIUM),
|
||||
"starpilotLateralState": (True, 100., 10),
|
||||
"selfdriveState": (True, 100., 10),
|
||||
"starpilotSelfdriveState": (True, 100., 10),
|
||||
"pandaStates": (True, 10., 1),
|
||||
"peripheralState": (True, 2., 1),
|
||||
"radarState": (True, 20., 5),
|
||||
"starpilotRadarState": (True, 20., 5),
|
||||
# Sparse, gated camera observations; freshness is qualified from payload
|
||||
# clocks, never inferred from a nominal service frequency.
|
||||
"spotMonitorState": (True, 0., 1),
|
||||
"narrowRoadEncodeIdx": (False, 20., 1),
|
||||
"radarTracks": (True, 20.),
|
||||
"sendcan": (True, 100., 139, QueueSize.MEDIUM),
|
||||
"logMessage": (True, 0., None, QueueSize.BIG),
|
||||
# Recorder-only transport carrying Event.logMessage, retained in qlog.
|
||||
"modelIdentity": (True, 0.2, 1),
|
||||
"errorLogMessage": (True, 0., 1, QueueSize.BIG),
|
||||
"extrinsicsCalibration": (True, 4., 4),
|
||||
"lateralTorqueParameters": (True, 4., 1),
|
||||
"autoSteeringLearning": (True, 4., 1),
|
||||
"lateralDelay": (True, 4., 1),
|
||||
"operatingSystemLog": (True, 0.),
|
||||
"carState": (True, 100., 10),
|
||||
"carControl": (True, 100., 10),
|
||||
"carOutput": (True, 100., 10),
|
||||
"longitudinalPlan": (True, 20., 10),
|
||||
"starpilotLongitudinalPlan": (True, 20., 10),
|
||||
"slcState": (True, 20., 10),
|
||||
"customReserved6": (False, 20., 10), # Historical starpilotPlan, replay only.
|
||||
"slcAction": (True, 0., 1),
|
||||
"slcCruiseEvent": (True, 0., 1),
|
||||
"slcDashboardObservation": (True, 100., 10),
|
||||
"slcVisionObservation": (True, 6., 6),
|
||||
"slcCruiseCommand": (True, 20., 1),
|
||||
"aolAxisState": (True, 100., 10),
|
||||
"aolSafetyWire": (True, 10., 10),
|
||||
"aolIntentWire": (True, 100., 10),
|
||||
"laneChangeAssistWire": (True, 20., 10),
|
||||
"mapdOut": (True, 20., 10),
|
||||
"mapdExtendedOut": (True, 1., 1),
|
||||
"mapdIn": (True, 1., 1),
|
||||
"lateralManeuverPlan": (True, 20.),
|
||||
"driverAssistance": (True, 20., 20),
|
||||
"procLog": (True, 0.5, 15, QueueSize.BIG),
|
||||
"gpsLocationExternal": (True, 10., 10),
|
||||
"starpilotCarState": (True, 0., 1),
|
||||
"gpsLocation": (True, 1., 1),
|
||||
"ubloxGnss": (True, 10.),
|
||||
"qcomGnss": (True, 2.),
|
||||
@@ -92,7 +118,6 @@ _services: dict[str, tuple] = {
|
||||
"livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
|
||||
"livestreamNarrowRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
|
||||
"livestreamCabinEncodeData": (False, 20., None, QueueSize.MEDIUM),
|
||||
"customReservedRawData0": (True, 0.),
|
||||
}
|
||||
SERVICE_LIST = {name: Service(*vals) for
|
||||
idx, (name, vals) in enumerate(_services.items())}
|
||||
|
||||
@@ -76,16 +76,28 @@ std::string ensure_params_path(const std::string &prefix, const std::string &pat
|
||||
|
||||
class FileLock {
|
||||
public:
|
||||
FileLock(const std::string &fn) {
|
||||
fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
|
||||
if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
|
||||
FileLock(const std::string &fn) : owner_pid_(getpid()) {
|
||||
fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT | O_CLOEXEC, 0775));
|
||||
locked_ = fd_ >= 0 && HANDLE_EINTR(flock(fd_, LOCK_EX)) == 0;
|
||||
if (!locked_) {
|
||||
LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
|
||||
}
|
||||
}
|
||||
~FileLock() { close(fd_); }
|
||||
~FileLock() {
|
||||
if (fd_ >= 0) {
|
||||
// flock follows the open file description across fork. Release explicitly
|
||||
// in the acquiring process even while children retain their copies.
|
||||
if (locked_ && owner_pid_ == getpid()) {
|
||||
HANDLE_EINTR(flock(fd_, LOCK_UN));
|
||||
}
|
||||
close(fd_);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const pid_t owner_pid_;
|
||||
int fd_ = -1;
|
||||
bool locked_ = false;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
@@ -8,7 +8,162 @@
|
||||
inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
{"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
|
||||
{"AdbEnabled", {PERSISTENT, BOOL}},
|
||||
{"BluetoothEnabled", {PERSISTENT, BOOL}},
|
||||
{"DisableOpenpilotLongitudinal", {PERSISTENT, BOOL, "0"}},
|
||||
{"AlwaysOnDM", {PERSISTENT, BOOL}},
|
||||
{"WarningImmediateVolume", {PERSISTENT, INT}},
|
||||
{"WarningSoftVolume", {PERSISTENT, INT}},
|
||||
{"RefuseVolume", {PERSISTENT, INT}},
|
||||
{"PromptDistractedVolume", {PERSISTENT, INT}},
|
||||
{"SoundPack", {PERSISTENT, STRING, "starpilot"}},
|
||||
{"EngageVolume", {PERSISTENT, INT}},
|
||||
{"DisengageVolume", {PERSISTENT, INT}},
|
||||
{"PromptVolume", {PERSISTENT, INT}},
|
||||
{"AlwaysOnLateral", {PERSISTENT, BOOL, "0"}},
|
||||
{"NostalgiaMode", {PERSISTENT, BOOL, "0"}},
|
||||
{"ToyotaAutoHold", {PERSISTENT, BOOL, "0"}},
|
||||
{"ToyotaRadarMode", {PERSISTENT, INT, "0"}},
|
||||
{"RemoteStartBootsComma", {PERSISTENT, BOOL, "0"}},
|
||||
{"HKGRemoteStartBootsComma", {PERSISTENT, BOOL, "0"}},
|
||||
{"IgnoreIgnitionLine", {PERSISTENT, BOOL, "0"}},
|
||||
{"TeslaWakeOnCAN", {PERSISTENT, BOOL, "0"}},
|
||||
{"VoltSNG", {PERSISTENT, BOOL, "0"}},
|
||||
{"GMAutoHold", {PERSISTENT, BOOL, "0"}},
|
||||
{"VoltOnePedalMode", {PERSISTENT, BOOL, "0"}},
|
||||
{"RadarRecoveryAlert", {PERSISTENT, BOOL, "0"}},
|
||||
{"HondaBoschARadar", {PERSISTENT, BOOL, "1"}},
|
||||
{"HondaGasFactorParams", {PERSISTENT, FLOAT, "1.0"}},
|
||||
{"HondaWindFactorParams", {PERSISTENT, FLOAT, "1.0"}},
|
||||
{"FordLateralMode", {PERSISTENT, INT, "0"}},
|
||||
{"FordHumanTurnDetection", {PERSISTENT, BOOL, "1"}},
|
||||
{"FordCurvatureBlendLow", {PERSISTENT, FLOAT, "0.4"}},
|
||||
{"FordCurvatureBlendHigh", {PERSISTENT, FLOAT, "0.4"}},
|
||||
{"FordCurvatureLaneChangeFactor", {PERSISTENT, FLOAT, "0.85"}},
|
||||
{"OnroadCustomizations", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"StarPilotFavoriteSlots", {PERSISTENT, JSON, R"([{"enabled":false,"show_onroad":false,"key":null,"label":""},{"enabled":false,"show_onroad":false,"key":null,"label":""},{"enabled":false,"show_onroad":false,"key":null,"label":""}])"}},
|
||||
{"StarPilotControllerBindings", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"UseStarPilotLongitudinalPlanner", {PERSISTENT, BOOL, "1"}},
|
||||
{"UseVOACC", {PERSISTENT, BOOL, "0"}},
|
||||
{"CustomPersonalities", {PERSISTENT, BOOL, "0"}},
|
||||
{"SettingsRetirementState", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"RetiredSettingsEvidence", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"QOLLongitudinal", {PERSISTENT, BOOL}},
|
||||
{"ReverseCruise", {PERSISTENT, BOOL, "0"}},
|
||||
{"ForceStops", {PERSISTENT, BOOL, "0"}},
|
||||
{"AlwaysAllowUploads", {PERSISTENT, BOOL, "0"}},
|
||||
{"UsageStatsState", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"UsageStatsStatus", {CLEAR_ON_MANAGER_START | DONT_LOG, JSON}},
|
||||
{"TeslaAOLScreenTap", {PERSISTENT, BOOL, "0"}},
|
||||
{"TeslaAOLDisengageOnBrake", {PERSISTENT, BOOL, "0"}},
|
||||
{"ForceStopDistanceOffset", {PERSISTENT, INT, "0"}},
|
||||
{"LeadApproachBuffer", {PERSISTENT, BOOL, "0"}},
|
||||
{"FasterLeadTakeoff", {PERSISTENT, BOOL, "0"}},
|
||||
{"SwitchbackModeCooldown", {PERSISTENT, FLOAT, "5.0"}},
|
||||
{"CustomCruise", {PERSISTENT, FLOAT, "1.0"}},
|
||||
{"CustomCruiseLong", {PERSISTENT, FLOAT, "5.0"}},
|
||||
{"LongitudinalPersonalityProfiles", {PERSISTENT | DONT_LOG, JSON, "{}"}},
|
||||
{"TrafficPersonalityProfile", {PERSISTENT, BOOL}},
|
||||
{"AggressivePersonalityProfile", {PERSISTENT, BOOL}},
|
||||
{"StandardPersonalityProfile", {PERSISTENT, BOOL}},
|
||||
{"RelaxedPersonalityProfile", {PERSISTENT, BOOL}},
|
||||
{"TrafficFollow", {PERSISTENT, FLOAT, "0.75"}},
|
||||
{"AggressiveFollow", {PERSISTENT, FLOAT, "1.25"}},
|
||||
{"AggressiveFollowHigh", {PERSISTENT, FLOAT, "1.0"}},
|
||||
{"StandardFollow", {PERSISTENT, FLOAT, "1.45"}},
|
||||
{"StandardFollowHigh", {PERSISTENT, FLOAT, "1.2"}},
|
||||
{"RelaxedFollow", {PERSISTENT, FLOAT, "1.6"}},
|
||||
{"RelaxedFollowHigh", {PERSISTENT, FLOAT, "1.4"}},
|
||||
{"TrafficJerkAcceleration", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"TrafficJerkDeceleration", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"TrafficJerkDanger", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"TrafficJerkSpeed", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"TrafficJerkSpeedDecrease", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"AggressiveJerkAcceleration", {PERSISTENT, FLOAT, "50.0"}},
|
||||
{"AggressiveJerkDeceleration", {PERSISTENT, FLOAT, "50.0"}},
|
||||
{"AggressiveJerkDanger", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"AggressiveJerkSpeed", {PERSISTENT, FLOAT, "50.0"}},
|
||||
{"AggressiveJerkSpeedDecrease", {PERSISTENT, FLOAT, "50.0"}},
|
||||
{"StandardJerkAcceleration", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"StandardJerkDeceleration", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"StandardJerkDanger", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"StandardJerkSpeed", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"StandardJerkSpeedDecrease", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"RelaxedJerkAcceleration", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"RelaxedJerkDeceleration", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"RelaxedJerkDanger", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"RelaxedJerkSpeed", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"RelaxedJerkSpeedDecrease", {PERSISTENT, FLOAT, "100.0"}},
|
||||
{"PauseAOLOnBrake", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"AolBrakePauseSpeedMps", {PERSISTENT, FLOAT}},
|
||||
{"LaneCentering", {PERSISTENT, BOOL, "0"}},
|
||||
{"LaneCenteringPauseOnSignal", {PERSISTENT, BOOL, "1"}},
|
||||
{"LaneCenterOffset", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"LaneCenteringE2EAuthority", {PERSISTENT, FLOAT, "1.0"}},
|
||||
{"LaneCenteringStrength", {PERSISTENT, FLOAT, "1.5"}},
|
||||
{"AndroidAutoEnabled", {PERSISTENT | DONT_LOG, BOOL, "0"}},
|
||||
{"AdvancedLateralTune", {PERSISTENT, BOOL}},
|
||||
{"LateralControllerSelection", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"SteeringLearningPreferences", {PERSISTENT, BYTES}},
|
||||
{"AutoSteeringLearningReset", {PERSISTENT, STRING}},
|
||||
{"AutomaticSteeringProfiles", {PERSISTENT | DONT_LOG, BYTES}},
|
||||
{"ForceAutoTuneOff", {PERSISTENT, BOOL, "0"}},
|
||||
{"SteerLatAccel", {PERSISTENT, FLOAT}},
|
||||
{"SteerLatAccelStock", {PERSISTENT, FLOAT}},
|
||||
{"SteerFriction", {PERSISTENT, FLOAT}},
|
||||
{"SteerFrictionStock", {PERSISTENT, FLOAT}},
|
||||
{"TorqueOverrideDocument", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"HideSpeed", {PERSISTENT, BOOL, "0"}},
|
||||
{"HideMaxSpeed", {PERSISTENT, BOOL, "0"}},
|
||||
{"HideSteeringWheel", {PERSISTENT, BOOL, "0"}},
|
||||
{"CameraView", {PERSISTENT, INT, "2"}},
|
||||
{"DriverCamera", {PERSISTENT, BOOL, "0"}},
|
||||
{"StoppedTimer", {PERSISTENT, BOOL, "0"}},
|
||||
{"StockConfidenceBallWidget", {PERSISTENT, BOOL, "0"}},
|
||||
{"EnableTorqueBarWidget", {PERSISTENT, BOOL, "1"}},
|
||||
{"HideLeadMarker", {PERSISTENT, BOOL, "1"}},
|
||||
{"LeadInfo", {PERSISTENT, BOOL, "0"}},
|
||||
{"LeadInfoMode", {PERSISTENT, INT, "2"}},
|
||||
{"BlindSpotMetrics", {PERSISTENT, BOOL, "1"}},
|
||||
{"SignalMetrics", {PERSISTENT, BOOL, "0"}},
|
||||
{"PIPPreviewEnabled", {PERSISTENT, BOOL, "0"}},
|
||||
{"PIPPreviewMask", {PERSISTENT | DONT_LOG, JSON, "{\"width\":1928,\"height\":1208,\"center_left\":[315,548],\"center_right\":[1571,539],\"crop_size\":580}"}},
|
||||
{"PIPPreviewShowOnBlinker", {PERSISTENT, BOOL, "0"}},
|
||||
{"PIPPreviewShowOnBSM", {PERSISTENT, BOOL, "0"}},
|
||||
{"PIPPreviewInvert", {PERSISTENT, BOOL, "0"}},
|
||||
{"StarPilotDisplayPreferencesEnabled", {PERSISTENT, BOOL}},
|
||||
{"ScreenBrightness", {PERSISTENT, INT}},
|
||||
{"ScreenBrightnessOnroad", {PERSISTENT, INT}},
|
||||
{"ScreenTimeout", {PERSISTENT, INT}},
|
||||
{"ScreenTimeoutOnroad", {PERSISTENT, INT}},
|
||||
{"TurnOffCommaScreen", {PERSISTENT, BOOL, "0"}},
|
||||
{"KeepScreenOnGpuLoading", {PERSISTENT, BOOL, "0"}},
|
||||
{"LKASButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"SonataHybridLKASButtonControlMigrated", {PERSISTENT, BOOL, "0"}},
|
||||
{"MainCruiseButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"PulseGlideSpeedDelta", {PERSISTENT, FLOAT, "5.0"}},
|
||||
{"DistanceButtonControl", {PERSISTENT, INT, "1"}},
|
||||
{"LongDistanceButtonControl", {PERSISTENT, INT, "5"}},
|
||||
{"VeryLongDistanceButtonControl", {PERSISTENT, INT, "6"}},
|
||||
{"ModeButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"LongModeButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"VeryLongModeButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"StarButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"LongStarButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"VeryLongStarButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"CancelButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"LongCancelButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"VeryLongCancelButtonControl", {PERSISTENT, INT, "0"}},
|
||||
{"LongPitch", {PERSISTENT, BOOL, "1"}},
|
||||
{"GMLongitudinalTune", {PERSISTENT, INT, "0"}},
|
||||
{"TruckTuning", {PERSISTENT, BOOL, "0"}},
|
||||
{"EVTuning", {PERSISTENT, BOOL}},
|
||||
{"LongitudinalMaxOutputAcceleration", {PERSISTENT, FLOAT, "4.0"}},
|
||||
{"TuningPreparationState", {PERSISTENT, JSON, ""}},
|
||||
{"PauseLateralSpeed", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"PauseLateralOnSignal", {PERSISTENT, BOOL, "0"}},
|
||||
{"LateralResumeDelay", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"TurnAssist", {PERSISTENT, BOOL, "1"}},
|
||||
{"GMPedalLongitudinal", {PERSISTENT, BOOL, "0"}},
|
||||
{"ApiCache_Device", {PERSISTENT, STRING}},
|
||||
{"ApiCache_FirehoseStats", {PERSISTENT, JSON}},
|
||||
{"AssistNowToken", {PERSISTENT, STRING}},
|
||||
@@ -39,8 +194,8 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
{"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
|
||||
{"DriverLockoutCount", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, INT, "0"}},
|
||||
{"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY, BOOL}},
|
||||
{"ExperimentalMode", {PERSISTENT, BOOL, "1"}},
|
||||
{"AlphaLongitudinalEnabled", {PERSISTENT, BOOL}},
|
||||
{"ExperimentalMode", {PERSISTENT, BOOL}},
|
||||
{"FirmwareQueryDone", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
|
||||
{"ForcePowerDown", {PERSISTENT, BOOL}},
|
||||
{"GitBranch", {PERSISTENT, STRING}},
|
||||
@@ -85,6 +240,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
{"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
|
||||
{"LocationFilterInitialState", {PERSISTENT, BYTES}},
|
||||
{"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
|
||||
{"LongitudinalManeuverPaddleMode", {PERSISTENT, STRING, "auto"}},
|
||||
{"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
|
||||
{"LongitudinalPersonality", {PERSISTENT, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
|
||||
{"NetworkMetered", {PERSISTENT, BOOL}},
|
||||
@@ -108,35 +264,94 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
|
||||
{"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
|
||||
{"Offroad_DriverMonitoringUncertain", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
|
||||
{"DriveStateSession", {CLEAR_ON_MANAGER_START | DONT_LOG, JSON}},
|
||||
{"DriveStateRequest", {CLEAR_ON_MANAGER_START | DONT_LOG, JSON}},
|
||||
{"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"OpenpilotEnabledToggle", {PERSISTENT, BOOL, "1"}},
|
||||
{"Offset1", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"Offset2", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"Offset3", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"Offset4", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"Offset5", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"Offset6", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"Offset7", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
|
||||
{"PairingEmail", {PERSISTENT, STRING}},
|
||||
{"ConnectProvider", {PERSISTENT, STRING, "comma"}},
|
||||
{"PairingProvider", {PERSISTENT, STRING}},
|
||||
{"PedalsOnUI", {PERSISTENT, BOOL, "0"}},
|
||||
{"PrimeType", {PERSISTENT, INT}},
|
||||
{"RainbowPath", {PERSISTENT, BOOL, "0"}},
|
||||
{"RecordAudio", {PERSISTENT, BOOL}},
|
||||
{"RecordFront", {PERSISTENT, BOOL}},
|
||||
{"RecordFrontLock", {PERSISTENT, BOOL}}, // for the internal fleet
|
||||
{"SecOCKey", {PERSISTENT | DONT_LOG, STRING}},
|
||||
{"SecOCKeys", {PERSISTENT | DONT_LOG, STRING}},
|
||||
{"LockDoors", {PERSISTENT, BOOL, "0"}},
|
||||
{"UnlockDoors", {PERSISTENT, BOOL, "0"}},
|
||||
{"ShowDebugInfo", {PERSISTENT, BOOL}},
|
||||
{"DeveloperSidebar", {PERSISTENT, BOOL, "0"}},
|
||||
{"ShowBrakeStatus", {PERSISTENT, BOOL, "0"}},
|
||||
{"HideDMIcon", {PERSISTENT, BOOL, "0"}},
|
||||
{"HidePersonalityAlerts", {PERSISTENT, BOOL, "0"}},
|
||||
{"ShowSpeedLimits", {PERSISTENT, BOOL, "1"}},
|
||||
{"SLCConfirmation", {PERSISTENT, BOOL, "0"}},
|
||||
{"SLCConfirmationHigher", {PERSISTENT, BOOL, "0"}},
|
||||
{"SLCConfirmationLower", {PERSISTENT, BOOL, "0"}},
|
||||
{"SLCFallback", {PERSISTENT, INT, "2"}},
|
||||
{"SLCOverride", {PERSISTENT, INT, "1"}},
|
||||
{"SLCQualifiedHistory", {PERSISTENT, JSON}},
|
||||
{"ConditionalModeConfig", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"ConditionalManualState", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"VehicleSelection", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"NAPPedalEnabled", {PERSISTENT, BOOL, "0"}},
|
||||
{"NAPRadarEnabled", {PERSISTENT, BOOL, "0"}},
|
||||
{"NAPRadarBehindNosecone", {PERSISTENT, BOOL, "0"}},
|
||||
{"SafeMode", {PERSISTENT | DONT_LOG, BOOL, "0"}},
|
||||
{"CurveSpeedController", {PERSISTENT | DONT_LOG, BOOL, "0"}},
|
||||
{"CurveSpeedControllerNoLead", {PERSISTENT | DONT_LOG, BOOL, "0"}},
|
||||
{"ShowCSCStatus", {PERSISTENT | DONT_LOG, BOOL, "0"}},
|
||||
{"CurvatureData", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"CurveComfortData", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"SLCOffsetSchedule", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"MapdShadowEnabled", {PERSISTENT | DONT_LOG, BOOL, "0"}},
|
||||
{"OffroadPowerPreferences", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"SentryMotionPreferences", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"VASMPreferences", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"LaneChangePreferences", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"SLCPriority1", {PERSISTENT, STRING, "Vision"}},
|
||||
{"SLCPriority2", {PERSISTENT, STRING, "Dashboard"}},
|
||||
{"SpeedLimitController", {PERSISTENT, BOOL, "0"}},
|
||||
{"RouteCount", {PERSISTENT, INT, "0"}},
|
||||
{"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
|
||||
{"SshEnabled", {PERSISTENT, BOOL}},
|
||||
{"UbloxAvailable", {PERSISTENT, BOOL}},
|
||||
{"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
|
||||
{"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}},
|
||||
{"UpdaterAutomaticDownloads", {PERSISTENT, BOOL}},
|
||||
{"UpdaterAvailableBranches", {PERSISTENT, STRING}},
|
||||
{"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
|
||||
{"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}},
|
||||
{"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
|
||||
{"UpdaterOSUpdate", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"UpdaterFastState", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"UpdaterLastCheckTime", {PERSISTENT, TIME}},
|
||||
{"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}},
|
||||
{"UpdaterLastFetchTime", {PERSISTENT, TIME}},
|
||||
{"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
|
||||
{"ChestnutActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
|
||||
{"ChestnutLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
|
||||
{"JetlinkMode", {PERSISTENT, INT, "0"}},
|
||||
{"JetlinkChargePhone", {PERSISTENT, BOOL, "0"}},
|
||||
{"JetlinkBigModel", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"JetlinkCatalog", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"JetlinkSpec", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"JetlinkPointers", {PERSISTENT | DONT_LOG, JSON}},
|
||||
{"JetlinkProgress", {CLEAR_ON_MANAGER_START | DONT_LOG, JSON}},
|
||||
{"JetlinkRuntime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | DONT_LOG, JSON}},
|
||||
{"Version", {PERSISTENT, STRING}},
|
||||
};
|
||||
|
||||
@@ -12,8 +12,10 @@ from openpilot.common.hardware.hw import Paths
|
||||
|
||||
|
||||
def get_file_handler():
|
||||
Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True)
|
||||
base_filename = os.path.join(Paths.swaglog_root(), "swaglog")
|
||||
from openpilot.starpilot.connect.provider import cloudlog_root
|
||||
directory = cloudlog_root()
|
||||
Path(directory).mkdir(parents=True, exist_ok=True)
|
||||
base_filename = os.path.join(directory, "swaglog")
|
||||
handler = SwaglogRotatingFileHandler(base_filename)
|
||||
return handler
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ from openpilot.common.params import Params
|
||||
from openpilot.common.realtime import config_realtime_process
|
||||
from openpilot.common.transformations.orientation import rot_from_euler, euler_from_rot
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.starpilot.schema_cache import get_cache, prewarm_cache_contracts, put_cache
|
||||
|
||||
MIN_SPEED_FILTER = 15 * CV.MPH_TO_MS
|
||||
MAX_VEL_ANGLE_STD = np.radians(0.25)
|
||||
@@ -63,13 +64,14 @@ def moving_avg_with_linear_decay(prev_mean: np.ndarray, new_val: np.ndarray, idx
|
||||
|
||||
class Calibrator:
|
||||
def __init__(self, param_put: bool = False):
|
||||
prewarm_cache_contracts(("CalibrationParams",))
|
||||
self.param_put = param_put
|
||||
|
||||
self.not_car = False
|
||||
|
||||
# Read saved calibration
|
||||
self.params = Params()
|
||||
calibration_params = self.params.get("CalibrationParams")
|
||||
calibration_params = get_cache(self.params, "CalibrationParams")
|
||||
rpy_init = RPY_INIT
|
||||
wide_from_device_euler = WIDE_FROM_DEVICE_EULER_INIT
|
||||
height = HEIGHT_INIT
|
||||
@@ -168,7 +170,7 @@ class Calibrator:
|
||||
|
||||
write_this_cycle = (self.idx == 0) and (self.block_idx % (INPUTS_WANTED//5) == 5)
|
||||
if self.param_put and write_this_cycle:
|
||||
self.params.put("CalibrationParams", self.get_msg(True).to_bytes())
|
||||
put_cache(self.params, "CalibrationParams", self.get_msg(True))
|
||||
|
||||
def handle_v_ego(self, v_ego: float) -> None:
|
||||
self.v_ego = v_ego
|
||||
|
||||
@@ -11,6 +11,7 @@ from opendbc.car.structs import car
|
||||
from openpilot.cereal.services import SERVICE_LIST
|
||||
from openpilot.common.constants import CV
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.starpilot.schema_cache import get_cache, prewarm_cache_contracts, put_cache
|
||||
from openpilot.common.realtime import config_realtime_process
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.selfdrive.locationd.helpers import PoseCalibrator, Pose, fft_next_good_size, parabolic_peak_interp
|
||||
@@ -362,10 +363,10 @@ class LateralLagEstimator:
|
||||
|
||||
|
||||
def retrieve_initial_lag(params: Params, CP: car.CarParams):
|
||||
last_lag_data = params.get("LiveDelay")
|
||||
last_carparams_data = params.get("CarParamsPrevRoute")
|
||||
last_lag_data = get_cache(params, "LiveDelay")
|
||||
last_carparams_data = get_cache(params, "CarParamsPrevRoute")
|
||||
|
||||
if last_lag_data is not None:
|
||||
if last_lag_data is not None and last_carparams_data is not None:
|
||||
try:
|
||||
with log.Event.from_bytes(last_lag_data) as last_lag_msg, car.CarParams.from_bytes(last_carparams_data) as last_CP:
|
||||
ld = last_lag_msg.lateralDelay
|
||||
@@ -385,6 +386,7 @@ def retrieve_initial_lag(params: Params, CP: car.CarParams):
|
||||
|
||||
|
||||
def main():
|
||||
prewarm_cache_contracts()
|
||||
config_realtime_process([0, 1, 2, 3], 5)
|
||||
|
||||
DEBUG = bool(int(os.getenv("DEBUG", "0")))
|
||||
@@ -417,4 +419,4 @@ def main():
|
||||
pm.send('lateralDelay', lag_msg_dat)
|
||||
|
||||
if sm.frame % 1200 == 0: # cache every 60 seconds
|
||||
params.put("LiveDelay", lag_msg_dat)
|
||||
put_cache(params, "LiveDelay", lag_msg)
|
||||
|
||||
@@ -7,6 +7,7 @@ import openpilot.cereal.messaging as messaging
|
||||
from openpilot.cereal import log
|
||||
from opendbc.car.structs import car
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.starpilot.schema_cache import get_cache, prewarm_cache_contracts, put_cache
|
||||
from openpilot.common.realtime import config_realtime_process, DT_MDL
|
||||
from openpilot.selfdrive.locationd.models.car_kf import CarKalman, ObservationKind, States
|
||||
from openpilot.selfdrive.locationd.models.constants import GENERATED_DIR
|
||||
@@ -201,8 +202,8 @@ def check_valid_with_hysteresis(current_valid: bool, val: float, threshold: floa
|
||||
|
||||
|
||||
def retrieve_initial_vehicle_params(params: Params, CP: car.CarParams, replay: bool, debug: bool):
|
||||
last_parameters_data = params.get("LiveParametersV2")
|
||||
last_carparams_data = params.get("CarParamsPrevRoute")
|
||||
last_parameters_data = get_cache(params, "LiveParametersV2")
|
||||
last_carparams_data = get_cache(params, "CarParamsPrevRoute")
|
||||
|
||||
steer_ratio, stiffness_factor, angle_offset_deg, p_initial = CP.steerRatio, 1.0, 0.0, None
|
||||
|
||||
@@ -243,6 +244,7 @@ def retrieve_initial_vehicle_params(params: Params, CP: car.CarParams, replay: b
|
||||
|
||||
|
||||
def main():
|
||||
prewarm_cache_contracts()
|
||||
config_realtime_process([0, 1, 2, 3], 5)
|
||||
|
||||
DEBUG = bool(int(os.getenv("DEBUG", "0")))
|
||||
@@ -270,7 +272,7 @@ def main():
|
||||
|
||||
msg_dat = msg.to_bytes()
|
||||
if sm.frame % 1200 == 0: # once a minute
|
||||
params.put("LiveParametersV2", msg_dat)
|
||||
put_cache(params, "LiveParametersV2", msg)
|
||||
|
||||
pm.send('vehicleParameters', msg_dat)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ from openpilot.common.test import OpenpilotTestCase
|
||||
import openpilot.cereal.messaging as messaging
|
||||
from openpilot.cereal import log
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.starpilot.schema_cache import put_cache
|
||||
from openpilot.selfdrive.locationd.calibrationd import Calibrator, INPUTS_NEEDED, INPUTS_WANTED, BLOCK_SIZE, MIN_SPEED_FILTER, \
|
||||
MAX_YAW_RATE_FILTER, SMOOTH_CYCLES, HEIGHT_INIT, MAX_ALLOWED_PITCH_SPREAD, MAX_ALLOWED_YAW_SPREAD
|
||||
|
||||
@@ -31,13 +32,25 @@ def process_messages(c, cam_odo_calib, cycles,
|
||||
[cam_odo_height_std, cam_odo_height_std, cam_odo_height_std])
|
||||
|
||||
class TestCalibrationd(OpenpilotTestCase):
|
||||
def test_unqualified_saved_params_are_not_loaded(self):
|
||||
msg = messaging.new_message('extrinsicsCalibration')
|
||||
msg.extrinsicsCalibration.validBlocks = 10
|
||||
msg.extrinsicsCalibration.rpyCalib = [0.0, 0.02, 0.01]
|
||||
raw = msg.to_bytes()
|
||||
params = Params()
|
||||
params.put("CalibrationParams", raw, block=True)
|
||||
c = Calibrator(param_put=True)
|
||||
|
||||
self.assertEqual(c.valid_blocks, 0)
|
||||
np.testing.assert_allclose(c.rpy, np.zeros(3))
|
||||
self.assertEqual(params.get("CalibrationParams"), raw)
|
||||
|
||||
def test_read_saved_params(self):
|
||||
msg = messaging.new_message('extrinsicsCalibration')
|
||||
msg.extrinsicsCalibration.validBlocks = random.randint(1, 10)
|
||||
msg.extrinsicsCalibration.rpyCalib = [random.random() for _ in range(3)]
|
||||
msg.extrinsicsCalibration.height = [random.random() for _ in range(1)]
|
||||
Params().put("CalibrationParams", msg.to_bytes(), block=True)
|
||||
put_cache(Params(), "CalibrationParams", msg, block=True)
|
||||
c = Calibrator(param_put=True)
|
||||
|
||||
np.testing.assert_allclose(msg.extrinsicsCalibration.rpyCalib, c.rpy)
|
||||
|
||||
@@ -9,10 +9,8 @@ from openpilot.cereal import messaging, log
|
||||
from opendbc.car.structs import car
|
||||
from openpilot.selfdrive.locationd.lagd import LateralLagEstimator, retrieve_initial_lag, masked_normalized_cross_correlation, \
|
||||
BLOCK_NUM_NEEDED, BLOCK_SIZE, MIN_OKAY_WINDOW_SEC, VERSION, MIN_LAG, MAX_LAG
|
||||
from openpilot.selfdrive.test.process_replay.migration import migrate, migrate_carParams
|
||||
from openpilot.selfdrive.locationd.test.test_locationd_scenarios import TEST_ROUTE
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.tools.lib.logreader import LogReader
|
||||
from openpilot.starpilot.schema_cache import put_cache
|
||||
from openpilot.common.hardware import PC
|
||||
|
||||
MAX_ERR_FRAMES = 1
|
||||
@@ -22,8 +20,7 @@ LAGD_MIN_LAG_FRAMES, LAGD_MAX_LAG_FRAMES = int(round(MIN_LAG / DT)), int(round(M
|
||||
|
||||
@cache
|
||||
def get_test_car_params():
|
||||
lr = migrate(LogReader(TEST_ROUTE), [migrate_carParams])
|
||||
return next(m for m in lr if m.which() == "carParams").carParams
|
||||
return car.CarParams.new_message(carFingerprint="cache-test-car", steerRatio=15.0)
|
||||
|
||||
|
||||
def process_messages(estimator, lag_frames, n_frames, vego=25.0, rejection_threshold=0.0):
|
||||
@@ -54,6 +51,20 @@ def process_messages(estimator, lag_frames, n_frames, vego=25.0, rejection_thres
|
||||
|
||||
|
||||
class TestLagd(OpenpilotTestCase):
|
||||
def test_unqualified_saved_params_are_not_loaded(self):
|
||||
params = Params()
|
||||
CP = get_test_car_params()
|
||||
msg = messaging.new_message('lateralDelay')
|
||||
msg.lateralDelay.lateralDelayEstimate = 0.15
|
||||
msg.lateralDelay.validBlocks = 5
|
||||
msg.lateralDelay.version = VERSION
|
||||
raw = msg.to_bytes()
|
||||
params.put("LiveDelay", raw, block=True)
|
||||
put_cache(params, "CarParamsPrevRoute", CP, block=True)
|
||||
|
||||
self.assertIsNone(retrieve_initial_lag(params, CP))
|
||||
self.assertEqual(params.get("LiveDelay"), raw)
|
||||
|
||||
def test_read_saved_params(self):
|
||||
params = Params()
|
||||
|
||||
@@ -63,8 +74,8 @@ class TestLagd(OpenpilotTestCase):
|
||||
msg.lateralDelay.lateralDelayEstimate = random.random()
|
||||
msg.lateralDelay.validBlocks = random.randint(1, 10)
|
||||
msg.lateralDelay.version = VERSION
|
||||
params.put("LiveDelay", msg.to_bytes(), block=True)
|
||||
params.put("CarParamsPrevRoute", CP.as_builder().to_bytes(), block=True)
|
||||
put_cache(params, "LiveDelay", msg, block=True)
|
||||
put_cache(params, "CarParamsPrevRoute", CP, block=True)
|
||||
|
||||
saved_lag_params = retrieve_initial_lag(params, CP)
|
||||
assert saved_lag_params is not None
|
||||
@@ -82,8 +93,8 @@ class TestLagd(OpenpilotTestCase):
|
||||
with subtests.test(msg=f"lateralDelay={msg_dict}"):
|
||||
msg = messaging.new_message('lateralDelay')
|
||||
msg.lateralDelay = msg_dict
|
||||
params.put("LiveDelay", msg.to_bytes(), block=True)
|
||||
params.put("CarParamsPrevRoute", CP.as_builder().to_bytes(), block=True)
|
||||
put_cache(params, "LiveDelay", msg, block=True)
|
||||
put_cache(params, "CarParamsPrevRoute", CP, block=True)
|
||||
assert retrieve_initial_lag(params, CP) is None
|
||||
|
||||
def test_ncc(self):
|
||||
|
||||
@@ -3,12 +3,11 @@ import numpy as np
|
||||
|
||||
from openpilot.common.test import OpenpilotTestCase
|
||||
from openpilot.cereal import messaging
|
||||
from opendbc.car.structs import car
|
||||
from openpilot.selfdrive.locationd.paramsd import retrieve_initial_vehicle_params
|
||||
from openpilot.selfdrive.locationd.models.car_kf import CarKalman
|
||||
from openpilot.selfdrive.locationd.test.test_locationd_scenarios import TEST_ROUTE
|
||||
from openpilot.selfdrive.test.process_replay.migration import migrate, migrate_carParams
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.tools.lib.logreader import LogReader
|
||||
from openpilot.starpilot.schema_cache import put_cache
|
||||
|
||||
|
||||
def get_random_vehicle_parameters(CP):
|
||||
@@ -21,15 +20,25 @@ def get_random_vehicle_parameters(CP):
|
||||
|
||||
|
||||
class TestParamsd(OpenpilotTestCase):
|
||||
def test_unqualified_saved_params_are_not_loaded(self):
|
||||
params = Params()
|
||||
CP = car.CarParams.new_message(carFingerprint="cache-test-car", steerRatio=15.0)
|
||||
msg = get_random_vehicle_parameters(CP)
|
||||
raw = msg.to_bytes()
|
||||
params.put("LiveParametersV2", raw, block=True)
|
||||
put_cache(params, "CarParamsPrevRoute", CP, block=True)
|
||||
|
||||
self.assertEqual(retrieve_initial_vehicle_params(params, CP, replay=True, debug=True), (CP.steerRatio, 1.0, 0.0, None))
|
||||
self.assertEqual(params.get("LiveParametersV2"), raw)
|
||||
|
||||
def test_read_saved_params(self):
|
||||
params = Params()
|
||||
|
||||
lr = migrate(LogReader(TEST_ROUTE), [migrate_carParams])
|
||||
CP = next(m for m in lr if m.which() == "carParams").carParams
|
||||
CP = car.CarParams.new_message(carFingerprint="cache-test-car", steerRatio=15.0)
|
||||
|
||||
msg = get_random_vehicle_parameters(CP)
|
||||
params.put("LiveParametersV2", msg.to_bytes(), block=True)
|
||||
params.put("CarParamsPrevRoute", CP.as_builder().to_bytes(), block=True)
|
||||
put_cache(params, "LiveParametersV2", msg, block=True)
|
||||
put_cache(params, "CarParamsPrevRoute", CP, block=True)
|
||||
|
||||
sr, sf, offset, p_init = retrieve_initial_vehicle_params(params, CP, replay=True, debug=True)
|
||||
np.testing.assert_allclose(sr, msg.vehicleParameters.steerRatio)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# Process replay
|
||||
|
||||
Process replay is a regression test designed to identify any changes in the output of a process. This test replays a segment through individual processes and compares the output to a known good replay. Each make is represented in the test with a segment.
|
||||
Process replay is a regression test designed to identify any changes in the output of a process. This test replays available segments through individual processes and compares the output to a known good replay.
|
||||
|
||||
All 16 currently selected recording inputs remain required in a full run. Volvo C1/V40 has no available recording. This exact platform is reported as an unrecorded gap in `coverage.json` and the GitHub job summary; it is not a passing replay. The interfaces job requires its existing host tests, and both native safety jobs require its existing C1 safety tests. New unrecorded brands, changed active Volvo platform scope, missing required test suites, and a newly available Volvo recording require coverage review and fail input accounting. Source/native tests and recorded replay outcomes remain separate evidence; neither this inventory nor the missing-input classification qualifies the fleet.
|
||||
|
||||
If the test fails, make sure that you didn't unintentionally change anything. If there are intentional changes, the reference logs will be updated.
|
||||
|
||||
@@ -43,7 +45,7 @@ To generate new logs:
|
||||
|
||||
`./test_processes.py`
|
||||
|
||||
Then, check in the new logs using git-lfs. Make sure to also update the `ref_commit` file to the current commit.
|
||||
Then, check in the new logs through the external test-data catalog. Make sure to also update the `ref_commit` file to the current commit.
|
||||
|
||||
## API
|
||||
|
||||
@@ -85,18 +87,49 @@ Supported processes:
|
||||
* modeld
|
||||
* dmonitoringmodeld
|
||||
|
||||
Certain processes may require an initial state, which is usually supplied within `Params` and persists from segment to segment (for example `CalibrationParams` or the learner cache keys like `LiveParametersV2`). The `custom_params` is a dictionary used to prepopulate `Params` with arbitrary values. The `get_custom_params_from_lr` helper is provided to fetch meaningful values from log files.
|
||||
Certain processes require saved initial state in `Params`, such as `CalibrationParams`
|
||||
or `LiveParametersV2`. Managed schema caches in `custom_params` must contain verified
|
||||
envelopes created by `openpilot.starpilot.schema_cache.put_cache` from current-schema
|
||||
messages whose origin is known. Synthetic current-schema fixtures can use this path.
|
||||
|
||||
```py
|
||||
from openpilot.selfdrive.test.process_replay import get_custom_params_from_lr
|
||||
Historical logs require source schema provenance and an explicit schema-specific
|
||||
conversion before their values can seed these caches. Reading old bytes through the
|
||||
current schema does not establish compatibility. `get_custom_params_from_lr` now
|
||||
refuses this operation, and automatic firmware `CarParamsCache` seeding also refuses
|
||||
unless a verified envelope was supplied. Configuration is checked before Params
|
||||
writes. An explicit current vehicle fingerprint can avoid firmware-cache seeding,
|
||||
but does not qualify the remaining logged messages.
|
||||
|
||||
previous_segment_lr = LogReader(...)
|
||||
current_segment_lr = LogReader(...)
|
||||
The checked-in replay recording catalog supplies a separate, reviewed firmware
|
||||
cache adapter. It pins each input and its independently published card reference,
|
||||
decodes the reference with its producing source schemas, checks firmware/VIN
|
||||
identity against the recording, and converts the reference CarParams into a
|
||||
verified current envelope. The current vehicle factory still recomputes control
|
||||
and safety settings. This adapter does not alter expected outputs or authorize
|
||||
implicit cache conversion for other historical recordings. Input, reference or
|
||||
schema changes require a new review; do not regenerate expected outputs to hide
|
||||
differences.
|
||||
|
||||
custom_params = get_custom_params_from_lr(previous_segment_lr, 'last')
|
||||
Default `ref_commit` and unlisted expected logs use the immutable official artifact
|
||||
revision. Fourteen exact segment/process pairs use reviewed StarPilot references in
|
||||
`refs/starpilot`, preserving intentional controller and capability changes. Their
|
||||
manifest pins inputs, comparison configurations, producing source snapshots and
|
||||
both official and reviewed output hashes. Every original comparison remains active;
|
||||
an arbitrary local output cannot override a default expectation. Deliberate alternate
|
||||
`ref_commit` and `--update-refs` retain their local reference behavior.
|
||||
|
||||
output_logs = replay_process_with_name('calibrationd', lr, custom_params=custom_params)
|
||||
```
|
||||
The EV6 and Bolt EUV control references include the default Turn Assist behavior;
|
||||
explicit `TurnAssist=False` independently restores every prior compared field.
|
||||
The legacy Corolla recording contains no native axis or safety acknowledgement,
|
||||
so its control reference requires both axes disabled and zero torque and acceleration
|
||||
even when stock enable is true. Actual Controls and native Toyota safety regressions
|
||||
verify fresh grants and missing acknowledgement denial. The four reference updates
|
||||
record the prior output hash, exact difference fields and counts, regression source
|
||||
hashes, and committed producing sources in the manifest. These records document the
|
||||
reviewed changes; they do not exclude any field from replay comparison.
|
||||
|
||||
Use `--output DIRECTORY` to place coverage, new logs and both diff reports outside
|
||||
the source tree. The predeploy runner supplies its results directory automatically.
|
||||
|
||||
Replaying processes that use VisionIPC (e.g. modeld, dmonitoringmodeld) require additional `frs` dictionary with camera states as keys and `FrameReader` objects as values.
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import argparse
|
||||
from openpilot.cereal import messaging, log
|
||||
from opendbc.car.structs import car
|
||||
from opendbc.car.fingerprints import MIGRATION
|
||||
from opendbc.car.toyota.values import EPS_SCALE, ToyotaSafetyFlags
|
||||
from opendbc.car.toyota.values import CAR as TOYOTA, EPS_SCALE, ToyotaSafetyFlags
|
||||
from opendbc.car.ford.values import CAR as FORD, FordFlags, FordSafetyFlags
|
||||
from opendbc.car.hyundai.values import HyundaiSafetyFlags
|
||||
from opendbc.car.gm.values import GMSafetyFlags
|
||||
@@ -308,8 +308,8 @@ def migrate_carOutput(msgs):
|
||||
def migrate_pandaStates(msgs):
|
||||
# TODO: safety param migration should be handled automatically
|
||||
safety_param_migration = {
|
||||
"TOYOTA_PRIUS": EPS_SCALE["TOYOTA_PRIUS"] | ToyotaSafetyFlags.STOCK_LONGITUDINAL,
|
||||
"TOYOTA_RAV4": EPS_SCALE["TOYOTA_RAV4"] | ToyotaSafetyFlags.ALT_BRAKE,
|
||||
"TOYOTA_PRIUS": EPS_SCALE[TOYOTA.TOYOTA_PRIUS] | ToyotaSafetyFlags.STOCK_LONGITUDINAL,
|
||||
"TOYOTA_RAV4": EPS_SCALE[TOYOTA.TOYOTA_RAV4] | ToyotaSafetyFlags.ALT_BRAKE,
|
||||
"KIA_EV6": HyundaiSafetyFlags.EV_GAS | HyundaiSafetyFlags.CANFD_LKA_STEER_MSG,
|
||||
"CHEVROLET_VOLT": GMSafetyFlags.EV,
|
||||
"CHEVROLET_BOLT_EUV": GMSafetyFlags.EV | GMSafetyFlags.HW_CAM,
|
||||
|
||||
@@ -21,6 +21,7 @@ from msgq.visionipc import VisionIpcClient, VisionIpcServer, get_endpoint_name a
|
||||
from opendbc.car.can_definitions import CanData
|
||||
from opendbc.car.car_helpers import get_car, interfaces
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.starpilot.schema_cache import CACHE_KEYS, get_cache, inspect_cache
|
||||
from openpilot.common.prefix import OpenpilotPrefix
|
||||
from openpilot.common.timeout import Timeout
|
||||
from openpilot.common.realtime import DT_CTRL
|
||||
@@ -175,6 +176,7 @@ class ProcessContainer:
|
||||
del os.environ[k]
|
||||
|
||||
def _setup_env(self, params_config: dict[str, Any], environ_config: dict[str, Any]):
|
||||
validate_cache_config(params_config)
|
||||
for k, v in environ_config.items():
|
||||
if len(v) != 0:
|
||||
os.environ[k] = v
|
||||
@@ -356,10 +358,10 @@ def get_car_params_callback(rc, pm, msgs, fingerprint):
|
||||
CP = CarInterface.get_non_essential_params(fingerprint)
|
||||
else:
|
||||
can_msgs = ([CanData(can.address, can.dat, can.src) for can in m.can] for m in msgs if m.which() == "can")
|
||||
cached_params_raw = params.get("CarParamsCache")
|
||||
cached_params_raw = get_cache(params, "CarParamsCache")
|
||||
assert next(can_msgs, None), "CAN messages are required for fingerprinting"
|
||||
assert os.environ.get("SKIP_FW_QUERY", False) or cached_params_raw is not None, \
|
||||
"CarParamsCache is required for fingerprinting. Make sure to keep carParams msgs in the logs."
|
||||
"Fingerprinting requires a verified CarParamsCache; convert source schemas or provide an explicit current vehicle fingerprint."
|
||||
|
||||
def can_recv(wait_for_one: bool = False) -> list[list[CanData]]:
|
||||
return [next(can_msgs, [])]
|
||||
@@ -584,35 +586,9 @@ def get_process_config(name: str) -> ProcessConfig:
|
||||
|
||||
|
||||
def get_custom_params_from_lr(lr: LogIterable, initial_state: str = "first") -> dict[str, Any]:
|
||||
"""
|
||||
Use this to get custom params dict based on provided logs.
|
||||
Useful when replaying following processes: calibrationd, paramsd, torqued
|
||||
The params may be based on first or last message of given type (carParams, extrinsicsCalibration, vehicleParameters, lateralTorqueParameters) in the logs.
|
||||
"""
|
||||
|
||||
car_params = [m for m in lr if m.which() == "carParams"]
|
||||
extrinsics_calibration = [m for m in lr if m.which() == "extrinsicsCalibration"]
|
||||
vehicle_parameters = [m for m in lr if m.which() == "vehicleParameters"]
|
||||
torque_parameters = [m for m in lr if m.which() == "lateralTorqueParameters"]
|
||||
|
||||
assert initial_state in ["first", "last"]
|
||||
msg_index = 0 if initial_state == "first" else -1
|
||||
|
||||
assert len(car_params) > 0, "carParams required for initial state of vehicleParameters and CarParamsPrevRoute"
|
||||
CP = car_params[msg_index].carParams
|
||||
|
||||
custom_params = {
|
||||
"CarParamsPrevRoute": CP.as_builder().to_bytes()
|
||||
}
|
||||
|
||||
if len(extrinsics_calibration) > 0:
|
||||
custom_params["CalibrationParams"] = extrinsics_calibration[msg_index].as_builder().to_bytes()
|
||||
if len(vehicle_parameters) > 0:
|
||||
custom_params["LiveParametersV2"] = vehicle_parameters[msg_index].as_builder().to_bytes()
|
||||
if len(torque_parameters) > 0:
|
||||
custom_params["LiveTorqueParameters"] = torque_parameters[msg_index].as_builder().to_bytes()
|
||||
|
||||
return custom_params
|
||||
"""Historical log bytes need explicit schema conversion before cache seeding."""
|
||||
raise ValueError("Log-derived cache seeding requires source schema provenance and a schema-specific conversion; " +
|
||||
"decoding a log with the current schema does not qualify its saved state")
|
||||
|
||||
|
||||
def replay_process_with_name(name: str | Iterable[str], lr: LogIterable, *args, **kwargs) -> list[capnp._DynamicStructReader]:
|
||||
@@ -731,6 +707,14 @@ def _replay_multi_process(
|
||||
return log_msgs
|
||||
|
||||
|
||||
def validate_cache_config(params_config: dict[str, Any]) -> None:
|
||||
for key in CACHE_KEYS:
|
||||
if key in params_config:
|
||||
raw = params_config[key]
|
||||
if not isinstance(raw, bytes) or inspect_cache(key, raw).status != "valid":
|
||||
raise ValueError(f"{key} requires a verified cache envelope; log-derived values need source schema provenance and conversion")
|
||||
|
||||
|
||||
def generate_params_config(lr=None, CP=None, fingerprint=None, custom_params=None) -> dict[str, Any]:
|
||||
params_dict = {
|
||||
"OpenpilotEnabledToggle": True,
|
||||
@@ -739,6 +723,7 @@ def generate_params_config(lr=None, CP=None, fingerprint=None, custom_params=Non
|
||||
}
|
||||
|
||||
if custom_params is not None:
|
||||
validate_cache_config(custom_params)
|
||||
params_dict.update(custom_params)
|
||||
if lr is not None:
|
||||
has_ublox = any(msg.which() == "ubloxGnss" for msg in lr)
|
||||
@@ -749,7 +734,9 @@ def generate_params_config(lr=None, CP=None, fingerprint=None, custom_params=Non
|
||||
if CP is not None:
|
||||
if fingerprint is None:
|
||||
if CP.fingerprintSource == "fw":
|
||||
params_dict["CarParamsCache"] = CP.as_builder().to_bytes()
|
||||
if "CarParamsCache" not in params_dict:
|
||||
raise ValueError("Automatic CarParamsCache seeding from logs is unqualified; provide an explicitly converted " +
|
||||
"cache envelope or replay with an explicit current vehicle fingerprint")
|
||||
|
||||
if CP.openpilotLongitudinalControl:
|
||||
params_dict["AlphaLongitudinalEnabled"] = True
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
# StarPilot runtime contracts
|
||||
|
||||
Feature-specific behavior belongs with its owner. The [UI guide](ui/README.md)
|
||||
describes presentation and input boundaries; the contracts below cover shared
|
||||
audio settings and retained-state migration.
|
||||
|
||||
## Alert levels and sound packs
|
||||
|
||||
`audio/alert_volume.py` defines seven audible-alert families. Saved integer
|
||||
`101`, or an absent value, means Auto. Manual levels range from 0–100%, except
|
||||
Soft and Immediate Warnings, which have a 25% minimum. The immediate warning,
|
||||
including a selfdrive timeout warning, still ramps to full volume. `soundd`
|
||||
validates saved levels and refreshes them outside its audio callback. Opening
|
||||
Settings does not create default values or reset other alert families.
|
||||
|
||||
`audio/sound_pack.py` loads the selected installed pack and falls back to the
|
||||
stock clip when a replacement is missing or invalid. It accepts bounded,
|
||||
uncompressed mono 48 kHz, 16-bit WAV files; decoded samples are immutable.
|
||||
`audio/downloads.py` installs catalogued packs with size/hash validation,
|
||||
bounded extraction and parked-state checks. Sound selection is independent of
|
||||
onroad colors and layouts. Code tests do not establish audible quality on the
|
||||
device.
|
||||
|
||||
## Settings preservation
|
||||
|
||||
`state_migration.py` separates exact recovery from importing preferences:
|
||||
|
||||
- `snapshot_settings(namespace, storage)` archives every raw value before cleanup,
|
||||
including unknown keys, empty files, binary caches and credentials. It uses the
|
||||
native Params lock, private permissions, content hashes and durable writes.
|
||||
- `restore_snapshot(snapshot, namespace)` verifies the entire archive before
|
||||
restoring exact bytes into an unused namespace. It refuses to overwrite an
|
||||
existing namespace. This is an offline recovery operation, not an upgrade.
|
||||
- `validate_bundle(bundle)` checks the versioned neutral settings format.
|
||||
`decode_bundle(encoded)` also rejects duplicate JSON members before validation.
|
||||
`stage_settings(bundle, storage, namespace)` saves it privately outside Params without
|
||||
activating any value. Pending settings have explicit types and raw-byte hashes;
|
||||
omitted settings have reasons. Archive and source hashes identify provenance,
|
||||
but a syntactically valid bundle is not evidence of feature compatibility.
|
||||
- `prepare_manager_start(params, storage)` runs before boot logging, clearing
|
||||
settings, defaults or registration. Initial startup only accepts an empty
|
||||
namespace or reviewed Boolean preferences. Other state is archived and startup
|
||||
stops for migration. Subsequent startup rejects unknown keys, malformed
|
||||
reviewed preferences and incompatible cache envelopes. Its profile marker
|
||||
identifies the namespace and schema epoch; each cache must pass its own
|
||||
schema and producer checks, even in a previously initialized namespace.
|
||||
|
||||
Event cache writers use version 2 envelopes. Their fingerprint covers the Event
|
||||
wrapper, common fields and the selected service's complete schema, so unrelated
|
||||
union members can change without invalidating saved calibration or learned values.
|
||||
Physical wrapper changes or changes to a stored service still require migration.
|
||||
Version 1 envelopes remain readable only when their full schema matches the
|
||||
current runtime; reads do not rewrite them. CarParams envelopes remain version 1.
|
||||
|
||||
Before activating an update, `prepare_manager_start(params, storage, dry_run=True)`
|
||||
checks the retained namespace with the candidate's compiled schemas and registry.
|
||||
It does not qualify a profile, create recovery files or alter saved values. A
|
||||
successful check describes that snapshot of state; normal startup still performs
|
||||
the admission checks before launching processes.
|
||||
|
||||
The initial Boolean allowlist is `AlwaysOnLateral`, `IsMetric`, `RecordAudio`,
|
||||
`RecordFront`, `RecordFrontLock`, `ShowDebugInfo`, `GsmMetered` and
|
||||
`OpenpilotEnabledToggle`. Saved values must be canonical `0` or `1` bytes.
|
||||
`AlwaysOnLateral` defaults to false for a fresh namespace; an existing true or
|
||||
false value is preserved. Registering this preference does not enable lateral
|
||||
control or claim that the feature has been ported.
|
||||
|
||||
The neutral bundle has `format: "starpilot-settings"`, integer `version: 1`,
|
||||
`source`, `preferences`, `pending` and `omitted`. Source metadata includes the full
|
||||
source revision, registry SHA-256 and raw snapshot SHA-256. Optional cache fallback
|
||||
requires a separate `cache_snapshot_sha256`. A raw snapshot hash is the SHA-256
|
||||
of sorted `{key, sha256, size}` entries encoded as JSON with sorted object keys,
|
||||
ASCII escaping and separators `(',', ':')`.
|
||||
|
||||
Never interpret archived vehicle parameters or logs using a different schema just
|
||||
because field numbers match. Binary caches, credentials, control-policy settings
|
||||
and pending feature settings each require their own migration policy. Staging
|
||||
alone never marks a profile ready. Stop services before export, restore or any
|
||||
future activation; older runtimes may write a separate cache outside the native
|
||||
Params lock. Keep private archives outside Params and source repositories.
|
||||
|
||||
## Schema-aware caches
|
||||
|
||||
`schema_cache.py` stores a versioned envelope in each managed cache value. Payload,
|
||||
producer contract, key, service, transitive compiled-schema fingerprint and content
|
||||
hash are written atomically through the existing native Params queue. Live
|
||||
`CarParams`, messaging and Panda wire formats remain unchanged.
|
||||
|
||||
`put_cache` requires a typed message from the current compiled producer schema.
|
||||
It preserves the caller's blocking/asynchronous write choice. `get_cache` only
|
||||
returns verified payload bytes; incompatible data stays untouched and is a cache
|
||||
miss. Startup archives and blocks incompatible existing state before processes
|
||||
can replace it. Fingerprints are prepared before realtime processing loops.
|
||||
|
||||
Managed keys are `CarParamsCache`, `CarParamsPersistent`, `CarParamsPrevRoute`,
|
||||
`CalibrationParams`, `LiveParametersV2`, `LiveTorqueParameters` and `LiveDelay`.
|
||||
The retired `LocationFilterInitialState` has no conversion contract and blocks
|
||||
startup if present. A numeric schema ID or successful decode is insufficient
|
||||
provenance: enum meanings and fields can change while IDs remain constant.
|
||||
|
||||
An envelope protects against accidental schema reuse; it is not a signature or
|
||||
an automatic semantic conversion. Producers must use current observations or
|
||||
verified cache data. Parsing unqualified historical bytes with today's schema
|
||||
and then wrapping them does not make those bytes compatible.
|
||||
|
||||
Historical-log cache seeding requires a source-schema conversion contract.
|
||||
Process replay rejects unqualified warm-start state before writing Params.
|
||||
Interactive replay only seeds raw live parameters in an explicit `replay-`
|
||||
namespace and does not seed persistent caches. Historical process-replay results
|
||||
remain unqualified until their source schemas and conversion fixtures are supplied.
|
||||
|
||||
Enveloped caches are incompatible with runtimes expecting bare serialized bytes.
|
||||
A downgrade requires an offline recovery or explicit conversion; switching Git
|
||||
branches alone is not a supported rollback. No deployment is qualified here.
|
||||
|
||||
Run native persistence checks after building the Params library:
|
||||
|
||||
```sh
|
||||
python -m unittest openpilot.starpilot.tests.test_state_migration -v
|
||||
```
|
||||
|
||||
## Host actuator boundary
|
||||
|
||||
`tests/test_control_host_boundary.py` exercises the actual upstream `Controls`
|
||||
constructor, longitudinal controller and native `carControl` publication in
|
||||
disposable Params and IPC namespaces. Synthetic messages pass through the real
|
||||
SubMaster input API. The two configurations deliberately supply a Model Y
|
||||
candidate with system or stock longitudinal ownership; they do not identify a
|
||||
physical vehicle or EPS firmware.
|
||||
|
||||
The checks warm the acceleration controller, then supply longitudinal override,
|
||||
lateral fault, disable, a continuing negative plan and explicit re-enable inputs.
|
||||
Inactive host longitudinal control must publish zero acceleration even with a
|
||||
nonzero plan or earlier positive output. The lateral-fault case checks the host
|
||||
activation flag while longitudinal control continues. Cancellation and override
|
||||
publication are checked independently. Native libraries and source hashes are
|
||||
recorded by the explicit probe.
|
||||
|
||||
This is a host input/output contract. It does not prove button or AOL policy,
|
||||
message freshness handling, emitted CAN, Panda permission, stock vehicle response
|
||||
or rearm behavior. Those need separate integrated evidence. A controller test
|
||||
which directly injects stale nonzero acceleration cannot assume this host
|
||||
neutralization step already ran.
|
||||
|
||||
## Modern Tesla integration
|
||||
|
||||
`tests/test_tesla_contract.py` checks known EPS firmware classification, controller
|
||||
command fields and counts, counter progression, override behavior and C safety
|
||||
decisions. Each test process compiles and loads its own temporary DEBUG safety
|
||||
library. Firmware identification and direct controller inputs are separate checks.
|
||||
|
||||
`tests/test_tesla_host_integration.py` joins actual parsed CAN state, the upstream
|
||||
Controls object, native `carControl` publication, the unchanged received command,
|
||||
the vehicle interface/controller and C safety hooks. Two known Model Y EPS profiles
|
||||
each exercise 120 consecutive logical frames. Override and disable each start with
|
||||
a positive host acceleration output and must neutralize it through the emitted CAN
|
||||
fields. Packet counts, counter continuity across transitions and native publication
|
||||
identity are asserted. Logical CAN time is separate from measured publication time.
|
||||
|
||||
Host enable, override and plan messages are supplied test inputs. Stock cruise CAN
|
||||
and native safety permission remain present during host disable; raw cancellation
|
||||
commands do not simulate the vehicle responding. These tests do not establish AOL,
|
||||
longitudinal-only policy, automatic rearm, physical hardware identity, release
|
||||
firmware behavior or qualification of a car. Direct controller calls still require
|
||||
the host to neutralize inactive acceleration. Sustained stock-ACC cancellation,
|
||||
full process coordination and physical response require separate evidence.
|
||||
|
||||
## CAN subscription contract
|
||||
|
||||
`tests/test_can_subscription_contract.py` exercises the actual upstream Python
|
||||
parser and packer with synthetic messages. A subscription frequency of `NaN`
|
||||
excludes that message from aggregate alive checks. Zero instead requires its
|
||||
presence and learns its frequency from received packets. Ported subscriptions
|
||||
must express the intended behavior explicitly for each message and vehicle.
|
||||
|
||||
Optional signals keep zero timestamps until first received and retain their last
|
||||
timestamps after reception stops. Aggregate `can_valid` therefore does not establish
|
||||
their freshness. Consumers must independently check freshness and applicability
|
||||
before using them. Continuing optional traffic also must not hide loss of a
|
||||
required message. These checks do not qualify message integrity algorithms,
|
||||
native safety, physical CAN or any vehicle.
|
||||
@@ -0,0 +1,326 @@
|
||||
"""Cloud-provider identity and a reboot-bound switch with recoverable state."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
import base64
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
|
||||
from openpilot.common.hardware.hw import Paths
|
||||
from openpilot.starpilot.storage import starpilot_storage_root
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Provider:
|
||||
name: str
|
||||
label: str
|
||||
api: str
|
||||
athena: str
|
||||
web: str
|
||||
upload_attribute: str
|
||||
|
||||
|
||||
PROVIDERS = {
|
||||
'comma': Provider('comma', 'comma connect', 'https://api.commadotai.com', 'wss://athena.comma.ai',
|
||||
'https://connect.comma.ai', 'user.upload'),
|
||||
'konik': Provider('konik', 'Konik', 'https://api.konik.ai', 'wss://athena.konik.ai',
|
||||
'https://stable.konik.ai', 'user.upload.konik'),
|
||||
}
|
||||
STATE_KEYS = ('DongleId', 'PrimeType', 'PairingProvider', 'PairingEmail', 'AthenadUploadQueue', 'AthenadRecentlyViewedRoutes',
|
||||
'ApiCache_Device', 'ApiCache_FirehoseStats')
|
||||
TRANSIENT_KEYS = ('AccessToken', 'LastAthenaPingTime')
|
||||
MAX_STATE_BYTES = 8 * 1024 * 1024
|
||||
ROUTE_MARKER = '.cloud-provider'
|
||||
OFFLINE = Provider('offline', 'Cloud Unavailable', '', '', '', 'user.upload.disabled')
|
||||
|
||||
|
||||
def root_path():
|
||||
return starpilot_storage_root() / 'connect'
|
||||
|
||||
|
||||
def _read(path, default=None):
|
||||
try:
|
||||
with path.open('rb') as stream:
|
||||
raw = stream.read(MAX_STATE_BYTES + 1)
|
||||
if len(raw) > MAX_STATE_BYTES:
|
||||
raise ValueError('Cloud state is too large')
|
||||
return json.loads(raw)
|
||||
except FileNotFoundError:
|
||||
return default
|
||||
|
||||
|
||||
def _write(path, value):
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix='.connect-', dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as stream:
|
||||
json.dump(value, stream, separators=(',', ':'))
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
parent = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(parent)
|
||||
finally:
|
||||
os.close(parent)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def configuration(root=None):
|
||||
value = _read((root or root_path()) / 'provider.json', {'active': 'comma', 'selected': 'comma', 'requestedBoot': ''})
|
||||
if (type(value) is not dict or set(value) != {'active', 'selected', 'requestedBoot'} or
|
||||
value['active'] not in PROVIDERS or value['selected'] not in PROVIDERS or type(value['requestedBoot']) is not str):
|
||||
raise ValueError('Cloud provider configuration is invalid')
|
||||
return value
|
||||
|
||||
|
||||
def active_provider():
|
||||
try:
|
||||
return PROVIDERS[configuration()['active']]
|
||||
except (OSError, ValueError, TypeError, KeyError):
|
||||
return OFFLINE
|
||||
|
||||
|
||||
def state_revision(value):
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
|
||||
def boot_id():
|
||||
try:
|
||||
return Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
||||
except FileNotFoundError:
|
||||
import psutil
|
||||
return f'host-{psutil.boot_time():.3f}'
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _exclusive(root):
|
||||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
fd = os.open(root / '.lock', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
yield
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def select_provider(name, revision, authorized, *, root=None, boot=boot_id):
|
||||
root = root or root_path()
|
||||
if name not in PROVIDERS:
|
||||
raise ValueError('Unknown cloud provider')
|
||||
with _exclusive(root):
|
||||
value = configuration(root)
|
||||
if not authorized() or state_revision(value) != revision:
|
||||
raise ValueError('Cloud selection changed. Refresh and try again.')
|
||||
value.update(selected=name, requestedBoot=boot())
|
||||
if not authorized():
|
||||
raise ValueError('Turn off the vehicle before changing cloud providers')
|
||||
_write(root / 'provider.json', value)
|
||||
return status(root)
|
||||
|
||||
|
||||
def status(root=None):
|
||||
value = configuration(root)
|
||||
return {**value, 'revision': state_revision(value), 'restartRequired': value['active'] != value['selected'],
|
||||
'providers': [{'id': p.name, 'label': p.label, 'url': p.web} for p in PROVIDERS.values()]}
|
||||
|
||||
|
||||
def _capture(params):
|
||||
values = {}
|
||||
for key in STATE_KEYS:
|
||||
path = Path(params.get_param_path(key))
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
if len(raw) > MAX_STATE_BYTES // 2:
|
||||
raise ValueError('Cloud preference is too large')
|
||||
values[key] = base64.b64encode(raw).decode()
|
||||
except FileNotFoundError:
|
||||
values[key] = None
|
||||
if len(json.dumps(values)) > MAX_STATE_BYTES:
|
||||
raise ValueError('Cloud snapshot is too large')
|
||||
return values
|
||||
|
||||
|
||||
def _restore(params, values):
|
||||
if type(values) is not dict or set(values) != set(STATE_KEYS):
|
||||
raise ValueError('Cloud identity snapshot is invalid')
|
||||
decoded = {key: None if value is None else base64.b64decode(value, validate=True) for key, value in values.items()}
|
||||
for key, raw in decoded.items():
|
||||
params.check_key(key)
|
||||
path = Path(params.get_param_path(key))
|
||||
if raw is None:
|
||||
params.remove(key)
|
||||
else:
|
||||
fd, temporary = tempfile.mkstemp(prefix='.cloud-', dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(raw)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
for key in TRANSIENT_KEYS:
|
||||
params.remove(key)
|
||||
fd = os.open(Path(params.get_param_path(STATE_KEYS[0])).parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def activate_at_boot(params, *, root=None, boot=boot_id):
|
||||
"""Called before managed services start; never changes provider in a live session."""
|
||||
root = root or root_path()
|
||||
if not (root / 'provider.json').exists() and not (root / 'switch.json').exists():
|
||||
params.put('ConnectProvider', 'comma', block=True)
|
||||
return
|
||||
with _exclusive(root):
|
||||
value = configuration(root)
|
||||
journal = _read(root / 'switch.json')
|
||||
if journal is None:
|
||||
if value['active'] == value['selected'] or value['requestedBoot'] == boot():
|
||||
params.put('ConnectProvider', value['active'], block=True)
|
||||
return
|
||||
_write(root / value['active'] / 'params.json', _capture(params))
|
||||
target = _read(root / value['selected'] / 'params.json', dict.fromkeys(STATE_KEYS))
|
||||
journal = {'target': value['selected'], 'values': target}
|
||||
_write(root / 'switch.json', journal)
|
||||
if type(journal) is not dict or set(journal) != {'target', 'values'} or journal['target'] not in PROVIDERS:
|
||||
raise ValueError('Cloud switch journal is invalid')
|
||||
_restore(params, journal['values'])
|
||||
params.put('ConnectProvider', journal['target'], block=True)
|
||||
value['active'] = journal['target']
|
||||
_write(root / 'provider.json', value)
|
||||
(root / 'switch.json').unlink()
|
||||
fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def konik_key_pair():
|
||||
"""Konik never receives the factory comma private key."""
|
||||
root = root_path()
|
||||
with _exclusive(root):
|
||||
path = root / 'konik' / 'identity.json'
|
||||
identity = _read(path)
|
||||
if identity is None:
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
identity = {
|
||||
'private': key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
serialization.NoEncryption()).decode(),
|
||||
'public': key.public_key().public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo).decode(),
|
||||
}
|
||||
_write(path, identity)
|
||||
return 'RS256', identity['private'], identity['public']
|
||||
|
||||
|
||||
def galaxy_device_id(params):
|
||||
"""Galaxy pairing belongs to the physical device, independent of its cloud."""
|
||||
if active_provider().name == 'comma':
|
||||
return params.get('DongleId') or ''
|
||||
try:
|
||||
saved = _read(root_path() / 'comma' / 'params.json', {})
|
||||
raw = saved.get('DongleId')
|
||||
if raw is not None:
|
||||
value = base64.b64decode(raw, validate=True).decode()
|
||||
if re.fullmatch(r'[a-f0-9]{16}', value):
|
||||
return value
|
||||
return (Path(Paths.persist_root()) / 'comma/dongle_id').read_text().strip()
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
return (params.get('DongleId') or '') if params.get('ConnectProvider') == 'comma' else ''
|
||||
|
||||
|
||||
def route_url(dongle, route, provider=None):
|
||||
provider = provider or active_provider()
|
||||
if not re.fullmatch(r'[a-f0-9]{16}', dongle or '') or not re.fullmatch(r'[a-f0-9]{8}--[a-f0-9]{10}|[0-9-]{20}', route or ''):
|
||||
return None
|
||||
return f'{provider.web}/{dongle}/{route}'
|
||||
|
||||
|
||||
def mark_recording(directory, provider_name):
|
||||
"""Commit ownership before creating derived recording data."""
|
||||
if provider_name not in PROVIDERS:
|
||||
raise ValueError('Cloud provider is unavailable')
|
||||
directory = Path(directory)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
marker = directory / ROUTE_MARKER
|
||||
if marker.exists():
|
||||
if recording_provider(directory) != provider_name:
|
||||
raise ValueError('Recording belongs to another cloud provider')
|
||||
return
|
||||
fd = os.open(marker, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(provider_name.encode('ascii'))
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
parent = os.open(directory, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(parent)
|
||||
finally:
|
||||
os.close(parent)
|
||||
|
||||
|
||||
def recording_provider(path, *, dir_fd=None):
|
||||
"""Untagged recordings predate provider selection and belong to comma."""
|
||||
marker = ROUTE_MARKER if dir_fd is not None else Path(path) / ROUTE_MARKER
|
||||
try:
|
||||
fd = os.open(marker, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=dir_fd)
|
||||
with os.fdopen(fd, 'rb') as stream:
|
||||
value = stream.read(32).decode('ascii')
|
||||
return value if value in PROVIDERS else None
|
||||
except FileNotFoundError:
|
||||
return 'comma'
|
||||
except (OSError, UnicodeError):
|
||||
return None
|
||||
|
||||
|
||||
def owns_recording(path, log_root, provider=None):
|
||||
provider = provider or active_provider().name
|
||||
try:
|
||||
relative = Path(path).resolve().relative_to(Path(log_root).resolve())
|
||||
if not relative.parts:
|
||||
return False
|
||||
directory = Path(log_root) / relative.parts[0]
|
||||
if len(relative.parts) > 1:
|
||||
marker = Path(path).parent / ('.cloud-' + Path(path).name)
|
||||
if marker.exists() or marker.is_symlink():
|
||||
fd = os.open(marker, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
with os.fdopen(fd, 'rb') as stream:
|
||||
return stream.read(32) == provider.encode('ascii')
|
||||
return recording_provider(directory) == provider
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def cloudlog_root():
|
||||
base = Path(Paths.swaglog_root())
|
||||
provider = active_provider().name
|
||||
return str(base if provider == 'comma' else base / provider)
|
||||
|
||||
|
||||
def recording_device_id(name, params):
|
||||
if name not in PROVIDERS:
|
||||
return None
|
||||
try:
|
||||
if active_provider().name == name:
|
||||
identity = params.get('DongleId')
|
||||
else:
|
||||
saved = _read(root_path() / name / 'params.json', {})
|
||||
raw = saved.get('DongleId')
|
||||
identity = base64.b64decode(raw, validate=True).decode() if raw else None
|
||||
if not identity and name == 'comma':
|
||||
identity = (Path(Paths.persist_root()) / 'comma/dongle_id').read_text().strip()
|
||||
return identity if re.fullmatch(r'[a-f0-9]{16}', identity or '') else None
|
||||
except (OSError, ValueError, TypeError, KeyError):
|
||||
return None
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Pure fresh offroad evidence shared by local saved-setting action owners.
|
||||
|
||||
Python deviceState stamps MONOTONIC; C++ pandad stamps BOOTTIME on Linux.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
|
||||
|
||||
DEVICE_TTL_NS = 1_500_000_000 # 2 Hz publisher, allowing one missed frame.
|
||||
PANDA_TTL_NS = 300_000_000 # 10 Hz publisher, allowing one missed frame.
|
||||
RESUME_SKEW_NS = 5_000_000
|
||||
|
||||
|
||||
def host_parked(environ=os.environ) -> bool:
|
||||
"""./dev galaxy, ./c3 and ./c4 have no car or drive-state publishers; they are always parked.
|
||||
|
||||
Only those launchers set SP_HOST_PARKED. Route replay (./onroad) keeps its recorded drive state.
|
||||
"""
|
||||
prefix = environ.get("OPENPILOT_PREFIX", "")
|
||||
return (environ.get("SP_HOST_PARKED") == "1" and environ.get("SP_HOST_RUNTIME") == "1" and
|
||||
bool(prefix) and prefix == environ.get("SP_HOST_PREFIX"))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParkedEvidence:
|
||||
manager_offroad: bool
|
||||
device_seen: bool
|
||||
device_alive: bool
|
||||
device_valid: bool
|
||||
device_started: bool
|
||||
device_stamp_mono_ns: int
|
||||
device_recv_mono_ns: int
|
||||
device_boot_minus_mono_at_recv_ns: int | None
|
||||
device_after_mono_ns: int # Collector must set a startup/resume barrier before receiving evidence.
|
||||
panda_seen: bool
|
||||
panda_alive: bool
|
||||
panda_valid: bool
|
||||
panda_stamp_boot_ns: int
|
||||
panda_recv_mono_ns: int
|
||||
panda_ignition: tuple[bool, ...]
|
||||
|
||||
|
||||
def fresh_offroad(evidence: ParkedEvidence, *, now_mono_ns: int, now_boot_ns: int) -> bool:
|
||||
"""Confirm effective offroad mode, including Force Offroad with ignition on."""
|
||||
e = evidence
|
||||
if (not e.manager_offroad or not all((e.device_seen, e.device_alive, e.device_valid,
|
||||
e.panda_seen, e.panda_alive, e.panda_valid)) or
|
||||
e.device_started or e.device_boot_minus_mono_at_recv_ns is None or
|
||||
not e.panda_ignition or
|
||||
min(e.device_stamp_mono_ns, e.panda_stamp_boot_ns, e.device_recv_mono_ns, e.panda_recv_mono_ns) <= 0 or
|
||||
e.device_stamp_mono_ns <= e.device_after_mono_ns):
|
||||
return False
|
||||
device_mono_age = now_mono_ns - e.device_stamp_mono_ns
|
||||
device_boot_age = now_boot_ns - (e.device_stamp_mono_ns + e.device_boot_minus_mono_at_recv_ns)
|
||||
panda_boot_age = now_boot_ns - e.panda_stamp_boot_ns
|
||||
device_receipt_age = now_mono_ns - e.device_recv_mono_ns
|
||||
panda_receipt_age = now_mono_ns - e.panda_recv_mono_ns
|
||||
return (0 <= device_mono_age <= DEVICE_TTL_NS and
|
||||
-RESUME_SKEW_NS <= device_boot_age <= DEVICE_TTL_NS and
|
||||
0 <= panda_boot_age <= PANDA_TTL_NS and
|
||||
0 <= device_receipt_age <= DEVICE_TTL_NS and
|
||||
0 <= panda_receipt_age <= PANDA_TTL_NS)
|
||||
|
||||
|
||||
def fresh_parked(evidence: ParkedEvidence, *, now_mono_ns: int, now_boot_ns: int) -> bool:
|
||||
"""Confirm fresh offroad mode with ignition off for operations that require it."""
|
||||
return not any(evidence.panda_ignition) and fresh_offroad(evidence, now_mono_ns=now_mono_ns, now_boot_ns=now_boot_ns)
|
||||
@@ -0,0 +1,81 @@
|
||||
"""One exact-source Params document replacement under the native Params lock."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from openpilot.starpilot.saved_source import read_saved
|
||||
|
||||
|
||||
LOCK_WAIT_SECONDS = .25
|
||||
|
||||
|
||||
def acquire_native_lock(fd: int) -> None:
|
||||
deadline = time.monotonic() + LOCK_WAIT_SECONDS
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
return
|
||||
except BlockingIOError:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise
|
||||
time.sleep(min(.01, remaining))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WriteResult:
|
||||
committed: bool
|
||||
verified: bool
|
||||
|
||||
|
||||
def commit_exact(params, *, key: str, max_bytes: int, raw: bytes, expected: bytes | None,
|
||||
authorized: Callable[[], bool], temp_prefix: str) -> WriteResult:
|
||||
"""Stage outside the lock; recheck authority and source before replacement."""
|
||||
if not authorized():
|
||||
return WriteResult(False, False)
|
||||
source, readable = read_saved(params, key, max_bytes)
|
||||
if not readable or source != expected:
|
||||
return WriteResult(False, False)
|
||||
temporary = None
|
||||
lock_fd = None
|
||||
committed = False
|
||||
try:
|
||||
destination = Path(params.get_param_path(key))
|
||||
root = destination.parent.parent
|
||||
with tempfile.NamedTemporaryFile(prefix=temp_prefix, dir=root, delete=False) as stage:
|
||||
temporary = stage.name
|
||||
stage.write(raw)
|
||||
stage.flush()
|
||||
os.fsync(stage.fileno())
|
||||
lock_fd = os.open(root / ".lock", os.O_CREAT | os.O_RDONLY, 0o775)
|
||||
acquire_native_lock(lock_fd)
|
||||
if not authorized():
|
||||
return WriteResult(False, False)
|
||||
source, readable = read_saved(params, key, max_bytes)
|
||||
if not readable or source != expected or not authorized():
|
||||
return WriteResult(False, False)
|
||||
os.replace(temporary, destination)
|
||||
temporary = None
|
||||
committed = True
|
||||
directory = os.open(destination.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
observed, readable = read_saved(params, key, max_bytes)
|
||||
return WriteResult(True, readable and observed == raw)
|
||||
except (OSError, ValueError, TypeError):
|
||||
return WriteResult(committed, False)
|
||||
finally:
|
||||
if lock_fd is not None:
|
||||
os.close(lock_fd)
|
||||
if temporary is not None:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Bounded saved-setting reads for the shared feature owner."""
|
||||
|
||||
import os
|
||||
import stat
|
||||
|
||||
|
||||
def read_saved(params, key: str, limit: int) -> tuple[bytes | None, bool]:
|
||||
"""Return complete bytes/absence and readability; never authorize a prefix."""
|
||||
path = params.get_param_path(key)
|
||||
flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
|
||||
try:
|
||||
fd = os.open(path, flags)
|
||||
except FileNotFoundError:
|
||||
return None, True
|
||||
except OSError:
|
||||
return b"", False
|
||||
try:
|
||||
before = os.fstat(fd)
|
||||
if not stat.S_ISREG(before.st_mode) or before.st_size > limit:
|
||||
return b"", False
|
||||
with os.fdopen(fd, "rb", closefd=False) as source:
|
||||
raw = source.read(limit + 1)
|
||||
after = os.fstat(fd)
|
||||
current = os.stat(path, follow_symlinks=False)
|
||||
if (len(raw) > limit or not stat.S_ISREG(after.st_mode) or
|
||||
(before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) !=
|
||||
(after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns) or
|
||||
(current.st_dev, current.st_ino) != (after.st_dev, after.st_ino)):
|
||||
return b"", False
|
||||
return raw, True
|
||||
except OSError:
|
||||
return b"", False
|
||||
finally:
|
||||
os.close(fd)
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Typed, versioned cache bytes. Live messages and native Params remain unchanged."""
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
|
||||
import capnp
|
||||
|
||||
from opendbc.car import structs as car
|
||||
from openpilot.cereal import log
|
||||
|
||||
MAGIC = b"SPCACHE\x01"
|
||||
MAX_HEADER_BYTES = 4096
|
||||
MAX_ENVELOPE_BYTES = 32 * 1024 * 1024
|
||||
MAX_PAYLOAD_BYTES = MAX_ENVELOPE_BYTES - MAX_HEADER_BYTES - len(MAGIC) - 4
|
||||
FINGERPRINT_FORMAT = "capnp-transitive-node-v1"
|
||||
EVENT_FINGERPRINT_FORMAT = "capnp-event-service-v2"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CacheContract:
|
||||
root: object
|
||||
service: str | None
|
||||
producer: str
|
||||
|
||||
|
||||
# Vehicle flag and safety namespaces can change without changing the Cap'n Proto
|
||||
# layout. Re-identify the vehicle instead of reusing flags from the old contract.
|
||||
CONTRACTS = {
|
||||
"CarParamsCache": CacheContract(car.CarParams, None, "identified-car-params-v2"),
|
||||
"CarParamsPersistent": CacheContract(car.CarParams, None, "identified-car-params-v2"),
|
||||
"CarParamsPrevRoute": CacheContract(car.CarParams, None, "identified-car-params-v2"),
|
||||
"CalibrationParams": CacheContract(log.Event, "extrinsicsCalibration", "calibration-state-v1"),
|
||||
"LiveParametersV2": CacheContract(log.Event, "vehicleParameters", "vehicle-learner-v1"),
|
||||
"LiveTorqueParameters": CacheContract(log.Event, "lateralTorqueParameters", "torque-learner-v1"),
|
||||
"LiveDelay": CacheContract(log.Event, "lateralDelay", "delay-learner-v1"),
|
||||
}
|
||||
CACHE_KEYS = frozenset(CONTRACTS)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CacheInspection:
|
||||
status: str
|
||||
reason: str = ""
|
||||
payload: bytes | None = None
|
||||
schema_sha256: str | None = None
|
||||
|
||||
|
||||
def _type_children(schema, descriptor):
|
||||
kind = next(iter(descriptor))
|
||||
if kind == "list":
|
||||
element = descriptor["list"]["elementType"]
|
||||
if next(iter(element)) in ("struct", "enum", "list"):
|
||||
yield from _type_children(schema.elementType, element)
|
||||
elif kind in ("struct", "enum"):
|
||||
yield schema
|
||||
|
||||
|
||||
def schema_fingerprint(root_schema):
|
||||
"""Hash complete compiled nodes and every referenced struct/enum, including groups.
|
||||
|
||||
Node serialization retains names, ordinals, layouts, defaults, union tags and
|
||||
annotations. Root IDs alone are insufficient. Display-name changes intentionally
|
||||
invalidate a cache as a conservative compatibility boundary.
|
||||
"""
|
||||
nodes = {}
|
||||
references = set()
|
||||
|
||||
def collect_references(value):
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
if key == "typeId":
|
||||
references.add(item)
|
||||
else:
|
||||
collect_references(item)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
collect_references(item)
|
||||
|
||||
pending = [root_schema]
|
||||
while pending:
|
||||
schema = pending.pop()
|
||||
node = schema.node
|
||||
if node.id in nodes:
|
||||
continue
|
||||
nodes[node.id] = node.as_builder().to_bytes()
|
||||
descriptor = node.to_dict()
|
||||
if "struct" not in descriptor:
|
||||
continue
|
||||
for field in descriptor["struct"].get("fields", []):
|
||||
if "group" in field:
|
||||
collect_references(field["group"])
|
||||
pending.append(schema.fields[field["name"]].schema)
|
||||
else:
|
||||
collect_references(field["slot"]["type"])
|
||||
if next(iter(field["slot"]["type"])) in ("struct", "enum", "list"):
|
||||
pending.extend(_type_children(schema.fields[field["name"]].schema, field["slot"]["type"]))
|
||||
if references - nodes.keys():
|
||||
raise ValueError("Compiled schema contains unresolved transitive type references")
|
||||
digest = hashlib.sha256(FINGERPRINT_FORMAT.encode())
|
||||
digest.update(struct.pack(">Q", root_schema.node.id))
|
||||
for node_id, encoded in sorted(nodes.items()):
|
||||
digest.update(struct.pack(">QI", node_id, len(encoded)))
|
||||
digest.update(encoded)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def event_service_fingerprint(root_schema, service):
|
||||
descriptor = root_schema.node.to_dict()
|
||||
selected = root_schema.fields[service].proto
|
||||
if selected.discriminantValue == 65535 or selected.which() != "slot" or selected.slot.type.which() != "struct":
|
||||
raise ValueError("Event caches require a struct union service")
|
||||
fields = descriptor["struct"].pop("fields")
|
||||
descriptor["struct"].pop("discriminantCount")
|
||||
descriptor.pop("nestedNodes")
|
||||
digest = hashlib.sha256(EVENT_FINGERPRINT_FORMAT.encode())
|
||||
digest.update(json.dumps(descriptor, sort_keys=True, separators=(",", ":")).encode())
|
||||
# Unselected union arms do not describe the persisted message. Keep the actual
|
||||
# wrapper layout and every field/type needed to decode this service.
|
||||
for field in fields:
|
||||
if field["discriminantValue"] != 65535 and field["name"] != service:
|
||||
continue
|
||||
compiled = root_schema.fields[field["name"]]
|
||||
field_node = compiled.proto.as_builder()
|
||||
field_node.codeOrder = 0 # Declaration order can move when another union arm is added.
|
||||
encoded = field_node.to_bytes()
|
||||
digest.update(struct.pack(">I", len(encoded)))
|
||||
digest.update(encoded)
|
||||
if "group" in field:
|
||||
children = (compiled.schema,)
|
||||
elif next(iter(field["slot"]["type"])) in ("struct", "enum", "list"):
|
||||
children = _type_children(compiled.schema, field["slot"]["type"])
|
||||
else:
|
||||
children = ()
|
||||
for child in children:
|
||||
digest.update(bytes.fromhex(schema_fingerprint(child)))
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
# Only process-owned compiled schemas are cached. Untrusted schemas are never
|
||||
# memoized by numeric ID: two parsers can use identical IDs for different types.
|
||||
_FINGERPRINTS = {}
|
||||
|
||||
|
||||
def _fingerprint(contract, legacy=False):
|
||||
root_name = "car_params" if contract.root is car.CarParams else "event"
|
||||
service = contract.service if not legacy else None
|
||||
key = (root_name, service)
|
||||
if key not in _FINGERPRINTS:
|
||||
_FINGERPRINTS[key] = (event_service_fingerprint(contract.root.schema, service) if service is not None
|
||||
else schema_fingerprint(contract.root.schema))
|
||||
return _FINGERPRINTS[key]
|
||||
|
||||
|
||||
def prewarm_cache_contracts(keys=CACHE_KEYS):
|
||||
"""Compute fingerprints during each producer/consumer process's startup."""
|
||||
result = {}
|
||||
for key in keys:
|
||||
contract = _contract(key)
|
||||
_fingerprint(contract, legacy=True)
|
||||
result[key] = _fingerprint(contract)
|
||||
return result
|
||||
|
||||
|
||||
def _contract(key):
|
||||
if key not in CONTRACTS:
|
||||
raise ValueError(f"No typed cache contract for {key}")
|
||||
return CONTRACTS[key]
|
||||
|
||||
|
||||
def _check_message(contract, message):
|
||||
if not isinstance(message, (capnp._DynamicStructBuilder, capnp._DynamicStructReader)):
|
||||
raise TypeError("Cache producers must supply a typed message, not serialized bytes")
|
||||
# Native schema equality identifies the actual compiled schema, not its type ID.
|
||||
# A schema loaded independently must not acquire current-producer provenance.
|
||||
if message.schema != contract.root.schema:
|
||||
raise ValueError("Cache message is not from the current compiled producer schema")
|
||||
if contract.service is not None and message.which() != contract.service:
|
||||
raise ValueError(f"Cache requires the {contract.service} service")
|
||||
|
||||
|
||||
def _header(contract, key, payload, version=None):
|
||||
if version is None:
|
||||
version = 2 if contract.service is not None else 1
|
||||
if version not in (1, 2) or (version == 2 and contract.service is None):
|
||||
raise ValueError("Unsupported cache contract version")
|
||||
return {
|
||||
"version": version, "key": key, "producer_contract": contract.producer,
|
||||
"fingerprint_format": EVENT_FINGERPRINT_FORMAT if version == 2 else FINGERPRINT_FORMAT,
|
||||
"schema_sha256": _fingerprint(contract, legacy=version == 1),
|
||||
"root_id": hex(contract.root.schema.node.id), "service": contract.service,
|
||||
"payload_bytes": len(payload), "payload_sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def put_cache(params, key, message, block=False):
|
||||
"""Queue one atomic envelope, preserving the caller's existing Params block mode.
|
||||
|
||||
The producer must create current-schema messages from current observations or a
|
||||
verified cache. Reinterpreting historical raw bytes with the current schema first
|
||||
cannot establish their origin and is not an authorized producer conversion.
|
||||
"""
|
||||
contract = _contract(key)
|
||||
_check_message(contract, message)
|
||||
reader = message.as_reader() if isinstance(message, capnp._DynamicStructBuilder) else message
|
||||
# Clone before serialization so callers can reuse a builder already serialized
|
||||
# for a live message, without changing its write flag or queued contents.
|
||||
payload = reader.as_builder().to_bytes()
|
||||
if len(payload) > MAX_PAYLOAD_BYTES:
|
||||
raise ValueError("Cache payload exceeds the size limit")
|
||||
header = json.dumps(_header(contract, key, payload), sort_keys=True, separators=(",", ":")).encode()
|
||||
envelope = MAGIC + struct.pack(">I", len(header)) + header + payload
|
||||
if len(header) > MAX_HEADER_BYTES or len(envelope) > MAX_ENVELOPE_BYTES:
|
||||
raise ValueError("Cache envelope exceeds the size limit")
|
||||
params.put(key, envelope, block=block)
|
||||
|
||||
|
||||
def _unique_members(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError("Duplicate cache metadata member")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def inspect_cache(key, raw):
|
||||
"""Check schema/producer/content provenance; no bytes are upgraded or activated."""
|
||||
contract = _contract(key)
|
||||
if raw is None:
|
||||
return CacheInspection("missing", "no stored value")
|
||||
if not isinstance(raw, bytes) or len(raw) < len(MAGIC) + 4 or not raw.startswith(MAGIC):
|
||||
return CacheInspection("incompatible", "missing versioned cache envelope")
|
||||
if len(raw) > MAX_ENVELOPE_BYTES:
|
||||
return CacheInspection("incompatible", "cache envelope exceeds the size limit")
|
||||
size = struct.unpack_from(">I", raw, len(MAGIC))[0]
|
||||
start = len(MAGIC) + 4
|
||||
if size > MAX_HEADER_BYTES or size == 0 or len(raw) < start + size:
|
||||
return CacheInspection("incompatible", "invalid envelope header length")
|
||||
payload = raw[start + size:]
|
||||
if not payload or len(payload) > MAX_PAYLOAD_BYTES:
|
||||
return CacheInspection("incompatible", "invalid cache payload length")
|
||||
try:
|
||||
header = json.loads(raw[start:start + size], object_pairs_hook=_unique_members)
|
||||
if not isinstance(header, dict) or type(header.get("version")) is not int or type(header.get("payload_bytes")) is not int:
|
||||
return CacheInspection("incompatible", "invalid cache metadata types")
|
||||
expected = _header(contract, key, payload, version=header["version"])
|
||||
if header != expected:
|
||||
return CacheInspection("incompatible", "cache metadata, schema or payload digest mismatch")
|
||||
with contract.root.from_bytes(payload, traversal_limit_in_words=MAX_PAYLOAD_BYTES // 8) as message:
|
||||
if contract.service is not None and message.which() != contract.service:
|
||||
return CacheInspection("incompatible", "cache payload has the wrong service")
|
||||
except (ValueError, UnicodeDecodeError, RecursionError, capnp.KjException) as error:
|
||||
return CacheInspection("incompatible", f"invalid cache encoding: {type(error).__name__}")
|
||||
return CacheInspection("valid", payload=payload, schema_sha256=expected["schema_sha256"])
|
||||
|
||||
|
||||
def get_cache(params, key):
|
||||
"""Return verified bytes or a cache miss; incompatible raw values are preserved."""
|
||||
return inspect_cache(key, params.get(key)).payload
|
||||
@@ -0,0 +1,456 @@
|
||||
"""Offline settings recovery and staging. Archived bytes are never an import policy."""
|
||||
import base64
|
||||
import binascii
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PREFERENCES = frozenset({
|
||||
"AlwaysOnLateral", "IsMetric", "RecordAudio", "RecordFront", "RecordFrontLock",
|
||||
"ShowDebugInfo", "GsmMetered", "OpenpilotEnabledToggle",
|
||||
})
|
||||
# Keys removed from the registry. Saved copies are archived at startup instead of blocking it.
|
||||
RETIRED_KEYS = frozenset({'ShareUsageStats'})
|
||||
MAX_VALUE_BYTES = 32 * 1024 * 1024
|
||||
MAX_SNAPSHOT_BYTES = 256 * 1024 * 1024
|
||||
MAX_KEYS = 4096
|
||||
OPERATIONAL_KEYS = frozenset({
|
||||
'DongleId', 'HardwareSerial', 'GithubSshKeys', 'GithubUsername', 'SshEnabled',
|
||||
'GsmApn', 'GsmMetered', 'GsmRoaming', 'NetworkMetered', 'BluetoothEnabled',
|
||||
'ConnectProvider', 'PairingProvider', 'PairingEmail', 'PrimeType', 'AssistNowToken', 'SecOCKey',
|
||||
})
|
||||
|
||||
|
||||
class MigrationRequired(RuntimeError):
|
||||
"""Existing state has not been qualified for this runtime."""
|
||||
|
||||
|
||||
def canonical_json(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode('utf-8')
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _key(value):
|
||||
if not isinstance(value, str) or not value or value in ('.', '..') or '/' in value or '\0' in value:
|
||||
raise ValueError("Invalid settings key")
|
||||
return value
|
||||
|
||||
|
||||
def _hash(value, length=64):
|
||||
if not isinstance(value, str) or re.fullmatch(f'[0-9a-f]{{{length}}}', value) is None:
|
||||
raise ValueError("Invalid source digest")
|
||||
|
||||
|
||||
def _private_dir(path):
|
||||
path = Path(path)
|
||||
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
info = path.lstat()
|
||||
if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077:
|
||||
raise ValueError("Recovery storage must be a private, owned directory")
|
||||
return path
|
||||
|
||||
|
||||
def _fsync_dir(path):
|
||||
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _write(path, data):
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd, 'wb') as output:
|
||||
output.write(data)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
|
||||
|
||||
def _atomic_write(path, data):
|
||||
fd, temporary = tempfile.mkstemp(prefix='.pending-', dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, 'wb') as output:
|
||||
output.write(data)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
os.replace(temporary, path)
|
||||
_fsync_dir(path.parent)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _params_lock(namespace, *, read_only=False):
|
||||
# Params writers use this same lock in the parent of the namespace symlink.
|
||||
flags = os.O_RDONLY | os.O_NOFOLLOW if read_only else os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW
|
||||
try:
|
||||
fd = os.open(namespace.parent / '.lock', flags, 0o600)
|
||||
except FileNotFoundError:
|
||||
if not read_only:
|
||||
raise
|
||||
yield
|
||||
return
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
raise ValueError("Invalid Params lock")
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
yield
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _read_value(path):
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
with os.fdopen(fd, 'rb') as source:
|
||||
if not stat.S_ISREG(os.fstat(source.fileno()).st_mode):
|
||||
raise ValueError("Settings namespace must contain only regular files")
|
||||
value = source.read(MAX_VALUE_BYTES + 1)
|
||||
if len(value) > MAX_VALUE_BYTES:
|
||||
raise ValueError("Settings value exceeds recovery limit")
|
||||
return value
|
||||
|
||||
|
||||
def _read_namespace(namespace):
|
||||
actual = namespace.resolve(strict=True)
|
||||
values = {}
|
||||
total = 0
|
||||
for entry in sorted(actual.iterdir()):
|
||||
key = _key(entry.name)
|
||||
value = _read_value(entry)
|
||||
total += len(value)
|
||||
values[key] = value
|
||||
if total > MAX_SNAPSHOT_BYTES or len(values) > MAX_KEYS:
|
||||
raise ValueError("Settings namespace exceeds recovery limit")
|
||||
return values
|
||||
|
||||
|
||||
def _inventory(values):
|
||||
return [{'key': key, 'sha256': digest(value), 'size': len(value)} for key, value in sorted(values.items())]
|
||||
|
||||
|
||||
def _outside_params(namespace, storage):
|
||||
if storage.resolve().is_relative_to(namespace.parent.resolve()) or storage.resolve().is_relative_to(namespace.resolve()):
|
||||
raise ValueError("Recovery storage must be outside Params")
|
||||
|
||||
|
||||
def _save_snapshot(values, storage):
|
||||
inventory = _inventory(values)
|
||||
snapshot_id = digest(canonical_json(inventory))
|
||||
storage = _private_dir(storage)
|
||||
destination = storage / snapshot_id
|
||||
if destination.exists():
|
||||
if load_snapshot(destination) != values:
|
||||
raise ValueError("Existing recovery snapshot does not match")
|
||||
return destination
|
||||
temporary = Path(tempfile.mkdtemp(prefix='.pending-', dir=storage))
|
||||
try:
|
||||
data_dir = _private_dir(temporary / 'values')
|
||||
for key, value in values.items():
|
||||
_write(data_dir / key, value)
|
||||
_write(temporary / 'manifest.json', canonical_json({'format': 'starpilot-raw-settings', 'version': 1, 'entries': inventory}))
|
||||
_fsync_dir(data_dir)
|
||||
_fsync_dir(temporary)
|
||||
os.rename(temporary, destination)
|
||||
_fsync_dir(storage)
|
||||
finally:
|
||||
if temporary.exists():
|
||||
shutil.rmtree(temporary)
|
||||
return destination
|
||||
|
||||
|
||||
def snapshot_settings(namespace, storage):
|
||||
"""Capture all raw values, including empty files, unknown keys and credentials.
|
||||
|
||||
Call with services stopped. The native Params lock coordinates cooperating
|
||||
writers, but cannot coordinate an older runtime's separate settings cache.
|
||||
"""
|
||||
namespace, storage = Path(namespace).absolute(), Path(storage).absolute()
|
||||
_outside_params(namespace, storage)
|
||||
with _params_lock(namespace):
|
||||
return _save_snapshot(_read_namespace(namespace), storage)
|
||||
|
||||
|
||||
def load_snapshot(snapshot):
|
||||
snapshot = Path(snapshot)
|
||||
manifest = json.loads(_read_value(snapshot / 'manifest.json'))
|
||||
if not isinstance(manifest, dict) or set(manifest) != {'format', 'version', 'entries'}:
|
||||
raise ValueError("Invalid recovery snapshot fields")
|
||||
if manifest['format'] != 'starpilot-raw-settings' or type(manifest['version']) is not int or manifest['version'] != 1:
|
||||
raise ValueError("Unsupported recovery snapshot")
|
||||
values = _read_namespace(snapshot / 'values')
|
||||
inventory = _inventory(values)
|
||||
if inventory != manifest['entries'] or digest(canonical_json(inventory)) != snapshot.name:
|
||||
raise ValueError("Recovery snapshot integrity failure")
|
||||
return values
|
||||
|
||||
|
||||
def restore_snapshot(snapshot, namespace):
|
||||
"""Restore exact bytes to a NEW, unused namespace; never overwrite live state."""
|
||||
values = load_snapshot(snapshot)
|
||||
namespace = Path(namespace).absolute()
|
||||
namespace.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
with _params_lock(namespace):
|
||||
if namespace.exists() or namespace.is_symlink():
|
||||
raise FileExistsError("Recovery requires an unused destination namespace")
|
||||
temporary = Path(tempfile.mkdtemp(prefix='.restored-', dir=namespace.parent))
|
||||
try:
|
||||
for key, value in values.items():
|
||||
_write(temporary / key, value)
|
||||
_fsync_dir(temporary)
|
||||
namespace.symlink_to(temporary, target_is_directory=True)
|
||||
_fsync_dir(namespace.parent)
|
||||
except BaseException:
|
||||
if not namespace.is_symlink():
|
||||
shutil.rmtree(temporary)
|
||||
raise
|
||||
return namespace
|
||||
|
||||
|
||||
def validate_bundle(bundle):
|
||||
if not isinstance(bundle, dict) or set(bundle) != {'format', 'version', 'source', 'preferences', 'pending', 'omitted'}:
|
||||
raise ValueError("Invalid settings bundle fields")
|
||||
if bundle['format'] != 'starpilot-settings' or type(bundle['version']) is not int or bundle['version'] != 1:
|
||||
raise ValueError("Unsupported settings bundle")
|
||||
source = bundle['source']
|
||||
required = {'revision', 'registry_sha256', 'snapshot_sha256'}
|
||||
if not isinstance(source, dict) or not required <= source.keys() or source.keys() - required - {'cache_snapshot_sha256'}:
|
||||
raise ValueError("Invalid settings source")
|
||||
for key, value in source.items():
|
||||
_hash(value, 40 if key == 'revision' else 64)
|
||||
preferences = bundle['preferences']
|
||||
if not isinstance(preferences, dict) or not preferences.keys() <= PREFERENCES or 'AlwaysOnLateral' not in preferences:
|
||||
raise ValueError("Unreviewed active preference")
|
||||
if any(type(value) is not bool for value in preferences.values()):
|
||||
raise ValueError("Preferences require canonical booleans")
|
||||
seen = set(preferences)
|
||||
for group in ('pending', 'omitted'):
|
||||
if not isinstance(bundle[group], list) or len(bundle[group]) > MAX_KEYS:
|
||||
raise ValueError("Invalid settings disposition list")
|
||||
for entry in bundle[group]:
|
||||
expected = {'key', 'type', 'raw_base64', 'sha256'} if group == 'pending' else {'key', 'reason'}
|
||||
if not isinstance(entry, dict) or set(entry) != expected:
|
||||
raise ValueError("Invalid settings disposition fields")
|
||||
key = _key(entry['key'])
|
||||
if key in seen:
|
||||
raise ValueError("Duplicate settings disposition")
|
||||
seen.add(key)
|
||||
if group == 'omitted':
|
||||
if not isinstance(entry['reason'], str) or not entry['reason']:
|
||||
raise ValueError("Missing settings omission reason")
|
||||
continue
|
||||
if not isinstance(entry['type'], str) or entry['type'] not in {'BOOL', 'INT', 'FLOAT', 'TIME', 'JSON', 'STRING'}:
|
||||
raise ValueError("Invalid pending settings type")
|
||||
if not isinstance(entry['raw_base64'], str) or len(entry['raw_base64']) > 4 * ((MAX_VALUE_BYTES + 2) // 3):
|
||||
raise ValueError("Invalid pending settings encoding size")
|
||||
try:
|
||||
value = base64.b64decode(entry['raw_base64'], validate=True)
|
||||
except (ValueError, binascii.Error) as error:
|
||||
raise ValueError("Invalid pending settings encoding") from error
|
||||
_hash(entry['sha256'])
|
||||
if len(value) > MAX_VALUE_BYTES or digest(value) != entry['sha256']:
|
||||
raise ValueError("Pending settings integrity failure")
|
||||
if len(canonical_json(bundle)) > MAX_SNAPSHOT_BYTES:
|
||||
raise ValueError("Settings bundle exceeds recovery limit")
|
||||
|
||||
|
||||
def decode_bundle(encoded):
|
||||
"""Parse serialized input without silently accepting duplicate JSON members."""
|
||||
if not isinstance(encoded, bytes) or len(encoded) > MAX_SNAPSHOT_BYTES:
|
||||
raise ValueError("Invalid settings bundle size or encoding")
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError("Duplicate settings JSON member")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
bundle = json.loads(encoded, object_pairs_hook=unique_object)
|
||||
validate_bundle(bundle)
|
||||
return bundle
|
||||
|
||||
|
||||
def stage_settings(bundle, storage, namespace):
|
||||
"""Persist a reviewed-format bundle outside Params. Nothing is activated."""
|
||||
validate_bundle(bundle)
|
||||
_outside_params(Path(namespace).absolute(), Path(storage).absolute())
|
||||
encoded = canonical_json(bundle)
|
||||
destination = _private_dir(Path(storage) / 'staged') / f'{digest(encoded)}.json'
|
||||
_atomic_write(destination, encoded)
|
||||
return destination
|
||||
|
||||
|
||||
def _migrate_first_start(params, namespace, storage, values, known, cache_keys, inspect_cache, *, retire_keys=None,
|
||||
retire_action='archive incompatible reconstructible vehicle cache'):
|
||||
actual = namespace.resolve(strict=True)
|
||||
snapshot = _save_snapshot(values, storage / 'snapshots')
|
||||
if load_snapshot(snapshot) != values:
|
||||
raise MigrationRequired('Raw recovery snapshot readback failed')
|
||||
prepared = dict(values)
|
||||
actions = {}
|
||||
for key, raw in values.items():
|
||||
if retire_keys is not None:
|
||||
if key in retire_keys:
|
||||
prepared.pop(key)
|
||||
actions[key] = ('archive incompatible reconstructible vehicle cache' if key in known else
|
||||
'archive unknown saved key; retire without interpretation')
|
||||
continue
|
||||
if key not in known or key not in OPERATIONAL_KEYS:
|
||||
prepared.pop(key)
|
||||
actions[key] = 'archive prior software state; initialize fresh defaults'
|
||||
elif getattr(params.get_type(key), 'name', None) == 'BOOL' and raw not in (b'0', b'1'):
|
||||
prepared.pop(key)
|
||||
actions[key] = 'archive invalid operational flag; initialize registry default'
|
||||
report = {'format': 'starpilot-first-start-migration', 'version': 2,
|
||||
'namespace': str(namespace), 'target': str(actual), 'snapshot': str(snapshot),
|
||||
'status': 'prepared', 'actions': actions, 'attempted': [], 'written': []}
|
||||
receipt = snapshot / 'migration.json'
|
||||
_atomic_write(receipt, canonical_json(report))
|
||||
try:
|
||||
if namespace.resolve(strict=True) != actual or _read_namespace(namespace) != values:
|
||||
raise MigrationRequired('Settings source changed before migration')
|
||||
for key in sorted(actions):
|
||||
if namespace.resolve(strict=True) != actual:
|
||||
raise MigrationRequired('Settings namespace changed during migration')
|
||||
report['attempted'].append(key)
|
||||
_atomic_write(receipt, canonical_json(report))
|
||||
if key in prepared:
|
||||
_atomic_write(actual / key, prepared[key])
|
||||
else:
|
||||
(actual / key).unlink()
|
||||
_fsync_dir(actual)
|
||||
report['written'].append(key)
|
||||
if namespace.resolve(strict=True) != actual or _read_namespace(namespace) != prepared:
|
||||
raise MigrationRequired('Migrated settings readback differs from prepared batch')
|
||||
report['status'] = 'migrated'
|
||||
_atomic_write(receipt, canonical_json(report))
|
||||
except Exception as error:
|
||||
report.update(status='failed', error=type(error).__name__)
|
||||
_atomic_write(receipt, canonical_json(report))
|
||||
raise MigrationRequired(f'Settings migration interrupted; raw recovery snapshot: {snapshot}') from error
|
||||
return prepared
|
||||
|
||||
|
||||
def _dom_handoff(namespace):
|
||||
path = namespace.parent / f'.starpilot-dom-handoff-{digest(str(namespace).encode())}.json'
|
||||
try:
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except OSError as error:
|
||||
raise MigrationRequired('Invalid Dom handoff marker') from error
|
||||
try:
|
||||
with os.fdopen(fd, 'rb') as source:
|
||||
info = os.fstat(source.fileno())
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077 or info.st_size > 4096:
|
||||
raise ValueError('Invalid Dom handoff file')
|
||||
raw = source.read(4097)
|
||||
record = json.loads(raw)
|
||||
if type(record) is not dict or set(record) != {'format', 'version', 'namespace', 'target', 'token'}:
|
||||
raise ValueError('Invalid Dom handoff fields')
|
||||
_hash(record['token'], 32)
|
||||
expected = {'format': 'starpilot-dom-handoff', 'version': 1, 'namespace': str(namespace),
|
||||
'target': str(namespace.resolve(strict=True)), 'token': record['token']}
|
||||
if type(record['version']) is not int or raw != canonical_json(expected):
|
||||
raise ValueError('Dom handoff does not bind this namespace')
|
||||
return record['token']
|
||||
except (OSError, ValueError, TypeError, KeyError) as error:
|
||||
raise MigrationRequired('Invalid Dom handoff marker') from error
|
||||
|
||||
|
||||
def prepare_manager_start(params, storage, *, dry_run=False, auto_migrate=False):
|
||||
from openpilot.starpilot.schema_cache import CACHE_KEYS, inspect_cache
|
||||
|
||||
namespace, storage = Path(params.get_param_path()).absolute(), Path(storage).absolute()
|
||||
_outside_params(namespace, storage)
|
||||
marker_dir = storage / 'profiles' if dry_run else _private_dir(storage / 'profiles')
|
||||
if dry_run and (marker_dir.exists() or marker_dir.is_symlink()):
|
||||
info = marker_dir.lstat()
|
||||
if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077:
|
||||
raise ValueError("Recovery storage must be a private, owned directory")
|
||||
marker = marker_dir / f'{digest(str(namespace).encode())}.json'
|
||||
with _params_lock(namespace, read_only=dry_run):
|
||||
values = _read_namespace(namespace)
|
||||
identity = {'version': 2, 'schema_epoch': 2, 'namespace': str(namespace), 'target': str(namespace.resolve())}
|
||||
handoff = _dom_handoff(namespace)
|
||||
consumed = None
|
||||
try:
|
||||
raw_identity = _read_value(marker)
|
||||
saved_identity = json.loads(raw_identity)
|
||||
candidate_identity = dict(identity)
|
||||
if type(saved_identity) is dict and 'consumed_dom_token' in saved_identity:
|
||||
_hash(saved_identity['consumed_dom_token'], 32)
|
||||
candidate_identity['consumed_dom_token'] = saved_identity['consumed_dom_token']
|
||||
initialized = raw_identity == canonical_json(candidate_identity)
|
||||
if initialized:
|
||||
consumed = candidate_identity.get('consumed_dom_token')
|
||||
except (OSError, ValueError, TypeError):
|
||||
initialized = False
|
||||
foreign_handoff = handoff is not None and handoff != consumed
|
||||
if consumed is not None:
|
||||
identity['consumed_dom_token'] = consumed
|
||||
known = {key.decode() if isinstance(key, bytes) else key for key in params.all_keys()}
|
||||
if auto_migrate and not dry_run and (foreign_handoff or not initialized and values):
|
||||
values = _migrate_first_start(params, namespace, storage, values, known, CACHE_KEYS, inspect_cache)
|
||||
migrated = True
|
||||
if foreign_handoff:
|
||||
identity["consumed_dom_token"] = handoff
|
||||
else:
|
||||
migrated = False
|
||||
unknown = set(values) - known
|
||||
retired = unknown & RETIRED_KEYS
|
||||
if auto_migrate and not dry_run and initialized and not foreign_handoff and retired:
|
||||
values = _migrate_first_start(params, namespace, storage, values, known, CACHE_KEYS, inspect_cache,
|
||||
retire_keys=retired, retire_action='archive setting removed from the registry')
|
||||
unknown = set(values) - known
|
||||
invalid_preferences = any(key in values and values[key] not in (b'0', b'1') for key in PREFERENCES)
|
||||
incompatible_keys = {key for key in CACHE_KEYS if key in values and inspect_cache(key, values[key]).status != 'valid'}
|
||||
invalid_booleans = any(getattr(params.get_type(key), 'name', None) == 'BOOL' and raw not in (b'0', b'1')
|
||||
for key, raw in values.items() if key in known)
|
||||
if (auto_migrate and not dry_run and initialized and not foreign_handoff and unknown and
|
||||
not invalid_preferences and not invalid_booleans and not incompatible_keys and
|
||||
'LocationFilterInitialState' not in values):
|
||||
values = _migrate_first_start(params, namespace, storage, values, known, CACHE_KEYS, inspect_cache,
|
||||
retire_keys=unknown)
|
||||
unknown = set()
|
||||
if (auto_migrate and not dry_run and initialized and incompatible_keys and
|
||||
incompatible_keys <= {'CarParamsCache', 'CarParamsPersistent', 'CarParamsPrevRoute'} and
|
||||
not unknown and not invalid_preferences and not invalid_booleans and 'LocationFilterInitialState' not in values):
|
||||
values = _migrate_first_start(params, namespace, storage, values, known, CACHE_KEYS, inspect_cache,
|
||||
retire_keys=incompatible_keys)
|
||||
incompatible_keys = set()
|
||||
incompatible_cache = bool(incompatible_keys)
|
||||
# This retired cache has no compatible producer or conversion.
|
||||
incompatible_cache |= 'LocationFilterInitialState' in values
|
||||
unqualified = (foreign_handoff and not migrated or unknown or invalid_preferences or incompatible_cache or
|
||||
(not initialized and not migrated and bool(set(values) - PREFERENCES)))
|
||||
if unqualified:
|
||||
if dry_run:
|
||||
reasons = []
|
||||
if foreign_handoff:
|
||||
reasons.append('unconsumed Dom handoff')
|
||||
if unknown:
|
||||
reasons.append('unknown saved keys')
|
||||
if invalid_preferences:
|
||||
reasons.append('invalid saved preferences')
|
||||
if incompatible_cache:
|
||||
reasons.append('incompatible retained cache')
|
||||
if not initialized and bool(set(values) - PREFERENCES):
|
||||
reasons.append('unqualified existing namespace')
|
||||
raise MigrationRequired('Settings require migration before startup: ' + ', '.join(reasons))
|
||||
snapshot = _save_snapshot(values, storage / 'snapshots')
|
||||
raise MigrationRequired(f"Settings require migration before startup; raw recovery snapshot: {snapshot}")
|
||||
if (not initialized or foreign_handoff) and not dry_run:
|
||||
_atomic_write(marker, canonical_json(identity))
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Writable StarPilot state location, separate from upstream read-only persist."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from openpilot.common.hardware import PC
|
||||
from openpilot.common.hardware.hw import Paths
|
||||
|
||||
|
||||
_PREFIX = re.compile(r"[A-Za-z0-9_-]{1,64}\Z")
|
||||
_DEVICE_ROOT = Path("/data/starpilot")
|
||||
|
||||
|
||||
def starpilot_storage_root() -> Path:
|
||||
"""Return the StarPilot-owned state root for this Params/IPC namespace.
|
||||
|
||||
Desktop retains its existing per-user persist path. On devices /persist is
|
||||
read-only; a named OPENPILOT_PREFIX gets a separate /data root so a desk
|
||||
instance cannot read or replace the normal instance's Galaxy credentials.
|
||||
"""
|
||||
if PC:
|
||||
return Path(Paths.persist_root()) / "starpilot"
|
||||
prefix = os.environ.get("OPENPILOT_PREFIX", "")
|
||||
if not prefix:
|
||||
return _DEVICE_ROOT
|
||||
if not _PREFIX.fullmatch(prefix):
|
||||
raise ValueError("Invalid OPENPILOT_PREFIX for StarPilot storage")
|
||||
return _DEVICE_ROOT.with_name(f"{_DEVICE_ROOT.name}-{prefix}")
|
||||
|
||||
|
||||
def galaxy_storage_root() -> Path:
|
||||
"""Keep the shared history and credential directory private, including older caches."""
|
||||
root = starpilot_storage_root() / "galaxy"
|
||||
try:
|
||||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if info.st_uid == os.geteuid() and not info.st_mode & 0o022 and info.st_mode & 0o077:
|
||||
os.fchmod(fd, 0o700)
|
||||
finally:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
# Credential owners still reject inaccessible or unsafe storage.
|
||||
pass
|
||||
return root
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"scope": "Synthetic messages encoded by actual historical Python schemas and pinned provider Go bindings; no live vehicle/map data or control qualification.",
|
||||
"historical": {
|
||||
"source_revision": "678af78347d9656bc2f5dacd4204b012db861fa7",
|
||||
"schema_sha256": "e8409cc6d4701406593fffdad889fa9380cb24933556fbe161b3266e90d2a831",
|
||||
"event_base64": "AAAAABYAAAAAAAAAAgABAAAOX6McAAAAjwAAAAAAAAAAAAAACAAFAB+Fm0FGlFZBAAAWQwAAAAAAAAAAAAAAAAAAAAAGAgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAdAAAAQgAAAB0AAAAaAAAACQAAAJoAAAAAAAAAAAAAAAAAAAAAAAAAU3ludGhldGljIG9sZCByb2FkAAAAAAAAT2xkIFdheQBUMQAAAAAAAA=="
|
||||
},
|
||||
"provider": {
|
||||
"source_revision": "7201c6b4b4ec1b0b9ea21daa8c05b80fdd7e01ee",
|
||||
"schema_sha256": "95914f6079c695bda03812e27737335d2f724792a6b6aa845d2823fd6ea93fad",
|
||||
"event_base64": "AAAAABoAAAAAAAAAAgABAADQtytqAAAAjwAAAAAAAAAAAAAACQAGAEaUVkEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAMAAAAAADLBPtxHwEAAAAAAAAAAAAAAAAAAAAAAAANAAAAwgAAAAAAAAAAAAAAAAAAAAAAAAANAAAAygAAAFN5bnRoZXRpYyBwcm92aWRlciByb2FkADIwIEAgKE1vLUZyIDA4OjAwLTA5OjAwKQAAAAAAAAAA"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
"""The compiled custom messages must continue to read their earlier layouts."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
|
||||
import capnp
|
||||
|
||||
from openpilot.cereal import custom
|
||||
from tools.ci.custom_schema_contract import CONTRACT, describe, validate
|
||||
|
||||
|
||||
class CustomSchemaEvolutionTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.baseline = json.loads(CONTRACT.read_text())
|
||||
|
||||
def test_native_custom_messages_preserve_committed_layout(self):
|
||||
self.assertEqual(validate(describe(custom, self.baseline['roots']), self.baseline), [])
|
||||
|
||||
def test_historical_map_roots_restored_and_development_axes_use_separate_roots(self):
|
||||
self.assertEqual(set(self.baseline['roots']), {
|
||||
'SlcState', 'SlcAction', 'SlcCruiseEvent', 'SlcDashboardObservation',
|
||||
'SlcCruiseCommand', 'AolAxisState', 'AolAxisState.SafetyWire', 'AolAxisState.IntentWire',
|
||||
'MapdOut', 'MapdExtendedOut', 'MapdIn',
|
||||
'AolAxisState.LaneChangeStatusWire', 'StarPilotModelDataV2', 'StarPilotRadarState',
|
||||
'StarPilotSelfdriveState', 'SpotMonitorState', 'StarPilotLateralState', 'StarPilotCarState',
|
||||
'StarPilotCarParams',
|
||||
})
|
||||
self.assertEqual(self.baseline['roots']['MapdExtendedOut'], str(0xa30662f84033036c))
|
||||
self.assertEqual(self.baseline['roots']['MapdIn'], str(0xc86a3d38d13eb3ef))
|
||||
self.assertEqual(self.baseline['roots']['SpotMonitorState'], str(0xcb9fd56c7057593a))
|
||||
self.assertNotIn(str(0xa30662f84033036c), (self.baseline['roots']['AolAxisState.SafetyWire'],
|
||||
self.baseline['roots']['AolAxisState.IntentWire']))
|
||||
|
||||
def test_lane_status_has_an_independent_wire_without_changing_aol_state(self):
|
||||
wire_id = self.baseline['roots']['AolAxisState.LaneChangeStatusWire']
|
||||
self.assertEqual(wire_id, str(0xa042ddb70aba3ea9))
|
||||
aol_fields = self.baseline['nodes'][self.baseline['roots']['AolAxisState']]['fields']
|
||||
self.assertEqual(set(aol_fields), {str(n) for n in range(12)})
|
||||
self.assertFalse(any(field['slot']['type'] == {'struct': {'typeId': int(wire_id)}} for field in aol_fields.values()))
|
||||
|
||||
def test_historical_lateral_fields_and_new_feedback_are_append_compatible(self):
|
||||
from openpilot.cereal import log
|
||||
source = """@0xbbaa06d64a5a9f42;
|
||||
struct HistoricalLateral @0xc2243c65e0340384 {
|
||||
active @0 :Bool;
|
||||
frictionThreshold @1 :Float32;
|
||||
frictionScale @2 :Float32;
|
||||
feedforward @3 :Float32;
|
||||
frictionJerk @4 :Float32;
|
||||
frictionJerkDeadzone @5 :Float32;
|
||||
lowSpeedFactor @6 :Float32;
|
||||
unwindDetected @7 :Bool;
|
||||
}
|
||||
"""
|
||||
with TemporaryDirectory() as directory:
|
||||
path = Path(directory) / 'historical.capnp'
|
||||
path.write_text(source)
|
||||
historical = capnp.SchemaParser().load(str(path))
|
||||
old_layout = describe(historical, ['HistoricalLateral'])
|
||||
current_layout = describe(custom, ['StarPilotLateralState'])
|
||||
old_layout['roots'] = current_layout['roots']
|
||||
self.assertEqual(validate(current_layout, old_layout), [])
|
||||
values = {'active': True, 'frictionThreshold': .25, 'frictionScale': .5, 'feedforward': .125,
|
||||
'frictionJerk': .75, 'frictionJerkDeadzone': .0625, 'lowSpeedFactor': 1.5, 'unwindDetected': True}
|
||||
old = historical.HistoricalLateral.new_message(**values)
|
||||
with custom.StarPilotLateralState.from_bytes(old.to_bytes()) as current:
|
||||
self.assertEqual(current.laneCentering.version, 0)
|
||||
self.assertTrue(all(getattr(current, key) == value for key, value in values.items()))
|
||||
new = custom.StarPilotLateralState.new_message(**values)
|
||||
new.laneCentering.version = 1
|
||||
with historical.HistoricalLateral.from_bytes(new.to_bytes()) as prior:
|
||||
self.assertEqual(prior.to_dict(), values)
|
||||
self.assertEqual(log.Event.schema.fields['starpilotLateralState'].proto.ordinal.explicit, 137)
|
||||
|
||||
def test_historical_selfdrive_alert_fields_and_new_ack_are_append_compatible(self):
|
||||
from openpilot.cereal import log
|
||||
|
||||
source = '''using Car = import "/car.capnp";
|
||||
@0xbbaa06d64a5a9f42;
|
||||
struct HistoricalSelfdrive @0xf416ec09499d9d19 {
|
||||
alertText1 @0 :Text;
|
||||
alertText2 @1 :Text;
|
||||
alertStatus @2 :AlertStatus;
|
||||
alertSize @3 :AlertSize;
|
||||
alertType @4 :Text;
|
||||
alertSound @5 :Car.CarControl.HUDControl.AudibleAlert;
|
||||
vEgo @6 :Float32;
|
||||
enum AlertStatus @0xc0f486ad93ed68c9 {
|
||||
normal @0; userPrompt @1; critical @2; starpilot @3;
|
||||
}
|
||||
enum AlertSize @0xe22723d973fc2afb {
|
||||
none @0; small @1; mid @2; full @3;
|
||||
}
|
||||
}
|
||||
'''
|
||||
with TemporaryDirectory() as directory:
|
||||
path = Path(directory) / 'historical.capnp'
|
||||
path.write_text(source)
|
||||
historical = capnp.SchemaParser().load(str(path), imports=[str(Path(__file__).parents[3] / 'opendbc_repo/opendbc/car')])
|
||||
old_layout = describe(historical, ['HistoricalSelfdrive'])
|
||||
current_layout = describe(custom, ['StarPilotSelfdriveState'])
|
||||
old_layout['roots'] = current_layout['roots']
|
||||
self.assertEqual(validate(current_layout, old_layout), [])
|
||||
values = {'alertText1': 'Keep hands on wheel', 'alertText2': 'Prompt', 'alertStatus': 'userPrompt',
|
||||
'alertSize': 'mid', 'alertType': 'test', 'alertSound': 'prompt', 'vEgo': 15.0}
|
||||
old = historical.HistoricalSelfdrive.new_message(**values)
|
||||
with custom.StarPilotSelfdriveState.from_bytes(old.to_bytes()) as current:
|
||||
self.assertEqual(current.alertText1, values['alertText1'])
|
||||
self.assertEqual(str(current.alertStatus), values['alertStatus'])
|
||||
self.assertEqual(str(current.alertSound), values['alertSound'])
|
||||
self.assertEqual(current.vEgo, values['vEgo'])
|
||||
self.assertEqual(current.conditionalModeAck.version, 0)
|
||||
new = custom.StarPilotSelfdriveState.new_message(**values)
|
||||
new.conditionalModeAck.version = 1
|
||||
with historical.HistoricalSelfdrive.from_bytes(new.to_bytes()) as prior:
|
||||
self.assertEqual(prior.to_dict(), values)
|
||||
self.assertEqual(log.Event.schema.fields['starpilotSelfdriveState'].proto.ordinal.explicit, 115)
|
||||
|
||||
def test_conditional_proposal_and_manual_receipt_are_appended_to_existing_roots(self):
|
||||
slc = custom.SlcState.new_message()
|
||||
slc.conditionalMode.version = 1
|
||||
slc.conditionalMode.choice = 'conditionalChill'
|
||||
slc.conditionalMode.settingsRevision = 7
|
||||
with custom.SlcState.from_bytes(slc.to_bytes()) as decoded:
|
||||
self.assertEqual(str(decoded.conditionalMode.choice), 'conditionalChill')
|
||||
self.assertEqual(decoded.conditionalMode.settingsRevision, 7)
|
||||
|
||||
event = custom.SlcCruiseEvent.new_message()
|
||||
event.kind = 'conditionalMode'
|
||||
event.manualMode.version = 1
|
||||
event.manualMode.choice = 'conditionalExperimental'
|
||||
event.manualMode.button = 'lkas'
|
||||
event.manualMode.press = 'long'
|
||||
event.manualMode.sourceCarStateMonoTime = 100
|
||||
event.manualMode.validUntilMonoTime = 200
|
||||
with custom.SlcCruiseEvent.from_bytes(event.to_bytes()) as decoded:
|
||||
self.assertEqual(str(decoded.kind), 'conditionalMode')
|
||||
self.assertEqual((str(decoded.manualMode.choice), str(decoded.manualMode.button), str(decoded.manualMode.press)),
|
||||
('conditionalExperimental', 'lkas', 'long'))
|
||||
self.assertEqual((decoded.manualMode.sourceCarStateMonoTime, decoded.manualMode.validUntilMonoTime), (100, 200))
|
||||
|
||||
def test_vision_appends_to_restored_model_slot_without_reinterpreting_history(self):
|
||||
from openpilot.cereal import log
|
||||
model_id = self.baseline['roots']['StarPilotModelDataV2']
|
||||
self.assertEqual(model_id, str(0x80ae746ee2596b11))
|
||||
field = log.Event.schema.fields['slcVisionObservation'].proto
|
||||
self.assertEqual(field.ordinal.explicit, 111)
|
||||
self.assertEqual(field.slot.type.struct.typeId, int(model_id))
|
||||
legacy_enum = self.baseline['nodes'][str(0xab5928774e6e64fc)]
|
||||
self.assertEqual(legacy_enum['values'], ['none', 'turnLeft', 'turnRight'])
|
||||
source = '''@0xeea51a25b1b2c3d4;
|
||||
struct HistoricalModel @0x80ae746ee2596b11 {
|
||||
turnDirection @0 :TurnDirection;
|
||||
enum TurnDirection @0xab5928774e6e64fc { none @0; turnLeft @1; turnRight @2; }
|
||||
}
|
||||
'''
|
||||
with TemporaryDirectory() as directory:
|
||||
path = Path(directory) / 'historical.capnp'
|
||||
path.write_text(source)
|
||||
historical = capnp.SchemaParser().load(str(path))
|
||||
old = historical.HistoricalModel.new_message(turnDirection='turnRight')
|
||||
with custom.StarPilotModelDataV2.from_bytes(old.to_bytes()) as current:
|
||||
self.assertEqual(str(current.turnDirection), 'turnRight')
|
||||
self.assertEqual(str(current.vision.status), 'unknown')
|
||||
self.assertEqual(current.vision.producerSessionId, '')
|
||||
new = custom.StarPilotModelDataV2.new_message(turnDirection='turnLeft')
|
||||
new.vision.status = 'valid'
|
||||
new.vision.speedMps = 20
|
||||
with historical.HistoricalModel.from_bytes(new.to_bytes()) as prior:
|
||||
self.assertEqual(str(prior.turnDirection), 'turnLeft')
|
||||
|
||||
def test_slc_preserves_historical_model_status_without_activating_a_limit(self):
|
||||
source = '''@0xbbaa06d64a5a9f41;
|
||||
struct HistoricalModelStatus @0xa1680744031fdb2d {
|
||||
slotId @0 :Text;
|
||||
slotName @1 :Text;
|
||||
variant @2 :Text;
|
||||
variantLabel @3 :Text;
|
||||
reason @4 :Text;
|
||||
wallTimeNanos @5 :UInt64;
|
||||
}
|
||||
'''
|
||||
with TemporaryDirectory() as directory:
|
||||
path = Path(directory) / 'historical.capnp'
|
||||
path.write_text(source)
|
||||
historical = capnp.SchemaParser().load(str(path))
|
||||
old_layout = describe(historical, ['HistoricalModelStatus'])
|
||||
current_layout = describe(custom, ['SlcState'])
|
||||
# Root renaming is permitted for a reserved slot; its wire meaning is not.
|
||||
old_layout['roots'] = current_layout['roots']
|
||||
self.assertEqual(validate(current_layout, old_layout), [])
|
||||
values = {'slotId': 'model-a', 'slotName': 'Model A', 'variant': 'small',
|
||||
'variantLabel': 'Small', 'reason': 'selected', 'wallTimeNanos': 123456789}
|
||||
old = historical.HistoricalModelStatus.new_message(**values)
|
||||
with custom.SlcState.from_bytes(old.to_bytes()) as current:
|
||||
for name, value in values.items():
|
||||
self.assertEqual(getattr(current, name), value)
|
||||
self.assertEqual(current.sessionId, '')
|
||||
self.assertEqual(current.frameMonoTime, 0)
|
||||
self.assertFalse(current.enabled)
|
||||
self.assertFalse(current.hasAccepted)
|
||||
self.assertFalse(current.hasCeiling)
|
||||
new = custom.SlcState.new_message(**values, sessionId='drive-a', frameMonoTime=987654321,
|
||||
enabled=True, hasCeiling=True, effectiveCap=20)
|
||||
with historical.HistoricalModelStatus.from_bytes(new.to_bytes()) as prior:
|
||||
self.assertEqual(prior.to_dict(), values)
|
||||
|
||||
def test_new_fields_and_enum_values_are_allowed(self):
|
||||
current = copy.deepcopy(self.baseline)
|
||||
for node in current['nodes'].values():
|
||||
if node['kind'] == 'enum':
|
||||
node['values'].append('additionalValue')
|
||||
else:
|
||||
node['fields']['999'] = {'name': 'additionalField'}
|
||||
node['data_words'] += 1
|
||||
node['pointers'] += 1
|
||||
self.assertEqual(validate(current, self.baseline), [])
|
||||
|
||||
def test_same_ids_cannot_hide_changed_field_meanings(self):
|
||||
struct_id = next(key for key, value in self.baseline['nodes'].items() if value['kind'] == 'struct')
|
||||
ordinal = next(iter(self.baseline['nodes'][struct_id]['fields']))
|
||||
for change in ('name', 'type', 'offset', 'default', 'union', 'removed'):
|
||||
with self.subTest(change=change):
|
||||
current = copy.deepcopy(self.baseline)
|
||||
fields = current['nodes'][struct_id]['fields']
|
||||
field = fields[ordinal]
|
||||
if change == 'name':
|
||||
field['name'] = 'unrelatedMeaning'
|
||||
elif change == 'type':
|
||||
field['slot']['type'] = {'bool': None}
|
||||
elif change == 'offset':
|
||||
field['slot']['offset'] += 1
|
||||
elif change == 'default':
|
||||
field['slot']['defaultValue'] = {'text': 'new default'}
|
||||
elif change == 'union':
|
||||
field['discriminantValue'] = 0
|
||||
else:
|
||||
del fields[ordinal]
|
||||
self.assertTrue(validate(current, self.baseline))
|
||||
|
||||
def test_reassigned_enum_values_or_custom_roots_are_rejected(self):
|
||||
current = copy.deepcopy(self.baseline)
|
||||
enum = next(node for node in current['nodes'].values() if node['kind'] == 'enum')
|
||||
enum['values'][0], enum['values'][1] = enum['values'][1], enum['values'][0]
|
||||
self.assertTrue(validate(current, self.baseline))
|
||||
current = copy.deepcopy(self.baseline)
|
||||
current['roots']['SlcState'] = '0'
|
||||
self.assertTrue(validate(current, self.baseline))
|
||||
|
||||
def test_actual_pointer_defaults_are_serializable_and_protected(self):
|
||||
source = '''@0xedeedaceefbcfbca;
|
||||
struct Child @0xb889677671233ef2 { number @0 :UInt32; }
|
||||
struct Sample @0xc889677671233ef3 {
|
||||
child @0 :Child = (number = CHILD_VALUE);
|
||||
values @1 :List(UInt16) = [1, LIST_VALUE];
|
||||
}
|
||||
'''
|
||||
with TemporaryDirectory() as directory:
|
||||
path = Path(directory) / 'fixture.capnp'
|
||||
contracts = []
|
||||
for child_value, list_value in ((7, 2), (8, 2), (7, 3), (7, 2)):
|
||||
path.write_text(source.replace('CHILD_VALUE', str(child_value)).replace('LIST_VALUE', str(list_value)))
|
||||
module = capnp.SchemaParser().load(str(path))
|
||||
# A separate parser permits independent revisions of the same type IDs.
|
||||
contracts.append(json.loads(json.dumps(describe(module, ['Sample']))))
|
||||
self.assertEqual(validate(contracts[3], contracts[0]), [])
|
||||
self.assertTrue(any('.child:' in issue for issue in validate(contracts[1], contracts[0])))
|
||||
self.assertTrue(any('.values:' in issue for issue in validate(contracts[2], contracts[0])))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Actual historical and Go-provider wire fixtures retain their field meanings."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
from openpilot.cereal import custom, log
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
FIXTURES = json.loads(Path(__file__).with_name('fixtures').joinpath('mapd_wire.json').read_text())
|
||||
IO_FIXTURES = json.loads((ROOT / 'mapd_repo/cereal/testdata/mapd_io_wire.json').read_text())
|
||||
|
||||
|
||||
class MapdSchemaTests(unittest.TestCase):
|
||||
def test_historical_status_and_input_payloads_preserve_meanings(self):
|
||||
with custom.MapdExtendedOut.from_bytes(base64.b64decode(IO_FIXTURES['historical']['extended_payload_base64'])) as status:
|
||||
progress = status.downloadProgress
|
||||
self.assertTrue(progress.active and progress.cancelled)
|
||||
self.assertEqual((progress.totalFiles, progress.downloadedFiles), (7, 3))
|
||||
self.assertEqual(list(progress.locations), ['historical-region'])
|
||||
self.assertEqual(progress.locationDetails[0].to_dict(), {'location': 'historical-region', 'totalFiles': 7, 'downloadedFiles': 3})
|
||||
self.assertEqual(status.settings, '{"historical":true}')
|
||||
self.assertEqual((status.path[0].latitude, status.path[0].longitude, status.path[0].targetVelocity), (36.25, -96.5, 8.5))
|
||||
self.assertAlmostEqual(status.path[0].curvature, -0.002)
|
||||
self.assertFalse(status._has('position'))
|
||||
self.assertEqual((status.loopRateAverage, status.loopRateMin), (0, 0))
|
||||
with custom.MapdIn.from_bytes(base64.b64decode(IO_FIXTURES['historical']['input_payload_base64'])) as request:
|
||||
self.assertEqual((str(request.type), request.float, request.str, request.bool, request.jsonPath),
|
||||
('setPressGasToOverrideSpeedLimit', 1.25, 'historical input', True, ''))
|
||||
|
||||
def test_actual_go_provider_extended_status_and_input(self):
|
||||
with log.Event.from_bytes(base64.b64decode(IO_FIXTURES['provider']['extended_event_base64'])) as event:
|
||||
self.assertEqual((event.which(), event.logMonoTime, event.valid), ('mapdExtendedOut', 780_000_000_000, True))
|
||||
status = event.mapdExtendedOut
|
||||
self.assertEqual((status.downloadProgress.totalFiles, status.downloadProgress.downloadedFiles), (10, 4))
|
||||
self.assertEqual(list(status.downloadProgress.locations), ['synthetic-region'])
|
||||
self.assertEqual(status.settings, '{"fixture":true}')
|
||||
self.assertEqual((status.path[0].latitude, status.path[0].longitude, status.path[0].targetVelocity), (35.15, -97.9, 12.5))
|
||||
self.assertAlmostEqual(status.path[0].curvature, 0.001)
|
||||
self.assertEqual((status.position.latitude, status.position.longitude), (35.151, -97.901))
|
||||
self.assertEqual((status.loopRateAverage, status.loopRateMin), (20, 18.5))
|
||||
with log.Event.from_bytes(base64.b64decode(IO_FIXTURES['provider']['input_event_base64'])) as event:
|
||||
self.assertEqual((event.which(), event.logMonoTime, event.valid), ('mapdIn', 780_000_000_000, True))
|
||||
request = event.mapdIn
|
||||
self.assertEqual((str(request.type), request.float, request.str, request.bool, request.jsonPath),
|
||||
('setJsonPathFloat', 2.75, 'synthetic input', True, 'curve.targetLatA'))
|
||||
|
||||
def test_actual_host_input_fixture_read_by_provider(self):
|
||||
event = log.Event.new_message(logMonoTime=790_000_000_000, valid=True)
|
||||
request = event.init('mapdIn')
|
||||
request.type, request.float, request.str, request.bool, request.jsonPath = ('setJsonPathText', 1.5, 'host fixture', True, 'display.units')
|
||||
self.assertEqual(event.to_bytes(), base64.b64decode(IO_FIXTURES['host']['input_event_base64']))
|
||||
|
||||
def test_historical_event_reads_with_appended_defaults(self):
|
||||
with log.Event.from_bytes(base64.b64decode(FIXTURES['historical']['event_base64'])) as event:
|
||||
self.assertEqual(event.which(), 'mapdOut')
|
||||
self.assertTrue(event.valid)
|
||||
self.assertEqual(event.logMonoTime, 123_000_000_000)
|
||||
road = event.mapdOut
|
||||
self.assertEqual((road.roadName, road.wayName, road.wayRef), ('Synthetic old road', 'Old Way', 'T1'))
|
||||
self.assertAlmostEqual(road.speedLimit, 19.44, places=4)
|
||||
self.assertAlmostEqual(road.nextSpeedLimit, 13.4112, places=4)
|
||||
self.assertEqual(road.nextSpeedLimitDistance, 150)
|
||||
self.assertTrue(road.tileLoaded)
|
||||
self.assertTrue(road.speedLimitAccepted)
|
||||
self.assertEqual(road.lanes, 2)
|
||||
self.assertEqual(str(road.roadContext), 'city')
|
||||
self.assertEqual(str(road.waySelectionType), 'current')
|
||||
self.assertEqual((road.wayId, str(road.highwayClass), road.conditionalSpeedLimit), (0, 'unknown', ''))
|
||||
|
||||
def test_provider_go_event_reads_new_fields_without_reinterpreting_old(self):
|
||||
with log.Event.from_bytes(base64.b64decode(FIXTURES['provider']['event_base64'])) as event:
|
||||
self.assertEqual(event.which(), 'mapdOut')
|
||||
self.assertTrue(event.valid)
|
||||
self.assertEqual(event.logMonoTime, 456_000_000_000)
|
||||
road = event.mapdOut
|
||||
self.assertEqual(road.roadName, 'Synthetic provider road')
|
||||
self.assertAlmostEqual(road.speedLimit, 13.4112, places=4)
|
||||
self.assertTrue(road.tileLoaded)
|
||||
self.assertEqual(str(road.roadContext), 'city')
|
||||
self.assertEqual(str(road.waySelectionType), 'possible')
|
||||
self.assertEqual(str(road.highwayClass), 'residential')
|
||||
self.assertEqual(road.wayId, 1_234_567_890_123)
|
||||
self.assertEqual(road.conditionalSpeedLimit, '20 @ (Mo-Fr 08:00-09:00)')
|
||||
# A fresh old-provider Event is still missing the v1 computation evidence.
|
||||
self.assertEqual(road.sampleVersion, 0)
|
||||
self.assertEqual(str(road.roadStatus), 'unknown')
|
||||
self.assertEqual((road.gpsMonoTime, road.computedMonoTime, road.producerSession), (0, 0, 0))
|
||||
|
||||
def test_historical_provider_v1_wire_remains_readable(self):
|
||||
fixture = json.loads((ROOT / 'mapd_repo/cereal/testdata/mapd_v1_wire.json').read_text())
|
||||
# Its schema hash records the original v1 producer. The current v2 reader
|
||||
# preserves that layout, but v1 timestamps no longer qualify as evidence.
|
||||
with log.Event.from_bytes(base64.b64decode(fixture['event_base64'])) as event:
|
||||
self.assertEqual(event.which(), 'mapdOut')
|
||||
self.assertTrue(event.valid)
|
||||
road = event.mapdOut
|
||||
self.assertEqual((road.sampleVersion, str(road.roadStatus), str(road.gpsSource)), (1, 'matchedLimit', 'external'))
|
||||
self.assertEqual((road.gpsMonoTime, road.computedMonoTime, event.logMonoTime),
|
||||
(455_950_000_000, 455_990_000_000, 456_000_000_000))
|
||||
self.assertEqual((road.sourceGeneration, road.producerSession), (3, 0x1234abcd5011))
|
||||
self.assertAlmostEqual(road.speedLimit, 13.4112, places=4)
|
||||
self.assertEqual(road.wayId, 1_234_567_890_123)
|
||||
|
||||
def test_provider_pin_and_enum_identities(self):
|
||||
pin = next(row for row in json.loads((ROOT / 'upstream-sync.json').read_text())['dependencies'] if row['path'] == 'mapd_repo')
|
||||
self.assertEqual(pin['commit'], FIXTURES['provider']['source_revision'])
|
||||
self.assertEqual(custom.MapdOut.schema.node.id, 0xa4f1eb3323f5f582)
|
||||
for name, type_id in (('roadContext', 0xabefa88b9563dbae), ('waySelectionType', 0xfa3e5ce74e25e82a),
|
||||
('highwayClass', 0xd18274dcdc63c41d)):
|
||||
self.assertEqual(custom.MapdOut.schema.fields[name].schema.node.id, type_id)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,309 @@
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import capnp
|
||||
|
||||
from opendbc.car import structs as car
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.starpilot import schema_cache as cache
|
||||
|
||||
|
||||
class TestSchemaCache(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.root = Path(self.temporary.name)
|
||||
self.params = Params(str(self.root / "params"))
|
||||
self.event_parsers = []
|
||||
cache.prewarm_cache_contracts()
|
||||
|
||||
def message(self, key):
|
||||
contract = cache.CONTRACTS[key]
|
||||
message = contract.root.new_message()
|
||||
if contract.service:
|
||||
message.init(contract.service)
|
||||
else:
|
||||
message.carFingerprint = "CACHE_TEST_FIXTURE"
|
||||
return message
|
||||
|
||||
def encoded(self, key="CarParamsPersistent"):
|
||||
cache.put_cache(self.params, key, self.message(key), block=True)
|
||||
return self.params.get(key)
|
||||
|
||||
@staticmethod
|
||||
def header_and_payload(raw):
|
||||
start = len(cache.MAGIC) + 4
|
||||
size = struct.unpack_from(">I", raw, len(cache.MAGIC))[0]
|
||||
return json.loads(raw[start:start + size]), raw[start + size:]
|
||||
|
||||
@staticmethod
|
||||
def repack(header, payload):
|
||||
raw = json.dumps(header, separators=(",", ":")).encode()
|
||||
return cache.MAGIC + struct.pack(">I", len(raw)) + raw + payload
|
||||
|
||||
def test_all_key_service_contracts_roundtrip_native_params(self):
|
||||
for key in cache.CACHE_KEYS:
|
||||
with self.subTest(key=key):
|
||||
raw = self.encoded(key)
|
||||
inspected = cache.inspect_cache(key, raw)
|
||||
self.assertEqual(inspected.status, "valid")
|
||||
self.assertEqual(cache.get_cache(self.params, key), inspected.payload)
|
||||
with cache.CONTRACTS[key].root.from_bytes(inspected.payload) as message:
|
||||
if cache.CONTRACTS[key].service:
|
||||
self.assertEqual(message.which(), cache.CONTRACTS[key].service)
|
||||
else:
|
||||
self.assertEqual(message.carFingerprint, "CACHE_TEST_FIXTURE")
|
||||
|
||||
def test_single_queued_write_preserves_block_argument(self):
|
||||
params = Mock()
|
||||
for block in (False, True):
|
||||
params.reset_mock()
|
||||
cache.put_cache(params, "CarParamsCache", self.message("CarParamsCache"), block=block)
|
||||
self.assertEqual(params.put.call_count, 1)
|
||||
self.assertEqual(params.put.call_args.kwargs, {"block": block})
|
||||
key, value = params.put.call_args.args
|
||||
self.assertEqual(cache.inspect_cache(key, value).status, "valid")
|
||||
|
||||
def test_native_async_write_and_capture_before_builder_mutation(self):
|
||||
message = self.message("CarParamsCache")
|
||||
cache.put_cache(self.params, "CarParamsCache", message, block=False)
|
||||
message.carFingerprint = "CHANGED_AFTER_QUEUE"
|
||||
raw = self.params.get("CarParamsCache", block=True)
|
||||
result = cache.inspect_cache("CarParamsCache", raw)
|
||||
with car.CarParams.from_bytes(result.payload) as restored:
|
||||
self.assertEqual(restored.carFingerprint, "CACHE_TEST_FIXTURE")
|
||||
|
||||
def test_already_serialized_builder_and_reader_can_be_used(self):
|
||||
message = self.message("CarParamsCache")
|
||||
original = message.to_bytes()
|
||||
cache.put_cache(self.params, "CarParamsCache", message, block=True)
|
||||
cache.put_cache(self.params, "CarParamsPersistent", message, block=True)
|
||||
with car.CarParams.from_bytes(original) as reader:
|
||||
cache.put_cache(self.params, "CarParamsPrevRoute", reader, block=True)
|
||||
self.assertEqual(cache.get_cache(self.params, "CarParamsCache"), original)
|
||||
self.assertEqual(cache.get_cache(self.params, "CarParamsPrevRoute"), original)
|
||||
|
||||
def test_raw_bytes_cannot_be_blessed_by_typed_writer(self):
|
||||
raw = self.message("CarParamsCache").to_bytes()
|
||||
for invalid in (raw, bytearray(raw), memoryview(raw), {}, None):
|
||||
with self.subTest(type=type(invalid)), self.assertRaises(TypeError):
|
||||
cache.put_cache(self.params, "CarParamsCache", invalid)
|
||||
self.assertIsNone(self.params.get("CarParamsCache"))
|
||||
|
||||
def test_wrong_root_and_service_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
cache.put_cache(self.params, "CalibrationParams", self.message("CarParamsCache"))
|
||||
with self.assertRaises(ValueError):
|
||||
cache.put_cache(self.params, "CalibrationParams", self.message("LiveDelay"))
|
||||
|
||||
def test_missing_and_unversioned_are_distinct_and_preserved(self):
|
||||
self.assertEqual(cache.inspect_cache("CarParamsCache", None).status, "missing")
|
||||
for raw in (b"", b"old bytes", self.message("CarParamsCache").to_bytes()):
|
||||
self.params.put("CarParamsCache", raw, block=True)
|
||||
self.assertEqual(cache.inspect_cache("CarParamsCache", raw).status, "incompatible")
|
||||
self.assertIsNone(cache.get_cache(self.params, "CarParamsCache"))
|
||||
# Native Params maps an empty value to None; inspect raw storage to prove
|
||||
# that a cache miss did not remove or replace even an empty file.
|
||||
self.assertEqual((Path(self.params.get_param_path()) / "CarParamsCache").read_bytes(), raw)
|
||||
|
||||
def test_cache_key_cannot_be_relabelled(self):
|
||||
raw = self.encoded()
|
||||
self.assertEqual(cache.inspect_cache("CarParamsPrevRoute", raw).status, "incompatible")
|
||||
with self.assertRaises(ValueError):
|
||||
cache.put_cache(self.params, "CarParams", self.message("CarParamsCache"))
|
||||
|
||||
def test_metadata_epoch_schema_and_digest_tampering_rejected(self):
|
||||
header, payload = self.header_and_payload(self.encoded())
|
||||
for key, value in (("version", 2), ("version", True), ("schema_sha256", "0" * 64),
|
||||
("root_id", "0x1"), ("producer_contract", "other"), ("payload_bytes", False),
|
||||
("fingerprint_format", "other"), ("extra", "field")):
|
||||
changed = copy.deepcopy(header)
|
||||
changed[key] = value
|
||||
self.assertEqual(cache.inspect_cache("CarParamsPersistent", self.repack(changed, payload)).status, "incompatible")
|
||||
damaged = payload[:-1] + bytes([payload[-1] ^ 1])
|
||||
self.assertEqual(cache.inspect_cache("CarParamsPersistent", self.repack(header, damaged)).status, "incompatible")
|
||||
|
||||
def test_duplicate_json_member_rejected(self):
|
||||
header, payload = self.header_and_payload(self.encoded())
|
||||
text = json.dumps(header)[:-1] + ', "version":1}'
|
||||
encoded = text.encode()
|
||||
raw = cache.MAGIC + struct.pack(">I", len(encoded)) + encoded + payload
|
||||
self.assertEqual(cache.inspect_cache("CarParamsPersistent", raw).status, "incompatible")
|
||||
|
||||
def test_wrong_event_service_rejected_even_with_consistent_digest(self):
|
||||
header, _ = self.header_and_payload(self.encoded("CalibrationParams"))
|
||||
payload = self.message("LiveDelay").to_bytes()
|
||||
header.update(payload_bytes=len(payload), payload_sha256=hashlib.sha256(payload).hexdigest())
|
||||
self.assertEqual(cache.inspect_cache("CalibrationParams", self.repack(header, payload)).status, "incompatible")
|
||||
|
||||
def test_malformed_capnp_rejected_even_with_consistent_digest(self):
|
||||
header, _ = self.header_and_payload(self.encoded())
|
||||
payload = b"not capnp"
|
||||
header.update(payload_bytes=len(payload), payload_sha256=hashlib.sha256(payload).hexdigest())
|
||||
self.assertEqual(cache.inspect_cache("CarParamsPersistent", self.repack(header, payload)).status, "incompatible")
|
||||
|
||||
def test_total_envelope_limit_includes_metadata(self):
|
||||
raw = self.encoded()
|
||||
params = Mock()
|
||||
with patch.object(cache, "MAX_ENVELOPE_BYTES", len(raw) - 1):
|
||||
self.assertEqual(cache.inspect_cache("CarParamsPersistent", raw).status, "incompatible")
|
||||
with self.assertRaises(ValueError):
|
||||
cache.put_cache(params, "CarParamsPersistent", self.message("CarParamsPersistent"))
|
||||
params.put.assert_not_called()
|
||||
self.assertLess(cache.MAX_PAYLOAD_BYTES + cache.MAX_HEADER_BYTES + len(cache.MAGIC) + 4, cache.MAX_ENVELOPE_BYTES + 1)
|
||||
|
||||
def test_invalid_or_truncated_header_length_rejected(self):
|
||||
for size, body in ((0, b""), (cache.MAX_HEADER_BYTES + 1, b"x"), (200, b"x")):
|
||||
raw = cache.MAGIC + struct.pack(">I", size) + body
|
||||
self.assertEqual(cache.inspect_cache("CarParamsCache", raw).status, "incompatible")
|
||||
|
||||
def schemas(self, members):
|
||||
parsers, modules = [], []
|
||||
for index, member in enumerate(members):
|
||||
directory = self.root / f"schema-{index}"
|
||||
directory.mkdir()
|
||||
path = directory / "fixture.capnp"
|
||||
path.write_text('@0xabcdefabcdefabcd; struct Root @0xabcdefabcdefabce { child @0 :Choice; ' +
|
||||
'enum Choice @0xabcdefabcdefabcf { ' + member + ' @0; } }')
|
||||
parser = capnp.SchemaParser()
|
||||
parsers.append(parser)
|
||||
modules.append(parser.load(str(path)))
|
||||
return parsers, modules
|
||||
|
||||
def test_same_root_id_and_bytes_different_nested_enum_differ(self):
|
||||
parsers, modules = self.schemas(("firstMeaning", "otherMeaning"))
|
||||
a, b = (module.Root.schema for module in modules)
|
||||
self.assertEqual(a.node.id, b.node.id)
|
||||
self.assertEqual(a.node.as_builder().to_bytes(), b.node.as_builder().to_bytes())
|
||||
self.assertNotEqual(a, b)
|
||||
self.assertNotEqual(cache.schema_fingerprint(a), cache.schema_fingerprint(b))
|
||||
self.assertEqual(len(parsers), 2) # keep parser ownership alive through assertions
|
||||
|
||||
def test_fingerprint_stable_across_schema_locations(self):
|
||||
parsers, modules = self.schemas(("sameMeaning", "sameMeaning"))
|
||||
self.assertEqual(cache.schema_fingerprint(modules[0].Root.schema), cache.schema_fingerprint(modules[1].Root.schema))
|
||||
self.assertEqual(len(parsers), 2)
|
||||
|
||||
def test_independent_copy_of_current_schema_is_not_current_producer(self):
|
||||
source = Path(__file__).resolve().parents[3] / "opendbc_repo/opendbc/car"
|
||||
destination = self.root / "separate-checkout"
|
||||
destination.mkdir()
|
||||
shutil.copy(source / "car.capnp", destination / "car.capnp")
|
||||
shutil.copytree(source / "include", destination / "include")
|
||||
parser = capnp.SchemaParser()
|
||||
module = parser.load(str(destination / "car.capnp"), imports=[str(destination)])
|
||||
self.assertEqual(cache.schema_fingerprint(module.CarParams.schema), cache.schema_fingerprint(car.CarParams.schema))
|
||||
self.assertEqual(module.CarParams.schema.node.id, car.CarParams.schema.node.id)
|
||||
with self.assertRaises(ValueError):
|
||||
cache.put_cache(self.params, "CarParamsCache", module.CarParams.new_message())
|
||||
|
||||
def test_compiled_fingerprints_stable_across_fresh_processes(self):
|
||||
code = 'import json; from openpilot.starpilot.schema_cache import prewarm_cache_contracts; print(json.dumps(prewarm_cache_contracts(), sort_keys=True))'
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
outputs = [subprocess.check_output([sys.executable, "-c", code], cwd=root, text=True) for _ in range(2)]
|
||||
self.assertEqual(outputs[0], outputs[1])
|
||||
self.assertEqual(json.loads(outputs[0]), cache.prewarm_cache_contracts())
|
||||
|
||||
def test_prewarm_optional_keys_and_no_recomputation(self):
|
||||
result = cache.prewarm_cache_contracts(("LiveDelay",))
|
||||
self.assertEqual(set(result), {"LiveDelay"})
|
||||
with patch.object(cache, "schema_fingerprint", side_effect=AssertionError("unexpected cold computation")):
|
||||
self.encoded("LiveDelay")
|
||||
self.assertIsNotNone(cache.get_cache(self.params, "LiveDelay"))
|
||||
|
||||
def event_schema(self, *, extra_arm="", extra_field="", unrelated="Text", selected="Float64", enum_name="first",
|
||||
valid_default="true", swap_tags=False):
|
||||
directory = Path(tempfile.mkdtemp(dir=self.root))
|
||||
path = directory / "event.capnp"
|
||||
first, second = (2, 1) if swap_tags else (1, 2)
|
||||
path.write_text(f'''@0xabcdefabcdefabd0;
|
||||
struct Event @0xabcdefabcdefabd1 {{
|
||||
logMonoTime @0 :UInt64;
|
||||
union {{ selected @{first} :Payload; unrelated @{second} :{unrelated}; {extra_arm} }}
|
||||
valid @3 :Bool = {valid_default}; {extra_field}
|
||||
}}
|
||||
struct Payload @0xabcdefabcdefabd2 {{ value @0 :{selected}; nested @1 :Nested; }}
|
||||
struct Nested @0xabcdefabcdefabd3 {{ values @0 :List(Float64); kind @1 :Kind; }}
|
||||
enum Kind @0xabcdefabcdefabd4 {{ {enum_name} @0; }}
|
||||
''')
|
||||
parser = capnp.SchemaParser()
|
||||
module = parser.load(str(path))
|
||||
self.event_parsers.append(parser)
|
||||
return module.Event
|
||||
|
||||
def event_contract(self, root):
|
||||
return patch.dict(cache.CONTRACTS, CalibrationParams=cache.CacheContract(root, "selected", "calibration-state-v1"))
|
||||
|
||||
def test_service_fingerprint_ignores_unrelated_union_changes(self):
|
||||
original = self.event_schema()
|
||||
for changed in (self.event_schema(unrelated="Data"), self.event_schema(extra_arm="added @4 :Text;")):
|
||||
self.assertNotEqual(cache.schema_fingerprint(original.schema), cache.schema_fingerprint(changed.schema))
|
||||
self.assertEqual(cache.event_service_fingerprint(original.schema, "selected"),
|
||||
cache.event_service_fingerprint(changed.schema, "selected"))
|
||||
|
||||
def test_service_fingerprint_rejects_selected_or_wrapper_changes(self):
|
||||
original = self.event_schema()
|
||||
for options in ({"selected": "Int64"}, {"enum_name": "otherMeaning"}, {"valid_default": "false"},
|
||||
{"swap_tags": True}, {"extra_field": "extra @4 :UInt64;"}):
|
||||
with self.subTest(options=options):
|
||||
changed = self.event_schema(**options)
|
||||
self.assertNotEqual(cache.event_service_fingerprint(original.schema, "selected"),
|
||||
cache.event_service_fingerprint(changed.schema, "selected"))
|
||||
|
||||
def test_unrelated_upgrade_restores_nondefault_typed_event_values(self):
|
||||
original, changed = self.event_schema(), self.event_schema(extra_arm="added @4 :Text;")
|
||||
message = original.new_message(logMonoTime=12345, valid=False,
|
||||
selected={"value": 1.25, "nested": {"values": [2.5, -3.0], "kind": "first"}})
|
||||
with self.event_contract(original), patch.dict(cache._FINGERPRINTS, clear=True):
|
||||
cache.put_cache(self.params, "CalibrationParams", message, block=True)
|
||||
raw = self.params.get("CalibrationParams")
|
||||
header, _ = self.header_and_payload(raw)
|
||||
self.assertEqual((header["version"], header["fingerprint_format"]), (2, cache.EVENT_FINGERPRINT_FORMAT))
|
||||
with self.event_contract(changed), patch.dict(cache._FINGERPRINTS, clear=True):
|
||||
payload = cache.get_cache(self.params, "CalibrationParams")
|
||||
self.assertIsNotNone(payload)
|
||||
with changed.from_bytes(payload) as restored:
|
||||
self.assertEqual(restored.to_dict(), message.to_dict())
|
||||
self.assertEqual(self.params.get("CalibrationParams"), raw)
|
||||
|
||||
def test_legacy_event_requires_full_current_schema_match(self):
|
||||
original, changed = self.event_schema(), self.event_schema(extra_arm="added @4 :Text;")
|
||||
message = original.new_message(selected={"value": 1.25})
|
||||
payload = message.to_bytes()
|
||||
with self.event_contract(original), patch.dict(cache._FINGERPRINTS, clear=True):
|
||||
header = cache._header(cache.CONTRACTS["CalibrationParams"], "CalibrationParams", payload, version=1)
|
||||
raw = self.repack(header, payload)
|
||||
self.assertEqual(cache.inspect_cache("CalibrationParams", raw).status, "valid")
|
||||
with self.event_contract(changed), patch.dict(cache._FINGERPRINTS, clear=True):
|
||||
self.assertEqual(cache.inspect_cache("CalibrationParams", raw).status, "incompatible")
|
||||
|
||||
def test_current_event_legacy_values_remain_readable_without_rewrite(self):
|
||||
for key in (key for key, contract in cache.CONTRACTS.items() if contract.service is not None):
|
||||
with self.subTest(key=key):
|
||||
payload = self.message(key).to_bytes()
|
||||
header = cache._header(cache.CONTRACTS[key], key, payload, version=1)
|
||||
raw = self.repack(header, payload)
|
||||
self.params.put(key, raw, block=True)
|
||||
self.assertEqual(cache.get_cache(self.params, key), payload)
|
||||
self.assertEqual(self.params.get(key), raw)
|
||||
|
||||
def test_service_cache_version_and_producer_cannot_be_relabelled(self):
|
||||
header, payload = self.header_and_payload(self.encoded("LiveDelay"))
|
||||
for key, value in (("version", 1), ("version", 3), ("version", True), ("producer_contract", "other"),
|
||||
("fingerprint_format", cache.FINGERPRINT_FORMAT), ("service", "vehicleParameters")):
|
||||
changed = {**header, key: value}
|
||||
with self.subTest(key=key, value=value):
|
||||
self.assertEqual(cache.inspect_cache("LiveDelay", self.repack(changed, payload)).status, "incompatible")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,105 @@
|
||||
"""Check additive evolution of the custom messages already recorded by this fork.
|
||||
|
||||
The source-only schema policy protects upstream and historical reservations.
|
||||
This separate native check protects the fields of the new populated slots,
|
||||
including compiled offsets, types, defaults, union tags and enum meanings.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
|
||||
CONTRACT = Path(__file__).with_name('custom-schema-contract.json')
|
||||
|
||||
|
||||
def _children(schema, descriptor):
|
||||
kind = next(iter(descriptor))
|
||||
if kind == 'list':
|
||||
element = descriptor['list']['elementType']
|
||||
if next(iter(element)) in ('struct', 'enum', 'list'):
|
||||
yield from _children(schema.elementType, element)
|
||||
elif kind in ('struct', 'enum'):
|
||||
yield schema
|
||||
|
||||
|
||||
def describe(module, roots):
|
||||
"""Read the actual compiled layouts; source spelling alone is insufficient."""
|
||||
def resolve(name):
|
||||
value = module
|
||||
for part in name.split('.'):
|
||||
value = getattr(value, part)
|
||||
return value.schema
|
||||
|
||||
pending = [resolve(name) for name in roots]
|
||||
result = {'roots': {name: str(resolve(name).node.id) for name in roots}, 'nodes': {}}
|
||||
while pending:
|
||||
schema = pending.pop()
|
||||
node = schema.node.to_dict()
|
||||
key = str(node['id'])
|
||||
if key in result['nodes']:
|
||||
continue
|
||||
if 'enum' in node:
|
||||
result['nodes'][key] = {'kind': 'enum', 'values': [e['name'] for e in node['enum']['enumerants']]}
|
||||
continue
|
||||
layout = node['struct']
|
||||
fields = {}
|
||||
for field in layout.get('fields', []):
|
||||
record = {k: v for k, v in field.items() if k not in ('codeOrder', 'annotations')}
|
||||
ordinal = str(field['ordinal'].get('explicit', 'group:' + field['name']))
|
||||
if 'group' in field:
|
||||
pending.append(schema.fields[field['name']].schema)
|
||||
else:
|
||||
record['slot'] = {k: v for k, v in field['slot'].items() if k != 'hadExplicitDefault'}
|
||||
descriptor = field['slot']['type']
|
||||
if next(iter(descriptor)) in ('struct', 'list', 'anyPointer'):
|
||||
# schema.Value stores these defaults as opaque pointers. Preserve the
|
||||
# value itself, including explicit pointer defaults, without expanding
|
||||
# it through the referenced type's potentially evolving field layout.
|
||||
value = schema.fields[field['name']].proto.slot.defaultValue
|
||||
record['slot']['defaultValue'] = {'schemaValueHex': value.as_builder().to_bytes().hex()}
|
||||
if next(iter(descriptor)) in ('struct', 'enum', 'list'):
|
||||
pending.extend(_children(schema.fields[field['name']].schema, descriptor))
|
||||
fields[ordinal] = record
|
||||
result['nodes'][key] = {'kind': 'struct', 'data_words': layout['dataWordCount'],
|
||||
'pointers': layout['pointerCount'], 'is_group': layout['isGroup'],
|
||||
'union_count': layout['discriminantCount'], 'union_offset': layout['discriminantOffset'],
|
||||
'fields': fields}
|
||||
return result
|
||||
|
||||
|
||||
def validate(current, baseline):
|
||||
"""Allow new fields/types/enum values; retain existing wire and API meaning."""
|
||||
errors = []
|
||||
for name, type_id in baseline['roots'].items():
|
||||
if current['roots'].get(name) != type_id:
|
||||
errors.append(f'{name}: existing custom root renamed, removed or reassigned')
|
||||
for type_id, old in baseline['nodes'].items():
|
||||
new = current['nodes'].get(type_id)
|
||||
if new is None or new['kind'] != old['kind']:
|
||||
errors.append(f'{type_id}: existing referenced type removed or changed kind')
|
||||
continue
|
||||
if old['kind'] == 'enum':
|
||||
if new['values'][:len(old['values'])] != old['values']:
|
||||
errors.append(f'{type_id}: existing enum ordinals or names changed')
|
||||
continue
|
||||
if new['data_words'] < old['data_words'] or new['pointers'] < old['pointers'] or new['is_group'] != old['is_group']:
|
||||
errors.append(f'{type_id}: existing struct storage changed incompatibly')
|
||||
if old['union_count'] and (new['union_count'] < old['union_count'] or new['union_offset'] != old['union_offset']):
|
||||
errors.append(f'{type_id}: existing union layout changed')
|
||||
for ordinal, field in old['fields'].items():
|
||||
if new['fields'].get(ordinal) != field:
|
||||
errors.append(f'{type_id}.{field["name"]}: existing field @{ordinal} changed or removed')
|
||||
return errors
|
||||
|
||||
|
||||
def main():
|
||||
from openpilot.cereal import custom
|
||||
|
||||
baseline = json.loads(CONTRACT.read_text())
|
||||
errors = validate(describe(custom, baseline['roots']), baseline)
|
||||
print(json.dumps({'passed': not errors, 'errors': errors}, indent=2))
|
||||
return int(bool(errors))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,200 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"upstream_commit": "3f06ed3bc7a4cc9c320be1f7d03dcd8a885a8e6f",
|
||||
"opendbc_commit": "35f7e0813462607ef1d703e52313e7571e31405d",
|
||||
"unchanged_schemas": {
|
||||
"openpilot/cereal/deprecated.capnp": "b3aa5c13de40001ed7376cef858500fb003f7698f96131cd0db18d878fa1af0d",
|
||||
"openpilot/cereal/include/c++.capnp": "fb306076cd38c27af1aed20fac6395e9a46fbe5b5df6a248b5f1b6845a079c44",
|
||||
"opendbc_repo/opendbc/car/car.capnp": "098a19d69e8d233ac653503b53761a94824f31812757a11a7ded1589fe1e0788",
|
||||
"opendbc_repo/opendbc/car/include/c++.capnp": "fb306076cd38c27af1aed20fac6395e9a46fbe5b5df6a248b5f1b6845a079c44",
|
||||
"opendbc_repo/opendbc/car/rlog.capnp": "76331a09f36764d08228ba1ed6c03419cc8cdfcc797db98c4d56fab52a6ac050"
|
||||
},
|
||||
"log_sha256": "bf39b822e577afdd6caacabde77261ac5c0929767c2f0e4ff7d5e5429e3d4c14",
|
||||
"custom_header": "using Cxx = import \"/include/c++.capnp\";\n$Cxx.namespace(\"cereal\");\nusing Car = import \"/car.capnp\";\n\n@0xb526ba661d550a59;\n\n# custom.capnp: a home for empty structs reserved for custom forks\n# These structs are guaranteed to remain reserved and empty in mainline\n# cereal, so use these if you want custom events in your fork.\n\n# DO rename the structs\n# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
|
||||
"historical_empty_slots": [
|
||||
5
|
||||
],
|
||||
"historical_restored": {
|
||||
"10": {
|
||||
"type_id": "0xcb9fd56c7057593a",
|
||||
"ordinal": 136,
|
||||
"field": "desiredCurvature @0 :Float32;"
|
||||
},
|
||||
"11": {
|
||||
"type_id": "0xc2243c65e0340384",
|
||||
"ordinal": 137,
|
||||
"field": "active @0 :Bool;"
|
||||
}
|
||||
},
|
||||
"reserved": [
|
||||
{
|
||||
"struct": "CustomReserved0",
|
||||
"type_id": "0x81c2f05a394cf4af",
|
||||
"field": "customReserved0",
|
||||
"ordinal": 107
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved1",
|
||||
"type_id": "0xaedffd8f31e7b55d",
|
||||
"field": "customReserved1",
|
||||
"ordinal": 108
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved2",
|
||||
"type_id": "0xf35cc4560bbf6ec2",
|
||||
"field": "customReserved2",
|
||||
"ordinal": 109
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved3",
|
||||
"type_id": "0xda96579883444c35",
|
||||
"field": "customReserved3",
|
||||
"ordinal": 110
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved4",
|
||||
"type_id": "0x80ae746ee2596b11",
|
||||
"field": "customReserved4",
|
||||
"ordinal": 111
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved5",
|
||||
"type_id": "0xa5cd762cd951a455",
|
||||
"field": "customReserved5",
|
||||
"ordinal": 112
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved6",
|
||||
"type_id": "0xf98d843bfd7004a3",
|
||||
"field": "customReserved6",
|
||||
"ordinal": 113
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved7",
|
||||
"type_id": "0xb86e6369214c01c8",
|
||||
"field": "customReserved7",
|
||||
"ordinal": 114
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved8",
|
||||
"type_id": "0xf416ec09499d9d19",
|
||||
"field": "customReserved8",
|
||||
"ordinal": 115
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved9",
|
||||
"type_id": "0xa1680744031fdb2d",
|
||||
"field": "customReserved9",
|
||||
"ordinal": 116
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved10",
|
||||
"type_id": "0xcb9fd56c7057593a",
|
||||
"field": "customReserved10",
|
||||
"ordinal": 136
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved11",
|
||||
"type_id": "0xc2243c65e0340384",
|
||||
"field": "customReserved11",
|
||||
"ordinal": 137
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved12",
|
||||
"type_id": "0x9ccdc8676701b412",
|
||||
"field": "customReserved12",
|
||||
"ordinal": 138
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved13",
|
||||
"type_id": "0xcd96dafb67a082d0",
|
||||
"field": "customReserved13",
|
||||
"ordinal": 139
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved14",
|
||||
"type_id": "0xb057204d7deadf3f",
|
||||
"field": "customReserved14",
|
||||
"ordinal": 140
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved15",
|
||||
"type_id": "0xbd443b539493bc68",
|
||||
"field": "customReserved15",
|
||||
"ordinal": 141
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved16",
|
||||
"type_id": "0xfc6241ed8877b611",
|
||||
"field": "customReserved16",
|
||||
"ordinal": 142
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved17",
|
||||
"type_id": "0xa30662f84033036c",
|
||||
"field": "customReserved17",
|
||||
"ordinal": 143
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved18",
|
||||
"type_id": "0xc86a3d38d13eb3ef",
|
||||
"field": "customReserved18",
|
||||
"ordinal": 144
|
||||
},
|
||||
{
|
||||
"struct": "CustomReserved19",
|
||||
"type_id": "0xa4f1eb3323f5f582",
|
||||
"field": "customReserved19",
|
||||
"ordinal": 145
|
||||
}
|
||||
],
|
||||
"raw_reserved": {
|
||||
"124": "customReservedRawData0",
|
||||
"125": "customReservedRawData1",
|
||||
"126": "customReservedRawData2"
|
||||
},
|
||||
"historical_prefixes": {
|
||||
"0": {
|
||||
"type_id": "0x81c2f05a394cf4af",
|
||||
"name": "StarPilotCarControl",
|
||||
"body": "\n hudControl @0 :HUDControl;\n steeringLimitInfo @1 :SteeringLimitInfo;\n\n struct HUDControl {\n audibleAlert @0 :AudibleAlert;\n\n enum AudibleAlert {\n none @0;\n\n engage @1;\n disengage @2;\n refuse @3;\n\n warningSoft @4;\n warningImmediate @5;\n\n prompt @6;\n promptRepeat @7;\n promptDistracted @8;\n\n # Random Events\n angry @9;\n continued @10;\n dejaVu @11;\n doc @12;\n fart @13;\n firefox @14;\n goat @15;\n hal9000 @16;\n mail @17;\n nessie @18;\n noice @19;\n startup @20;\n thisIsFine @21;\n uwu @22;\n }\n }\n\n struct SteeringLimitInfo {\n valid @0 :Bool;\n modelLimitErrorDeg @1 :Float32;\n resumeLimitErrorDeg @2 :Float32;\n cooperativeLimitErrorDeg @3 :Float32;\n cooperativeOffsetDeg @4 :Float32;\n monoTime @5 :UInt64;\n combinedLimitErrorDeg @6 :Float32;\n }\n",
|
||||
"source_commit": "c2109496a72e85ca702ecef7aff53ff45adcfb8c",
|
||||
"source_path": "cereal/custom.capnp"
|
||||
},
|
||||
"3": {
|
||||
"type_id": "0xda96579883444c35",
|
||||
"name": "StarPilotLongitudinalPlan",
|
||||
"body": "\n version @0 :UInt16;\n sourcePlanMonoTime @1 :UInt64;\n modelMonoTime @2 :UInt64;\n driveStartMonoTime @3 :UInt64;\n forceStopHolding @4 :Bool;\n"
|
||||
},
|
||||
"2": {
|
||||
"type_id": "0xf35cc4560bbf6ec2",
|
||||
"name": "StarPilotCarState",
|
||||
"body": "\n accelPressed @0 :Bool;\n alwaysOnLateralEnabled @1 :Bool;\n brakeLights @2 :Bool;\n dashboardSpeedLimit @3 :Float32;\n decelPressed @4 :Bool;\n distancePressed @5 :Bool;\n distanceLongPressed @6 :Bool;\n distanceVeryLongPressed @7 :Bool;\n ecoGear @8 :Bool;\n forceCoast @9 :Bool;\n isParked @10 :Bool;\n pauseLateral @11 :Bool;\n pauseLongitudinal @12 :Bool;\n sportGear @13 :Bool;\n trafficModeEnabled @14 :Bool;\n gasStack @15 :Bool; # Compatibility with older StarPilot payloads\n modePressed @16 :Bool;\n customPressed @17 :Bool;\n alwaysOnLateralAllowed @18 :Bool;\n dashboardStopSign @19 :UInt8; # 0 = no signal / platform doesn't publish\n cancelPressed @20 :Bool;\n cancelLongPressed @21 :Bool;\n cancelVeryLongPressed @22 :Bool;\n pedalMaxRegen @23 :Bool; # pedal at max regen, driver should use brake for more decel\n pedalLongActive @24 :Bool; # Pre-AP pedal longitudinal mode is active (enableLongControl)\n teslaCCEngaged @25 :Bool; # rising edge of stock Tesla CC engaging (no-pedal mode)\n teslaCCDisengaged @26 :Bool; # falling edge of stock Tesla CC\n teslaCCNotArmed @27 :Bool; # lateral engaged but DI_cruiseState != STANDBY/ENABLED\n accelHardCruise @28 :Bool; # current/releasing accel cruise button came from GM hard-press signal\n decelHardCruise @29 :Bool; # current/releasing decel cruise button came from GM hard-press signal\n pulseAndGlide @30 :Bool; # developer-only wheel-button pulse-and-glide mode is enabled",
|
||||
"source_commit": "6de77049b3c3dc77c7275e9ff724a96f9e72a19d",
|
||||
"source_path": "cereal/custom.capnp"
|
||||
},
|
||||
"6": {
|
||||
"type_id": "0xf98d843bfd7004a3",
|
||||
"name": "CustomReserved6",
|
||||
"body": "\naccelerationJerk @0 :Float32; cscControllingSpeed @1 :Bool; cscSpeed @2 :Float32;\ncscTraining @3 :Bool; dangerFactor @4 :Float32; dangerJerk @5 :Float32;\ndesiredFollowDistance @6 :Int64; experimentalMode @7 :Bool; forcingStop @8 :Bool;\nforcingStopLength @9 :Float32; starpilotEvents @10 :AnyPointer; starpilotToggles @11 :Text;\nincreasedStoppedDistance @12 :Float32; lateralCheck @13 :Bool; laneWidthLeft @14 :Float32;\nlaneWidthRight @15 :Float32; maxAcceleration @16 :Float32; minAcceleration @17 :Float32;\nredLight @18 :Bool; roadCurvature @19 :Float32;\nslcMapSpeedLimit @20 :Float32;\nslcMapboxSpeedLimit @21 :Float32; slcNextSpeedLimit @22 :Float32; slcOverriddenSpeed @23 :Float32;\nslcSpeedLimit @24 :Float32; slcSpeedLimitOffset @25 :Float32; slcSpeedLimitSource @26 :Text;\n",
|
||||
"source_commit": "77b44257a0a2a6e574d8e375ab61004be21b7a6e",
|
||||
"source_path": "openpilot/cereal/custom.capnp"
|
||||
},
|
||||
"1": {
|
||||
"type_id": "0xaedffd8f31e7b55d",
|
||||
"name": "StarPilotCarParams",
|
||||
"body": "\n alternativeExperience @0 :Int16;\n canUsePedal @1 :Bool;\n canUseSDSU @2 :Bool;\n flags @3 :UInt32;\n isHDA2 @4 :Bool;\n openpilotLongitudinalControlDisabled @5 :Bool;\n safetyConfigs @6 :List(SafetyConfig);\n pcmCruiseSpeed @7 :Bool = true;\n redneckCruiseAvailable @8 :Bool;\n\n struct SafetyConfig {\n safetyParam @0 :UInt16;\n }\n\n learning @9 :AutoSteeringLearning;\n\n struct AutoSteeringLearning {\n version @0 :UInt16;\n status @1 :Status;\n reason @2 :Text;\n identity @3 :Identity;\n session @4 :Text;\n sequence @5 :UInt64;\n observedMonoTime @6 :UInt64; # MONOTONIC, separate from Event BOOTTIME.\n validUntilMonoTime @7 :UInt64;\n sourceWindowStartMonoTime @8 :UInt64;\n sourceWindowEndMonoTime @9 :UInt64;\n family @10 :Family;\n latAccelFactor @11 :Float64;\n acceptancePolicyId @12 :Text;\n trainWindows @13 :UInt32;\n heldoutWindows @14 :UInt32;\n trainBaselineLoss @15 :Float64;\n trainCandidateLoss @16 :Float64;\n heldoutBaselineLoss @17 :Float64;\n heldoutCandidateLoss @18 :Float64;\n\n enum Status { unavailable @0; collecting @1; candidate @2; rejected @3; qualifiedFit @4; }\n enum Family { none @0; linearFactor @1; }\n struct Identity {\n vehicle @0 :Text;\n brand @1 :Text;\n controller @2 :Text;\n policy @3 :Text;\n profile @4 :Text;\n conversionVersion @5 :Text;\n firmwareSha256 @6 :Text;\n basis @7 :List(Float64);\n baselineSha256 @8 :Text;\n preferenceSha256 @9 :Text;\n resetGeneration @10 :Text;\n controllerRevision @11 :Text;\n }\n }\n",
|
||||
"historical_source_commit": "61fd19556f985ef462bda9c103396108ea69ad14",
|
||||
"historical_source_path": "cereal/custom.capnp",
|
||||
"historical_body_sha256": "d95c3b15a6e51608a8157bc3adc965d3914d4ae94364f9ea605c757cc7924ec8",
|
||||
"learning_source_commit": "c204f0b165db7769bd795768e4e25019748ee12c",
|
||||
"learning_source_path": "openpilot/cereal/custom.capnp"
|
||||
}
|
||||
},
|
||||
"reviewed_enum_additions": {
|
||||
"opendbc_repo/opendbc/car/car.capnp": {
|
||||
"enum": "SafetyModel",
|
||||
"field": "teslaPreap @39;"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Enforce cereal's reserved-event contract against the pinned upstream schemas.
|
||||
|
||||
See openpilot/cereal/README.md, Custom forks. When syncing upstream, review and
|
||||
update schema-policy.json alongside upstream-sync.json. Custom fields belong
|
||||
inside reserved custom.capnp structs; log.capnp may rename only their event
|
||||
aliases and the reserved raw-data aliases, preserving ordinals and type IDs.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
POLICY = Path(__file__).with_name('schema-policy.json')
|
||||
DECLARATION = re.compile(r'^struct (\w+) @(0x[0-9a-fA-F]+)\s*\{', re.MULTILINE)
|
||||
SCHEMA_ROOTS = ('openpilot/cereal', 'opendbc_repo/opendbc/car')
|
||||
|
||||
|
||||
def sha256(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def custom_structs(source):
|
||||
# Mask comments and strings before counting braces; quoted defaults may
|
||||
# contain braces without ending a message definition.
|
||||
masked = re.sub(r'"(?:\\.|[^"\\])*"|\#[^\n]*', lambda m: ' ' * len(m[0]), source)
|
||||
definitions, bodies, spans = {}, {}, []
|
||||
cursor = 0
|
||||
for match in DECLARATION.finditer(masked):
|
||||
if match.start() < cursor:
|
||||
continue
|
||||
depth, end = 1, match.end()
|
||||
while depth and end < len(masked):
|
||||
depth += (masked[end] == '{') - (masked[end] == '}')
|
||||
end += 1
|
||||
if depth:
|
||||
raise ValueError('Unclosed custom struct')
|
||||
name, type_id = match.groups()
|
||||
if type_id.lower() in definitions or name in definitions.values():
|
||||
raise ValueError('Duplicate custom type ID or name')
|
||||
definitions[type_id.lower()] = name
|
||||
bodies[type_id.lower()] = source[match.end():end - 1]
|
||||
spans.append((match.start(), end))
|
||||
cursor = end
|
||||
remainder = source
|
||||
for start, end in reversed(spans):
|
||||
remainder = remainder[:start] + remainder[end:]
|
||||
return definitions, bodies, remainder
|
||||
|
||||
|
||||
def validate(schemas, policy, sync):
|
||||
errors = []
|
||||
expected_paths = {*policy['unchanged_schemas'], 'openpilot/cereal/custom.capnp', 'openpilot/cereal/log.capnp'}
|
||||
for path in sorted(set(schemas) - expected_paths):
|
||||
errors.append(f'{path}: schema has no explicit policy coverage')
|
||||
for path in sorted(expected_paths - set(schemas)):
|
||||
errors.append(f'{path}: policy-covered schema is missing')
|
||||
if expected_paths - set(schemas):
|
||||
return errors
|
||||
dependencies = {entry['path']: entry['commit'] for entry in sync['dependencies']}
|
||||
if sync['upstream']['commit'] != policy['upstream_commit'] or dependencies.get('opendbc_repo') != policy['opendbc_commit']:
|
||||
errors.append('Schema policy must be reviewed against the current upstream sync pins')
|
||||
for path, expected in policy['unchanged_schemas'].items():
|
||||
reviewed = policy.get('reviewed_enum_additions', {}).get(path)
|
||||
candidate = schemas[path]
|
||||
if reviewed:
|
||||
source = candidate.decode()
|
||||
pattern = rf'(enum {re.escape(reviewed["enum"])}\s*\{{)([^}}]*)(\}})'
|
||||
match = re.search(pattern, source)
|
||||
field = reviewed['field']
|
||||
if match is None or len(re.findall(rf'(?m)^\s*{re.escape(field)}\s*$', match[2])) != 1:
|
||||
errors.append(f'{path}: reviewed enum addition is missing or changed')
|
||||
else:
|
||||
body = re.sub(rf'(?m)^[ \t]*{re.escape(field)}[ \t]*\n', '', match[2])
|
||||
source = source[:match.start(2)] + body + source[match.end(2):]
|
||||
candidate = source.encode()
|
||||
if sha256(candidate) != expected:
|
||||
errors.append(f'{path}: upstream-owned schema changed; use custom.capnp reserved structs')
|
||||
try:
|
||||
definitions, bodies, remainder = custom_structs(schemas['openpilot/cereal/custom.capnp'].decode())
|
||||
except (UnicodeDecodeError, ValueError) as error:
|
||||
return [*errors, f'custom.capnp: {error}']
|
||||
expected_ids = {slot['type_id'] for slot in policy['reserved']}
|
||||
if set(definitions) != expected_ids:
|
||||
errors.append('custom.capnp: reserved type IDs were removed/changed or unreserved top-level types were added')
|
||||
reserved_by_name = {slot['struct']: slot for slot in policy['reserved']}
|
||||
for index in policy['historical_empty_slots']:
|
||||
original_name = f'CustomReserved{index}'
|
||||
type_id = reserved_by_name[original_name]['type_id']
|
||||
body = re.sub(r'#[^\n]*', '', bodies.get(type_id, ''))
|
||||
if definitions.get(type_id) != original_name or body.strip():
|
||||
errors.append(f'custom.capnp: historical slot {index} must retain its original name and empty body')
|
||||
for index, restored in policy.get('historical_restored', {}).items():
|
||||
type_id = restored['type_id']
|
||||
body = re.sub(r'#[^\n]*', '', bodies.get(type_id, ''))
|
||||
field = restored['field']
|
||||
if not re.search(rf'(?m)^\s*{re.escape(field)}\s*$', body):
|
||||
errors.append(f'custom.capnp: historical slot {index} must preserve {field}')
|
||||
slot = next((entry for entry in policy['reserved'] if entry['type_id'] == type_id), None)
|
||||
if slot is None or slot['ordinal'] != restored['ordinal']:
|
||||
errors.append(f'custom.capnp: historical slot {index} type ID or event ordinal changed')
|
||||
def canonical(text):
|
||||
return re.sub(r'\s+', '', re.sub(r'#[^\n]*', '', text))
|
||||
|
||||
for restored in policy.get('historical_prefixes', {}).values():
|
||||
if (definitions.get(restored['type_id']) != restored['name'] or
|
||||
not canonical(bodies.get(restored['type_id'], '')).startswith(canonical(restored['body']))):
|
||||
errors.append('custom.capnp: historical occupied prefix changed')
|
||||
if remainder.strip() != policy['custom_header'].strip():
|
||||
errors.append('custom.capnp: edits outside the reserved structs')
|
||||
log = schemas['openpilot/cereal/log.capnp'].decode()
|
||||
for slot in policy['reserved']:
|
||||
name = definitions.get(slot['type_id'])
|
||||
if name is None:
|
||||
continue
|
||||
# Changing the name is permitted; changing its type or ordinal is not.
|
||||
pattern = rf'(?m)^(\s*)\w+ @{slot["ordinal"]} :Custom\.{re.escape(name)};'
|
||||
log, count = re.subn(pattern, rf'\g<1>{slot["field"]} @{slot["ordinal"]} :Custom.{slot["struct"]};', log)
|
||||
if count != 1:
|
||||
errors.append(f'log.capnp: reserved event @{slot["ordinal"]} no longer points to its original custom type ID')
|
||||
for ordinal, name in policy['raw_reserved'].items():
|
||||
log, count = re.subn(rf'(?m)^(\s*)\w+ @{ordinal} :Data;', rf'\g<1>{name} @{ordinal} :Data;', log)
|
||||
if count != 1:
|
||||
errors.append(f'log.capnp: reserved raw event @{ordinal} changed type or ordinal')
|
||||
if policy.get('reviewed_log_additions') or policy.get('reviewed_log_ordinal_remaps'):
|
||||
errors.append('log.capnp: core schema exceptions are not permitted; use custom.capnp reserved structs')
|
||||
if sha256(log.encode()) != policy['log_sha256']:
|
||||
errors.append('log.capnp: changes outside the permitted reserved-event aliases')
|
||||
return errors
|
||||
|
||||
|
||||
def read_inputs(root=ROOT):
|
||||
policy = json.loads(POLICY.read_text())
|
||||
paths = {path.relative_to(root).as_posix() for schema_root in SCHEMA_ROOTS for path in (root / schema_root).rglob('*.capnp')}
|
||||
return {path: (root / path).read_bytes() for path in paths}, policy, json.loads((root / 'upstream-sync.json').read_text())
|
||||
|
||||
|
||||
def main():
|
||||
errors = validate(*read_inputs())
|
||||
print(json.dumps({'passed': not errors, 'errors': errors}, indent=2))
|
||||
return int(bool(errors))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Compatibility regressions the reserved-event policy must catch in CI."""
|
||||
|
||||
import copy
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
|
||||
from tools.ci.schema_policy import read_inputs, validate
|
||||
|
||||
|
||||
class SchemaPolicyTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.schemas, self.policy, self.sync = read_inputs()
|
||||
|
||||
def test_current_fork_uses_only_reserved_schema_slots(self):
|
||||
self.assertEqual(validate(self.schemas, self.policy, self.sync), [])
|
||||
|
||||
def test_adding_a_car_state_field_is_rejected(self):
|
||||
name = 'opendbc_repo/opendbc/car/car.capnp'
|
||||
self.schemas[name] = self.schemas[name].replace(b'struct CarState {', b'struct CarState {\n forkValue @62 :Float32;')
|
||||
self.assertTrue(any('car.capnp' in error for error in validate(self.schemas, self.policy, self.sync)))
|
||||
|
||||
def test_other_opendbc_schemas_are_pinned(self):
|
||||
for name in ('opendbc_repo/opendbc/car/rlog.capnp', 'opendbc_repo/opendbc/car/include/c++.capnp'):
|
||||
with self.subTest(name=name):
|
||||
schemas = self.schemas.copy()
|
||||
schemas[name] += b'\n# unexpected upstream schema edit\n'
|
||||
self.assertTrue(any(name in error for error in validate(schemas, self.policy, self.sync)))
|
||||
|
||||
def test_new_schema_under_either_owned_root_requires_policy(self):
|
||||
for name in ('openpilot/cereal/new.capnp', 'opendbc_repo/opendbc/car/new.capnp'):
|
||||
with self.subTest(name=name):
|
||||
schemas = self.schemas.copy()
|
||||
schemas[name] = b'@0xdeadbeefdeadbeef;\n'
|
||||
self.assertTrue(any(name in error for error in validate(schemas, self.policy, self.sync)))
|
||||
|
||||
def test_input_discovery_includes_new_schema(self):
|
||||
with TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name, data in self.schemas.items():
|
||||
path = root / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
(root / 'upstream-sync.json').write_text('{"dependencies": []}')
|
||||
new_schema = root / 'opendbc_repo/opendbc/car/new.capnp'
|
||||
new_schema.write_bytes(b'@0xdeadbeefdeadbeef;\n')
|
||||
discovered, _, _ = read_inputs(root)
|
||||
self.assertIn('opendbc_repo/opendbc/car/new.capnp', discovered)
|
||||
|
||||
def test_reserved_custom_extension_and_alias_are_allowed(self):
|
||||
# This source-policy test permits extension/aliasing; the separate compiled
|
||||
# contract protects the existing message's API and field layout.
|
||||
original = b'struct AolAxisState @0xfc6241ed8877b611 {'
|
||||
self.assertIn(original, self.schemas['openpilot/cereal/custom.capnp'])
|
||||
self.assertIn(b'aolAxisState @142 :Custom.AolAxisState;', self.schemas['openpilot/cereal/log.capnp'])
|
||||
self.schemas['openpilot/cereal/custom.capnp'] = self.schemas['openpilot/cereal/custom.capnp'].replace(
|
||||
original, b'struct FixtureState @0xfc6241ed8877b611 {\n note @12 :Text = "{fixture}";')
|
||||
self.schemas['openpilot/cereal/log.capnp'] = self.schemas['openpilot/cereal/log.capnp'].replace(
|
||||
b'aolAxisState @142 :Custom.AolAxisState;', b'fixtureState @142 :Custom.FixtureState;')
|
||||
self.assertEqual(validate(self.schemas, self.policy, self.sync), [])
|
||||
|
||||
def test_restored_map_event_cannot_be_retargeted(self):
|
||||
self.assertFalse({17, 18, 19}.intersection(self.policy['historical_empty_slots']))
|
||||
schemas = self.schemas.copy()
|
||||
schemas['openpilot/cereal/log.capnp'] = schemas['openpilot/cereal/log.capnp'].replace(
|
||||
b'mapdExtendedOut @143 :Custom.MapdExtendedOut;', b'mapdExtendedOut @143 :Custom.MapdIn;')
|
||||
self.assertTrue(any('event @143' in error for error in validate(schemas, self.policy, self.sync)))
|
||||
|
||||
def test_aol_raw_slots_remain_data_with_original_ordinals(self):
|
||||
for old, new in ((b'aolSafetyWire @124 :Data;', b'aolSafetyWire @124 :Custom.AolAxisState;'),
|
||||
(b'aolIntentWire @125 :Data;', b'aolIntentWire @126 :Data;')):
|
||||
with self.subTest(old=old):
|
||||
schemas = self.schemas.copy()
|
||||
self.assertIn(old, schemas['openpilot/cereal/log.capnp'])
|
||||
schemas['openpilot/cereal/log.capnp'] = schemas['openpilot/cereal/log.capnp'].replace(old, new)
|
||||
self.assertTrue(validate(schemas, self.policy, self.sync))
|
||||
|
||||
def test_historical_slot_rename_and_field_are_rejected(self):
|
||||
path = 'openpilot/cereal/custom.capnp'
|
||||
reserved_by_name = {slot['struct']: slot for slot in self.policy['reserved']}
|
||||
for index in self.policy['historical_empty_slots']:
|
||||
name = f'CustomReserved{index}'
|
||||
type_id = reserved_by_name[name]['type_id']
|
||||
original = f'struct {name} @{type_id} {{'.encode()
|
||||
self.assertIn(original, self.schemas[path])
|
||||
for replacement in (f'struct DifferentEvent @{type_id} {{'.encode(), original + b'\n newValue @0 :UInt32;'):
|
||||
with self.subTest(index=index, replacement=replacement):
|
||||
schemas = self.schemas.copy()
|
||||
schemas[path] = schemas[path].replace(original, replacement)
|
||||
self.assertTrue(any(f'historical slot {index}' in error for error in validate(schemas, self.policy, self.sync)))
|
||||
|
||||
def test_upstream_plan_and_event_enums_cannot_be_extended_or_remapped(self):
|
||||
path = 'openpilot/cereal/log.capnp'
|
||||
self.assertIn(b'accelBoost @40 :Float32;', self.schemas[path])
|
||||
self.assertNotIn(b'forceStopHolding', self.schemas[path])
|
||||
self.assertNotIn(b'teslaCCEngaged', self.schemas[path])
|
||||
mutations = (
|
||||
(b'accelBoost @40 :Float32;', b'accelBoost @41 :Float32;'),
|
||||
(b'accelBoost @40 :Float32;', b'accelBoost @40 :Float32;\n forceStopHolding @41 :Bool;'),
|
||||
(b'enum EventName @0x91f1992a1f77fb03 {', b'enum EventName @0x91f1992a1f77fb03 {\n teslaCCEngaged @105;'),
|
||||
)
|
||||
for original, replacement in mutations:
|
||||
with self.subTest(replacement=replacement):
|
||||
schemas = self.schemas.copy()
|
||||
self.assertIn(original, schemas[path])
|
||||
schemas[path] = schemas[path].replace(original, replacement)
|
||||
self.assertTrue(validate(schemas, self.policy, self.sync))
|
||||
policy = copy.deepcopy(self.policy)
|
||||
policy['reviewed_log_additions'] = [{'scope': 'struct LongitudinalPlan', 'fields': ['forceStopHolding @41 :Bool;']}]
|
||||
self.assertTrue(validate(schemas, policy, self.sync))
|
||||
|
||||
def test_historical_slot_comments_do_not_count_as_fields(self):
|
||||
path = 'openpilot/cereal/custom.capnp'
|
||||
schemas = self.schemas.copy()
|
||||
original = b'struct CustomReserved5 @0xa5cd762cd951a455 {'
|
||||
self.assertIn(original, schemas[path])
|
||||
schemas[path] = schemas[path].replace(
|
||||
original, original + b'\n # Past wire use is reserved.\n')
|
||||
self.assertEqual(validate(schemas, self.policy, self.sync), [])
|
||||
|
||||
def test_automatic_learning_keeps_reserved_identity_and_allows_append(self):
|
||||
self.assertNotIn(1, self.policy['historical_empty_slots'])
|
||||
slot = self.policy['reserved'][1]
|
||||
self.assertEqual(slot, {'struct': 'CustomReserved1', 'type_id': '0xaedffd8f31e7b55d',
|
||||
'field': 'customReserved1', 'ordinal': 108})
|
||||
prefix = self.policy['historical_prefixes']['1']
|
||||
self.assertEqual((prefix['name'], prefix['type_id']), ('StarPilotCarParams', slot['type_id']))
|
||||
self.assertIn(b'struct StarPilotCarParams @0xaedffd8f31e7b55d {', self.schemas['openpilot/cereal/custom.capnp'])
|
||||
self.assertIn(b'autoSteeringLearning @108 :Custom.StarPilotCarParams;', self.schemas['openpilot/cereal/log.capnp'])
|
||||
self.assertIn(b'learning @9 :AutoSteeringLearning;', self.schemas['openpilot/cereal/custom.capnp'])
|
||||
self.assertIn(b'pcmCruiseSpeed @7 :Bool = true;', self.schemas['openpilot/cereal/custom.capnp'])
|
||||
schemas = self.schemas.copy()
|
||||
original = b' controllerRevision @11 :Text;\n }\n }\n}'
|
||||
self.assertIn(original, schemas['openpilot/cereal/custom.capnp'])
|
||||
schemas['openpilot/cereal/custom.capnp'] = schemas['openpilot/cereal/custom.capnp'].replace(
|
||||
original, b' controllerRevision @11 :Text;\n }\n }\n futureStatus @10 :UInt32;\n}')
|
||||
self.assertEqual(validate(schemas, self.policy, self.sync), [])
|
||||
|
||||
def test_automatic_learning_payload_and_event_cannot_be_reinterpreted(self):
|
||||
path = 'openpilot/cereal/custom.capnp'
|
||||
for before, after in (
|
||||
(b'struct StarPilotCarParams @0xaedffd8f31e7b55d {', b'struct UnrelatedFit @0xaedffd8f31e7b55d {'),
|
||||
(b'0xaedffd8f31e7b55d', b'0xaedffd8f31e7b55e'),
|
||||
(b'alternativeExperience @0 :Int16;', b'alternativeExperience @0 :UInt16;'),
|
||||
(b'canUsePedal @1 :Bool;', b'canUsePedal @2 :Bool;'),
|
||||
(b'flags @3 :UInt32;', b'flags @3 :Bool;'),
|
||||
(b'pcmCruiseSpeed @7 :Bool = true;', b'pcmCruiseSpeed @7 :Bool = false;'),
|
||||
(b'redneckCruiseAvailable @8 :Bool;', b''),
|
||||
(b'safetyParam @0 :UInt16;', b'safetyParam @0 :UInt32;'),
|
||||
(b'learning @9 :AutoSteeringLearning;', b'learning @10 :AutoSteeringLearning;'),
|
||||
(b'version @0 :UInt16;', b'version @0 :UInt32;'),
|
||||
(b'latAccelFactor @11 :Float64;', b'latAccelFactor @11 :Float32;'),
|
||||
(b'latAccelFactor @11 :Float64;', b'unrelatedFactor @11 :Float64;'),
|
||||
(b'heldoutCandidateLoss @18 :Float64;', b'heldoutCandidateLoss @19 :Float64;'),
|
||||
(b'acceptancePolicyId @12 :Text;', b''),
|
||||
(b'qualifiedFit @4;', b'qualifiedFit @5;'),
|
||||
(b'linearFactor @1;', b'linearFactor @2;'),
|
||||
(b'vehicle @0 :Text;', b'vehicle @0 :UInt64;'),
|
||||
(b'controllerRevision @11 :Text;', b'controllerRevision @11 :Text = "changed";'),
|
||||
):
|
||||
with self.subTest(after=after):
|
||||
schemas = self.schemas.copy()
|
||||
head, marker, tail = schemas[path].partition(b'struct StarPilotCarParams @0xaedffd8f31e7b55d {')
|
||||
body, following, remainder = tail.partition(b'\nstruct StarPilotCarState')
|
||||
self.assertTrue(marker and following)
|
||||
occupied = marker + body
|
||||
self.assertIn(before, occupied)
|
||||
schemas[path] = head + occupied.replace(before, after) + following + remainder
|
||||
self.assertTrue(validate(schemas, self.policy, self.sync))
|
||||
path = 'openpilot/cereal/log.capnp'
|
||||
original = b'autoSteeringLearning @108 :Custom.StarPilotCarParams;'
|
||||
for replacement in (b'autoSteeringLearning @109 :Custom.StarPilotCarParams;',
|
||||
b'autoSteeringLearning @108 :Custom.CustomReserved5;'):
|
||||
with self.subTest(replacement=replacement):
|
||||
schemas = self.schemas.copy()
|
||||
self.assertIn(original, schemas[path])
|
||||
schemas[path] = schemas[path].replace(original, replacement)
|
||||
self.assertTrue(any('event @108' in error for error in validate(schemas, self.policy, self.sync)))
|
||||
|
||||
def test_restored_lateral_field_keeps_original_wire_meaning(self):
|
||||
self.assertNotIn(10, self.policy['historical_empty_slots'])
|
||||
path = 'openpilot/cereal/custom.capnp'
|
||||
original = b'desiredCurvature @0 :Float32;'
|
||||
self.assertIn(original, self.schemas[path])
|
||||
for changed in (b'desiredCurvature @0 :UInt32;', b'desiredCurvature @2 :Float32;'):
|
||||
with self.subTest(changed=changed):
|
||||
schemas = self.schemas.copy()
|
||||
schemas[path] = schemas[path].replace(original, changed)
|
||||
self.assertTrue(any('historical slot 10' in error for error in validate(schemas, self.policy, self.sync)))
|
||||
|
||||
def test_custom_id_reuse_event_retargeting_and_core_edits_are_rejected(self):
|
||||
cases = (
|
||||
('openpilot/cereal/custom.capnp', b'0xa4f1eb3323f5f582', b'0xc86a3d38d13eb3ef'),
|
||||
('openpilot/cereal/log.capnp', b'@145 :Custom.MapdOut', b'@145 :Custom.MapdIn'),
|
||||
('openpilot/cereal/log.capnp', b'@145 :Custom.MapdOut', b'@146 :Custom.MapdOut'),
|
||||
('openpilot/cereal/log.capnp', b'struct Event {', b'struct Event {\n forkValue @200 :UInt32;'),
|
||||
)
|
||||
for path, before, after in cases:
|
||||
with self.subTest(after=after):
|
||||
source = copy.copy(self.schemas)
|
||||
self.assertIn(before, source[path])
|
||||
source[path] = source[path].replace(before, after)
|
||||
self.assertTrue(validate(source, self.policy, self.sync))
|
||||
|
||||
def test_new_upstream_pin_requires_schema_contract_review(self):
|
||||
self.sync['upstream']['commit'] = '0' * 40
|
||||
self.assertTrue(validate(self.schemas, self.policy, self.sync))
|
||||
|
||||
def test_reviewed_native_enum_is_only_permitted_core_extension(self):
|
||||
path = 'opendbc_repo/opendbc/car/car.capnp'
|
||||
self.assertEqual(self.policy['reviewed_enum_additions'], {path: {'enum': 'SafetyModel', 'field': 'teslaPreap @39;'}})
|
||||
for before, after in ((b'teslaPreap @39;', b'teslaPreap @40;'),
|
||||
(b'teslaPreap @39;', b'teslaPreap @39;\n extraFork @40;'),
|
||||
(b'teslaPreap @39;', b'teslaPreap @39;\n extraBool @62 :Bool;')):
|
||||
changed = copy.copy(self.schemas)
|
||||
self.assertIn(before, changed[path])
|
||||
changed[path] = changed[path].replace(before, after)
|
||||
self.assertTrue(validate(changed, self.policy, self.sync))
|
||||
|
||||
def test_navigation_keeps_full_historical_control_prefix(self):
|
||||
path = 'openpilot/cereal/custom.capnp'
|
||||
for before, after in ((b'hudControl @0 :HUDControl;', b'hudControl @0 :Text;'),
|
||||
(b'cooperativeOffsetDeg @4 :Float32;', b'cooperativeOffsetDeg @4 :Bool;'),
|
||||
(b'uwu @22;', b'uwu @23;')):
|
||||
changed = copy.copy(self.schemas)
|
||||
self.assertIn(before, changed[path])
|
||||
changed[path] = changed[path].replace(before, after)
|
||||
self.assertTrue(validate(changed, self.policy, self.sync))
|
||||
|
||||
def test_longitudinal_companion_keeps_wire_prefix_and_allows_append(self):
|
||||
path = 'openpilot/cereal/custom.capnp'
|
||||
original = b'forceStopHolding @4 :Bool;'
|
||||
self.assertIn(original, self.schemas[path])
|
||||
appended = self.schemas.copy()
|
||||
appended[path] = appended[path].replace(original, original + b'\n futureStatus @5 :UInt32;')
|
||||
self.assertEqual(validate(appended, self.policy, self.sync), [])
|
||||
for replacement in (b'', b'forceStopHolding @5 :Bool;', b'forceStopHolding @4 :UInt32;',
|
||||
b'changedHold @4 :Bool;'):
|
||||
with self.subTest(replacement=replacement):
|
||||
changed = self.schemas.copy()
|
||||
changed[path] = changed[path].replace(original, replacement)
|
||||
self.assertTrue(any('occupied prefix changed' in error for error in validate(changed, self.policy, self.sync)))
|
||||
|
||||
def test_replay_slc_prefix_preserves_fields_and_allows_append(self):
|
||||
self.assertNotIn(6, self.policy['historical_empty_slots'])
|
||||
path = 'openpilot/cereal/custom.capnp'
|
||||
original = b'slcSpeedLimitSource @26 :Text;'
|
||||
self.assertIn(original, self.schemas[path])
|
||||
appended = self.schemas.copy()
|
||||
appended[path] = appended[path].replace(original, original + b'\n futureStatus @27 :UInt32;')
|
||||
self.assertEqual(validate(appended, self.policy, self.sync), [])
|
||||
for before, after in ((b'accelerationJerk @0 :Float32;', b'accelerationJerk @0 :UInt32;'),
|
||||
(b'starpilotEvents @10 :AnyPointer;', b'starpilotEvents @10 :Text;'),
|
||||
(b'slcSpeedLimit @24 :Float32;', b'slcSpeedLimit @25 :Float32;'),
|
||||
(original, b''),
|
||||
(b'struct CustomReserved6 @0xf98d843bfd7004a3 {',
|
||||
b'struct DifferentEvent @0xf98d843bfd7004a3 {')):
|
||||
with self.subTest(after=after):
|
||||
changed = self.schemas.copy()
|
||||
self.assertIn(before, changed[path])
|
||||
changed[path] = changed[path].replace(before, after)
|
||||
self.assertTrue(any('occupied prefix changed' in error for error in validate(changed, self.policy, self.sync)))
|
||||
|
||||
def test_replay_slc_event_keeps_original_type_id_and_ordinal(self):
|
||||
path = 'openpilot/cereal/log.capnp'
|
||||
original = b'customReserved6 @113 :Custom.CustomReserved6;'
|
||||
self.assertIn(original, self.schemas[path])
|
||||
for after in (b'customReserved6 @113 :Custom.CustomReserved5;',
|
||||
b'customReserved6 @114 :Custom.CustomReserved6;'):
|
||||
with self.subTest(after=after):
|
||||
changed = self.schemas.copy()
|
||||
changed[path] = changed[path].replace(original, after)
|
||||
self.assertTrue(validate(changed, self.policy, self.sync))
|
||||
@@ -3,20 +3,20 @@ import sys
|
||||
|
||||
from opendbc.car.structs import car
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.tools.lib.route import Route
|
||||
from openpilot.tools.lib.logreader import LogReader
|
||||
from openpilot.starpilot.schema_cache import put_cache
|
||||
|
||||
|
||||
def main(args: list[str]) -> None:
|
||||
if args:
|
||||
raise SystemExit("Route-derived CarParams require source schema provenance and explicit conversion before cache seeding")
|
||||
CP = car.CarParams.new_message()
|
||||
CP.openpilotLongitudinalControl = True
|
||||
CP.alphaLongitudinalAvailable = False
|
||||
params = Params()
|
||||
params.put("CarParams", CP.to_bytes(), block=True)
|
||||
for key in ("CarParamsCache", "CarParamsPersistent"):
|
||||
put_cache(params, key, CP, block=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
CP = None
|
||||
if len(sys.argv) > 1:
|
||||
r = Route(sys.argv[1])
|
||||
cps = [m for m in LogReader(r.qlog_paths()[0]) if m.which() == 'carParams']
|
||||
CP = cps[0].carParams.as_builder()
|
||||
else:
|
||||
CP = car.CarParams.new_message()
|
||||
CP.openpilotLongitudinalControl = True
|
||||
CP.alphaLongitudinalAvailable = False
|
||||
|
||||
cp_bytes = CP.to_bytes()
|
||||
for p in ("CarParams", "CarParamsCache", "CarParamsPersistent"):
|
||||
Params().put(p, cp_bytes, block=True)
|
||||
main(sys.argv[1:])
|
||||
|
||||
Reference in New Issue
Block a user