mawei/StarPilot
1
0
Fork 0
mirror of https://github.com/firestar5683/StarPilot.git synced 2026-10-28 00:53:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Preserve Cereal Fork Compatibility

Browse Source
This commit is contained in:
firestar5683
2026-10-19 00:47:34 -05:00
parent 6fba491e77
commit 73003793a0
7 changed files with 6144 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openpilot/cereal/custom.capnp
+9 -9
View File
@@ -1117,7 +1117,7 @@ struct MapdOut @0xa4f1eb3323f5f582 {
# Automatic FLM adds children only to existing reserved fork carriers.
# All evidence clocks are explicitly MONOTONIC; default/version0 is invalid.
struct LearningControlEvidence {
struct LearningControlEvidence @0xf2447170bb68f358 {
version @0 :UInt16;
identity @1 :Text;
generation @2 :UInt64;
@@ -1181,7 +1181,7 @@ struct LearningControlEvidence {
}
struct StageValue { name @0 :Text; value @1 :Float64; }
}
struct LearningAppliedEvidence {
struct LearningAppliedEvidence @0xbab0f51a74ffee6e {
version @0 :UInt16;
controlMonoTime @1 :UInt64;
sourceStateMonoTime @2 :UInt64;
@@ -1199,7 +1199,7 @@ struct LearningAppliedEvidence {
carrierFrame @13 :UInt64;
carrierObservedMonoTime @14 :UInt64;
}
struct AutomaticRuntimeProfile {
struct AutomaticRuntimeProfile @0xef38d302825641e3 {
version @0 :UInt16;
identity @1 :Text;
sequence @2 :UInt64;
@@ -1263,7 +1263,7 @@ struct AutomaticRuntimeProfile {
}
}
struct AutomaticLearningSourceReceipt {
struct AutomaticLearningSourceReceipt @0x9fe2d23e75805149 {
version @0 :UInt16;
enabled @1 :Bool;
manual @2 :Bool;
@@ -1277,7 +1277,7 @@ struct AutomaticLearningSourceReceipt {
persistentSourceEpoch @10 :Text;
}
struct LearningApplication {
struct LearningApplication @0x82b6c8ea6a6efabd {
version @0 :UInt16;
identity @1 :Text;
profileSequence @2 :UInt64;
@@ -1293,7 +1293,7 @@ struct LearningApplication {
}
struct FordLearningEvidence {
struct FordLearningEvidence @0xd1dfbadb6a811fee {
version @0 :UInt16;
identity @1 :Text;
generation @2 :UInt64;
@@ -1317,7 +1317,7 @@ struct FordLearningEvidence {
appliedProfileValidUntilMonoTime @20 :UInt64;
}
struct FordLearningEvent {
struct FordLearningEvent @0xb276dede9e6b2cc4 {
eventMonoTime @0 :UInt64;
sequence @1 :UInt64;
kind @2 :Text;
@@ -1351,7 +1351,7 @@ struct FordLearningEvent {
}
struct CentreApplicationWitness {
struct CentreApplicationWitness @0xab0b33cac682523e {
version @0 :UInt16;
identity @1 :Text;
sourceEpoch @2 :Text;
@@ -1366,7 +1366,7 @@ struct CentreApplicationWitness {
actualFriction @11 :Float64;
}
struct LearningTorqueProjection {
struct LearningTorqueProjection @0xf078f562cbce49ad {
version @0 :UInt16;
family @1 :Text;
fingerprint @2 :Text;
openpilot/starpilot/tests/test_custom_schema_evolution.py
+50
View File
@@ -27,6 +27,9 @@ class CustomSchemaEvolutionTest(unittest.TestCase):
'AolAxisState.LaneChangeStatusWire', 'StarPilotModelDataV2', 'StarPilotRadarState',
'StarPilotSelfdriveState', 'SpotMonitorState', 'StarPilotLateralState', 'StarPilotCarState',
'StarPilotCarParams',
'LearningControlEvidence', 'LearningAppliedEvidence', 'AutomaticRuntimeProfile',
'AutomaticLearningSourceReceipt', 'LearningApplication', 'FordLearningEvidence',
'FordLearningEvent', 'CentreApplicationWitness', 'LearningTorqueProjection',
})
self.assertEqual(self.baseline['roots']['MapdExtendedOut'], str(0xa30662f84033036c))
self.assertEqual(self.baseline['roots']['MapdIn'], str(0xc86a3d38d13eb3ef))
@@ -270,6 +273,53 @@ struct Sample @0xc889677671233ef3 {
self.assertTrue(any('.child:' in issue for issue in validate(contracts[1], contracts[0])))
self.assertTrue(any('.values:' in issue for issue in validate(contracts[2], contracts[0])))
def test_binary_data_defaults_are_serializable_lossless_and_protected(self):
source = '''@0xedeedaceefbcfbca;
struct DataDefaults @0xc889677671233ef4 {
payload @0 :Data = 0x"DATA_DEFAULT";
implicitPayload @1 :Data;
}
'''
with TemporaryDirectory() as directory:
path = Path(directory) / 'data-defaults.capnp'
contracts = []
parsers = []
for hex_value in ('0001feff', '0002feff', '0001feff'):
path.write_text(source.replace('DATA_DEFAULT', hex_value))
parser = capnp.SchemaParser()
parsers.append(parser)
module = parser.load(str(path))
contract = describe(module, ['DataDefaults'])
# Neither an implicit empty Data default nor an explicit binary default
# may leak raw bytes into the committed JSON contract.
contract = json.loads(json.dumps(contract))
fields = contract['nodes'][str(0xc889677671233ef4)]['fields']
for ordinal in ('0', '1'):
self.assertEqual(set(fields[ordinal]['slot']['defaultValue']), {'schemaValueHex'})
contracts.append(contract)
message = module.DataDefaults.new_message()
with module.DataDefaults.from_bytes(message.to_bytes()) as reader:
self.assertEqual(reader.payload, bytes.fromhex(hex_value))
self.assertEqual(reader.implicitPayload, b'')
self.assertEqual(validate(contracts[2], contracts[0]), [])
self.assertTrue(any('.payload:' in issue for issue in validate(contracts[1], contracts[0])))
def test_preexisting_payload_helper_root_ids_are_preserved(self):
expected = {
'LearningControlEvidence': 0xf2447170bb68f358,
'LearningAppliedEvidence': 0xbab0f51a74ffee6e,
'AutomaticRuntimeProfile': 0xef38d302825641e3,
'AutomaticLearningSourceReceipt': 0x9fe2d23e75805149,
'LearningApplication': 0x82b6c8ea6a6efabd,
'FordLearningEvidence': 0xd1dfbadb6a811fee,
'FordLearningEvent': 0xb276dede9e6b2cc4,
'CentreApplicationWitness': 0xab0b33cac682523e,
'LearningTorqueProjection': 0xf078f562cbce49ad,
}
for name, type_id in expected.items():
self.assertEqual(self.baseline['roots'][name], str(type_id))
self.assertEqual(getattr(custom, name).schema.node.id, type_id)
if __name__ == '__main__':
unittest.main()
tools/ci/custom-schema-contract.json
+5598 -1
View File
File diff suppressed because it is too large Load Diff
tools/ci/custom_schema_contract.py
+2 -2
View File
@@ -51,9 +51,9 @@ def describe(module, roots):
else:
record['slot'] = {k: v for k, v in field['slot'].items() if k != 'hadExplicitDefault'}
descriptor = field['slot']['type']
if next(iter(descriptor)) in ('struct', 'list', 'anyPointer'):
if next(iter(descriptor)) in ('struct', 'list', 'anyPointer', 'data'):
# schema.Value stores these defaults as opaque pointers. Preserve the
# value itself, including explicit pointer defaults, without expanding
# value itself, including explicit Data/pointer defaults, without expanding
# it through the referenced type's potentially evolving field layout.
value = schema.fields[field['name']].proto.slot.defaultValue
record['slot']['defaultValue'] = {'schemaValueHex': value.as_builder().to_bytes().hex()}
tools/ci/schema-policy.json
+117 -1
View File
@@ -10,7 +10,7 @@
"opendbc_repo/opendbc/car/rlog.capnp": "76331a09f36764d08228ba1ed6c03419cc8cdfcc797db98c4d56fab52a6ac050"
},
"log_sha256": "bf39b822e577afdd6caacabde77261ac5c0929767c2f0e4ff7d5e5429e3d4c14",
"custom_header": "using Cxx = import \"/include/c++.capnp\";\n$Cxx.namespace(\"cereal\");\nusing Car = import \"/car.capnp\";\n\n@0xb526ba661d550a59;\n\n# custom.capnp: a home for empty structs reserved for custom forks\n# These structs are guaranteed to remain reserved and empty in mainline\n# cereal, so use these if you want custom events in your fork.\n\n# DO rename the structs\n# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"custom_header": "using Cxx = import \"/include/c++.capnp\";\n$Cxx.namespace(\"cereal\");\nusing Car = import \"/car.capnp\";\n\n@0xb526ba661d550a59;\n\n# custom.capnp: a home for empty structs reserved for custom forks\n# These structs are guaranteed to remain reserved and empty in mainline\n# cereal, so use these if you want custom events in your fork.\n\n# DO rename the structs\n# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n# Automatic FLM adds children only to existing reserved fork carriers.\n# All evidence clocks are explicitly MONOTONIC; default/version0 is invalid.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
"historical_empty_slots": [
5
],
@@ -196,5 +196,121 @@
"enum": "SafetyModel",
"field": "teslaPreap @39;"
}
},
"reviewed_payload_children": {
"LearningControlEvidence": {
"type_id": "0xf2447170bb68f358",
"body": "\n version @0 :UInt16;\n identity @1 :Text;\n generation @2 :UInt64;\n controlMonoTime @3 :UInt64;\n sourceStateMonoTime @4 :UInt64;\n domain @5 :Text;\n stageRevision @6 :Text;\n responseRevision @7 :Text;\n requested @8 :Float64;\n features @9 :List(Float64);\n baselinePrediction @10 :Float64;\n enabled @11 :Bool;\n manual @12 :Bool;\n active @13 :Bool;\n valid @14 :Bool;\n override @15 :Bool;\n fault @16 :Bool;\n saturated @17 :Bool;\n limited @18 :Bool;\n modelWobble @19 :Bool;\n controllerChatter @20 :Bool;\n stageSchemaRevision @21 :Text;\n stageValues @22 :List(StageValue);\n appliedCoordinates @23 :List(Float64);\n sourceEpoch @24 :Text;\n sourceReceiptMonoTime @25 :UInt64;\n controllerMode @26 :Text;\n policy @27 :Text;\n baselineRevision @28 :Text;\n baselineValues @29 :List(Float64);\n controllerRevision @30 :Text;\n replaySnapshot @31 :List(NamedValue);\n replayInputs @32 :List(NamedValue);\n snapshotSequence @33 :UInt64;\n turnAssist @34 :Bool;\n appliedParameterNames @35 :List(Text);\n nativeRequired @36 :Bool;\n nativeObservedMonoTime @37 :UInt64;\n nativeValidUntilMonoTime @38 :UInt64;\n nativeAllowed @39 :Bool;\n nativeIdentity @40 :Text;\n appliedProfileSequence @41 :UInt64;\n appliedProfileObservedMonoTime @42 :UInt64;\n appliedProfileValidUntilMonoTime @43 :UInt64;\n effectTrace @44 :FullEffectTrace;\n effectPrepareMonoTime @45 :UInt64; # Exact MONOTONIC owner tick, not carControl publication.\n struct FullEffectTrace {\n version @0 :UInt16;\n envelope @1 :AutomaticRuntimeProfile.FullSurfaceEnvelope;\n controlMonoTime @2 :UInt64;\n previousMonoTime @3 :UInt64;\n qualified @4 :Bool;\n before @5 :List(StageValue);\n target @6 :List(StageValue);\n applied @7 :List(StageValue);\n appliedFullProfileSha256 @8 :Text;\n }\n struct NamedValue {\n name @0 :Text;\n union { number @1 :Float64; text @2 :Text; boolean @3 :Bool; }\n }\n struct StageValue { name @0 :Text; value @1 :Float64; }\n",
"owners": [
{
"scope": "StarPilotLateralState",
"field": "learningEvidence",
"ordinal": 9,
"target": "LearningControlEvidence",
"declaration": "learningEvidence@9:LearningControlEvidence"
}
]
},
"LearningAppliedEvidence": {
"type_id": "0xbab0f51a74ffee6e",
"body": "\n version @0 :UInt16;\n controlMonoTime @1 :UInt64;\n sourceStateMonoTime @2 :UInt64;\n appliedMonoTime @3 :UInt64;\n domain @4 :Text;\n applied @5 :Float64;\n valid @6 :Bool;\n active @7 :Bool;\n override @8 :Bool;\n fault @9 :Bool;\n limited @10 :Bool;\n nativeAllowed @11 :Bool;\n projection @12 :LearningTorqueProjection;\n # Carrier continuity only; never a command/physical/native lease timestamp.\n carrierFrame @13 :UInt64;\n carrierObservedMonoTime @14 :UInt64;\n",
"owners": [
{
"scope": "StarPilotCarState",
"field": "appliedEvidence",
"ordinal": 34,
"target": "LearningAppliedEvidence",
"declaration": "appliedEvidence@34:LearningAppliedEvidence"
}
]
},
"AutomaticRuntimeProfile": {
"type_id": "0xef38d302825641e3",
"body": "\n version @0 :UInt16;\n identity @1 :Text;\n sequence @2 :UInt64;\n observedMonoTime @3 :UInt64;\n validUntilMonoTime @4 :UInt64;\n policy @5 :Text;\n parameters @6 :List(Parameter);\n coefficients @7 :List(Float64);\n trainIds @8 :List(Text);\n heldoutIds @9 :List(Text);\n baselineLoss @10 :Float64;\n candidateLoss @11 :Float64;\n cohortImprovements @12 :List(CohortImprovement);\n replay @13 :ReplayValidation;\n fullSurface @14 :FullSurfaceEnvelope;\n struct FullSurfaceEnvelope {\n version @0 :UInt16;\n manifestSha256 @1 :Text;\n fullOriginalAnchor @2 :Text;\n trialAnchor @3 :Text;\n contextKey @4 :Text;\n cpSha256 @5 :Text;\n controllerRevision @6 :Text;\n sourceEpoch @7 :Text;\n resetGeneration @8 :Text;\n selectedBaselineRevision @9 :Text;\n priorFullProfileSha256 @10 :Text;\n fullProfileSha256 @11 :Text;\n priorCoordinates @12 :List(Coordinate);\n coordinates @13 :List(Coordinate);\n trialNames @14 :List(Text);\n persistentSourceEpoch @15 :Text;\n operation @16 :Text;\n historyFullProfileSha256 @17 :Text;\n restoreTransactionSha256 @18 :Text;\n struct Coordinate {\n name @0 :Text;\n value @1 :Float64;\n }\n }\n struct ReplayValidation {\n baselineRevision @0 :Text;\n physicalModelRevision @1 :Text;\n noSaturationRegression @2 :Bool;\n noSlewRegression @3 :Bool;\n worstCohortImprovement @4 :Float64;\n nativeBaselineValues @5 :List(Float64);\n }\n struct Parameter {\n name @0 :Text;\n stageRevision @1 :Text;\n low @2 :Float64;\n high @3 :Float64;\n maxStep @4 :Float64;\n slewPerS @5 :Float64;\n excitationFloor @6 :Float64 = 0.01;\n }\n struct CohortImprovement {\n name @0 :Text;\n lowerBound @1 :Float64;\n }\n",
"owners": [
{
"scope": "StarPilotCarParams.AutoSteeringLearning",
"field": "runtimeProfile",
"ordinal": 19,
"target": "AutomaticRuntimeProfile",
"declaration": "runtimeProfile@19:AutomaticRuntimeProfile"
}
]
},
"AutomaticLearningSourceReceipt": {
"type_id": "0x9fe2d23e75805149",
"body": "\n version @0 :UInt16;\n enabled @1 :Bool;\n manual @2 :Bool;\n resetGeneration @3 :Text;\n sourceEpoch @4 :Text;\n observedMonoTime @5 :UInt64;\n validUntilMonoTime @6 :UInt64;\n cpSha256 @7 :Text;\n calibrationBasis @8 :List(Float64);\n calibrationGeneration @9 :Text;\n persistentSourceEpoch @10 :Text;\n",
"owners": [
{
"scope": "StarPilotCarParams.AutoSteeringLearning",
"field": "sourceReceipt",
"ordinal": 20,
"target": "AutomaticLearningSourceReceipt",
"declaration": "sourceReceipt@20:AutomaticLearningSourceReceipt"
}
]
},
"LearningApplication": {
"type_id": "0x82b6c8ea6a6efabd",
"body": "\n version @0 :UInt16;\n identity @1 :Text;\n profileSequence @2 :UInt64;\n observedMonoTime @3 :UInt64;\n status @4 :Text;\n actualCoordinates @5 :List(Float64);\n reason @6 :Text;\n baselineRevision @7 :Text;\n physicalModelRevision @8 :Text;\n effectManifestSha256 @9 :Text;\n effectOriginalAnchor @10 :Text;\n effectAppliedProfileSha256 @11 :Text;\n",
"owners": [
{
"scope": "StarPilotCarState",
"field": "learningApplication",
"ordinal": 36,
"target": "LearningApplication",
"declaration": "learningApplication@36:LearningApplication"
},
{
"scope": "StarPilotLateralState",
"field": "learningApplication",
"ordinal": 10,
"target": "LearningApplication",
"declaration": "learningApplication@10:LearningApplication"
}
]
},
"FordLearningEvidence": {
"type_id": "0xd1dfbadb6a811fee",
"body": "\n version @0 :UInt16;\n identity @1 :Text;\n generation @2 :UInt64;\n sourceEpoch @3 :Text;\n baselineRevision @4 :Text;\n baselineValues @5 :List(Float64);\n controllerRevision @6 :Text;\n cpSha256 @7 :Text;\n controllerMode @8 :Text;\n policy @9 :Text;\n conversionRevision @10 :Text;\n stageRevision @11 :Text;\n sourceReceiptMonoTime @12 :UInt64;\n sourceValidUntilMonoTime @13 :UInt64;\n enabled @14 :Bool;\n manual @15 :Bool;\n rows @16 :List(FordLearningEvent);\n dropped @17 :UInt64;\n appliedProfileSequence @18 :UInt64;\n appliedProfileObservedMonoTime @19 :UInt64;\n appliedProfileValidUntilMonoTime @20 :UInt64;\n",
"owners": [
{
"scope": "StarPilotCarState",
"field": "fordLearningEvidence",
"ordinal": 35,
"target": "FordLearningEvidence",
"declaration": "fordLearningEvidence@35:FordLearningEvidence"
}
]
},
"FordLearningEvent": {
"type_id": "0xb276dede9e6b2cc4",
"body": "\n eventMonoTime @0 :UInt64;\n sequence @1 :UInt64;\n kind @2 :Text;\n inputs @3 :List(LearningControlEvidence.NamedValue);\n appliedCoefficient @4 :Float64;\n actualPending @5 :List(Float64);\n actualCommand @6 :List(Float64);\n controlMonoTime @7 :UInt64;\n pendingControlMonoTime @8 :UInt64;\n announcedControlMonoTime @9 :UInt64;\n carStateMonoTime @10 :UInt64;\n nativeAllowed @11 :Bool;\n fault @12 :Bool;\n override @13 :Bool;\n limited @14 :Bool;\n hasWirePath @15 :Bool;\n actualWirePath @16 :Float64;\n checkpoint @17 :List(LearningControlEvidence.NamedValue);\n checkpointSequence @18 :UInt64;\n sourceReceiptMonoTime @19 :UInt64;\n sourceValidUntilMonoTime @20 :UInt64;\n appliedProfileSequence @21 :UInt64;\n appliedProfileObservedMonoTime @22 :UInt64;\n appliedProfileValidUntilMonoTime @23 :UInt64;\n nativeIdentity @24 :Text;\n nativeObservedMonoTime @25 :UInt64;\n nativeValidUntilMonoTime @26 :UInt64;\n nativeValidatedMonoTime @27 :UInt64;\n nativeCommandMonoTime @28 :UInt64;\n announcedCoefficient @29 :Float64;\n",
"owners": []
},
"CentreApplicationWitness": {
"type_id": "0xab0b33cac682523e",
"body": "\n version @0 :UInt16;\n identity @1 :Text;\n sourceEpoch @2 :Text;\n controlMonoTime @3 :UInt64;\n profileSequence @4 :UInt64;\n actualCoordinates @5 :List(Float64);\n baselineRevision @6 :Text;\n physicalModelRevision @7 :Text;\n originalObservedMonoTime @8 :UInt64;\n originalValidUntilMonoTime @9 :UInt64;\n baselineFriction @10 :Float64;\n actualFriction @11 :Float64;\n",
"owners": [
{
"scope": "StarPilotCarParams.AutoSteeringLearning",
"field": "centreApplicationWitness",
"ordinal": 21,
"target": "CentreApplicationWitness",
"declaration": "centreApplicationWitness@21:CentreApplicationWitness"
}
]
},
"LearningTorqueProjection": {
"type_id": "0xf078f562cbce49ad",
"body": "\n version @0 :UInt16;\n family @1 :Text;\n fingerprint @2 :Text;\n flags @3 :UInt64;\n codecId @4 :Text;\n mappingKind @5 :Text;\n frame @6 :UInt64;\n requestedTorque @7 :Float64;\n codecInputTorque @8 :Float64;\n unlimitedRaw @9 :Int32;\n finalRaw @10 :Int32;\n effectiveMax @11 :Float64;\n lookupX @12 :List(Float64);\n lookupY @13 :List(Float64);\n limiterKind @14 :Text;\n previousTorque @15 :Float64;\n measuredTorque @16 :Float64;\n limits @17 :List(LearningControlEvidence.StageValue);\n shape @18 :List(LearningControlEvidence.NamedValue);\n steerRequested @19 :Bool;\n canAddress @20 :UInt32;\n canBus @21 :UInt16;\n canData @22 :Data;\n reportedTorque @23 :Float64;\n unlimitedRawPresent @24 :Bool;\n",
"owners": []
}
},
"reviewed_payload_children_source": {
"commit": "1ef8cafbdf0ad8dc26137fc5f6c5a0dfabd731be",
"schema_sha256": "a693b33bdd68230776acc89ee88cfa4be357d62ef2c099ba7ab071683b3f3170",
"generated_header_sha256": "765c60549108ce6d4a2e7e8b7e16349be60bfd9ae0440da35b66771666fd3e03",
"reason": "Existing payload children of reserved carriers; no additional Event reservations. Exact bodies and carrier edges are immutable."
}
}
tools/ci/schema_policy.py
+204 -8
View File
@@ -3,7 +3,9 @@
See openpilot/cereal/README.md, Custom forks. When syncing upstream, review and
update schema-policy.json alongside upstream-sync.json. Custom fields belong
inside reserved custom.capnp structs; log.capnp may rename only their event
inside reserved custom.capnp structs or the exact reviewed payload children
already shipped by this fork; those children grant no Event slots and their
existing bodies and reserved-carrier references are immutable. log.capnp may rename only their event
aliases and the reserved raw-data aliases, preserving ordinals and type IDs.
"""
@@ -51,6 +53,190 @@ def custom_structs(source):
return definitions, bodies, remainder
def source_tokens(source):
"""Tokenize the reviewed schema subset, preserving quoted defaults exactly."""
return re.findall(r'"(?:\\.|[^"\\])*"|[A-Za-z_][A-Za-z_0-9]*|0x[0-9a-fA-F]+|[0-9]+|[^\s]',
re.sub(r'"(?:\\.|[^"\\])*"|#[^\n]*',
lambda match: match[0] if match[0].startswith('"') else '', source))
def source_scope(source, kind='struct'):
"""Parse fields, enum values and nested scopes; unsupported syntax fails shut.
This is a source identity contract, not a replacement for compiled wire-layout
checks. In particular no annotations, aliases, groups or implicit unions are
accepted by this subset without a separate grammar/contract review.
"""
tokens = source_tokens(source)
position = 0
def parse(scope_kind, nested=False):
nonlocal position
fields, children, unions = [], {}, []
while position < len(tokens) and tokens[position] != '}':
word = tokens[position]
if word in ('struct', 'enum', 'union'):
position += 1
if word == 'union':
name, type_id = '', None
else:
if position >= len(tokens) or not re.fullmatch(r'[A-Za-z_]\w*', tokens[position]):
raise ValueError('Invalid nested type name')
name, type_id = tokens[position], None
position += 1
if position < len(tokens) and tokens[position] == '@':
position += 1
if position >= len(tokens) or not re.fullmatch(r'0x[0-9a-fA-F]+', tokens[position]):
raise ValueError('Invalid nested type ID')
type_id = tokens[position].lower()
position += 1
if position >= len(tokens) or tokens[position] != '{' or scope_kind == 'enum':
raise ValueError('Unsupported nested declaration')
position += 1
child = {'kind': word, 'type_id': type_id, 'body': parse(word, True)}
if word == 'union':
unions.append(child)
elif name in children:
raise ValueError('Duplicate nested type name')
else:
children[name] = child
continue
start = position
while position < len(tokens) and tokens[position] not in (';', '{', '}'):
position += 1
if position >= len(tokens) or tokens[position] != ';':
raise ValueError('Unsupported or unterminated member declaration')
member = tokens[start:position]
position += 1
if (len(member) < 3 or not re.fullmatch(r'[A-Za-z_]\w*', member[0]) or
member[1] != '@' or not member[2].isdigit()):
raise ValueError('Unsupported member declaration')
if scope_kind == 'enum':
if len(member) != 3:
raise ValueError('Unsupported enumerant declaration')
else:
if len(member) < 5 or member[3] != ':':
raise ValueError('Unsupported field declaration')
type_end = member.index('=') if '=' in member else len(member)
descriptor = ''.join(member[4:type_end])
# No aliases/annotations/inline defaults can silently affect resolution.
type_pattern = r'[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*'
while descriptor.startswith('List(') and descriptor.endswith(')'):
descriptor = descriptor[5:-1]
if not re.fullmatch(type_pattern, descriptor):
raise ValueError('Unsupported field type')
if type_end < len(member):
default = ''.join(member[type_end + 1:])
if not re.fullmatch(r'"(?:\\.|[^"\\])*"|true|false|[-+]?[0-9]+(?:\.[0-9]+)?', default):
raise ValueError('Unsupported field default')
fields.append(member)
if nested:
if position >= len(tokens):
raise ValueError('Unclosed nested declaration')
position += 1
elif position != len(tokens):
raise ValueError('Unexpected closing brace')
names = [field[0] for field in fields]
ordinals = [int(field[2]) for field in fields]
# Anonymous-union fields share their containing struct's ordinal space.
for union in unions:
names.extend(field[0] for field in union['body']['fields'])
ordinals.extend(int(field[2]) for field in union['body']['fields'])
if len(names) != len(set(names)) or len(ordinals) != len(set(ordinals)):
raise ValueError('Duplicate member name or ordinal')
if set(names) & set(children):
raise ValueError('Field shadows nested type')
return {'fields': fields, 'children': children, 'unions': unions}
return parse(kind)
def preserves_scope(current, historical):
"""Allow recursive appends while preserving every historical declaration."""
old_fields, new_fields = historical['fields'], current['fields']
if new_fields[:len(old_fields)] != old_fields or current['unions'] != historical['unions']:
return False
if old_fields and any(int(field[2]) <= max(int(old[2]) for old in old_fields)
for field in new_fields[len(old_fields):]):
return False
for name, old in historical['children'].items():
new = current['children'].get(name)
if (new is None or new['kind'] != old['kind'] or new['type_id'] != old['type_id'] or
not preserves_scope(new['body'], old['body'])):
return False
return True
def payload_child_contract(definitions, bodies, reviewed):
"""The nine shipped payload helpers are not additional Event reservations."""
errors, trees, ids = [], {}, set()
for type_id, name in definitions.items():
trees[name] = source_scope(bodies[type_id])
ids.add(type_id)
for name, helper in reviewed.items():
type_id = helper['type_id']
if (definitions.get(type_id) != name or
trees.get(name) != source_scope(helper['body'])):
errors.append(f'custom.capnp: reviewed payload child {name} changed')
scopes = {}
def visit(name, tree):
scopes[name] = tree
for child_name, child in tree['children'].items():
child_id = child['type_id']
if child_id:
if child_id in ids:
raise ValueError('Duplicate explicit nested type ID')
ids.add(child_id)
visit(name + '.' + child_name, child['body'])
for name, tree in trees.items():
visit(name, tree)
def resolved(scope, descriptor):
while descriptor.startswith('List(') and descriptor.endswith(')'):
descriptor = descriptor[5:-1]
parts = scope.split('.')
while parts:
candidate = '.'.join(parts) + '.' + descriptor
if candidate in scopes:
return candidate
parts.pop()
return descriptor if descriptor in scopes else None
incoming = {name: [] for name in reviewed}
# Lexical nesting alone never makes a type reachable. Only an
# actual field reference from an already-reachable scope does.
graph = {scope: set() for scope in scopes}
for scope, tree in scopes.items():
fields = tree['fields'] + [field for union in tree['unions'] for field in union['body']['fields']]
for field in fields:
end = field.index('=') if '=' in field else len(field)
target = resolved(scope, ''.join(field[4:end]))
if target is None:
continue
source_root, target_root = scope.split('.')[0], target.split('.')[0]
graph[scope].add(target)
if target_root in reviewed and source_root not in reviewed:
incoming[target_root].append({'scope': scope, 'field': field[0], 'ordinal': int(field[2]),
'target': target, 'declaration': ''.join(field)})
def ordered(entries):
return sorted(entries, key=lambda entry: (entry['scope'], entry['ordinal']))
for name, helper in reviewed.items():
if ordered(incoming[name]) != ordered(helper['owners']):
errors.append(f'custom.capnp: reviewed payload child {name} carrier references changed')
pending, reachable = list(set(trees) - set(reviewed)), set()
while pending:
name = pending.pop()
if name in reachable:
continue
reachable.add(name)
pending.extend(graph[name] - reachable)
if set(reviewed) - reachable:
errors.append('custom.capnp: reviewed payload children must remain reachable from reserved carriers')
return errors
def validate(schemas, policy, sync):
errors = []
expected_paths = {*policy['unchanged_schemas'], 'openpilot/cereal/custom.capnp', 'openpilot/cereal/log.capnp'}
@@ -83,7 +269,9 @@ def validate(schemas, policy, sync):
definitions, bodies, remainder = custom_structs(schemas['openpilot/cereal/custom.capnp'].decode())
except (UnicodeDecodeError, ValueError) as error:
return [*errors, f'custom.capnp: {error}']
expected_ids = {slot['type_id'] for slot in policy['reserved']}
reviewed_children = policy.get('reviewed_payload_children', {})
expected_ids = ({slot['type_id'] for slot in policy['reserved']} |
{helper['type_id'] for helper in reviewed_children.values()})
if set(definitions) != expected_ids:
errors.append('custom.capnp: reserved type IDs were removed/changed or unreserved top-level types were added')
reserved_by_name = {slot['struct']: slot for slot in policy['reserved']}
@@ -102,13 +290,21 @@ def validate(schemas, policy, sync):
slot = next((entry for entry in policy['reserved'] if entry['type_id'] == type_id), None)
if slot is None or slot['ordinal'] != restored['ordinal']:
errors.append(f'custom.capnp: historical slot {index} type ID or event ordinal changed')
def canonical(text):
return re.sub(r'\s+', '', re.sub(r'#[^\n]*', '', text))
for restored in policy.get('historical_prefixes', {}).values():
if (definitions.get(restored['type_id']) != restored['name'] or
not canonical(bodies.get(restored['type_id'], '')).startswith(canonical(restored['body']))):
errors.append('custom.capnp: historical occupied prefix changed')
try:
preserved = (definitions.get(restored['type_id']) == restored['name'] and
preserves_scope(source_scope(bodies.get(restored['type_id'], '')), source_scope(restored['body'])))
except ValueError as error:
# Invalid syntax/duplicate ordinals inside an occupied historical scope
# also violate that scope's contract; retain this diagnostic boundary.
errors.append(f'custom.capnp: historical occupied prefix changed: {error}')
else:
if not preserved:
errors.append('custom.capnp: historical occupied prefix changed')
try:
errors.extend(payload_child_contract(definitions, bodies, reviewed_children))
except ValueError as error:
errors.append(f'custom.capnp: source contract rejected: {error}')
if remainder.strip() != policy['custom_header'].strip():
errors.append('custom.capnp: edits outside the reserved structs')
log = schemas['openpilot/cereal/log.capnp'].decode()
tools/ci/tests/test_schema_policy.py
+164 -1
View File
@@ -54,7 +54,13 @@ class SchemaPolicyTest(unittest.TestCase):
self.assertIn(original, self.schemas['openpilot/cereal/custom.capnp'])
self.assertIn(b'aolAxisState @142 :Custom.AolAxisState;', self.schemas['openpilot/cereal/log.capnp'])
self.schemas['openpilot/cereal/custom.capnp'] = self.schemas['openpilot/cereal/custom.capnp'].replace(
original, b'struct FixtureState @0xfc6241ed8877b611 {\n note @12 :Text = "{fixture}";')
original, b'struct FixtureState @0xfc6241ed8877b611 {')
# AolAxisState already occupies @0..17. Append a genuinely fresh ordinal;
# the parser must continue rejecting the old duplicate-@12 fixture.
occupied_tail = b'intentIntegrityEpoch @17 :UInt64;'
self.assertIn(occupied_tail, self.schemas['openpilot/cereal/custom.capnp'])
self.schemas['openpilot/cereal/custom.capnp'] = self.schemas['openpilot/cereal/custom.capnp'].replace(
occupied_tail, occupied_tail + b'\n note @18 :Text = "{fixture}";', 1)
self.schemas['openpilot/cereal/log.capnp'] = self.schemas['openpilot/cereal/log.capnp'].replace(
b'aolAxisState @142 :Custom.AolAxisState;', b'fixtureState @142 :Custom.FixtureState;')
self.assertEqual(validate(self.schemas, self.policy, self.sync), [])
@@ -271,3 +277,160 @@ class SchemaPolicyTest(unittest.TestCase):
changed = self.schemas.copy()
changed[path] = changed[path].replace(original, after)
self.assertTrue(validate(changed, self.policy, self.sync))
def test_recursive_historical_appends_preserve_existing_scopes(self):
path = 'openpilot/cereal/custom.capnp'
mutations = (
(b'controllerRevision @11 :Text;', b'controllerRevision @11 :Text;\n futureIdentity @12 :Text;'),
(b'qualifiedFit @4;', b'qualifiedFit @4; futureStatus @5;'),
(b'runtimeProfile @2;', b'runtimeProfile @2; futureFamily @3;'),
)
for before, after in mutations:
with self.subTest(after=after):
changed = self.schemas.copy()
changed[path] = changed[path].replace(before, after, 1)
self.assertEqual(validate(changed, self.policy, self.sync), [])
def test_reviewed_payload_children_have_exact_existing_ids_and_bodies(self):
expected = {
'LearningControlEvidence': '0xf2447170bb68f358',
'LearningAppliedEvidence': '0xbab0f51a74ffee6e',
'AutomaticRuntimeProfile': '0xef38d302825641e3',
'AutomaticLearningSourceReceipt': '0x9fe2d23e75805149',
'LearningApplication': '0x82b6c8ea6a6efabd',
'FordLearningEvidence': '0xd1dfbadb6a811fee',
'FordLearningEvent': '0xb276dede9e6b2cc4',
'CentreApplicationWitness': '0xab0b33cac682523e',
'LearningTorqueProjection': '0xf078f562cbce49ad',
}
self.assertEqual({name: child['type_id'] for name, child in self.policy['reviewed_payload_children'].items()}, expected)
path = 'openpilot/cereal/custom.capnp'
for name, type_id in expected.items():
declaration = f'struct {name} @{type_id} {{'.encode()
original = self.policy['reviewed_payload_children'][name]['body'].encode()
for body in (original.replace(b'@0', b'@99', 1), original + b'\n futureChild @99 :Text;',
original.replace(b'UInt16', b'UInt32', 1) if b'UInt16' in original else original.replace(b'UInt64', b'UInt32', 1)):
with self.subTest(name=name, body=body[:60]):
changed = self.schemas.copy()
changed[path] = changed[path].replace(declaration + original, declaration + body, 1)
self.assertTrue(any('reviewed payload child' in error for error in validate(changed, self.policy, self.sync)))
for replacement in (f'struct {name} {{'.encode(), f'struct {name} @0xf000000000000001 {{'.encode(),
f'struct Other{name} @{type_id} {{'.encode()):
with self.subTest(name=name, replacement=replacement):
changed = self.schemas.copy()
changed[path] = changed[path].replace(declaration, replacement, 1)
self.assertTrue(validate(changed, self.policy, self.sync))
def test_helper_nested_union_defaults_and_field_names_are_immutable(self):
path = 'openpilot/cereal/custom.capnp'
for before, after in (
(b'union { number @1 :Float64; text @2 :Text; boolean @3 :Bool; }',
b'number @1 :Float64; text @2 :Text; boolean @3 :Bool;'),
(b'excitationFloor @6 :Float64 = 0.01;', b'excitationFloor @6 :Float64 = 0.02;'),
(b'struct FullEffectTrace {', b'struct FullEffectTrace @0xc9c9b325ca50aaf7 {'),
(b'struct NamedValue {', b'struct OtherNamedValue {'),
(b'effectPrepareMonoTime @45 :UInt64;', b''),
(b'effectPrepareMonoTime @45 :UInt64;', b'unrelatedTime @45 :UInt64;'),
):
with self.subTest(after=after):
self.assertIn(before, self.schemas[path])
changed = self.schemas.copy()
changed[path] = changed[path].replace(before, after, 1)
self.assertTrue(any('reviewed payload child' in error for error in validate(changed, self.policy, self.sync)))
def test_helpers_cannot_gain_new_carriers_or_change_pointer_shape(self):
path = 'openpilot/cereal/custom.capnp'
cases = (
(b'active @0 :Bool;\n frictionThreshold', b'foreignChild @99 :LearningControlEvidence;\n active @0 :Bool;\n frictionThreshold'),
(b'learningEvidence @9 :LearningControlEvidence;', b'learningEvidence @9 :List(LearningControlEvidence);'),
(b'learningApplication @36 :LearningApplication;', b'learningApplication @36 :LearningControlEvidence;'),
(b'learningEvidence @9 :LearningControlEvidence;', b''),
)
for before, after in cases:
with self.subTest(after=after):
self.assertIn(before, self.schemas[path])
changed = self.schemas.copy()
changed[path] = changed[path].replace(before, after, 1)
self.assertTrue(any('carrier references changed' in error for error in validate(changed, self.policy, self.sync)))
def test_unknown_helper_alias_and_helper_event_retargeting_are_rejected(self):
path = 'openpilot/cereal/custom.capnp'
for addition in (b'\nstruct UnknownHelper @0xf000000000000001 {}\n',
b'\nstruct UnknownImplicitHelper {}\n',
b'\nusing ExtraAlias = LearningControlEvidence;\n'):
changed = self.schemas.copy()
changed[path] += addition
self.assertTrue(validate(changed, self.policy, self.sync))
changed = self.schemas.copy()
changed['openpilot/cereal/log.capnp'] = changed['openpilot/cereal/log.capnp'].replace(
b'autoSteeringLearning @108 :Custom.StarPilotCarParams;',
b'autoSteeringLearning @108 :Custom.AutomaticRuntimeProfile;')
self.assertTrue(any('event @108' in error for error in validate(changed, self.policy, self.sync)))
def test_recursive_historical_parser_rejects_shadowing_and_unknown_syntax(self):
path = 'openpilot/cereal/custom.capnp'
for after in (b'controllerRevision @11 :Text;\n shadow @11 :UInt64;',
b'controllerRevision @11 :Text;\n using Hidden = Text;',
b'controllerRevision @11 :Text;\n payload @12 :union { x @13 :Text; }'):
changed = self.schemas.copy()
changed[path] = changed[path].replace(b'controllerRevision @11 :Text;', after, 1)
self.assertTrue(validate(changed, self.policy, self.sync))
def test_recursive_source_contract_preserves_quoted_default_content(self):
from tools.ci.schema_policy import preserves_scope, source_scope
original = 'label @0 :Text = "{# original value }"; enum Mode { first @0; }'
historical = source_scope(original)
appended = source_scope(original.replace('first @0;', 'first @0; second @1;'))
self.assertTrue(preserves_scope(appended, historical))
for replacement in ('{# originalvalue }', '{# changed value }', '{# original value}'):
changed = source_scope(original.replace('{# original value }', replacement))
self.assertFalse(preserves_scope(changed, historical))
def test_helpers_in_unreferenced_nested_owner_are_not_reachable(self):
path = 'openpilot/cereal/custom.capnp'
original = b'learning @9 :AutoSteeringLearning;'
self.assertIn(original, self.schemas[path])
# Keep the existing nested owner and its exact three helper fields intact.
# Its enclosing reserved root has no field pointing to that scope anymore;
# merely visiting its lexical declaration must not grant helper ownership.
for replacement in (b'', b'learning @9 :Text;'):
with self.subTest(replacement=replacement):
changed = self.schemas.copy()
changed[path] = changed[path].replace(original, replacement, 1)
errors = validate(changed, self.policy, self.sync)
self.assertTrue(any('must remain reachable from reserved carriers' in error for error in errors))
self.assertTrue(any('historical occupied prefix changed' in error for error in errors))
def test_literal_learning_field_keeps_name_ordinal_and_nested_type(self):
path = 'openpilot/cereal/custom.capnp'
original = b'learning @9 :AutoSteeringLearning;'
self.assertIn(original, self.schemas[path])
for replacement in (b'otherLearning @9 :AutoSteeringLearning;',
b'learning @10 :AutoSteeringLearning;',
b'learning @9 :List(AutoSteeringLearning);',
b'learning @9 :Text;', b''):
with self.subTest(replacement=replacement):
changed = self.schemas.copy()
changed[path] = changed[path].replace(original, replacement, 1)
self.assertTrue(any('historical occupied prefix changed' in error
for error in validate(changed, self.policy, self.sync)))
def test_duplicate_historical_ordinal_reports_occupied_prefix_violation(self):
path = 'openpilot/cereal/custom.capnp'
original = b'slcSpeedLimit @24 :Float32;'
self.assertIn(original, self.schemas[path])
self.assertIn(b'slcSpeedLimitOffset @25 :Float32;', self.schemas[path])
changed = self.schemas.copy()
changed[path] = changed[path].replace(original, b'slcSpeedLimit @25 :Float32;', 1)
errors = validate(changed, self.policy, self.sync)
self.assertTrue(any('historical occupied prefix changed: Duplicate member name or ordinal' in error for error in errors))
def test_duplicate_existing_aol_ordinal_remains_rejected(self):
path = 'openpilot/cereal/custom.capnp'
original = b'struct AolAxisState @0xfc6241ed8877b611 {'
self.assertIn(b'optionalSetRelease @12 :Bool;', self.schemas[path])
changed = self.schemas.copy()
changed[path] = changed[path].replace(original, original + b'\n duplicate @12 :Text;', 1)
self.assertTrue(any('Duplicate member name or ordinal' in error for error in validate(changed, self.policy, self.sync)))