diff --git a/.gitignore b/.gitignore index 757b67c705..af96cafb86 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,5 @@ openpilot/selfdrive/modeld/models/*.unchunked /jetlink_repo/jetlink.egg-info/ # Local Google Play resolver build output (source and lockfile are versioned). + +openpilot/starpilot/flm/_primitive_walk.cpp diff --git a/SConstruct b/SConstruct index 441d127381..487525c871 100644 --- a/SConstruct +++ b/SConstruct @@ -329,6 +329,7 @@ SConscript([ 'openpilot/selfdrive/ui/SConscript', 'openpilot/starpilot/audio/native/SConscript', 'openpilot/starpilot/models/runtime_bridge/SConscript', + 'openpilot/starpilot/flm/SConscript', ]) # Replay is also the isolated on-device viewer engine. diff --git a/openpilot/cereal/custom.capnp b/openpilot/cereal/custom.capnp index f06c34ad21..da3c58a141 100644 --- a/openpilot/cereal/custom.capnp +++ b/openpilot/cereal/custom.capnp @@ -213,6 +213,7 @@ struct StarPilotCarState @0xf35cc4560bbf6ec2 { appliedEvidence @34 :LearningAppliedEvidence; fordLearningEvidence @35 :FordLearningEvidence; learningApplication @36 :LearningApplication; + feedbackPublication @37 :FeedbackPublication; struct Gps { sourceMonoTime @0 :UInt64; # CAN observation converted to CLOCK_MONOTONIC @@ -229,6 +230,26 @@ struct StarPilotCarState @0xf35cc4560bbf6ec2 { hasFix @11 :Bool; vNED @12 :List(Float32); } + + struct FeedbackPublication { + version @0 :UInt16; + cardSession @1 :Data; # Exactly16 bytes, never default-empty proof. + publicationSequence @2 :UInt64; + publicationMonoTime @3 :UInt64; + sourceCarStateMonoTime @4 :UInt64; # Held AppliedEvidence.state_ns. + outputKnown @5 :Bool; + outputControlMonoTime @6 :UInt64; + outputApplyMonoTime @7 :UInt64; + outputNativeIdentity @8 :Text; # Optional reference only; child verifies original native proof. + outputCarrierSequence @9 :UInt64; # Held AppliedEvidence.carrier_frame, NOT a local apply serial. + reportedTorque @10 :Float32; + outputValid @11 :Bool; # Exact CO Event.valid, not held evidence validity. + coBodySha256 @12 :Data; # Exactly32 bytes. + applyEvidenceSha256 @13 :Data; # Exactly32 bytes when known; empty when unknown. + sourceEpoch @14 :Text; + cpSha256 @15 :Text; + carrierObservedMonoTime @16 :UInt64; # Exact held carrier receipt; never CO/apply relabel. + } } struct StarPilotLongitudinalPlan @0xda96579883444c35 { @@ -618,7 +639,11 @@ struct StarPilotModelDataV2 @0x80ae746ee2596b11 { } } -struct CustomReserved5 @0xa5cd762cd951a455 { +struct StarPilotSteeringTracking @0xa5cd762cd951a455 { + version @0 :UInt16; + sourceReceipt @1 :AutomaticLearningSourceReceipt; + trial @2 :LiveTrackingTrial; + restore @3 :LiveTrackingRestore; } struct CustomReserved6 @0xf98d843bfd7004a3 { @@ -803,6 +828,8 @@ struct StarPilotLateralState @0xc2243c65e0340384 { laneCentering @8 :LaneCentering; learningEvidence @9 :LearningControlEvidence; learningApplication @10 :LearningApplication; + feedbackConsumption @11 :FeedbackConsumption; + trackingApplication @12 :LiveTrackingApplication; struct LaneCentering { version @0 :UInt16; @@ -813,6 +840,25 @@ struct StarPilotLateralState @0xc2243c65e0340384 { appliedCorrection @5 :Float32; reason @6 :Text; } + + struct FeedbackConsumption { + version @0 :UInt16; + controlMonoTime @1 :UInt64; + consumedPublicationMonoTime @2 :UInt64; + consumedRecvFrame @3 :UInt64; + consumedReceiptMonoTime @4 :UInt64; + consumedSeen @5 :Bool; + consumedValid @6 :Bool; + coBodySha256 @7 :Data; # Exactly32 bytes. + reportedTorque @8 :Float32; + requestTorque @9 :Float32; + latActive @10 :Bool; + inputSafetyLimited @11 :Bool; + publishedSafetyLimited @12 :Bool; + priorPublishControlMonoTime @13 :UInt64; + sourceEpoch @14 :Text; + cpSha256 @15 :Text; + } } struct SlcCruiseCommand @0xbd443b539493bc68 { @@ -1196,6 +1242,8 @@ struct LearningControlEvidence @0xf2447170bb68f358 { appliedProfileValidUntilMonoTime @43 :UInt64; effectTrace @44 :FullEffectTrace; effectPrepareMonoTime @45 :UInt64; # Exact MONOTONIC owner tick, not carControl publication. + vehicleModelAnchor @46 :List(Float64); # Actual owner stable (sR,cF,cR), empty on old senders. + liveAppliedCoordinates @47 :List(Float64); # Actual same-frame live full30 hooks, never offered target. struct FullEffectTrace { version @0 :UInt16; envelope @1 :AutomaticRuntimeProfile.FullSurfaceEnvelope; @@ -1425,3 +1473,50 @@ struct LearningTorqueProjection @0xf078f562cbce49ad { reportedTorque @23 :Float64; unlimitedRawPresent @24 :Bool; } + +# Live tracking trials are a separate protocol, never ReplayValidation. +struct LiveTrackingBinding @0x9f853f7eb4110c72 { + identity @0 :Text; cpSha256 @1 :Text; controllerRevision @2 :Text; + baselineRevision @3 :Text; sourceEpoch @4 :Text; resetGeneration @5 :Text; + calibrationGeneration @6 :Text; vehicleModelGeneration @7 :Text; + nativeIdentity @8 :Text; physicalModelRevision @9 :Text; persistentSourceEpoch @10 :Text; + nativeRequired @11 :Bool; +} +struct LiveTrackingTrial @0xe6f67224bc0ee1a3 { + version @0 :UInt16; protocol @1 :Text; binding @2 :LiveTrackingBinding; + trialId @3 :Text; sequence @4 :UInt64; phase @5 :Text; group @6 :Text; + names @7 :List(Text); baseline @8 :List(Coordinate); target @9 :List(Coordinate); + baselineSha256 @10 :Text; targetSha256 @11 :Text; + startedMonoTime @12 :UInt64; observedMonoTime @13 :UInt64; validUntilMonoTime @14 :UInt64; + responseWitness @15 :LiveTrackingResponseWitness; + targetSequence @16 :UInt64; deadlineMonoTime @17 :UInt64; + struct Coordinate { name @0 :Text; value @1 :Float64; } +} +struct LiveTrackingApplication @0xdb85b982067f1bc1 { + version @0 :UInt16; binding @1 :LiveTrackingBinding; trialId @2 :Text; + sequence @3 :UInt64; controlMonoTime @4 :UInt64; status @5 :Text; + applied @6 :List(LiveTrackingTrial.Coordinate); appliedSha256 @7 :Text; + confirmedTargetSha256 @8 :Text; reason @9 :Text; targetSequence @10 :UInt64; +} + +struct LiveTrackingRestore @0xef6257bed9318012 { + version @0 :UInt16; protocol @1 :Text; binding @2 :LiveTrackingBinding; + historyScope @3 :Text; historyTrialId @4 :Text; target @5 :List(LiveTrackingTrial.Coordinate); + targetSha256 @6 :Text; sequence @7 :UInt64; observedMonoTime @8 :UInt64; + validUntilMonoTime @9 :UInt64; responseWitness @10 :LiveTrackingResponseWitness; +} + +struct LiveTrackingResponseCertificate @0xbbb8f17940692312 { + version @0 :Text; modelRevision @1 :Text; lagNs @2 :UInt64; + slope @3 :Float64; intercept @4 :Float64; commandLow @5 :Float64; + commandHigh @6 :Float64; heldoutRmse @7 :Float64; + trainBlockIds @8 :List(Text); heldoutBlockIds @9 :List(Text); + sourceInventorySha256 @10 :Text; identificationSha256 @11 :Text; identifiedMonoTime @12 :UInt64; + provenance @13 :Text; parentModelRevision @14 :Text; requalificationSha256 @15 :Text; +} +struct LiveTrackingResponseWitness @0xdaa1a9f0d8968e31 { + certificate @0 :LiveTrackingResponseCertificate; + lastControlMonoTime @1 :UInt64; lastApplyMonoTime @2 :UInt64; lastPoseMonoTime @3 :UInt64; + windowStartMonoTime @4 :UInt64; independentPoses @5 :UInt16; + residualRmse @6 :Float64; validUntilMonoTime @7 :UInt64; +} diff --git a/openpilot/cereal/gen/cpp/custom.capnp.c++ b/openpilot/cereal/gen/cpp/custom.capnp.c++ index 2231c9f9d5..98e2dc2865 100644 --- a/openpilot/cereal/gen/cpp/custom.capnp.c++ +++ b/openpilot/cereal/gen/cpp/custom.capnp.c++ @@ -2208,287 +2208,299 @@ const ::capnp::_::RawSchema s_af3712fe799c76ce = { 0, 12, i_af3712fe799c76ce, nullptr, nullptr, { &s_af3712fe799c76ce, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<627> b_f35cc4560bbf6ec2 = { +static const ::capnp::_::AlignedData<649> b_f35cc4560bbf6ec2 = { { 0, 0, 0, 0, 5, 0, 6, 0, 194, 110, 191, 11, 86, 196, 92, 243, 13, 0, 0, 0, 1, 0, 3, 0, 89, 10, 85, 29, 102, 186, 38, 181, - 4, 0, 7, 0, 0, 0, 0, 0, + 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 250, 0, 0, 0, - 33, 0, 0, 0, 23, 0, 0, 0, + 33, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 31, 8, 0, 0, + 61, 0, 0, 0, 87, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 117, 115, 116, 111, 109, 46, 99, 97, 112, 110, 112, 58, 83, 116, 97, 114, 80, 105, 108, 111, 116, 67, 97, 114, 83, 116, 97, 116, 101, 0, 0, - 4, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 1, 0, 1, 0, 45, 201, 117, 88, 170, 25, 141, 145, - 1, 0, 0, 0, 34, 0, 0, 0, + 9, 0, 0, 0, 34, 0, 0, 0, + 147, 2, 64, 226, 130, 227, 146, 222, + 5, 0, 0, 0, 162, 0, 0, 0, 71, 112, 115, 0, 0, 0, 0, 0, - 148, 0, 0, 0, 3, 0, 4, 0, + 70, 101, 101, 100, 98, 97, 99, 107, + 80, 117, 98, 108, 105, 99, 97, 116, + 105, 111, 110, 0, 0, 0, 0, 0, + 152, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 3, 0, 0, 106, 0, 0, 0, + 25, 4, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 252, 3, 0, 0, 3, 0, 1, 0, - 8, 4, 0, 0, 2, 0, 1, 0, + 24, 4, 0, 0, 3, 0, 1, 0, + 36, 4, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 4, 0, 0, 186, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 8, 4, 0, 0, 3, 0, 1, 0, - 20, 4, 0, 0, 2, 0, 1, 0, - 2, 0, 0, 0, 2, 0, 0, 0, - 0, 0, 1, 0, 2, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 17, 4, 0, 0, 98, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 16, 4, 0, 0, 3, 0, 1, 0, - 28, 4, 0, 0, 2, 0, 1, 0, - 3, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 3, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 25, 4, 0, 0, 162, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 28, 4, 0, 0, 3, 0, 1, 0, - 40, 4, 0, 0, 2, 0, 1, 0, - 4, 0, 0, 0, 3, 0, 0, 0, - 0, 0, 1, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 37, 4, 0, 0, 106, 0, 0, 0, + 33, 4, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4, 0, 0, 3, 0, 1, 0, 48, 4, 0, 0, 2, 0, 1, 0, - 5, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 1, 0, 5, 0, 0, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 4, 0, 0, 130, 0, 0, 0, + 45, 4, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4, 0, 0, 3, 0, 1, 0, 56, 4, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 5, 0, 0, 0, - 0, 0, 1, 0, 6, 0, 0, 0, + 3, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4, 0, 0, 3, 0, 1, 0, 68, 4, 0, 0, 2, 0, 1, 0, - 7, 0, 0, 0, 6, 0, 0, 0, - 0, 0, 1, 0, 7, 0, 0, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 65, 4, 0, 0, 194, 0, 0, 0, + 65, 4, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 68, 4, 0, 0, 3, 0, 1, 0, - 80, 4, 0, 0, 2, 0, 1, 0, - 8, 0, 0, 0, 7, 0, 0, 0, - 0, 0, 1, 0, 8, 0, 0, 0, + 64, 4, 0, 0, 3, 0, 1, 0, + 76, 4, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 77, 4, 0, 0, 66, 0, 0, 0, + 73, 4, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4, 0, 0, 3, 0, 1, 0, 84, 4, 0, 0, 2, 0, 1, 0, - 9, 0, 0, 0, 8, 0, 0, 0, - 0, 0, 1, 0, 9, 0, 0, 0, + 6, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 4, 0, 0, 90, 0, 0, 0, + 81, 4, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 80, 4, 0, 0, 3, 0, 1, 0, - 92, 4, 0, 0, 2, 0, 1, 0, - 10, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 1, 0, 10, 0, 0, 0, + 84, 4, 0, 0, 3, 0, 1, 0, + 96, 4, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 89, 4, 0, 0, 74, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 88, 4, 0, 0, 3, 0, 1, 0, - 100, 4, 0, 0, 2, 0, 1, 0, - 11, 0, 0, 0, 10, 0, 0, 0, - 0, 0, 1, 0, 11, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 97, 4, 0, 0, 106, 0, 0, 0, + 93, 4, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 4, 0, 0, 3, 0, 1, 0, 108, 4, 0, 0, 2, 0, 1, 0, - 12, 0, 0, 0, 11, 0, 0, 0, - 0, 0, 1, 0, 12, 0, 0, 0, + 8, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 105, 4, 0, 0, 146, 0, 0, 0, + 105, 4, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 4, 0, 0, 3, 0, 1, 0, + 112, 4, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 4, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 4, 0, 0, 3, 0, 1, 0, 120, 4, 0, 0, 2, 0, 1, 0, - 13, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 1, 0, 13, 0, 0, 0, + 10, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 4, 0, 0, 82, 0, 0, 0, + 117, 4, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 4, 0, 0, 3, 0, 1, 0, 128, 4, 0, 0, 2, 0, 1, 0, - 14, 0, 0, 0, 13, 0, 0, 0, - 0, 0, 1, 0, 14, 0, 0, 0, + 11, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 125, 4, 0, 0, 154, 0, 0, 0, + 125, 4, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 128, 4, 0, 0, 3, 0, 1, 0, - 140, 4, 0, 0, 2, 0, 1, 0, - 15, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 1, 0, 15, 0, 0, 0, + 124, 4, 0, 0, 3, 0, 1, 0, + 136, 4, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 11, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 137, 4, 0, 0, 74, 0, 0, 0, + 133, 4, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 4, 0, 0, 3, 0, 1, 0, 148, 4, 0, 0, 2, 0, 1, 0, - 16, 0, 0, 0, 15, 0, 0, 0, - 0, 0, 1, 0, 16, 0, 0, 0, + 13, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 145, 4, 0, 0, 98, 0, 0, 0, + 145, 4, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 4, 0, 0, 3, 0, 1, 0, 156, 4, 0, 0, 2, 0, 1, 0, - 17, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 1, 0, 17, 0, 0, 0, + 14, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 4, 0, 0, 114, 0, 0, 0, + 153, 4, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 4, 0, 0, 3, 0, 1, 0, - 164, 4, 0, 0, 2, 0, 1, 0, - 18, 0, 0, 0, 17, 0, 0, 0, - 0, 0, 1, 0, 18, 0, 0, 0, + 156, 4, 0, 0, 3, 0, 1, 0, + 168, 4, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 4, 0, 0, 186, 0, 0, 0, + 165, 4, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 4, 0, 0, 3, 0, 1, 0, 176, 4, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 15, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 173, 4, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 4, 0, 0, 3, 0, 1, 0, + 184, 4, 0, 0, 2, 0, 1, 0, + 17, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 4, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 180, 4, 0, 0, 3, 0, 1, 0, + 192, 4, 0, 0, 2, 0, 1, 0, + 18, 0, 0, 0, 17, 0, 0, 0, + 0, 0, 1, 0, 18, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 189, 4, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 192, 4, 0, 0, 3, 0, 1, 0, + 204, 4, 0, 0, 2, 0, 1, 0, 19, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 4, 0, 0, 146, 0, 0, 0, + 201, 4, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 176, 4, 0, 0, 3, 0, 1, 0, - 188, 4, 0, 0, 2, 0, 1, 0, + 204, 4, 0, 0, 3, 0, 1, 0, + 216, 4, 0, 0, 2, 0, 1, 0, 20, 0, 0, 0, 18, 0, 0, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 185, 4, 0, 0, 114, 0, 0, 0, + 213, 4, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 184, 4, 0, 0, 3, 0, 1, 0, - 196, 4, 0, 0, 2, 0, 1, 0, + 212, 4, 0, 0, 3, 0, 1, 0, + 224, 4, 0, 0, 2, 0, 1, 0, 21, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 193, 4, 0, 0, 146, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 196, 4, 0, 0, 3, 0, 1, 0, - 208, 4, 0, 0, 2, 0, 1, 0, - 22, 0, 0, 0, 20, 0, 0, 0, - 0, 0, 1, 0, 22, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 205, 4, 0, 0, 178, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 208, 4, 0, 0, 3, 0, 1, 0, - 220, 4, 0, 0, 2, 0, 1, 0, - 23, 0, 0, 0, 21, 0, 0, 0, - 0, 0, 1, 0, 23, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 217, 4, 0, 0, 114, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 216, 4, 0, 0, 3, 0, 1, 0, - 228, 4, 0, 0, 2, 0, 1, 0, - 24, 0, 0, 0, 22, 0, 0, 0, - 0, 0, 1, 0, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 225, 4, 0, 0, 130, 0, 0, 0, + 221, 4, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 4, 0, 0, 3, 0, 1, 0, 236, 4, 0, 0, 2, 0, 1, 0, - 25, 0, 0, 0, 23, 0, 0, 0, - 0, 0, 1, 0, 25, 0, 0, 0, + 22, 0, 0, 0, 20, 0, 0, 0, + 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 4, 0, 0, 122, 0, 0, 0, + 233, 4, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 4, 0, 0, 3, 0, 1, 0, - 244, 4, 0, 0, 2, 0, 1, 0, - 26, 0, 0, 0, 64, 0, 0, 0, - 0, 0, 1, 0, 26, 0, 0, 0, + 236, 4, 0, 0, 3, 0, 1, 0, + 248, 4, 0, 0, 2, 0, 1, 0, + 23, 0, 0, 0, 21, 0, 0, 0, + 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 4, 0, 0, 146, 0, 0, 0, + 245, 4, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 4, 0, 0, 3, 0, 1, 0, 0, 5, 0, 0, 2, 0, 1, 0, - 27, 0, 0, 0, 65, 0, 0, 0, - 0, 0, 1, 0, 27, 0, 0, 0, + 24, 0, 0, 0, 22, 0, 0, 0, + 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 4, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 4, 0, 0, 3, 0, 1, 0, 8, 5, 0, 0, 2, 0, 1, 0, - 28, 0, 0, 0, 66, 0, 0, 0, - 0, 0, 1, 0, 28, 0, 0, 0, + 25, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 1, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 5, 0, 0, 130, 0, 0, 0, + 5, 5, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 3, 0, 1, 0, 16, 5, 0, 0, 2, 0, 1, 0, - 29, 0, 0, 0, 67, 0, 0, 0, - 0, 0, 1, 0, 29, 0, 0, 0, + 26, 0, 0, 0, 64, 0, 0, 0, + 0, 0, 1, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 5, 0, 0, 130, 0, 0, 0, + 13, 5, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 12, 5, 0, 0, 3, 0, 1, 0, - 24, 5, 0, 0, 2, 0, 1, 0, - 30, 0, 0, 0, 68, 0, 0, 0, - 0, 0, 1, 0, 30, 0, 0, 0, + 16, 5, 0, 0, 3, 0, 1, 0, + 28, 5, 0, 0, 2, 0, 1, 0, + 27, 0, 0, 0, 65, 0, 0, 0, + 0, 0, 1, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 5, 0, 0, 114, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 20, 5, 0, 0, 3, 0, 1, 0, - 32, 5, 0, 0, 2, 0, 1, 0, - 31, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 31, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 29, 5, 0, 0, 34, 0, 0, 0, + 25, 5, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 5, 0, 0, 3, 0, 1, 0, 36, 5, 0, 0, 2, 0, 1, 0, - 32, 0, 0, 0, 69, 0, 0, 0, - 0, 0, 1, 0, 32, 0, 0, 0, + 28, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 1, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 5, 0, 0, 226, 0, 0, 0, + 33, 5, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 5, 0, 0, 3, 0, 1, 0, + 44, 5, 0, 0, 2, 0, 1, 0, + 29, 0, 0, 0, 67, 0, 0, 0, + 0, 0, 1, 0, 29, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 5, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 5, 0, 0, 3, 0, 1, 0, 52, 5, 0, 0, 2, 0, 1, 0, - 33, 0, 0, 0, 2, 0, 0, 0, - 0, 0, 1, 0, 33, 0, 0, 0, + 30, 0, 0, 0, 68, 0, 0, 0, + 0, 0, 1, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 5, 0, 0, 186, 0, 0, 0, + 49, 5, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 48, 5, 0, 0, 3, 0, 1, 0, + 60, 5, 0, 0, 2, 0, 1, 0, + 31, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 31, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 57, 5, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 5, 0, 0, 3, 0, 1, 0, 64, 5, 0, 0, 2, 0, 1, 0, + 32, 0, 0, 0, 69, 0, 0, 0, + 0, 0, 1, 0, 32, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 61, 5, 0, 0, 226, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 5, 0, 0, 3, 0, 1, 0, + 80, 5, 0, 0, 2, 0, 1, 0, + 33, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 33, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 5, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 5, 0, 0, 3, 0, 1, 0, + 92, 5, 0, 0, 2, 0, 1, 0, 34, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 61, 5, 0, 0, 130, 0, 0, 0, + 89, 5, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 60, 5, 0, 0, 3, 0, 1, 0, - 72, 5, 0, 0, 2, 0, 1, 0, + 88, 5, 0, 0, 3, 0, 1, 0, + 100, 5, 0, 0, 2, 0, 1, 0, 35, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 5, 0, 0, 170, 0, 0, 0, + 97, 5, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 5, 0, 0, 3, 0, 1, 0, - 84, 5, 0, 0, 2, 0, 1, 0, + 100, 5, 0, 0, 3, 0, 1, 0, + 112, 5, 0, 0, 2, 0, 1, 0, 36, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 5, 0, 0, 162, 0, 0, 0, + 109, 5, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 84, 5, 0, 0, 3, 0, 1, 0, - 96, 5, 0, 0, 2, 0, 1, 0, + 112, 5, 0, 0, 3, 0, 1, 0, + 124, 5, 0, 0, 2, 0, 1, 0, + 37, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 37, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 121, 5, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 5, 0, 0, 3, 0, 1, 0, + 136, 5, 0, 0, 2, 0, 1, 0, 97, 99, 99, 101, 108, 80, 114, 101, 115, 115, 101, 100, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, @@ -2833,6 +2845,16 @@ static const ::capnp::_::AlignedData<627> b_f35cc4560bbf6ec2 = { 189, 250, 110, 106, 234, 200, 182, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 101, 101, 100, 98, 97, 99, 107, + 80, 117, 98, 108, 105, 99, 97, 116, + 105, 111, 110, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 147, 2, 64, 226, 130, 227, 146, 222, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -2844,12 +2866,13 @@ static const ::capnp::_::RawSchema* const d_f35cc4560bbf6ec2[] = { &s_918d19aa5875c92d, &s_bab0f51a74ffee6e, &s_d1dfbadb6a811fee, + &s_de92e382e2400293, }; -static const uint16_t m_f35cc4560bbf6ec2[] = {28, 0, 18, 1, 34, 2, 21, 20, 22, 17, 3, 19, 29, 4, 6, 5, 7, 8, 9, 35, 15, 31, 10, 32, 36, 16, 11, 12, 24, 23, 30, 33, 13, 26, 25, 27, 14}; -static const uint16_t i_f35cc4560bbf6ec2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36}; +static const uint16_t m_f35cc4560bbf6ec2[] = {28, 0, 18, 1, 34, 2, 21, 20, 22, 17, 3, 19, 29, 4, 6, 5, 7, 8, 37, 9, 35, 15, 31, 10, 32, 36, 16, 11, 12, 24, 23, 30, 33, 13, 26, 25, 27, 14}; +static const uint16_t i_f35cc4560bbf6ec2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37}; const ::capnp::_::RawSchema s_f35cc4560bbf6ec2 = { - 0xf35cc4560bbf6ec2, b_f35cc4560bbf6ec2.words, 627, d_f35cc4560bbf6ec2, m_f35cc4560bbf6ec2, - 4, 37, i_f35cc4560bbf6ec2, nullptr, nullptr, { &s_f35cc4560bbf6ec2, nullptr, nullptr, 0, 0, nullptr }, false + 0xf35cc4560bbf6ec2, b_f35cc4560bbf6ec2.words, 649, d_f35cc4560bbf6ec2, m_f35cc4560bbf6ec2, + 5, 38, i_f35cc4560bbf6ec2, nullptr, nullptr, { &s_f35cc4560bbf6ec2, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<232> b_918d19aa5875c92d = { @@ -3095,6 +3118,318 @@ const ::capnp::_::RawSchema s_918d19aa5875c92d = { 0, 13, i_918d19aa5875c92d, nullptr, nullptr, { &s_918d19aa5875c92d, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<301> b_de92e382e2400293 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 147, 2, 64, 226, 130, 227, 146, 222, + 31, 0, 0, 0, 1, 0, 8, 0, + 194, 110, 191, 11, 86, 196, 92, 243, + 6, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 154, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 191, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 115, 116, 111, 109, 46, 99, + 97, 112, 110, 112, 58, 83, 116, 97, + 114, 80, 105, 108, 111, 116, 67, 97, + 114, 83, 116, 97, 116, 101, 46, 70, + 101, 101, 100, 98, 97, 99, 107, 80, + 117, 98, 108, 105, 99, 97, 116, 105, + 111, 110, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 68, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 1, 0, 0, 3, 0, 1, 0, + 212, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 1, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 1, 0, 0, 3, 0, 1, 0, + 220, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 217, 1, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 220, 1, 0, 0, 3, 0, 1, 0, + 232, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 1, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 1, 0, 0, 3, 0, 1, 0, + 244, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 241, 1, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 1, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 252, 1, 0, 0, 3, 0, 1, 0, + 8, 2, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 2, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 2, 0, 0, 3, 0, 1, 0, + 20, 2, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 2, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 20, 2, 0, 0, 3, 0, 1, 0, + 32, 2, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 2, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 2, 0, 0, 3, 0, 1, 0, + 44, 2, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 2, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 2, 0, 0, 3, 0, 1, 0, + 56, 2, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 53, 2, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 52, 2, 0, 0, 3, 0, 1, 0, + 64, 2, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 17, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 61, 2, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 60, 2, 0, 0, 3, 0, 1, 0, + 72, 2, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 2, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 2, 0, 0, 3, 0, 1, 0, + 80, 2, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 2, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 2, 0, 0, 3, 0, 1, 0, + 92, 2, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 2, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 88, 2, 0, 0, 3, 0, 1, 0, + 100, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 2, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 2, 0, 0, 3, 0, 1, 0, + 108, 2, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 2, 0, 0, 194, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 2, 0, 0, 3, 0, 1, 0, + 120, 2, 0, 0, 2, 0, 1, 0, + 118, 101, 114, 115, 105, 111, 110, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 97, 114, 100, 83, 101, 115, 115, + 105, 111, 110, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 117, 98, 108, 105, 99, 97, 116, + 105, 111, 110, 83, 101, 113, 117, 101, + 110, 99, 101, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 117, 98, 108, 105, 99, 97, 116, + 105, 111, 110, 77, 111, 110, 111, 84, + 105, 109, 101, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 111, 117, 114, 99, 101, 67, 97, + 114, 83, 116, 97, 116, 101, 77, 111, + 110, 111, 84, 105, 109, 101, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 117, 116, 112, 117, 116, 75, 110, + 111, 119, 110, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 117, 116, 112, 117, 116, 67, 111, + 110, 116, 114, 111, 108, 77, 111, 110, + 111, 84, 105, 109, 101, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 117, 116, 112, 117, 116, 65, 112, + 112, 108, 121, 77, 111, 110, 111, 84, + 105, 109, 101, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 117, 116, 112, 117, 116, 78, 97, + 116, 105, 118, 101, 73, 100, 101, 110, + 116, 105, 116, 121, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 117, 116, 112, 117, 116, 67, 97, + 114, 114, 105, 101, 114, 83, 101, 113, + 117, 101, 110, 99, 101, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 112, 111, 114, 116, 101, 100, + 84, 111, 114, 113, 117, 101, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 117, 116, 112, 117, 116, 86, 97, + 108, 105, 100, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 66, 111, 100, 121, 83, 104, + 97, 50, 53, 54, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 112, 112, 108, 121, 69, 118, 105, + 100, 101, 110, 99, 101, 83, 104, 97, + 50, 53, 54, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 111, 117, 114, 99, 101, 69, 112, + 111, 99, 104, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 112, 83, 104, 97, 50, 53, 54, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 97, 114, 114, 105, 101, 114, 79, + 98, 115, 101, 114, 118, 101, 100, 77, + 111, 110, 111, 84, 105, 109, 101, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_de92e382e2400293 = b_de92e382e2400293.words; +#if !CAPNP_LITE +static const uint16_t m_de92e382e2400293[] = {13, 1, 16, 12, 15, 7, 9, 6, 5, 8, 11, 3, 2, 10, 4, 14, 0}; +static const uint16_t i_de92e382e2400293[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; +const ::capnp::_::RawSchema s_de92e382e2400293 = { + 0xde92e382e2400293, b_de92e382e2400293.words, 301, nullptr, m_de92e382e2400293, + 0, 17, i_de92e382e2400293, nullptr, nullptr, { &s_de92e382e2400293, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE static const ::capnp::_::AlignedData<101> b_da96579883444c35 = { { 0, 0, 0, 0, 5, 0, 6, 0, 53, 76, 68, 131, 152, 87, 150, 218, @@ -3207,17 +3542,17 @@ const ::capnp::_::RawSchema s_da96579883444c35 = { 0, 5, i_da96579883444c35, nullptr, nullptr, { &s_da96579883444c35, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<875> b_a1680744031fdb2d = { +static const ::capnp::_::AlignedData<962> b_a1680744031fdb2d = { { 0, 0, 0, 0, 5, 0, 6, 0, 45, 219, 31, 3, 68, 7, 104, 161, - 13, 0, 0, 0, 1, 0, 15, 0, + 13, 0, 0, 0, 1, 0, 17, 0, 89, 10, 85, 29, 102, 186, 38, 181, - 20, 0, 7, 0, 0, 0, 0, 0, + 21, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 178, 0, 0, 0, 29, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 0, 0, 0, 47, 11, 0, 0, + 117, 0, 0, 0, 71, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 117, 115, 116, 111, 109, 46, 99, @@ -3247,364 +3582,399 @@ static const ::capnp::_::AlignedData<875> b_a1680744031fdb2d = { 67, 111, 110, 100, 105, 116, 105, 111, 110, 97, 108, 77, 111, 100, 101, 83, 116, 97, 116, 101, 0, 0, 0, 0, - 204, 0, 0, 0, 3, 0, 4, 0, + 224, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 133, 5, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 128, 5, 0, 0, 3, 0, 1, 0, - 140, 5, 0, 0, 2, 0, 1, 0, - 1, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 137, 5, 0, 0, 74, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 136, 5, 0, 0, 3, 0, 1, 0, - 148, 5, 0, 0, 2, 0, 1, 0, - 2, 0, 0, 0, 2, 0, 0, 0, - 0, 0, 1, 0, 2, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 145, 5, 0, 0, 66, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 140, 5, 0, 0, 3, 0, 1, 0, - 152, 5, 0, 0, 2, 0, 1, 0, - 3, 0, 0, 0, 3, 0, 0, 0, - 0, 0, 1, 0, 3, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 149, 5, 0, 0, 106, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 148, 5, 0, 0, 3, 0, 1, 0, - 160, 5, 0, 0, 2, 0, 1, 0, - 4, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 1, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 157, 5, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 152, 5, 0, 0, 3, 0, 1, 0, - 164, 5, 0, 0, 2, 0, 1, 0, - 5, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 5, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 161, 5, 0, 0, 114, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 160, 5, 0, 0, 3, 0, 1, 0, - 172, 5, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 5, 0, 0, 0, - 0, 0, 1, 0, 6, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 169, 5, 0, 0, 82, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 168, 5, 0, 0, 3, 0, 1, 0, - 180, 5, 0, 0, 2, 0, 1, 0, - 7, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 7, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 177, 5, 0, 0, 114, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 176, 5, 0, 0, 3, 0, 1, 0, - 188, 5, 0, 0, 2, 0, 1, 0, - 8, 0, 0, 0, 128, 0, 0, 0, - 0, 0, 1, 0, 8, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 185, 5, 0, 0, 66, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 180, 5, 0, 0, 3, 0, 1, 0, - 192, 5, 0, 0, 2, 0, 1, 0, - 9, 0, 0, 0, 129, 0, 0, 0, - 0, 0, 1, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 189, 5, 0, 0, 98, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 188, 5, 0, 0, 3, 0, 1, 0, - 200, 5, 0, 0, 2, 0, 1, 0, - 10, 0, 0, 0, 6, 0, 0, 0, - 0, 0, 1, 0, 10, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 197, 5, 0, 0, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 196, 5, 0, 0, 3, 0, 1, 0, - 208, 5, 0, 0, 2, 0, 1, 0, - 11, 0, 0, 0, 7, 0, 0, 0, - 0, 0, 1, 0, 11, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 205, 5, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 200, 5, 0, 0, 3, 0, 1, 0, - 212, 5, 0, 0, 2, 0, 1, 0, - 12, 0, 0, 0, 5, 0, 0, 0, - 0, 0, 1, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 209, 5, 0, 0, 90, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 208, 5, 0, 0, 3, 0, 1, 0, - 220, 5, 0, 0, 2, 0, 1, 0, - 13, 0, 0, 0, 6, 0, 0, 0, - 0, 0, 1, 0, 13, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 217, 5, 0, 0, 154, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 220, 5, 0, 0, 3, 0, 1, 0, - 232, 5, 0, 0, 2, 0, 1, 0, - 14, 0, 0, 0, 130, 0, 0, 0, - 0, 0, 1, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 229, 5, 0, 0, 98, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 228, 5, 0, 0, 3, 0, 1, 0, - 240, 5, 0, 0, 2, 0, 1, 0, - 15, 0, 0, 0, 7, 0, 0, 0, - 0, 0, 1, 0, 15, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 237, 5, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 232, 5, 0, 0, 3, 0, 1, 0, - 244, 5, 0, 0, 2, 0, 1, 0, - 16, 0, 0, 0, 8, 0, 0, 0, - 0, 0, 1, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 241, 5, 0, 0, 106, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 240, 5, 0, 0, 3, 0, 1, 0, - 252, 5, 0, 0, 2, 0, 1, 0, - 17, 0, 0, 0, 131, 0, 0, 0, - 0, 0, 1, 0, 17, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 249, 5, 0, 0, 90, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 248, 5, 0, 0, 3, 0, 1, 0, - 4, 6, 0, 0, 2, 0, 1, 0, - 18, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 1, 0, 18, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 1, 6, 0, 0, 146, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 4, 6, 0, 0, 3, 0, 1, 0, - 16, 6, 0, 0, 2, 0, 1, 0, - 19, 0, 0, 0, 132, 0, 0, 0, - 0, 0, 1, 0, 19, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 13, 6, 0, 0, 90, 0, 0, 0, + 17, 6, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 6, 0, 0, 3, 0, 1, 0, 24, 6, 0, 0, 2, 0, 1, 0, - 20, 0, 0, 0, 5, 0, 0, 0, - 0, 0, 1, 0, 20, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 6, 0, 0, 90, 0, 0, 0, + 21, 6, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 6, 0, 0, 3, 0, 1, 0, 32, 6, 0, 0, 2, 0, 1, 0, - 21, 0, 0, 0, 6, 0, 0, 0, - 0, 0, 1, 0, 21, 0, 0, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 6, 0, 0, 122, 0, 0, 0, + 29, 6, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28, 6, 0, 0, 3, 0, 1, 0, - 40, 6, 0, 0, 2, 0, 1, 0, - 22, 0, 0, 0, 8, 0, 0, 0, - 0, 0, 1, 0, 22, 0, 0, 0, + 24, 6, 0, 0, 3, 0, 1, 0, + 36, 6, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 6, 0, 0, 58, 0, 0, 0, + 33, 6, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 6, 0, 0, 3, 0, 1, 0, 44, 6, 0, 0, 2, 0, 1, 0, - 23, 0, 0, 0, 7, 0, 0, 0, - 0, 0, 1, 0, 23, 0, 0, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 6, 0, 0, 138, 0, 0, 0, + 41, 6, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 6, 0, 0, 3, 0, 1, 0, + 48, 6, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 6, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 6, 0, 0, 3, 0, 1, 0, 56, 6, 0, 0, 2, 0, 1, 0, - 24, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 1, 0, 24, 0, 0, 0, + 6, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 6, 0, 0, 106, 0, 0, 0, + 53, 6, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 6, 0, 0, 3, 0, 1, 0, 64, 6, 0, 0, 2, 0, 1, 0, - 25, 0, 0, 0, 10, 0, 0, 0, - 0, 0, 1, 0, 25, 0, 0, 0, + 7, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 61, 6, 0, 0, 194, 0, 0, 0, + 61, 6, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 60, 6, 0, 0, 3, 0, 1, 0, + 72, 6, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 128, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 6, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 6, 0, 0, 3, 0, 1, 0, 76, 6, 0, 0, 2, 0, 1, 0, - 26, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 1, 0, 26, 0, 0, 0, + 9, 0, 0, 0, 129, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 73, 6, 0, 0, 114, 0, 0, 0, + 73, 6, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 6, 0, 0, 3, 0, 1, 0, 84, 6, 0, 0, 2, 0, 1, 0, - 27, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 1, 0, 27, 0, 0, 0, + 10, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 6, 0, 0, 186, 0, 0, 0, + 81, 6, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 6, 0, 0, 3, 0, 1, 0, + 92, 6, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 6, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 6, 0, 0, 3, 0, 1, 0, 96, 6, 0, 0, 2, 0, 1, 0, - 28, 0, 0, 0, 10, 0, 0, 0, - 0, 0, 1, 0, 28, 0, 0, 0, + 12, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 93, 6, 0, 0, 202, 0, 0, 0, + 93, 6, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 100, 6, 0, 0, 3, 0, 1, 0, - 112, 6, 0, 0, 2, 0, 1, 0, - 29, 0, 0, 0, 11, 0, 0, 0, - 0, 0, 1, 0, 29, 0, 0, 0, + 92, 6, 0, 0, 3, 0, 1, 0, + 104, 6, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 109, 6, 0, 0, 82, 0, 0, 0, + 101, 6, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 108, 6, 0, 0, 3, 0, 1, 0, - 120, 6, 0, 0, 2, 0, 1, 0, - 30, 0, 0, 0, 11, 0, 0, 0, - 0, 0, 1, 0, 30, 0, 0, 0, + 104, 6, 0, 0, 3, 0, 1, 0, + 116, 6, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 6, 0, 0, 114, 0, 0, 0, + 113, 6, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 6, 0, 0, 3, 0, 1, 0, + 124, 6, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 121, 6, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 6, 0, 0, 3, 0, 1, 0, 128, 6, 0, 0, 2, 0, 1, 0, - 31, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 1, 0, 31, 0, 0, 0, + 16, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 125, 6, 0, 0, 50, 0, 0, 0, + 125, 6, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 6, 0, 0, 3, 0, 1, 0, - 132, 6, 0, 0, 2, 0, 1, 0, - 32, 0, 0, 0, 13, 0, 0, 0, - 0, 0, 1, 0, 32, 0, 0, 0, + 124, 6, 0, 0, 3, 0, 1, 0, + 136, 6, 0, 0, 2, 0, 1, 0, + 17, 0, 0, 0, 131, 0, 0, 0, + 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 129, 6, 0, 0, 130, 0, 0, 0, + 133, 6, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 128, 6, 0, 0, 3, 0, 1, 0, - 140, 6, 0, 0, 2, 0, 1, 0, - 33, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 1, 0, 33, 0, 0, 0, + 132, 6, 0, 0, 3, 0, 1, 0, + 144, 6, 0, 0, 2, 0, 1, 0, + 18, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 137, 6, 0, 0, 98, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 136, 6, 0, 0, 3, 0, 1, 0, - 148, 6, 0, 0, 2, 0, 1, 0, - 34, 0, 0, 0, 15, 0, 0, 0, - 0, 0, 1, 0, 34, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 145, 6, 0, 0, 122, 0, 0, 0, + 141, 6, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 6, 0, 0, 3, 0, 1, 0, 156, 6, 0, 0, 2, 0, 1, 0, - 35, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 1, 0, 35, 0, 0, 0, + 19, 0, 0, 0, 132, 0, 0, 0, + 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 6, 0, 0, 122, 0, 0, 0, + 153, 6, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 6, 0, 0, 3, 0, 1, 0, 164, 6, 0, 0, 2, 0, 1, 0, - 36, 0, 0, 0, 17, 0, 0, 0, - 0, 0, 1, 0, 36, 0, 0, 0, + 20, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 6, 0, 0, 114, 0, 0, 0, + 161, 6, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 6, 0, 0, 3, 0, 1, 0, 172, 6, 0, 0, 2, 0, 1, 0, - 37, 0, 0, 0, 17, 0, 0, 0, - 0, 0, 1, 0, 37, 0, 0, 0, + 21, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 6, 0, 0, 186, 0, 0, 0, + 169, 6, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 168, 6, 0, 0, 3, 0, 1, 0, + 180, 6, 0, 0, 2, 0, 1, 0, + 22, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 22, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 177, 6, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 6, 0, 0, 3, 0, 1, 0, 184, 6, 0, 0, 2, 0, 1, 0, - 38, 0, 0, 0, 133, 0, 0, 0, - 0, 0, 1, 0, 38, 0, 0, 0, + 23, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 181, 6, 0, 0, 210, 0, 0, 0, + 181, 6, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 6, 0, 0, 3, 0, 1, 0, - 200, 6, 0, 0, 2, 0, 1, 0, - 39, 0, 0, 0, 134, 0, 0, 0, - 0, 0, 1, 0, 39, 0, 0, 0, + 184, 6, 0, 0, 3, 0, 1, 0, + 196, 6, 0, 0, 2, 0, 1, 0, + 24, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 6, 0, 0, 138, 0, 0, 0, + 193, 6, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 6, 0, 0, 3, 0, 1, 0, - 212, 6, 0, 0, 2, 0, 1, 0, - 40, 0, 0, 0, 135, 0, 0, 0, - 0, 0, 1, 0, 40, 0, 0, 0, + 192, 6, 0, 0, 3, 0, 1, 0, + 204, 6, 0, 0, 2, 0, 1, 0, + 25, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 6, 0, 0, 170, 0, 0, 0, + 201, 6, 0, 0, 194, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 6, 0, 0, 3, 0, 1, 0, + 216, 6, 0, 0, 2, 0, 1, 0, + 26, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 6, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 6, 0, 0, 3, 0, 1, 0, 224, 6, 0, 0, 2, 0, 1, 0, - 41, 0, 0, 0, 18, 0, 0, 0, - 0, 0, 1, 0, 41, 0, 0, 0, + 27, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 6, 0, 0, 122, 0, 0, 0, + 221, 6, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 220, 6, 0, 0, 3, 0, 1, 0, - 248, 6, 0, 0, 2, 0, 1, 0, - 42, 0, 0, 0, 24, 0, 0, 0, - 0, 0, 1, 0, 42, 0, 0, 0, + 224, 6, 0, 0, 3, 0, 1, 0, + 236, 6, 0, 0, 2, 0, 1, 0, + 28, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 6, 0, 0, 178, 0, 0, 0, + 233, 6, 0, 0, 202, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 240, 6, 0, 0, 3, 0, 1, 0, + 252, 6, 0, 0, 2, 0, 1, 0, + 29, 0, 0, 0, 11, 0, 0, 0, + 0, 0, 1, 0, 29, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 249, 6, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 6, 0, 0, 3, 0, 1, 0, 4, 7, 0, 0, 2, 0, 1, 0, - 43, 0, 0, 0, 136, 0, 0, 0, - 0, 0, 1, 0, 43, 0, 0, 0, + 30, 0, 0, 0, 11, 0, 0, 0, + 0, 0, 1, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 7, 0, 0, 202, 0, 0, 0, + 1, 7, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 7, 0, 0, 3, 0, 1, 0, - 20, 7, 0, 0, 2, 0, 1, 0, - 44, 0, 0, 0, 25, 0, 0, 0, - 0, 0, 1, 0, 44, 0, 0, 0, + 0, 7, 0, 0, 3, 0, 1, 0, + 12, 7, 0, 0, 2, 0, 1, 0, + 31, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 1, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 7, 0, 0, 170, 0, 0, 0, + 9, 7, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 7, 0, 0, 3, 0, 1, 0, + 16, 7, 0, 0, 2, 0, 1, 0, + 32, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 32, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 7, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 7, 0, 0, 3, 0, 1, 0, + 24, 7, 0, 0, 2, 0, 1, 0, + 33, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 33, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 7, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 7, 0, 0, 3, 0, 1, 0, 32, 7, 0, 0, 2, 0, 1, 0, - 45, 0, 0, 0, 137, 0, 0, 0, - 0, 0, 1, 0, 45, 0, 0, 0, + 34, 0, 0, 0, 15, 0, 0, 0, + 0, 0, 1, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 7, 0, 0, 194, 0, 0, 0, + 29, 7, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 32, 7, 0, 0, 3, 0, 1, 0, - 44, 7, 0, 0, 2, 0, 1, 0, - 46, 0, 0, 0, 26, 0, 0, 0, - 0, 0, 1, 0, 46, 0, 0, 0, + 28, 7, 0, 0, 3, 0, 1, 0, + 40, 7, 0, 0, 2, 0, 1, 0, + 35, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 7, 0, 0, 210, 0, 0, 0, + 37, 7, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 48, 7, 0, 0, 3, 0, 1, 0, - 60, 7, 0, 0, 2, 0, 1, 0, - 47, 0, 0, 0, 138, 0, 0, 0, - 0, 0, 1, 0, 47, 0, 0, 0, + 36, 7, 0, 0, 3, 0, 1, 0, + 48, 7, 0, 0, 2, 0, 1, 0, + 36, 0, 0, 0, 17, 0, 0, 0, + 0, 0, 1, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 57, 7, 0, 0, 234, 0, 0, 0, + 45, 7, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 64, 7, 0, 0, 3, 0, 1, 0, - 76, 7, 0, 0, 2, 0, 1, 0, - 48, 0, 0, 0, 19, 0, 0, 0, - 0, 0, 1, 0, 48, 0, 0, 0, + 44, 7, 0, 0, 3, 0, 1, 0, + 56, 7, 0, 0, 2, 0, 1, 0, + 37, 0, 0, 0, 17, 0, 0, 0, + 0, 0, 1, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 73, 7, 0, 0, 114, 0, 0, 0, + 53, 7, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 56, 7, 0, 0, 3, 0, 1, 0, + 68, 7, 0, 0, 2, 0, 1, 0, + 38, 0, 0, 0, 133, 0, 0, 0, + 0, 0, 1, 0, 38, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 7, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 7, 0, 0, 3, 0, 1, 0, 84, 7, 0, 0, 2, 0, 1, 0, - 49, 0, 0, 0, 139, 0, 0, 0, - 0, 0, 1, 0, 49, 0, 0, 0, + 39, 0, 0, 0, 134, 0, 0, 0, + 0, 0, 1, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 7, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 7, 0, 0, 3, 0, 1, 0, 96, 7, 0, 0, 2, 0, 1, 0, - 50, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 1, 0, 50, 0, 0, 0, + 40, 0, 0, 0, 135, 0, 0, 0, + 0, 0, 1, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 93, 7, 0, 0, 194, 0, 0, 0, + 93, 7, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 7, 0, 0, 3, 0, 1, 0, 108, 7, 0, 0, 2, 0, 1, 0, + 41, 0, 0, 0, 18, 0, 0, 0, + 0, 0, 1, 0, 41, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 7, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 7, 0, 0, 3, 0, 1, 0, + 132, 7, 0, 0, 2, 0, 1, 0, + 42, 0, 0, 0, 24, 0, 0, 0, + 0, 0, 1, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 7, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 7, 0, 0, 3, 0, 1, 0, + 144, 7, 0, 0, 2, 0, 1, 0, + 43, 0, 0, 0, 136, 0, 0, 0, + 0, 0, 1, 0, 43, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 7, 0, 0, 202, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 148, 7, 0, 0, 3, 0, 1, 0, + 160, 7, 0, 0, 2, 0, 1, 0, + 44, 0, 0, 0, 25, 0, 0, 0, + 0, 0, 1, 0, 44, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 7, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 7, 0, 0, 3, 0, 1, 0, + 172, 7, 0, 0, 2, 0, 1, 0, + 45, 0, 0, 0, 137, 0, 0, 0, + 0, 0, 1, 0, 45, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 169, 7, 0, 0, 194, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 7, 0, 0, 3, 0, 1, 0, + 184, 7, 0, 0, 2, 0, 1, 0, + 46, 0, 0, 0, 26, 0, 0, 0, + 0, 0, 1, 0, 46, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 7, 0, 0, 210, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 7, 0, 0, 3, 0, 1, 0, + 200, 7, 0, 0, 2, 0, 1, 0, + 47, 0, 0, 0, 138, 0, 0, 0, + 0, 0, 1, 0, 47, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 7, 0, 0, 234, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 7, 0, 0, 3, 0, 1, 0, + 216, 7, 0, 0, 2, 0, 1, 0, + 48, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 1, 0, 48, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 7, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 212, 7, 0, 0, 3, 0, 1, 0, + 224, 7, 0, 0, 2, 0, 1, 0, + 49, 0, 0, 0, 139, 0, 0, 0, + 0, 0, 1, 0, 49, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 221, 7, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 224, 7, 0, 0, 3, 0, 1, 0, + 236, 7, 0, 0, 2, 0, 1, 0, + 50, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 233, 7, 0, 0, 194, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 7, 0, 0, 3, 0, 1, 0, + 248, 7, 0, 0, 2, 0, 1, 0, + 51, 0, 0, 0, 140, 0, 0, 0, + 0, 0, 1, 0, 51, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 245, 7, 0, 0, 202, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 252, 7, 0, 0, 3, 0, 1, 0, + 8, 8, 0, 0, 2, 0, 1, 0, + 52, 0, 0, 0, 15, 0, 0, 0, + 0, 0, 1, 0, 52, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 8, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 8, 0, 0, 3, 0, 1, 0, + 20, 8, 0, 0, 2, 0, 1, 0, + 53, 0, 0, 0, 27, 0, 0, 0, + 0, 0, 1, 0, 53, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 8, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 20, 8, 0, 0, 3, 0, 1, 0, + 32, 8, 0, 0, 2, 0, 1, 0, + 54, 0, 0, 0, 20, 0, 0, 0, + 0, 0, 1, 0, 54, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 8, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 8, 0, 0, 3, 0, 1, 0, + 44, 8, 0, 0, 2, 0, 1, 0, + 55, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 0, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 8, 0, 0, 210, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 48, 8, 0, 0, 3, 0, 1, 0, + 60, 8, 0, 0, 2, 0, 1, 0, 115, 108, 111, 116, 73, 100, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4081,6 +4451,58 @@ static const ::capnp::_::AlignedData<875> b_a1680744031fdb2d = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 117, 116, 111, 109, 97, 116, 105, + 99, 65, 100, 111, 112, 116, 105, 111, + 110, 80, 101, 110, 100, 105, 110, 103, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 101, 110, 100, 105, 110, 103, 83, + 105, 110, 99, 101, 77, 111, 110, 111, + 84, 105, 109, 101, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 115, 116, 82, 101, 99, 101, + 105, 118, 101, 100, 83, 112, 101, 101, + 100, 76, 105, 109, 105, 116, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 115, 116, 82, 101, 99, 101, + 105, 118, 101, 100, 83, 111, 117, 114, + 99, 101, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 115, 116, 82, 101, 99, 101, + 105, 118, 101, 100, 65, 100, 106, 117, + 115, 116, 101, 100, 76, 105, 109, 105, + 116, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -4093,11 +4515,11 @@ static const ::capnp::_::RawSchema* const d_a1680744031fdb2d[] = { &s_f58695a93b568caf, &s_fb41e1a738d832cb, }; -static const uint16_t m_a1680744031fdb2d[] = {42, 35, 13, 23, 24, 50, 29, 30, 32, 31, 20, 9, 40, 16, 37, 8, 7, 14, 43, 17, 38, 19, 45, 47, 39, 10, 15, 48, 44, 36, 18, 46, 21, 4, 49, 6, 0, 1, 11, 26, 27, 25, 41, 28, 12, 22, 34, 33, 2, 3, 5}; -static const uint16_t i_a1680744031fdb2d[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50}; +static const uint16_t m_a1680744031fdb2d[] = {42, 35, 13, 23, 24, 51, 50, 29, 30, 32, 31, 20, 9, 40, 16, 37, 8, 7, 14, 43, 17, 38, 19, 45, 47, 39, 55, 54, 53, 10, 15, 48, 44, 52, 36, 18, 46, 21, 4, 49, 6, 0, 1, 11, 26, 27, 25, 41, 28, 12, 22, 34, 33, 2, 3, 5}; +static const uint16_t i_a1680744031fdb2d[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55}; const ::capnp::_::RawSchema s_a1680744031fdb2d = { - 0xa1680744031fdb2d, b_a1680744031fdb2d.words, 875, d_a1680744031fdb2d, m_a1680744031fdb2d, - 5, 51, i_a1680744031fdb2d, nullptr, nullptr, { &s_a1680744031fdb2d, nullptr, nullptr, 0, 0, nullptr }, false + 0xa1680744031fdb2d, b_a1680744031fdb2d.words, 962, d_a1680744031fdb2d, m_a1680744031fdb2d, + 5, 56, i_a1680744031fdb2d, nullptr, nullptr, { &s_a1680744031fdb2d, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<81> b_a7940385bedd1d4c = { @@ -8142,30 +8564,100 @@ const ::capnp::_::RawSchema s_b45fd9e3634d228f = { }; #endif // !CAPNP_LITE CAPNP_DEFINE_ENUM(Stream_b45fd9e3634d228f, b45fd9e3634d228f); -static const ::capnp::_::AlignedData<17> b_a5cd762cd951a455 = { +static const ::capnp::_::AlignedData<80> b_a5cd762cd951a455 = { { 0, 0, 0, 0, 5, 0, 6, 0, 85, 164, 81, 217, 44, 118, 205, 165, - 13, 0, 0, 0, 1, 0, 0, 0, + 13, 0, 0, 0, 1, 0, 1, 0, 89, 10, 85, 29, 102, 186, 38, 181, - 0, 0, 7, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 234, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 58, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 117, 115, 116, 111, 109, 46, 99, - 97, 112, 110, 112, 58, 67, 117, 115, - 116, 111, 109, 82, 101, 115, 101, 114, - 118, 101, 100, 53, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 1, 0, } + 97, 112, 110, 112, 58, 83, 116, 97, + 114, 80, 105, 108, 111, 116, 83, 116, + 101, 101, 114, 105, 110, 103, 84, 114, + 97, 99, 107, 105, 110, 103, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 0, 0, 0, 3, 0, 1, 0, + 104, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 0, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 0, 0, 0, 3, 0, 1, 0, + 120, 0, 0, 0, 2, 0, 1, 0, + 118, 101, 114, 115, 105, 111, 110, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 111, 117, 114, 99, 101, 82, 101, + 99, 101, 105, 112, 116, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 73, 81, 128, 117, 62, 210, 226, 159, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 114, 105, 97, 108, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 163, 225, 14, 188, 36, 114, 246, 230, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 115, 116, 111, 114, 101, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 18, 128, 49, 217, 190, 87, 98, 239, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } }; ::capnp::word const* const bp_a5cd762cd951a455 = b_a5cd762cd951a455.words; #if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_a5cd762cd951a455[] = { + &s_9fe2d23e75805149, + &s_e6f67224bc0ee1a3, + &s_ef6257bed9318012, +}; +static const uint16_t m_a5cd762cd951a455[] = {3, 1, 2, 0}; +static const uint16_t i_a5cd762cd951a455[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_a5cd762cd951a455 = { - 0xa5cd762cd951a455, b_a5cd762cd951a455.words, 17, nullptr, nullptr, - 0, 0, nullptr, nullptr, nullptr, { &s_a5cd762cd951a455, nullptr, nullptr, 0, 0, nullptr }, false + 0xa5cd762cd951a455, b_a5cd762cd951a455.words, 80, d_a5cd762cd951a455, m_a5cd762cd951a455, + 3, 4, i_a5cd762cd951a455, nullptr, nullptr, { &s_a5cd762cd951a455, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<464> b_f98d843bfd7004a3 = { @@ -10601,17 +11093,17 @@ const ::capnp::_::RawSchema s_f187d96cea4f09df = { }; #endif // !CAPNP_LITE CAPNP_DEFINE_ENUM(Status_f187d96cea4f09df, f187d96cea4f09df); -static const ::capnp::_::AlignedData<202> b_c2243c65e0340384 = { +static const ::capnp::_::AlignedData<241> b_c2243c65e0340384 = { { 0, 0, 0, 0, 5, 0, 6, 0, 132, 3, 52, 224, 101, 60, 36, 194, 13, 0, 0, 0, 1, 0, 4, 0, 89, 10, 85, 29, 102, 186, 38, 181, - 3, 0, 7, 0, 0, 0, 0, 0, + 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, - 37, 0, 0, 0, 23, 0, 0, 0, + 37, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 111, 2, 0, 0, + 69, 0, 0, 0, 223, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 117, 115, 116, 111, 109, 46, 99, @@ -10619,89 +11111,108 @@ static const ::capnp::_::AlignedData<202> b_c2243c65e0340384 = { 114, 80, 105, 108, 111, 116, 76, 97, 116, 101, 114, 97, 108, 83, 116, 97, 116, 101, 0, 0, 0, 0, 0, 0, - 4, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 1, 0, 1, 0, 25, 125, 103, 99, 66, 31, 247, 136, - 1, 0, 0, 0, 114, 0, 0, 0, + 9, 0, 0, 0, 114, 0, 0, 0, + 182, 96, 107, 166, 93, 61, 105, 197, + 9, 0, 0, 0, 162, 0, 0, 0, 76, 97, 110, 101, 67, 101, 110, 116, 101, 114, 105, 110, 103, 0, 0, 0, - 44, 0, 0, 0, 3, 0, 4, 0, + 70, 101, 101, 100, 98, 97, 99, 107, + 67, 111, 110, 115, 117, 109, 112, 116, + 105, 111, 110, 0, 0, 0, 0, 0, + 52, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 1, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 32, 1, 0, 0, 3, 0, 1, 0, - 44, 1, 0, 0, 2, 0, 1, 0, - 1, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 41, 1, 0, 0, 146, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 44, 1, 0, 0, 3, 0, 1, 0, - 56, 1, 0, 0, 2, 0, 1, 0, - 2, 0, 0, 0, 2, 0, 0, 0, - 0, 0, 1, 0, 2, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 53, 1, 0, 0, 114, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 52, 1, 0, 0, 3, 0, 1, 0, - 64, 1, 0, 0, 2, 0, 1, 0, - 3, 0, 0, 0, 3, 0, 0, 0, - 0, 0, 1, 0, 3, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 61, 1, 0, 0, 98, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 60, 1, 0, 0, 3, 0, 1, 0, - 72, 1, 0, 0, 2, 0, 1, 0, - 4, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 1, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 69, 1, 0, 0, 106, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 68, 1, 0, 0, 3, 0, 1, 0, - 80, 1, 0, 0, 2, 0, 1, 0, - 5, 0, 0, 0, 5, 0, 0, 0, - 0, 0, 1, 0, 5, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 77, 1, 0, 0, 170, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 80, 1, 0, 0, 3, 0, 1, 0, - 92, 1, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 6, 0, 0, 0, - 0, 0, 1, 0, 6, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 89, 1, 0, 0, 122, 0, 0, 0, + 93, 1, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 1, 0, 0, 3, 0, 1, 0, 100, 1, 0, 0, 2, 0, 1, 0, - 7, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 7, 0, 0, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 1, 0, 0, 122, 0, 0, 0, + 97, 1, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 1, 0, 0, 3, 0, 1, 0, - 108, 1, 0, 0, 2, 0, 1, 0, - 8, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 8, 0, 0, 0, + 100, 1, 0, 0, 3, 0, 1, 0, + 112, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 105, 1, 0, 0, 114, 0, 0, 0, + 109, 1, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 104, 1, 0, 0, 3, 0, 1, 0, - 116, 1, 0, 0, 2, 0, 1, 0, - 9, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 9, 0, 0, 0, + 108, 1, 0, 0, 3, 0, 1, 0, + 120, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 113, 1, 0, 0, 138, 0, 0, 0, + 117, 1, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 1, 0, 0, 3, 0, 1, 0, 128, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 1, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 1, 0, 0, 3, 0, 1, 0, + 136, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 1, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 136, 1, 0, 0, 3, 0, 1, 0, + 148, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 145, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 144, 1, 0, 0, 3, 0, 1, 0, + 156, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 153, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 152, 1, 0, 0, 3, 0, 1, 0, + 164, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 161, 1, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 1, 0, 0, 3, 0, 1, 0, + 172, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 169, 1, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 1, 0, 0, 3, 0, 1, 0, + 184, 1, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 125, 1, 0, 0, 162, 0, 0, 0, + 181, 1, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 128, 1, 0, 0, 3, 0, 1, 0, - 140, 1, 0, 0, 2, 0, 1, 0, + 184, 1, 0, 0, 3, 0, 1, 0, + 196, 1, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 1, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 1, 0, 0, 3, 0, 1, 0, + 208, 1, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 1, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 1, 0, 0, 3, 0, 1, 0, + 220, 1, 0, 0, 2, 0, 1, 0, 97, 99, 116, 105, 118, 101, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10801,6 +11312,26 @@ static const ::capnp::_::AlignedData<202> b_c2243c65e0340384 = { 189, 250, 110, 106, 234, 200, 182, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 101, 101, 100, 98, 97, 99, 107, + 67, 111, 110, 115, 117, 109, 112, 116, + 105, 111, 110, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 182, 96, 107, 166, 93, 61, 105, 197, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 114, 97, 99, 107, 105, 110, 103, + 65, 112, 112, 108, 105, 99, 97, 116, + 105, 111, 110, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 193, 27, 127, 6, 130, 185, 133, 219, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -10810,13 +11341,15 @@ static const ::capnp::_::AlignedData<202> b_c2243c65e0340384 = { static const ::capnp::_::RawSchema* const d_c2243c65e0340384[] = { &s_82b6c8ea6a6efabd, &s_88f71f4263677d19, + &s_c5693d5da66b60b6, + &s_db85b982067f1bc1, &s_f2447170bb68f358, }; -static const uint16_t m_c2243c65e0340384[] = {0, 3, 4, 5, 2, 1, 8, 10, 9, 6, 7}; -static const uint16_t i_c2243c65e0340384[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; +static const uint16_t m_c2243c65e0340384[] = {0, 11, 3, 4, 5, 2, 1, 8, 10, 9, 6, 12, 7}; +static const uint16_t i_c2243c65e0340384[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; const ::capnp::_::RawSchema s_c2243c65e0340384 = { - 0xc2243c65e0340384, b_c2243c65e0340384.words, 202, d_c2243c65e0340384, m_c2243c65e0340384, - 3, 11, i_c2243c65e0340384, nullptr, nullptr, { &s_c2243c65e0340384, nullptr, nullptr, 0, 0, nullptr }, false + 0xc2243c65e0340384, b_c2243c65e0340384.words, 241, d_c2243c65e0340384, m_c2243c65e0340384, + 5, 13, i_c2243c65e0340384, nullptr, nullptr, { &s_c2243c65e0340384, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<134> b_88f71f4263677d19 = { @@ -10964,17 +11497,312 @@ const ::capnp::_::RawSchema s_88f71f4263677d19 = { 0, 7, i_88f71f4263677d19, nullptr, nullptr, { &s_88f71f4263677d19, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<249> b_bd443b539493bc68 = { +static const ::capnp::_::AlignedData<284> b_c5693d5da66b60b6 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 182, 96, 107, 166, 93, 61, 105, 197, + 35, 0, 0, 0, 1, 0, 7, 0, + 132, 3, 52, 224, 101, 60, 36, 194, + 3, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 186, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 135, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 115, 116, 111, 109, 46, 99, + 97, 112, 110, 112, 58, 83, 116, 97, + 114, 80, 105, 108, 111, 116, 76, 97, + 116, 101, 114, 97, 108, 83, 116, 97, + 116, 101, 46, 70, 101, 101, 100, 98, + 97, 99, 107, 67, 111, 110, 115, 117, + 109, 112, 116, 105, 111, 110, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 64, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 177, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 1, 0, 0, 3, 0, 1, 0, + 184, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 180, 1, 0, 0, 3, 0, 1, 0, + 192, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 189, 1, 0, 0, 226, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 1, 0, 0, 3, 0, 1, 0, + 208, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 1, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 1, 0, 0, 3, 0, 1, 0, + 220, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 217, 1, 0, 0, 194, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 220, 1, 0, 0, 3, 0, 1, 0, + 232, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 1, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 228, 1, 0, 0, 3, 0, 1, 0, + 240, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 17, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 1, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 248, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 245, 1, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 252, 1, 0, 0, 3, 0, 1, 0, + 8, 2, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 2, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 18, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 2, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 2, 0, 0, 3, 0, 1, 0, + 24, 2, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 24, 2, 0, 0, 3, 0, 1, 0, + 36, 2, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 20, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 2, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 2, 0, 0, 3, 0, 1, 0, + 48, 2, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 2, 0, 0, 226, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 52, 2, 0, 0, 3, 0, 1, 0, + 64, 2, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 61, 2, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 60, 2, 0, 0, 3, 0, 1, 0, + 72, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 2, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 2, 0, 0, 3, 0, 1, 0, + 80, 2, 0, 0, 2, 0, 1, 0, + 118, 101, 114, 115, 105, 111, 110, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 116, 114, 111, 108, 77, + 111, 110, 111, 84, 105, 109, 101, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 115, 117, 109, 101, 100, + 80, 117, 98, 108, 105, 99, 97, 116, + 105, 111, 110, 77, 111, 110, 111, 84, + 105, 109, 101, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 115, 117, 109, 101, 100, + 82, 101, 99, 118, 70, 114, 97, 109, + 101, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 115, 117, 109, 101, 100, + 82, 101, 99, 101, 105, 112, 116, 77, + 111, 110, 111, 84, 105, 109, 101, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 115, 117, 109, 101, 100, + 83, 101, 101, 110, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 115, 117, 109, 101, 100, + 86, 97, 108, 105, 100, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 66, 111, 100, 121, 83, 104, + 97, 50, 53, 54, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 112, 111, 114, 116, 101, 100, + 84, 111, 114, 113, 117, 101, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 113, 117, 101, 115, 116, 84, + 111, 114, 113, 117, 101, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 116, 65, 99, 116, 105, 118, + 101, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 112, 117, 116, 83, 97, 102, + 101, 116, 121, 76, 105, 109, 105, 116, + 101, 100, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 117, 98, 108, 105, 115, 104, 101, + 100, 83, 97, 102, 101, 116, 121, 76, + 105, 109, 105, 116, 101, 100, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 114, 105, 111, 114, 80, 117, 98, + 108, 105, 115, 104, 67, 111, 110, 116, + 114, 111, 108, 77, 111, 110, 111, 84, + 105, 109, 101, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 111, 117, 114, 99, 101, 69, 112, + 111, 99, 104, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 112, 83, 104, 97, 50, 53, 54, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_c5693d5da66b60b6 = b_c5693d5da66b60b6.words; +#if !CAPNP_LITE +static const uint16_t m_c5693d5da66b60b6[] = {7, 2, 4, 3, 5, 6, 1, 15, 11, 10, 13, 12, 8, 9, 14, 0}; +static const uint16_t i_c5693d5da66b60b6[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; +const ::capnp::_::RawSchema s_c5693d5da66b60b6 = { + 0xc5693d5da66b60b6, b_c5693d5da66b60b6.words, 284, nullptr, m_c5693d5da66b60b6, + 0, 16, i_c5693d5da66b60b6, nullptr, nullptr, { &s_c5693d5da66b60b6, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<264> b_bd443b539493bc68 = { { 0, 0, 0, 0, 5, 0, 6, 0, 104, 188, 147, 148, 83, 59, 68, 189, 13, 0, 0, 0, 1, 0, 10, 0, 89, 10, 85, 29, 102, 186, 38, 181, - 2, 0, 7, 0, 0, 0, 0, 0, + 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 23, 3, 0, 0, + 41, 0, 0, 0, 79, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 117, 115, 116, 111, 109, 46, 99, @@ -10985,105 +11813,112 @@ static const ::capnp::_::AlignedData<249> b_bd443b539493bc68 = { 80, 234, 168, 178, 131, 27, 213, 148, 1, 0, 0, 0, 42, 0, 0, 0, 75, 105, 110, 100, 0, 0, 0, 0, - 56, 0, 0, 0, 3, 0, 4, 0, + 60, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 121, 1, 0, 0, 42, 0, 0, 0, + 149, 1, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 1, 0, 0, 3, 0, 1, 0, - 128, 1, 0, 0, 2, 0, 1, 0, + 144, 1, 0, 0, 3, 0, 1, 0, + 156, 1, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 125, 1, 0, 0, 82, 0, 0, 0, + 153, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 124, 1, 0, 0, 3, 0, 1, 0, - 136, 1, 0, 0, 2, 0, 1, 0, + 152, 1, 0, 0, 3, 0, 1, 0, + 164, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 133, 1, 0, 0, 82, 0, 0, 0, + 161, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 132, 1, 0, 0, 3, 0, 1, 0, - 144, 1, 0, 0, 2, 0, 1, 0, + 160, 1, 0, 0, 3, 0, 1, 0, + 172, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 141, 1, 0, 0, 74, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 140, 1, 0, 0, 3, 0, 1, 0, - 152, 1, 0, 0, 2, 0, 1, 0, - 4, 0, 0, 0, 3, 0, 0, 0, - 0, 0, 1, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 149, 1, 0, 0, 90, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 148, 1, 0, 0, 3, 0, 1, 0, - 160, 1, 0, 0, 2, 0, 1, 0, - 5, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 1, 0, 5, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 157, 1, 0, 0, 122, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 156, 1, 0, 0, 3, 0, 1, 0, - 168, 1, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 6, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 165, 1, 0, 0, 194, 0, 0, 0, + 169, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 1, 0, 0, 3, 0, 1, 0, 180, 1, 0, 0, 2, 0, 1, 0, - 7, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 7, 0, 0, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 1, 0, 0, 114, 0, 0, 0, + 177, 1, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 1, 0, 0, 3, 0, 1, 0, 188, 1, 0, 0, 2, 0, 1, 0, - 8, 0, 0, 0, 5, 0, 0, 0, - 0, 0, 1, 0, 8, 0, 0, 0, + 5, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 185, 1, 0, 0, 186, 0, 0, 0, + 185, 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 1, 0, 0, 3, 0, 1, 0, - 200, 1, 0, 0, 2, 0, 1, 0, - 9, 0, 0, 0, 6, 0, 0, 0, - 0, 0, 1, 0, 9, 0, 0, 0, + 184, 1, 0, 0, 3, 0, 1, 0, + 196, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 1, 0, 0, 202, 0, 0, 0, + 193, 1, 0, 0, 194, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 1, 0, 0, 3, 0, 1, 0, + 208, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 1, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 1, 0, 0, 3, 0, 1, 0, 216, 1, 0, 0, 2, 0, 1, 0, - 10, 0, 0, 0, 7, 0, 0, 0, - 0, 0, 1, 0, 10, 0, 0, 0, + 8, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 213, 1, 0, 0, 122, 0, 0, 0, + 213, 1, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 1, 0, 0, 3, 0, 1, 0, - 224, 1, 0, 0, 2, 0, 1, 0, - 11, 0, 0, 0, 8, 0, 0, 0, - 0, 0, 1, 0, 11, 0, 0, 0, + 216, 1, 0, 0, 3, 0, 1, 0, + 228, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 1, 0, 0, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 220, 1, 0, 0, 3, 0, 1, 0, - 232, 1, 0, 0, 2, 0, 1, 0, - 12, 0, 0, 0, 18, 0, 0, 0, - 0, 0, 1, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 229, 1, 0, 0, 162, 0, 0, 0, + 225, 1, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 1, 0, 0, 3, 0, 1, 0, 244, 1, 0, 0, 2, 0, 1, 0, - 13, 0, 0, 0, 19, 0, 0, 0, - 0, 0, 1, 0, 13, 0, 0, 0, + 10, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 1, 0, 0, 82, 0, 0, 0, + 241, 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 1, 0, 0, 3, 0, 1, 0, 252, 1, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 249, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 248, 1, 0, 0, 3, 0, 1, 0, + 4, 2, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 18, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 2, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 2, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 2, 0, 0, 3, 0, 1, 0, + 24, 2, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 28, 2, 0, 0, 2, 0, 1, 0, 107, 105, 110, 100, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 80, 234, 168, 178, 131, 27, 213, 148, @@ -11212,6 +12047,14 @@ static const ::capnp::_::AlignedData<249> b_bd443b539493bc68 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 111, 117, 114, 99, 101, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -11220,14 +12063,14 @@ static const ::capnp::_::AlignedData<249> b_bd443b539493bc68 = { static const ::capnp::_::RawSchema* const d_bd443b539493bc68[] = { &s_94d51b83b2a8ea50, }; -static const uint16_t m_bd443b539493bc68[] = {3, 2, 4, 12, 11, 10, 0, 5, 1, 7, 8, 6, 9, 13}; -static const uint16_t i_bd443b539493bc68[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}; +static const uint16_t m_bd443b539493bc68[] = {3, 2, 4, 12, 11, 10, 0, 5, 1, 14, 7, 8, 6, 9, 13}; +static const uint16_t i_bd443b539493bc68[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; const ::capnp::_::RawSchema s_bd443b539493bc68 = { - 0xbd443b539493bc68, b_bd443b539493bc68.words, 249, d_bd443b539493bc68, m_bd443b539493bc68, - 1, 14, i_bd443b539493bc68, nullptr, nullptr, { &s_bd443b539493bc68, nullptr, nullptr, 0, 0, nullptr }, false + 0xbd443b539493bc68, b_bd443b539493bc68.words, 264, d_bd443b539493bc68, m_bd443b539493bc68, + 1, 15, i_bd443b539493bc68, nullptr, nullptr, { &s_bd443b539493bc68, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<30> b_94d51b83b2a8ea50 = { +static const ::capnp::_::AlignedData<36> b_94d51b83b2a8ea50 = { { 0, 0, 0, 0, 5, 0, 6, 0, 80, 234, 168, 178, 131, 27, 213, 148, 30, 0, 0, 0, 2, 0, 0, 0, @@ -11237,7 +12080,7 @@ static const ::capnp::_::AlignedData<30> b_94d51b83b2a8ea50 = { 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 55, 0, 0, 0, + 33, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 117, 115, 116, 111, 109, 46, 99, @@ -11246,25 +12089,31 @@ static const ::capnp::_::AlignedData<30> b_94d51b83b2a8ea50 = { 109, 109, 97, 110, 100, 46, 75, 105, 110, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 8, 0, 0, 0, 1, 0, 2, 0, + 12, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 0, 0, 0, 66, 0, 0, 0, + 29, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 202, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 110, 107, 110, 111, 119, 110, 0, 97, 100, 111, 112, 116, 65, 99, 99, 101, 112, 116, 101, 100, 72, 105, 103, 104, 101, 114, 76, 105, 109, 105, 116, - 0, 0, 0, 0, 0, 0, 0, 0, } + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 100, 111, 112, 116, 65, 99, 99, + 101, 112, 116, 101, 100, 76, 105, 109, + 105, 116, 0, 0, 0, 0, 0, 0, } }; ::capnp::word const* const bp_94d51b83b2a8ea50 = b_94d51b83b2a8ea50.words; #if !CAPNP_LITE -static const uint16_t m_94d51b83b2a8ea50[] = {1, 0}; +static const uint16_t m_94d51b83b2a8ea50[] = {1, 2, 0}; const ::capnp::_::RawSchema s_94d51b83b2a8ea50 = { - 0x94d51b83b2a8ea50, b_94d51b83b2a8ea50.words, 30, nullptr, m_94d51b83b2a8ea50, - 0, 2, nullptr, nullptr, nullptr, { &s_94d51b83b2a8ea50, nullptr, nullptr, 0, 0, nullptr }, false + 0x94d51b83b2a8ea50, b_94d51b83b2a8ea50.words, 36, nullptr, m_94d51b83b2a8ea50, + 0, 3, nullptr, nullptr, nullptr, { &s_94d51b83b2a8ea50, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE CAPNP_DEFINE_ENUM(Kind_94d51b83b2a8ea50, 94d51b83b2a8ea50); @@ -14644,17 +15493,17 @@ const ::capnp::_::RawSchema s_d18274dcdc63c41d = { }; #endif // !CAPNP_LITE CAPNP_DEFINE_ENUM(HighwayClass_d18274dcdc63c41d, d18274dcdc63c41d); -static const ::capnp::_::AlignedData<807> b_f2447170bb68f358 = { +static const ::capnp::_::AlignedData<849> b_f2447170bb68f358 = { { 0, 0, 0, 0, 5, 0, 6, 0, 88, 243, 104, 187, 112, 113, 68, 242, 13, 0, 0, 0, 1, 0, 14, 0, 89, 10, 85, 29, 102, 186, 38, 181, - 19, 0, 7, 0, 0, 0, 0, 0, + 21, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 0, 0, 0, 23, 10, 0, 0, + 81, 0, 0, 0, 135, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 117, 115, 116, 111, 109, 46, 99, @@ -14675,329 +15524,343 @@ static const ::capnp::_::AlignedData<807> b_f2447170bb68f358 = { 117, 101, 0, 0, 0, 0, 0, 0, 83, 116, 97, 103, 101, 86, 97, 108, 117, 101, 0, 0, 0, 0, 0, 0, - 184, 0, 0, 0, 3, 0, 4, 0, + 192, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 4, 0, 0, 66, 0, 0, 0, + 49, 5, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 244, 4, 0, 0, 3, 0, 1, 0, - 0, 5, 0, 0, 2, 0, 1, 0, + 44, 5, 0, 0, 3, 0, 1, 0, + 56, 5, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 4, 0, 0, 74, 0, 0, 0, + 53, 5, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 252, 4, 0, 0, 3, 0, 1, 0, - 8, 5, 0, 0, 2, 0, 1, 0, + 52, 5, 0, 0, 3, 0, 1, 0, + 64, 5, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 5, 0, 0, 90, 0, 0, 0, + 61, 5, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 5, 0, 0, 3, 0, 1, 0, - 16, 5, 0, 0, 2, 0, 1, 0, + 60, 5, 0, 0, 3, 0, 1, 0, + 72, 5, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 5, 0, 0, 130, 0, 0, 0, + 69, 5, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 12, 5, 0, 0, 3, 0, 1, 0, - 24, 5, 0, 0, 2, 0, 1, 0, + 68, 5, 0, 0, 3, 0, 1, 0, + 80, 5, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 5, 0, 0, 162, 0, 0, 0, + 77, 5, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 24, 5, 0, 0, 3, 0, 1, 0, - 36, 5, 0, 0, 2, 0, 1, 0, + 80, 5, 0, 0, 3, 0, 1, 0, + 92, 5, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 5, 0, 0, 58, 0, 0, 0, + 89, 5, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28, 5, 0, 0, 3, 0, 1, 0, - 40, 5, 0, 0, 2, 0, 1, 0, + 84, 5, 0, 0, 3, 0, 1, 0, + 96, 5, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 5, 0, 0, 114, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 36, 5, 0, 0, 3, 0, 1, 0, - 48, 5, 0, 0, 2, 0, 1, 0, - 7, 0, 0, 0, 3, 0, 0, 0, - 0, 0, 1, 0, 7, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 45, 5, 0, 0, 138, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 48, 5, 0, 0, 3, 0, 1, 0, - 60, 5, 0, 0, 2, 0, 1, 0, - 8, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 1, 0, 8, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 57, 5, 0, 0, 82, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 56, 5, 0, 0, 3, 0, 1, 0, - 68, 5, 0, 0, 2, 0, 1, 0, - 9, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 1, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 65, 5, 0, 0, 74, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 64, 5, 0, 0, 3, 0, 1, 0, - 92, 5, 0, 0, 2, 0, 1, 0, - 10, 0, 0, 0, 5, 0, 0, 0, - 0, 0, 1, 0, 10, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 89, 5, 0, 0, 154, 0, 0, 0, + 93, 5, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 5, 0, 0, 3, 0, 1, 0, 104, 5, 0, 0, 2, 0, 1, 0, - 11, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 1, 0, 11, 0, 0, 0, + 7, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 101, 5, 0, 0, 66, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 96, 5, 0, 0, 3, 0, 1, 0, - 108, 5, 0, 0, 2, 0, 1, 0, - 12, 0, 0, 0, 17, 0, 0, 0, - 0, 0, 1, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 105, 5, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 100, 5, 0, 0, 3, 0, 1, 0, - 112, 5, 0, 0, 2, 0, 1, 0, - 13, 0, 0, 0, 18, 0, 0, 0, - 0, 0, 1, 0, 13, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 109, 5, 0, 0, 58, 0, 0, 0, + 101, 5, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 5, 0, 0, 3, 0, 1, 0, 116, 5, 0, 0, 2, 0, 1, 0, - 14, 0, 0, 0, 19, 0, 0, 0, - 0, 0, 1, 0, 14, 0, 0, 0, + 8, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 113, 5, 0, 0, 50, 0, 0, 0, + 113, 5, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 108, 5, 0, 0, 3, 0, 1, 0, - 120, 5, 0, 0, 2, 0, 1, 0, - 15, 0, 0, 0, 20, 0, 0, 0, - 0, 0, 1, 0, 15, 0, 0, 0, + 112, 5, 0, 0, 3, 0, 1, 0, + 124, 5, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 5, 0, 0, 74, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 116, 5, 0, 0, 3, 0, 1, 0, - 128, 5, 0, 0, 2, 0, 1, 0, - 16, 0, 0, 0, 21, 0, 0, 0, - 0, 0, 1, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 125, 5, 0, 0, 50, 0, 0, 0, + 121, 5, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 5, 0, 0, 3, 0, 1, 0, - 132, 5, 0, 0, 2, 0, 1, 0, - 17, 0, 0, 0, 22, 0, 0, 0, - 0, 0, 1, 0, 17, 0, 0, 0, + 148, 5, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 129, 5, 0, 0, 82, 0, 0, 0, + 145, 5, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 128, 5, 0, 0, 3, 0, 1, 0, - 140, 5, 0, 0, 2, 0, 1, 0, - 18, 0, 0, 0, 23, 0, 0, 0, - 0, 0, 1, 0, 18, 0, 0, 0, + 148, 5, 0, 0, 3, 0, 1, 0, + 160, 5, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 137, 5, 0, 0, 66, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 132, 5, 0, 0, 3, 0, 1, 0, - 144, 5, 0, 0, 2, 0, 1, 0, - 19, 0, 0, 0, 24, 0, 0, 0, - 0, 0, 1, 0, 19, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 141, 5, 0, 0, 98, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 140, 5, 0, 0, 3, 0, 1, 0, - 152, 5, 0, 0, 2, 0, 1, 0, - 20, 0, 0, 0, 25, 0, 0, 0, - 0, 0, 1, 0, 20, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 149, 5, 0, 0, 146, 0, 0, 0, + 157, 5, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 5, 0, 0, 3, 0, 1, 0, 164, 5, 0, 0, 2, 0, 1, 0, - 21, 0, 0, 0, 5, 0, 0, 0, - 0, 0, 1, 0, 21, 0, 0, 0, + 12, 0, 0, 0, 17, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 5, 0, 0, 162, 0, 0, 0, + 161, 5, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 156, 5, 0, 0, 3, 0, 1, 0, + 168, 5, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 18, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 165, 5, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 5, 0, 0, 3, 0, 1, 0, + 172, 5, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 169, 5, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 5, 0, 0, 3, 0, 1, 0, 176, 5, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 20, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 173, 5, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 5, 0, 0, 3, 0, 1, 0, + 184, 5, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 21, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 5, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 5, 0, 0, 3, 0, 1, 0, + 188, 5, 0, 0, 2, 0, 1, 0, + 17, 0, 0, 0, 22, 0, 0, 0, + 0, 0, 1, 0, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 5, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 5, 0, 0, 3, 0, 1, 0, + 196, 5, 0, 0, 2, 0, 1, 0, + 18, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 1, 0, 18, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 5, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 5, 0, 0, 3, 0, 1, 0, + 200, 5, 0, 0, 2, 0, 1, 0, + 19, 0, 0, 0, 24, 0, 0, 0, + 0, 0, 1, 0, 19, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 5, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 5, 0, 0, 3, 0, 1, 0, + 208, 5, 0, 0, 2, 0, 1, 0, + 20, 0, 0, 0, 25, 0, 0, 0, + 0, 0, 1, 0, 20, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 5, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 5, 0, 0, 3, 0, 1, 0, + 220, 5, 0, 0, 2, 0, 1, 0, + 21, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 21, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 217, 5, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 220, 5, 0, 0, 3, 0, 1, 0, + 232, 5, 0, 0, 2, 0, 1, 0, 22, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 5, 0, 0, 98, 0, 0, 0, + 229, 5, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 172, 5, 0, 0, 3, 0, 1, 0, - 200, 5, 0, 0, 2, 0, 1, 0, + 228, 5, 0, 0, 3, 0, 1, 0, + 0, 6, 0, 0, 2, 0, 1, 0, 23, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 5, 0, 0, 154, 0, 0, 0, + 253, 5, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 5, 0, 0, 3, 0, 1, 0, - 228, 5, 0, 0, 2, 0, 1, 0, + 0, 6, 0, 0, 3, 0, 1, 0, + 28, 6, 0, 0, 2, 0, 1, 0, 24, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 5, 0, 0, 98, 0, 0, 0, + 25, 6, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 5, 0, 0, 3, 0, 1, 0, - 236, 5, 0, 0, 2, 0, 1, 0, + 24, 6, 0, 0, 3, 0, 1, 0, + 36, 6, 0, 0, 2, 0, 1, 0, 25, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 5, 0, 0, 178, 0, 0, 0, + 33, 6, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 236, 5, 0, 0, 3, 0, 1, 0, - 248, 5, 0, 0, 2, 0, 1, 0, + 36, 6, 0, 0, 3, 0, 1, 0, + 48, 6, 0, 0, 2, 0, 1, 0, 26, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 5, 0, 0, 122, 0, 0, 0, + 45, 6, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 244, 5, 0, 0, 3, 0, 1, 0, - 0, 6, 0, 0, 2, 0, 1, 0, + 44, 6, 0, 0, 3, 0, 1, 0, + 56, 6, 0, 0, 2, 0, 1, 0, 27, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 5, 0, 0, 58, 0, 0, 0, + 53, 6, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 248, 5, 0, 0, 3, 0, 1, 0, - 4, 6, 0, 0, 2, 0, 1, 0, + 48, 6, 0, 0, 3, 0, 1, 0, + 60, 6, 0, 0, 2, 0, 1, 0, 28, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 6, 0, 0, 138, 0, 0, 0, + 57, 6, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 6, 0, 0, 3, 0, 1, 0, - 16, 6, 0, 0, 2, 0, 1, 0, + 60, 6, 0, 0, 3, 0, 1, 0, + 72, 6, 0, 0, 2, 0, 1, 0, 29, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 6, 0, 0, 122, 0, 0, 0, + 69, 6, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 12, 6, 0, 0, 3, 0, 1, 0, - 40, 6, 0, 0, 2, 0, 1, 0, + 68, 6, 0, 0, 3, 0, 1, 0, + 96, 6, 0, 0, 2, 0, 1, 0, 30, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 6, 0, 0, 154, 0, 0, 0, + 93, 6, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 40, 6, 0, 0, 3, 0, 1, 0, - 52, 6, 0, 0, 2, 0, 1, 0, + 96, 6, 0, 0, 3, 0, 1, 0, + 108, 6, 0, 0, 2, 0, 1, 0, 31, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 6, 0, 0, 122, 0, 0, 0, + 105, 6, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 48, 6, 0, 0, 3, 0, 1, 0, - 76, 6, 0, 0, 2, 0, 1, 0, + 104, 6, 0, 0, 3, 0, 1, 0, + 132, 6, 0, 0, 2, 0, 1, 0, 32, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 73, 6, 0, 0, 106, 0, 0, 0, + 129, 6, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 6, 0, 0, 3, 0, 1, 0, - 100, 6, 0, 0, 2, 0, 1, 0, + 128, 6, 0, 0, 3, 0, 1, 0, + 156, 6, 0, 0, 2, 0, 1, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 6, 0, 0, 138, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 100, 6, 0, 0, 3, 0, 1, 0, - 112, 6, 0, 0, 2, 0, 1, 0, - 34, 0, 0, 0, 26, 0, 0, 0, - 0, 0, 1, 0, 34, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 109, 6, 0, 0, 90, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 108, 6, 0, 0, 3, 0, 1, 0, - 120, 6, 0, 0, 2, 0, 1, 0, - 35, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 1, 0, 35, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 117, 6, 0, 0, 178, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 120, 6, 0, 0, 3, 0, 1, 0, - 148, 6, 0, 0, 2, 0, 1, 0, - 36, 0, 0, 0, 27, 0, 0, 0, - 0, 0, 1, 0, 36, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 145, 6, 0, 0, 122, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 144, 6, 0, 0, 3, 0, 1, 0, - 156, 6, 0, 0, 2, 0, 1, 0, - 37, 0, 0, 0, 8, 0, 0, 0, - 0, 0, 1, 0, 37, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 153, 6, 0, 0, 186, 0, 0, 0, + 153, 6, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 6, 0, 0, 3, 0, 1, 0, 168, 6, 0, 0, 2, 0, 1, 0, - 38, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 1, 0, 38, 0, 0, 0, + 34, 0, 0, 0, 26, 0, 0, 0, + 0, 0, 1, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 165, 6, 0, 0, 202, 0, 0, 0, + 165, 6, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 172, 6, 0, 0, 3, 0, 1, 0, - 184, 6, 0, 0, 2, 0, 1, 0, - 39, 0, 0, 0, 28, 0, 0, 0, - 0, 0, 1, 0, 39, 0, 0, 0, + 164, 6, 0, 0, 3, 0, 1, 0, + 176, 6, 0, 0, 2, 0, 1, 0, + 35, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 181, 6, 0, 0, 114, 0, 0, 0, + 173, 6, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 6, 0, 0, 3, 0, 1, 0, - 192, 6, 0, 0, 2, 0, 1, 0, - 40, 0, 0, 0, 17, 0, 0, 0, - 0, 0, 1, 0, 40, 0, 0, 0, + 176, 6, 0, 0, 3, 0, 1, 0, + 204, 6, 0, 0, 2, 0, 1, 0, + 36, 0, 0, 0, 27, 0, 0, 0, + 0, 0, 1, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 189, 6, 0, 0, 122, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 188, 6, 0, 0, 3, 0, 1, 0, - 200, 6, 0, 0, 2, 0, 1, 0, - 41, 0, 0, 0, 10, 0, 0, 0, - 0, 0, 1, 0, 41, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 197, 6, 0, 0, 186, 0, 0, 0, + 201, 6, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 6, 0, 0, 3, 0, 1, 0, 212, 6, 0, 0, 2, 0, 1, 0, - 42, 0, 0, 0, 11, 0, 0, 0, - 0, 0, 1, 0, 42, 0, 0, 0, + 37, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 6, 0, 0, 250, 0, 0, 0, + 209, 6, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 6, 0, 0, 3, 0, 1, 0, - 228, 6, 0, 0, 2, 0, 1, 0, - 43, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 1, 0, 43, 0, 0, 0, + 212, 6, 0, 0, 3, 0, 1, 0, + 224, 6, 0, 0, 2, 0, 1, 0, + 38, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 6, 0, 0, 10, 1, 0, 0, + 221, 6, 0, 0, 202, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 228, 6, 0, 0, 3, 0, 1, 0, + 240, 6, 0, 0, 2, 0, 1, 0, + 39, 0, 0, 0, 28, 0, 0, 0, + 0, 0, 1, 0, 39, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 6, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 6, 0, 0, 3, 0, 1, 0, 248, 6, 0, 0, 2, 0, 1, 0, - 44, 0, 0, 0, 18, 0, 0, 0, - 0, 0, 1, 0, 44, 0, 0, 0, + 40, 0, 0, 0, 17, 0, 0, 0, + 0, 0, 1, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 6, 0, 0, 98, 0, 0, 0, + 245, 6, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 6, 0, 0, 3, 0, 1, 0, 0, 7, 0, 0, 2, 0, 1, 0, - 45, 0, 0, 0, 13, 0, 0, 0, - 0, 0, 1, 0, 45, 0, 0, 0, + 41, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 6, 0, 0, 178, 0, 0, 0, + 253, 6, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 1, 0, 12, 7, 0, 0, 2, 0, 1, 0, + 42, 0, 0, 0, 11, 0, 0, 0, + 0, 0, 1, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 7, 0, 0, 250, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 7, 0, 0, 3, 0, 1, 0, + 28, 7, 0, 0, 2, 0, 1, 0, + 43, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 1, 0, 43, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 7, 0, 0, 10, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 7, 0, 0, 3, 0, 1, 0, + 48, 7, 0, 0, 2, 0, 1, 0, + 44, 0, 0, 0, 18, 0, 0, 0, + 0, 0, 1, 0, 44, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 7, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 7, 0, 0, 3, 0, 1, 0, + 56, 7, 0, 0, 2, 0, 1, 0, + 45, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 45, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 53, 7, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 56, 7, 0, 0, 3, 0, 1, 0, + 68, 7, 0, 0, 2, 0, 1, 0, + 46, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 1, 0, 46, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 7, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 7, 0, 0, 3, 0, 1, 0, + 96, 7, 0, 0, 2, 0, 1, 0, + 47, 0, 0, 0, 20, 0, 0, 0, + 0, 0, 1, 0, 47, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 93, 7, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 7, 0, 0, 3, 0, 1, 0, + 124, 7, 0, 0, 2, 0, 1, 0, 118, 101, 114, 115, 105, 111, 110, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -15450,6 +16313,34 @@ static const ::capnp::_::AlignedData<807> b_f2447170bb68f358 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 101, 104, 105, 99, 108, 101, 77, + 111, 100, 101, 108, 65, 110, 99, 104, + 111, 114, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 105, 118, 101, 65, 112, 112, 108, + 105, 101, 100, 67, 111, 111, 114, 100, + 105, 110, 97, 116, 101, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -15460,11 +16351,11 @@ static const ::capnp::_::RawSchema* const d_f2447170bb68f358[] = { &s_ddd8eb0f1a3fb843, &s_e7b005eb931864bc, }; -static const uint16_t m_f2447170bb68f358[] = {13, 23, 35, 42, 41, 43, 10, 28, 29, 3, 20, 26, 30, 5, 45, 44, 11, 16, 9, 2, 1, 18, 12, 19, 39, 40, 37, 36, 38, 15, 27, 32, 31, 8, 7, 17, 33, 24, 25, 4, 6, 21, 22, 34, 14, 0}; -static const uint16_t i_f2447170bb68f358[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45}; +static const uint16_t m_f2447170bb68f358[] = {13, 23, 35, 42, 41, 43, 10, 28, 29, 3, 20, 26, 30, 5, 45, 44, 11, 16, 9, 2, 1, 18, 47, 12, 19, 39, 40, 37, 36, 38, 15, 27, 32, 31, 8, 7, 17, 33, 24, 25, 4, 6, 21, 22, 34, 14, 46, 0}; +static const uint16_t i_f2447170bb68f358[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47}; const ::capnp::_::RawSchema s_f2447170bb68f358 = { - 0xf2447170bb68f358, b_f2447170bb68f358.words, 807, d_f2447170bb68f358, m_f2447170bb68f358, - 3, 46, i_f2447170bb68f358, nullptr, nullptr, { &s_f2447170bb68f358, nullptr, nullptr, 0, 0, nullptr }, false + 0xf2447170bb68f358, b_f2447170bb68f358.words, 849, d_f2447170bb68f358, m_f2447170bb68f358, + 3, 48, i_f2447170bb68f358, nullptr, nullptr, { &s_f2447170bb68f358, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<176> b_c9c9b325ca50aaf7 = { @@ -19210,6 +20101,1517 @@ const ::capnp::_::RawSchema s_f078f562cbce49ad = { 2, 25, i_f078f562cbce49ad, nullptr, nullptr, { &s_f078f562cbce49ad, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<217> b_9f853f7eb4110c72 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 114, 12, 17, 180, 126, 63, 133, 159, + 13, 0, 0, 0, 1, 0, 1, 0, + 89, 10, 85, 29, 102, 186, 38, 181, + 11, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 167, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 115, 116, 111, 109, 46, 99, + 97, 112, 110, 112, 58, 76, 105, 118, + 101, 84, 114, 97, 99, 107, 105, 110, + 103, 66, 105, 110, 100, 105, 110, 103, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 48, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 1, 0, 0, 3, 0, 1, 0, + 76, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 73, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 1, 0, 0, 3, 0, 1, 0, + 84, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 1, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 84, 1, 0, 0, 3, 0, 1, 0, + 96, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 93, 1, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 1, 0, 0, 3, 0, 1, 0, + 108, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 1, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 1, 0, 0, 3, 0, 1, 0, + 116, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 1, 0, 0, 3, 0, 1, 0, + 124, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 121, 1, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 1, 0, 0, 3, 0, 1, 0, + 136, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 1, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 136, 1, 0, 0, 3, 0, 1, 0, + 148, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 145, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 144, 1, 0, 0, 3, 0, 1, 0, + 156, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 153, 1, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 156, 1, 0, 0, 3, 0, 1, 0, + 168, 1, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 165, 1, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 168, 1, 0, 0, 3, 0, 1, 0, + 180, 1, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 177, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 1, 0, 0, 3, 0, 1, 0, + 188, 1, 0, 0, 2, 0, 1, 0, + 105, 100, 101, 110, 116, 105, 116, 121, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 112, 83, 104, 97, 50, 53, 54, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 116, 114, 111, 108, 108, + 101, 114, 82, 101, 118, 105, 115, 105, + 111, 110, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 97, 115, 101, 108, 105, 110, 101, + 82, 101, 118, 105, 115, 105, 111, 110, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 111, 117, 114, 99, 101, 69, 112, + 111, 99, 104, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 115, 101, 116, 71, 101, 110, + 101, 114, 97, 116, 105, 111, 110, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 97, 108, 105, 98, 114, 97, 116, + 105, 111, 110, 71, 101, 110, 101, 114, + 97, 116, 105, 111, 110, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 101, 104, 105, 99, 108, 101, 77, + 111, 100, 101, 108, 71, 101, 110, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 97, 116, 105, 118, 101, 73, 100, + 101, 110, 116, 105, 116, 121, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 104, 121, 115, 105, 99, 97, 108, + 77, 111, 100, 101, 108, 82, 101, 118, + 105, 115, 105, 111, 110, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 101, 114, 115, 105, 115, 116, 101, + 110, 116, 83, 111, 117, 114, 99, 101, + 69, 112, 111, 99, 104, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 97, 116, 105, 118, 101, 82, 101, + 113, 117, 105, 114, 101, 100, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_9f853f7eb4110c72 = b_9f853f7eb4110c72.words; +#if !CAPNP_LITE +static const uint16_t m_9f853f7eb4110c72[] = {3, 6, 2, 1, 0, 8, 11, 10, 9, 5, 4, 7}; +static const uint16_t i_9f853f7eb4110c72[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; +const ::capnp::_::RawSchema s_9f853f7eb4110c72 = { + 0x9f853f7eb4110c72, b_9f853f7eb4110c72.words, 217, nullptr, m_9f853f7eb4110c72, + 0, 12, i_9f853f7eb4110c72, nullptr, nullptr, { &s_9f853f7eb4110c72, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<318> b_e6f67224bc0ee1a3 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 163, 225, 14, 188, 36, 114, 246, 230, + 13, 0, 0, 0, 1, 0, 7, 0, + 89, 10, 85, 29, 102, 186, 38, 181, + 11, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 250, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 247, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 115, 116, 111, 109, 46, 99, + 97, 112, 110, 112, 58, 76, 105, 118, + 101, 84, 114, 97, 99, 107, 105, 110, + 103, 84, 114, 105, 97, 108, 0, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 38, 9, 194, 58, 95, 55, 55, 236, + 1, 0, 0, 0, 90, 0, 0, 0, + 67, 111, 111, 114, 100, 105, 110, 97, + 116, 101, 0, 0, 0, 0, 0, 0, + 72, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 233, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 228, 1, 0, 0, 3, 0, 1, 0, + 240, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 248, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 245, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 240, 1, 0, 0, 3, 0, 1, 0, + 252, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 249, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 252, 1, 0, 0, 3, 0, 1, 0, + 8, 2, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 2, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2, 0, 0, 3, 0, 1, 0, + 12, 2, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 2, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 2, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 2, 0, 0, 3, 0, 1, 0, + 36, 2, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 2, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 57, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 52, 2, 0, 0, 3, 0, 1, 0, + 80, 2, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 2, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 76, 2, 0, 0, 3, 0, 1, 0, + 88, 2, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 85, 2, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 84, 2, 0, 0, 3, 0, 1, 0, + 96, 2, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 93, 2, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 2, 0, 0, 3, 0, 1, 0, + 104, 2, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 2, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 2, 0, 0, 3, 0, 1, 0, + 116, 2, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 2, 0, 0, 3, 0, 1, 0, + 128, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 2, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 2, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 144, 2, 0, 0, 2, 0, 1, 0, + 17, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 2, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 144, 2, 0, 0, 3, 0, 1, 0, + 156, 2, 0, 0, 2, 0, 1, 0, + 118, 101, 114, 115, 105, 111, 110, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 114, 111, 116, 111, 99, 111, 108, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 105, 110, 100, 105, 110, 103, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 114, 12, 17, 180, 126, 63, 133, 159, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 114, 105, 97, 108, 73, 100, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 101, 113, 117, 101, 110, 99, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 104, 97, 115, 101, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 103, 114, 111, 117, 112, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 97, 109, 101, 115, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 97, 115, 101, 108, 105, 110, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 38, 9, 194, 58, 95, 55, 55, 236, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 97, 114, 103, 101, 116, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 38, 9, 194, 58, 95, 55, 55, 236, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 97, 115, 101, 108, 105, 110, 101, + 83, 104, 97, 50, 53, 54, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 97, 114, 103, 101, 116, 83, 104, + 97, 50, 53, 54, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 114, 116, 101, 100, 77, + 111, 110, 111, 84, 105, 109, 101, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 98, 115, 101, 114, 118, 101, 100, + 77, 111, 110, 111, 84, 105, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 105, 100, 85, 110, 116, + 105, 108, 77, 111, 110, 111, 84, 105, + 109, 101, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 115, 112, 111, 110, 115, 101, + 87, 105, 116, 110, 101, 115, 115, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 49, 142, 150, 216, 240, 169, 161, 218, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 97, 114, 103, 101, 116, 83, 101, + 113, 117, 101, 110, 99, 101, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 101, 97, 100, 108, 105, 110, 101, + 77, 111, 110, 111, 84, 105, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_e6f67224bc0ee1a3 = b_e6f67224bc0ee1a3.words; +#if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_e6f67224bc0ee1a3[] = { + &s_9f853f7eb4110c72, + &s_daa1a9f0d8968e31, + &s_ec37375f3ac20926, +}; +static const uint16_t m_e6f67224bc0ee1a3[] = {8, 10, 2, 17, 6, 7, 13, 5, 1, 15, 4, 12, 9, 16, 11, 3, 14, 0}; +static const uint16_t i_e6f67224bc0ee1a3[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}; +const ::capnp::_::RawSchema s_e6f67224bc0ee1a3 = { + 0xe6f67224bc0ee1a3, b_e6f67224bc0ee1a3.words, 318, d_e6f67224bc0ee1a3, m_e6f67224bc0ee1a3, + 3, 18, i_e6f67224bc0ee1a3, nullptr, nullptr, { &s_e6f67224bc0ee1a3, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<50> b_ec37375f3ac20926 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 38, 9, 194, 58, 95, 55, 55, 236, + 31, 0, 0, 0, 1, 0, 1, 0, + 163, 225, 14, 188, 36, 114, 246, 230, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 82, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 115, 116, 111, 109, 46, 99, + 97, 112, 110, 112, 58, 76, 105, 118, + 101, 84, 114, 97, 99, 107, 105, 110, + 103, 84, 114, 105, 97, 108, 46, 67, + 111, 111, 114, 100, 105, 110, 97, 116, + 101, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_ec37375f3ac20926 = b_ec37375f3ac20926.words; +#if !CAPNP_LITE +static const uint16_t m_ec37375f3ac20926[] = {0, 1}; +static const uint16_t i_ec37375f3ac20926[] = {0, 1}; +const ::capnp::_::RawSchema s_ec37375f3ac20926 = { + 0xec37375f3ac20926, b_ec37375f3ac20926.words, 50, nullptr, m_ec37375f3ac20926, + 0, 2, i_ec37375f3ac20926, nullptr, nullptr, { &s_ec37375f3ac20926, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<194> b_db85b982067f1bc1 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 193, 27, 127, 6, 130, 185, 133, 219, + 13, 0, 0, 0, 1, 0, 4, 0, + 89, 10, 85, 29, 102, 186, 38, 181, + 7, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 42, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 111, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 115, 116, 111, 109, 46, 99, + 97, 112, 110, 112, 58, 76, 105, 118, + 101, 84, 114, 97, 99, 107, 105, 110, + 103, 65, 112, 112, 108, 105, 99, 97, + 116, 105, 111, 110, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 44, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 1, 0, 0, 3, 0, 1, 0, + 44, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 1, 0, 0, 3, 0, 1, 0, + 48, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 1, 0, 0, 3, 0, 1, 0, + 52, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 48, 1, 0, 0, 3, 0, 1, 0, + 60, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 57, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 56, 1, 0, 0, 3, 0, 1, 0, + 68, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 60, 1, 0, 0, 3, 0, 1, 0, + 72, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 1, 0, 0, 3, 0, 1, 0, + 92, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 1, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 88, 1, 0, 0, 3, 0, 1, 0, + 100, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 1, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 1, 0, 0, 3, 0, 1, 0, + 112, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 1, 0, 0, 3, 0, 1, 0, + 116, 1, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 1, 0, 0, 3, 0, 1, 0, + 124, 1, 0, 0, 2, 0, 1, 0, + 118, 101, 114, 115, 105, 111, 110, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 105, 110, 100, 105, 110, 103, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 114, 12, 17, 180, 126, 63, 133, 159, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 114, 105, 97, 108, 73, 100, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 101, 113, 117, 101, 110, 99, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 116, 114, 111, 108, 77, + 111, 110, 111, 84, 105, 109, 101, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 117, 115, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 112, 112, 108, 105, 101, 100, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 38, 9, 194, 58, 95, 55, 55, 236, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 112, 112, 108, 105, 101, 100, 83, + 104, 97, 50, 53, 54, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 102, 105, 114, 109, 101, + 100, 84, 97, 114, 103, 101, 116, 83, + 104, 97, 50, 53, 54, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 97, 115, 111, 110, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 97, 114, 103, 101, 116, 83, 101, + 113, 117, 101, 110, 99, 101, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_db85b982067f1bc1 = b_db85b982067f1bc1.words; +#if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_db85b982067f1bc1[] = { + &s_9f853f7eb4110c72, + &s_ec37375f3ac20926, +}; +static const uint16_t m_db85b982067f1bc1[] = {6, 7, 1, 8, 4, 9, 3, 5, 10, 2, 0}; +static const uint16_t i_db85b982067f1bc1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; +const ::capnp::_::RawSchema s_db85b982067f1bc1 = { + 0xdb85b982067f1bc1, b_db85b982067f1bc1.words, 194, d_db85b982067f1bc1, m_db85b982067f1bc1, + 2, 11, i_db85b982067f1bc1, nullptr, nullptr, { &s_db85b982067f1bc1, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<198> b_ef6257bed9318012 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 18, 128, 49, 217, 190, 87, 98, 239, + 13, 0, 0, 0, 1, 0, 4, 0, + 89, 10, 85, 29, 102, 186, 38, 181, + 7, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 111, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 115, 116, 111, 109, 46, 99, + 97, 112, 110, 112, 58, 76, 105, 118, + 101, 84, 114, 97, 99, 107, 105, 110, + 103, 82, 101, 115, 116, 111, 114, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 44, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 1, 0, 0, 3, 0, 1, 0, + 44, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 1, 0, 0, 3, 0, 1, 0, + 52, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 1, 0, 0, 3, 0, 1, 0, + 56, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 53, 1, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 52, 1, 0, 0, 3, 0, 1, 0, + 64, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 61, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 60, 1, 0, 0, 3, 0, 1, 0, + 72, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 1, 0, 0, 3, 0, 1, 0, + 92, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 1, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 88, 1, 0, 0, 3, 0, 1, 0, + 100, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 1, 0, 0, 3, 0, 1, 0, + 108, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 1, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 1, 0, 0, 3, 0, 1, 0, + 120, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 1, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 1, 0, 0, 3, 0, 1, 0, + 132, 1, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 1, 0, 0, 3, 0, 1, 0, + 140, 1, 0, 0, 2, 0, 1, 0, + 118, 101, 114, 115, 105, 111, 110, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 114, 111, 116, 111, 99, 111, 108, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 105, 110, 100, 105, 110, 103, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 114, 12, 17, 180, 126, 63, 133, 159, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 105, 115, 116, 111, 114, 121, 83, + 99, 111, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 105, 115, 116, 111, 114, 121, 84, + 114, 105, 97, 108, 73, 100, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 97, 114, 103, 101, 116, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 38, 9, 194, 58, 95, 55, 55, 236, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 97, 114, 103, 101, 116, 83, 104, + 97, 50, 53, 54, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 101, 113, 117, 101, 110, 99, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 98, 115, 101, 114, 118, 101, 100, + 77, 111, 110, 111, 84, 105, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 105, 100, 85, 110, 116, + 105, 108, 77, 111, 110, 111, 84, 105, + 109, 101, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 115, 112, 111, 110, 115, 101, + 87, 105, 116, 110, 101, 115, 115, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 49, 142, 150, 216, 240, 169, 161, 218, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_ef6257bed9318012 = b_ef6257bed9318012.words; +#if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_ef6257bed9318012[] = { + &s_9f853f7eb4110c72, + &s_daa1a9f0d8968e31, + &s_ec37375f3ac20926, +}; +static const uint16_t m_ef6257bed9318012[] = {2, 3, 4, 8, 1, 10, 7, 5, 6, 9, 0}; +static const uint16_t i_ef6257bed9318012[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; +const ::capnp::_::RawSchema s_ef6257bed9318012 = { + 0xef6257bed9318012, b_ef6257bed9318012.words, 198, d_ef6257bed9318012, m_ef6257bed9318012, + 3, 11, i_ef6257bed9318012, nullptr, nullptr, { &s_ef6257bed9318012, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<286> b_bbb8f17940692312 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 18, 35, 105, 64, 121, 241, 184, 187, + 13, 0, 0, 0, 1, 0, 7, 0, + 89, 10, 85, 29, 102, 186, 38, 181, + 9, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 106, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 135, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 115, 116, 111, 109, 46, 99, + 97, 112, 110, 112, 58, 76, 105, 118, + 101, 84, 114, 97, 99, 107, 105, 110, + 103, 82, 101, 115, 112, 111, 110, 115, + 101, 67, 101, 114, 116, 105, 102, 105, + 99, 97, 116, 101, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 64, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 177, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 1, 0, 0, 3, 0, 1, 0, + 184, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 1, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 180, 1, 0, 0, 3, 0, 1, 0, + 192, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 189, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 1, 0, 0, 3, 0, 1, 0, + 196, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 1, 0, 0, 3, 0, 1, 0, + 200, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 1, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 1, 0, 0, 3, 0, 1, 0, + 208, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 1, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 1, 0, 0, 3, 0, 1, 0, + 216, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 1, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 212, 1, 0, 0, 3, 0, 1, 0, + 224, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 221, 1, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 220, 1, 0, 0, 3, 0, 1, 0, + 232, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 1, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 228, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 252, 1, 0, 0, 3, 0, 1, 0, + 24, 2, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 2, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 24, 2, 0, 0, 3, 0, 1, 0, + 36, 2, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 2, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 2, 0, 0, 3, 0, 1, 0, + 48, 2, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 48, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 57, 2, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 56, 2, 0, 0, 3, 0, 1, 0, + 68, 2, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 2, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 2, 0, 0, 3, 0, 1, 0, + 80, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 2, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 2, 0, 0, 3, 0, 1, 0, + 92, 2, 0, 0, 2, 0, 1, 0, + 118, 101, 114, 115, 105, 111, 110, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 111, 100, 101, 108, 82, 101, 118, + 105, 115, 105, 111, 110, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 103, 78, 115, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 108, 111, 112, 101, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 101, 114, 99, 101, 112, + 116, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 109, 109, 97, 110, 100, 76, + 111, 119, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 109, 109, 97, 110, 100, 72, + 105, 103, 104, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 101, 108, 100, 111, 117, 116, 82, + 109, 115, 101, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 114, 97, 105, 110, 66, 108, 111, + 99, 107, 73, 100, 115, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 101, 108, 100, 111, 117, 116, 66, + 108, 111, 99, 107, 73, 100, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 111, 117, 114, 99, 101, 73, 110, + 118, 101, 110, 116, 111, 114, 121, 83, + 104, 97, 50, 53, 54, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 100, 101, 110, 116, 105, 102, 105, + 99, 97, 116, 105, 111, 110, 83, 104, + 97, 50, 53, 54, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 100, 101, 110, 116, 105, 102, 105, + 101, 100, 77, 111, 110, 111, 84, 105, + 109, 101, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 114, 111, 118, 101, 110, 97, 110, + 99, 101, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 97, 114, 101, 110, 116, 77, 111, + 100, 101, 108, 82, 101, 118, 105, 115, + 105, 111, 110, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 113, 117, 97, 108, 105, 102, + 105, 99, 97, 116, 105, 111, 110, 83, + 104, 97, 50, 53, 54, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_bbb8f17940692312 = b_bbb8f17940692312.words; +#if !CAPNP_LITE +static const uint16_t m_bbb8f17940692312[] = {6, 5, 9, 7, 11, 12, 4, 2, 1, 14, 13, 15, 3, 10, 8, 0}; +static const uint16_t i_bbb8f17940692312[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; +const ::capnp::_::RawSchema s_bbb8f17940692312 = { + 0xbbb8f17940692312, b_bbb8f17940692312.words, 286, nullptr, m_bbb8f17940692312, + 0, 16, i_bbb8f17940692312, nullptr, nullptr, { &s_bbb8f17940692312, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<154> b_daa1a9f0d8968e31 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 49, 142, 150, 216, 240, 169, 161, 218, + 13, 0, 0, 0, 1, 0, 7, 0, + 89, 10, 85, 29, 102, 186, 38, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 74, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 199, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 115, 116, 111, 109, 46, 99, + 97, 112, 110, 112, 58, 76, 105, 118, + 101, 84, 114, 97, 99, 107, 105, 110, + 103, 82, 101, 115, 112, 111, 110, 115, + 101, 87, 105, 116, 110, 101, 115, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 32, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 217, 0, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 220, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 0, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 244, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 241, 0, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 244, 0, 0, 0, 3, 0, 1, 0, + 0, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 0, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 3, 0, 1, 0, + 12, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 1, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 1, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 20, 1, 0, 0, 3, 0, 1, 0, + 32, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 1, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 1, 0, 0, 3, 0, 1, 0, + 44, 1, 0, 0, 2, 0, 1, 0, + 99, 101, 114, 116, 105, 102, 105, 99, + 97, 116, 101, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 18, 35, 105, 64, 121, 241, 184, 187, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 115, 116, 67, 111, 110, 116, + 114, 111, 108, 77, 111, 110, 111, 84, + 105, 109, 101, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 115, 116, 65, 112, 112, 108, + 121, 77, 111, 110, 111, 84, 105, 109, + 101, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 115, 116, 80, 111, 115, 101, + 77, 111, 110, 111, 84, 105, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 119, 105, 110, 100, 111, 119, 83, 116, + 97, 114, 116, 77, 111, 110, 111, 84, + 105, 109, 101, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 100, 101, 112, 101, 110, 100, + 101, 110, 116, 80, 111, 115, 101, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 115, 105, 100, 117, 97, 108, + 82, 109, 115, 101, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 105, 100, 85, 110, 116, + 105, 108, 77, 111, 110, 111, 84, 105, + 109, 101, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_daa1a9f0d8968e31 = b_daa1a9f0d8968e31.words; +#if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_daa1a9f0d8968e31[] = { + &s_bbb8f17940692312, +}; +static const uint16_t m_daa1a9f0d8968e31[] = {0, 5, 2, 1, 3, 6, 7, 4}; +static const uint16_t i_daa1a9f0d8968e31[] = {0, 1, 2, 3, 4, 5, 6, 7}; +const ::capnp::_::RawSchema s_daa1a9f0d8968e31 = { + 0xdaa1a9f0d8968e31, b_daa1a9f0d8968e31.words, 154, d_daa1a9f0d8968e31, m_daa1a9f0d8968e31, + 1, 8, i_daa1a9f0d8968e31, nullptr, nullptr, { &s_daa1a9f0d8968e31, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE } // namespace schemas } // namespace capnp @@ -19385,6 +21787,18 @@ constexpr ::capnp::_::RawSchema const* StarPilotCarState::Gps::_capnpPrivate::sc #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE +// StarPilotCarState::FeedbackPublication +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t StarPilotCarState::FeedbackPublication::_capnpPrivate::dataWordSize; +constexpr uint16_t StarPilotCarState::FeedbackPublication::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind StarPilotCarState::FeedbackPublication::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* StarPilotCarState::FeedbackPublication::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + // StarPilotLongitudinalPlan #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr uint16_t StarPilotLongitudinalPlan::_capnpPrivate::dataWordSize; @@ -19589,15 +22003,15 @@ constexpr ::capnp::_::RawSchema const* StarPilotModelDataV2::VisionObservation:: #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE -// CustomReserved5 +// StarPilotSteeringTracking #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr uint16_t CustomReserved5::_capnpPrivate::dataWordSize; -constexpr uint16_t CustomReserved5::_capnpPrivate::pointerCount; +constexpr uint16_t StarPilotSteeringTracking::_capnpPrivate::dataWordSize; +constexpr uint16_t StarPilotSteeringTracking::_capnpPrivate::pointerCount; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr ::capnp::Kind CustomReserved5::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* CustomReserved5::_capnpPrivate::schema; +constexpr ::capnp::Kind StarPilotSteeringTracking::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* StarPilotSteeringTracking::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -19757,6 +22171,18 @@ constexpr ::capnp::_::RawSchema const* StarPilotLateralState::LaneCentering::_ca #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE +// StarPilotLateralState::FeedbackConsumption +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t StarPilotLateralState::FeedbackConsumption::_capnpPrivate::dataWordSize; +constexpr uint16_t StarPilotLateralState::FeedbackConsumption::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind StarPilotLateralState::FeedbackConsumption::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* StarPilotLateralState::FeedbackConsumption::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + // SlcCruiseCommand #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr uint16_t SlcCruiseCommand::_capnpPrivate::dataWordSize; @@ -20117,6 +22543,90 @@ constexpr ::capnp::_::RawSchema const* LearningTorqueProjection::_capnpPrivate:: #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE +// LiveTrackingBinding +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t LiveTrackingBinding::_capnpPrivate::dataWordSize; +constexpr uint16_t LiveTrackingBinding::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind LiveTrackingBinding::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* LiveTrackingBinding::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + +// LiveTrackingTrial +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t LiveTrackingTrial::_capnpPrivate::dataWordSize; +constexpr uint16_t LiveTrackingTrial::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind LiveTrackingTrial::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* LiveTrackingTrial::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + +// LiveTrackingTrial::Coordinate +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t LiveTrackingTrial::Coordinate::_capnpPrivate::dataWordSize; +constexpr uint16_t LiveTrackingTrial::Coordinate::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind LiveTrackingTrial::Coordinate::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* LiveTrackingTrial::Coordinate::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + +// LiveTrackingApplication +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t LiveTrackingApplication::_capnpPrivate::dataWordSize; +constexpr uint16_t LiveTrackingApplication::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind LiveTrackingApplication::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* LiveTrackingApplication::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + +// LiveTrackingRestore +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t LiveTrackingRestore::_capnpPrivate::dataWordSize; +constexpr uint16_t LiveTrackingRestore::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind LiveTrackingRestore::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* LiveTrackingRestore::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + +// LiveTrackingResponseCertificate +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t LiveTrackingResponseCertificate::_capnpPrivate::dataWordSize; +constexpr uint16_t LiveTrackingResponseCertificate::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind LiveTrackingResponseCertificate::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* LiveTrackingResponseCertificate::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + +// LiveTrackingResponseWitness +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t LiveTrackingResponseWitness::_capnpPrivate::dataWordSize; +constexpr uint16_t LiveTrackingResponseWitness::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind LiveTrackingResponseWitness::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* LiveTrackingResponseWitness::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + } // namespace diff --git a/openpilot/cereal/gen/cpp/custom.capnp.h b/openpilot/cereal/gen/cpp/custom.capnp.h index 63d908b0ef..896e74aca1 100644 --- a/openpilot/cereal/gen/cpp/custom.capnp.h +++ b/openpilot/cereal/gen/cpp/custom.capnp.h @@ -76,6 +76,7 @@ CAPNP_DECLARE_ENUM(Family, b03bf8f7c878937a); CAPNP_DECLARE_SCHEMA(af3712fe799c76ce); CAPNP_DECLARE_SCHEMA(f35cc4560bbf6ec2); CAPNP_DECLARE_SCHEMA(918d19aa5875c92d); +CAPNP_DECLARE_SCHEMA(de92e382e2400293); CAPNP_DECLARE_SCHEMA(da96579883444c35); CAPNP_DECLARE_SCHEMA(a1680744031fdb2d); CAPNP_DECLARE_SCHEMA(a7940385bedd1d4c); @@ -254,11 +255,13 @@ enum class Status_f187d96cea4f09df: uint16_t { CAPNP_DECLARE_ENUM(Status, f187d96cea4f09df); CAPNP_DECLARE_SCHEMA(c2243c65e0340384); CAPNP_DECLARE_SCHEMA(88f71f4263677d19); +CAPNP_DECLARE_SCHEMA(c5693d5da66b60b6); CAPNP_DECLARE_SCHEMA(bd443b539493bc68); CAPNP_DECLARE_SCHEMA(94d51b83b2a8ea50); enum class Kind_94d51b83b2a8ea50: uint16_t { UNKNOWN, ADOPT_ACCEPTED_HIGHER_LIMIT, + ADOPT_ACCEPTED_LIMIT, }; CAPNP_DECLARE_ENUM(Kind, 94d51b83b2a8ea50); CAPNP_DECLARE_SCHEMA(fc6241ed8877b611); @@ -425,6 +428,13 @@ CAPNP_DECLARE_SCHEMA(d1dfbadb6a811fee); CAPNP_DECLARE_SCHEMA(b276dede9e6b2cc4); CAPNP_DECLARE_SCHEMA(ab0b33cac682523e); CAPNP_DECLARE_SCHEMA(f078f562cbce49ad); +CAPNP_DECLARE_SCHEMA(9f853f7eb4110c72); +CAPNP_DECLARE_SCHEMA(e6f67224bc0ee1a3); +CAPNP_DECLARE_SCHEMA(ec37375f3ac20926); +CAPNP_DECLARE_SCHEMA(db85b982067f1bc1); +CAPNP_DECLARE_SCHEMA(ef6257bed9318012); +CAPNP_DECLARE_SCHEMA(bbb8f17940692312); +CAPNP_DECLARE_SCHEMA(daa1a9f0d8968e31); } // namespace schemas } // namespace capnp @@ -634,9 +644,10 @@ struct StarPilotCarState { class Builder; class Pipeline; struct Gps; + struct FeedbackPublication; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(f35cc4560bbf6ec2, 3, 4) + CAPNP_DECLARE_STRUCT_HEADER(f35cc4560bbf6ec2, 3, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } #endif // !CAPNP_LITE @@ -658,6 +669,21 @@ struct StarPilotCarState::Gps { }; }; +struct StarPilotCarState::FeedbackPublication { + FeedbackPublication() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(de92e382e2400293, 8, 6) + #if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } + #endif // !CAPNP_LITE + }; +}; + struct StarPilotLongitudinalPlan { StarPilotLongitudinalPlan() = delete; @@ -686,7 +712,7 @@ struct SlcState { struct ConditionalModeState; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(a1680744031fdb2d, 15, 20) + CAPNP_DECLARE_STRUCT_HEADER(a1680744031fdb2d, 17, 21) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } #endif // !CAPNP_LITE @@ -950,15 +976,15 @@ struct StarPilotModelDataV2::VisionObservation { }; }; -struct CustomReserved5 { - CustomReserved5() = delete; +struct StarPilotSteeringTracking { + StarPilotSteeringTracking() = delete; class Reader; class Builder; class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(a5cd762cd951a455, 0, 0) + CAPNP_DECLARE_STRUCT_HEADER(a5cd762cd951a455, 1, 3) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } #endif // !CAPNP_LITE @@ -1152,9 +1178,10 @@ struct StarPilotLateralState { class Builder; class Pipeline; struct LaneCentering; + struct FeedbackConsumption; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(c2243c65e0340384, 4, 3) + CAPNP_DECLARE_STRUCT_HEADER(c2243c65e0340384, 4, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } #endif // !CAPNP_LITE @@ -1176,6 +1203,21 @@ struct StarPilotLateralState::LaneCentering { }; }; +struct StarPilotLateralState::FeedbackConsumption { + FeedbackConsumption() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(c5693d5da66b60b6, 7, 3) + #if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } + #endif // !CAPNP_LITE + }; +}; + struct SlcCruiseCommand { SlcCruiseCommand() = delete; @@ -1186,7 +1228,7 @@ struct SlcCruiseCommand { struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(bd443b539493bc68, 10, 2) + CAPNP_DECLARE_STRUCT_HEADER(bd443b539493bc68, 10, 3) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } #endif // !CAPNP_LITE @@ -1412,7 +1454,7 @@ struct LearningControlEvidence { struct StageValue; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(f2447170bb68f358, 14, 19) + CAPNP_DECLARE_STRUCT_HEADER(f2447170bb68f358, 14, 21) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } #endif // !CAPNP_LITE @@ -1669,6 +1711,112 @@ struct LearningTorqueProjection { }; }; +struct LiveTrackingBinding { + LiveTrackingBinding() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(9f853f7eb4110c72, 1, 11) + #if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } + #endif // !CAPNP_LITE + }; +}; + +struct LiveTrackingTrial { + LiveTrackingTrial() = delete; + + class Reader; + class Builder; + class Pipeline; + struct Coordinate; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(e6f67224bc0ee1a3, 7, 11) + #if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } + #endif // !CAPNP_LITE + }; +}; + +struct LiveTrackingTrial::Coordinate { + Coordinate() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(ec37375f3ac20926, 1, 1) + #if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } + #endif // !CAPNP_LITE + }; +}; + +struct LiveTrackingApplication { + LiveTrackingApplication() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(db85b982067f1bc1, 4, 7) + #if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } + #endif // !CAPNP_LITE + }; +}; + +struct LiveTrackingRestore { + LiveTrackingRestore() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(ef6257bed9318012, 4, 7) + #if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } + #endif // !CAPNP_LITE + }; +}; + +struct LiveTrackingResponseCertificate { + LiveTrackingResponseCertificate() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(bbb8f17940692312, 7, 9) + #if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } + #endif // !CAPNP_LITE + }; +}; + +struct LiveTrackingResponseWitness { + LiveTrackingResponseWitness() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(daa1a9f0d8968e31, 7, 1) + #if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; } + #endif // !CAPNP_LITE + }; +}; + // ======================================================================================= class StarPilotCarControl::Reader { @@ -3280,6 +3428,9 @@ public: inline bool hasLearningApplication() const; inline ::cereal::LearningApplication::Reader getLearningApplication() const; + inline bool hasFeedbackPublication() const; + inline ::cereal::StarPilotCarState::FeedbackPublication::Reader getFeedbackPublication() const; + private: ::capnp::_::StructReader _reader; template @@ -3435,6 +3586,13 @@ public: inline void adoptLearningApplication(::capnp::Orphan< ::cereal::LearningApplication>&& value); inline ::capnp::Orphan< ::cereal::LearningApplication> disownLearningApplication(); + inline bool hasFeedbackPublication(); + inline ::cereal::StarPilotCarState::FeedbackPublication::Builder getFeedbackPublication(); + inline void setFeedbackPublication( ::cereal::StarPilotCarState::FeedbackPublication::Reader value); + inline ::cereal::StarPilotCarState::FeedbackPublication::Builder initFeedbackPublication(); + inline void adoptFeedbackPublication(::capnp::Orphan< ::cereal::StarPilotCarState::FeedbackPublication>&& value); + inline ::capnp::Orphan< ::cereal::StarPilotCarState::FeedbackPublication> disownFeedbackPublication(); + private: ::capnp::_::StructBuilder _builder; template @@ -3457,6 +3615,7 @@ public: inline ::cereal::LearningAppliedEvidence::Pipeline getAppliedEvidence(); inline ::cereal::FordLearningEvidence::Pipeline getFordLearningEvidence(); inline ::cereal::LearningApplication::Pipeline getLearningApplication(); + inline ::cereal::StarPilotCarState::FeedbackPublication::Pipeline getFeedbackPublication(); private: ::capnp::AnyPointer::Pipeline _typeless; friend class ::capnp::PipelineHook; @@ -3607,6 +3766,192 @@ private: }; #endif // !CAPNP_LITE +class StarPilotCarState::FeedbackPublication::Reader { +public: + typedef FeedbackPublication Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion() const; + + inline bool hasCardSession() const; + inline ::capnp::Data::Reader getCardSession() const; + + inline ::uint64_t getPublicationSequence() const; + + inline ::uint64_t getPublicationMonoTime() const; + + inline ::uint64_t getSourceCarStateMonoTime() const; + + inline bool getOutputKnown() const; + + inline ::uint64_t getOutputControlMonoTime() const; + + inline ::uint64_t getOutputApplyMonoTime() const; + + inline bool hasOutputNativeIdentity() const; + inline ::capnp::Text::Reader getOutputNativeIdentity() const; + + inline ::uint64_t getOutputCarrierSequence() const; + + inline float getReportedTorque() const; + + inline bool getOutputValid() const; + + inline bool hasCoBodySha256() const; + inline ::capnp::Data::Reader getCoBodySha256() const; + + inline bool hasApplyEvidenceSha256() const; + inline ::capnp::Data::Reader getApplyEvidenceSha256() const; + + inline bool hasSourceEpoch() const; + inline ::capnp::Text::Reader getSourceEpoch() const; + + inline bool hasCpSha256() const; + inline ::capnp::Text::Reader getCpSha256() const; + + inline ::uint64_t getCarrierObservedMonoTime() const; + +private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class StarPilotCarState::FeedbackPublication::Builder { +public: + typedef FeedbackPublication Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) {} + inline explicit Builder(::capnp::_::StructBuilder base): _builder(base) {} + inline operator Reader() const { return Reader(_builder.asReader()); } + inline Reader asReader() const { return *this; } + + inline ::capnp::MessageSize totalSize() const { return asReader().totalSize(); } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { return asReader().toString(); } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion(); + inline void setVersion( ::uint16_t value); + + inline bool hasCardSession(); + inline ::capnp::Data::Builder getCardSession(); + inline void setCardSession( ::capnp::Data::Reader value); + inline ::capnp::Data::Builder initCardSession(unsigned int size); + inline void adoptCardSession(::capnp::Orphan< ::capnp::Data>&& value); + inline ::capnp::Orphan< ::capnp::Data> disownCardSession(); + + inline ::uint64_t getPublicationSequence(); + inline void setPublicationSequence( ::uint64_t value); + + inline ::uint64_t getPublicationMonoTime(); + inline void setPublicationMonoTime( ::uint64_t value); + + inline ::uint64_t getSourceCarStateMonoTime(); + inline void setSourceCarStateMonoTime( ::uint64_t value); + + inline bool getOutputKnown(); + inline void setOutputKnown(bool value); + + inline ::uint64_t getOutputControlMonoTime(); + inline void setOutputControlMonoTime( ::uint64_t value); + + inline ::uint64_t getOutputApplyMonoTime(); + inline void setOutputApplyMonoTime( ::uint64_t value); + + inline bool hasOutputNativeIdentity(); + inline ::capnp::Text::Builder getOutputNativeIdentity(); + inline void setOutputNativeIdentity( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initOutputNativeIdentity(unsigned int size); + inline void adoptOutputNativeIdentity(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownOutputNativeIdentity(); + + inline ::uint64_t getOutputCarrierSequence(); + inline void setOutputCarrierSequence( ::uint64_t value); + + inline float getReportedTorque(); + inline void setReportedTorque(float value); + + inline bool getOutputValid(); + inline void setOutputValid(bool value); + + inline bool hasCoBodySha256(); + inline ::capnp::Data::Builder getCoBodySha256(); + inline void setCoBodySha256( ::capnp::Data::Reader value); + inline ::capnp::Data::Builder initCoBodySha256(unsigned int size); + inline void adoptCoBodySha256(::capnp::Orphan< ::capnp::Data>&& value); + inline ::capnp::Orphan< ::capnp::Data> disownCoBodySha256(); + + inline bool hasApplyEvidenceSha256(); + inline ::capnp::Data::Builder getApplyEvidenceSha256(); + inline void setApplyEvidenceSha256( ::capnp::Data::Reader value); + inline ::capnp::Data::Builder initApplyEvidenceSha256(unsigned int size); + inline void adoptApplyEvidenceSha256(::capnp::Orphan< ::capnp::Data>&& value); + inline ::capnp::Orphan< ::capnp::Data> disownApplyEvidenceSha256(); + + inline bool hasSourceEpoch(); + inline ::capnp::Text::Builder getSourceEpoch(); + inline void setSourceEpoch( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initSourceEpoch(unsigned int size); + inline void adoptSourceEpoch(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownSourceEpoch(); + + inline bool hasCpSha256(); + inline ::capnp::Text::Builder getCpSha256(); + inline void setCpSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initCpSha256(unsigned int size); + inline void adoptCpSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownCpSha256(); + + inline ::uint64_t getCarrierObservedMonoTime(); + inline void setCarrierObservedMonoTime( ::uint64_t value); + +private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class StarPilotCarState::FeedbackPublication::Pipeline { +public: + typedef FeedbackPublication Pipelines; + + inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) {} + +private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class StarPilotLongitudinalPlan::Reader { public: typedef StarPilotLongitudinalPlan Reads; @@ -3842,6 +4187,17 @@ public: inline ::uint64_t getCommandActionSequenceId() const; + inline bool getAutomaticAdoptionPending() const; + + inline ::uint64_t getPendingSinceMonoTime() const; + + inline float getLastReceivedSpeedLimit() const; + + inline bool hasLastReceivedSource() const; + inline ::capnp::Text::Reader getLastReceivedSource() const; + + inline float getLastReceivedAdjustedLimit() const; + private: ::capnp::_::StructReader _reader; template @@ -4103,6 +4459,25 @@ public: inline ::uint64_t getCommandActionSequenceId(); inline void setCommandActionSequenceId( ::uint64_t value); + inline bool getAutomaticAdoptionPending(); + inline void setAutomaticAdoptionPending(bool value); + + inline ::uint64_t getPendingSinceMonoTime(); + inline void setPendingSinceMonoTime( ::uint64_t value); + + inline float getLastReceivedSpeedLimit(); + inline void setLastReceivedSpeedLimit(float value); + + inline bool hasLastReceivedSource(); + inline ::capnp::Text::Builder getLastReceivedSource(); + inline void setLastReceivedSource( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initLastReceivedSource(unsigned int size); + inline void adoptLastReceivedSource(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownLastReceivedSource(); + + inline float getLastReceivedAdjustedLimit(); + inline void setLastReceivedAdjustedLimit(float value); + private: ::capnp::_::StructBuilder _builder; template @@ -6276,9 +6651,9 @@ private: }; #endif // !CAPNP_LITE -class CustomReserved5::Reader { +class StarPilotSteeringTracking::Reader { public: - typedef CustomReserved5 Reads; + typedef StarPilotSteeringTracking Reads; Reader() = default; inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} @@ -6293,6 +6668,17 @@ public: } #endif // !CAPNP_LITE + inline ::uint16_t getVersion() const; + + inline bool hasSourceReceipt() const; + inline ::cereal::AutomaticLearningSourceReceipt::Reader getSourceReceipt() const; + + inline bool hasTrial() const; + inline ::cereal::LiveTrackingTrial::Reader getTrial() const; + + inline bool hasRestore() const; + inline ::cereal::LiveTrackingRestore::Reader getRestore() const; + private: ::capnp::_::StructReader _reader; template @@ -6305,9 +6691,9 @@ private: friend class ::capnp::Orphanage; }; -class CustomReserved5::Builder { +class StarPilotSteeringTracking::Builder { public: - typedef CustomReserved5 Builds; + typedef StarPilotSteeringTracking Builds; Builder() = delete; // Deleted to discourage incorrect usage. // You can explicitly initialize to nullptr instead. @@ -6321,6 +6707,30 @@ public: inline ::kj::StringTree toString() const { return asReader().toString(); } #endif // !CAPNP_LITE + inline ::uint16_t getVersion(); + inline void setVersion( ::uint16_t value); + + inline bool hasSourceReceipt(); + inline ::cereal::AutomaticLearningSourceReceipt::Builder getSourceReceipt(); + inline void setSourceReceipt( ::cereal::AutomaticLearningSourceReceipt::Reader value); + inline ::cereal::AutomaticLearningSourceReceipt::Builder initSourceReceipt(); + inline void adoptSourceReceipt(::capnp::Orphan< ::cereal::AutomaticLearningSourceReceipt>&& value); + inline ::capnp::Orphan< ::cereal::AutomaticLearningSourceReceipt> disownSourceReceipt(); + + inline bool hasTrial(); + inline ::cereal::LiveTrackingTrial::Builder getTrial(); + inline void setTrial( ::cereal::LiveTrackingTrial::Reader value); + inline ::cereal::LiveTrackingTrial::Builder initTrial(); + inline void adoptTrial(::capnp::Orphan< ::cereal::LiveTrackingTrial>&& value); + inline ::capnp::Orphan< ::cereal::LiveTrackingTrial> disownTrial(); + + inline bool hasRestore(); + inline ::cereal::LiveTrackingRestore::Builder getRestore(); + inline void setRestore( ::cereal::LiveTrackingRestore::Reader value); + inline ::cereal::LiveTrackingRestore::Builder initRestore(); + inline void adoptRestore(::capnp::Orphan< ::cereal::LiveTrackingRestore>&& value); + inline ::capnp::Orphan< ::cereal::LiveTrackingRestore> disownRestore(); + private: ::capnp::_::StructBuilder _builder; template @@ -6331,14 +6741,17 @@ private: }; #if !CAPNP_LITE -class CustomReserved5::Pipeline { +class StarPilotSteeringTracking::Pipeline { public: - typedef CustomReserved5 Pipelines; + typedef StarPilotSteeringTracking Pipelines; inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) : _typeless(kj::mv(typeless)) {} + inline ::cereal::AutomaticLearningSourceReceipt::Pipeline getSourceReceipt(); + inline ::cereal::LiveTrackingTrial::Pipeline getTrial(); + inline ::cereal::LiveTrackingRestore::Pipeline getRestore(); private: ::capnp::AnyPointer::Pipeline _typeless; friend class ::capnp::PipelineHook; @@ -7867,6 +8280,12 @@ public: inline bool hasLearningApplication() const; inline ::cereal::LearningApplication::Reader getLearningApplication() const; + inline bool hasFeedbackConsumption() const; + inline ::cereal::StarPilotLateralState::FeedbackConsumption::Reader getFeedbackConsumption() const; + + inline bool hasTrackingApplication() const; + inline ::cereal::LiveTrackingApplication::Reader getTrackingApplication() const; + private: ::capnp::_::StructReader _reader; template @@ -7940,6 +8359,20 @@ public: inline void adoptLearningApplication(::capnp::Orphan< ::cereal::LearningApplication>&& value); inline ::capnp::Orphan< ::cereal::LearningApplication> disownLearningApplication(); + inline bool hasFeedbackConsumption(); + inline ::cereal::StarPilotLateralState::FeedbackConsumption::Builder getFeedbackConsumption(); + inline void setFeedbackConsumption( ::cereal::StarPilotLateralState::FeedbackConsumption::Reader value); + inline ::cereal::StarPilotLateralState::FeedbackConsumption::Builder initFeedbackConsumption(); + inline void adoptFeedbackConsumption(::capnp::Orphan< ::cereal::StarPilotLateralState::FeedbackConsumption>&& value); + inline ::capnp::Orphan< ::cereal::StarPilotLateralState::FeedbackConsumption> disownFeedbackConsumption(); + + inline bool hasTrackingApplication(); + inline ::cereal::LiveTrackingApplication::Builder getTrackingApplication(); + inline void setTrackingApplication( ::cereal::LiveTrackingApplication::Reader value); + inline ::cereal::LiveTrackingApplication::Builder initTrackingApplication(); + inline void adoptTrackingApplication(::capnp::Orphan< ::cereal::LiveTrackingApplication>&& value); + inline ::capnp::Orphan< ::cereal::LiveTrackingApplication> disownTrackingApplication(); + private: ::capnp::_::StructBuilder _builder; template @@ -7961,6 +8394,8 @@ public: inline ::cereal::StarPilotLateralState::LaneCentering::Pipeline getLaneCentering(); inline ::cereal::LearningControlEvidence::Pipeline getLearningEvidence(); inline ::cereal::LearningApplication::Pipeline getLearningApplication(); + inline ::cereal::StarPilotLateralState::FeedbackConsumption::Pipeline getFeedbackConsumption(); + inline ::cereal::LiveTrackingApplication::Pipeline getTrackingApplication(); private: ::capnp::AnyPointer::Pipeline _typeless; friend class ::capnp::PipelineHook; @@ -8080,6 +8515,172 @@ private: }; #endif // !CAPNP_LITE +class StarPilotLateralState::FeedbackConsumption::Reader { +public: + typedef FeedbackConsumption Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion() const; + + inline ::uint64_t getControlMonoTime() const; + + inline ::uint64_t getConsumedPublicationMonoTime() const; + + inline ::uint64_t getConsumedRecvFrame() const; + + inline ::uint64_t getConsumedReceiptMonoTime() const; + + inline bool getConsumedSeen() const; + + inline bool getConsumedValid() const; + + inline bool hasCoBodySha256() const; + inline ::capnp::Data::Reader getCoBodySha256() const; + + inline float getReportedTorque() const; + + inline float getRequestTorque() const; + + inline bool getLatActive() const; + + inline bool getInputSafetyLimited() const; + + inline bool getPublishedSafetyLimited() const; + + inline ::uint64_t getPriorPublishControlMonoTime() const; + + inline bool hasSourceEpoch() const; + inline ::capnp::Text::Reader getSourceEpoch() const; + + inline bool hasCpSha256() const; + inline ::capnp::Text::Reader getCpSha256() const; + +private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class StarPilotLateralState::FeedbackConsumption::Builder { +public: + typedef FeedbackConsumption Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) {} + inline explicit Builder(::capnp::_::StructBuilder base): _builder(base) {} + inline operator Reader() const { return Reader(_builder.asReader()); } + inline Reader asReader() const { return *this; } + + inline ::capnp::MessageSize totalSize() const { return asReader().totalSize(); } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { return asReader().toString(); } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion(); + inline void setVersion( ::uint16_t value); + + inline ::uint64_t getControlMonoTime(); + inline void setControlMonoTime( ::uint64_t value); + + inline ::uint64_t getConsumedPublicationMonoTime(); + inline void setConsumedPublicationMonoTime( ::uint64_t value); + + inline ::uint64_t getConsumedRecvFrame(); + inline void setConsumedRecvFrame( ::uint64_t value); + + inline ::uint64_t getConsumedReceiptMonoTime(); + inline void setConsumedReceiptMonoTime( ::uint64_t value); + + inline bool getConsumedSeen(); + inline void setConsumedSeen(bool value); + + inline bool getConsumedValid(); + inline void setConsumedValid(bool value); + + inline bool hasCoBodySha256(); + inline ::capnp::Data::Builder getCoBodySha256(); + inline void setCoBodySha256( ::capnp::Data::Reader value); + inline ::capnp::Data::Builder initCoBodySha256(unsigned int size); + inline void adoptCoBodySha256(::capnp::Orphan< ::capnp::Data>&& value); + inline ::capnp::Orphan< ::capnp::Data> disownCoBodySha256(); + + inline float getReportedTorque(); + inline void setReportedTorque(float value); + + inline float getRequestTorque(); + inline void setRequestTorque(float value); + + inline bool getLatActive(); + inline void setLatActive(bool value); + + inline bool getInputSafetyLimited(); + inline void setInputSafetyLimited(bool value); + + inline bool getPublishedSafetyLimited(); + inline void setPublishedSafetyLimited(bool value); + + inline ::uint64_t getPriorPublishControlMonoTime(); + inline void setPriorPublishControlMonoTime( ::uint64_t value); + + inline bool hasSourceEpoch(); + inline ::capnp::Text::Builder getSourceEpoch(); + inline void setSourceEpoch( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initSourceEpoch(unsigned int size); + inline void adoptSourceEpoch(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownSourceEpoch(); + + inline bool hasCpSha256(); + inline ::capnp::Text::Builder getCpSha256(); + inline void setCpSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initCpSha256(unsigned int size); + inline void adoptCpSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownCpSha256(); + +private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class StarPilotLateralState::FeedbackConsumption::Pipeline { +public: + typedef FeedbackConsumption Pipelines; + + inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) {} + +private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class SlcCruiseCommand::Reader { public: typedef SlcCruiseCommand Reads; @@ -8127,6 +8728,9 @@ public: inline float getTargetMps() const; + inline bool hasSource() const; + inline ::capnp::Text::Reader getSource() const; + private: ::capnp::_::StructReader _reader; template @@ -8205,6 +8809,13 @@ public: inline float getTargetMps(); inline void setTargetMps(float value); + inline bool hasSource(); + inline ::capnp::Text::Builder getSource(); + inline void setSource( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initSource(unsigned int size); + inline void adoptSource(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownSource(); + private: ::capnp::_::StructBuilder _builder; template @@ -9974,6 +10585,12 @@ public: inline ::uint64_t getEffectPrepareMonoTime() const; + inline bool hasVehicleModelAnchor() const; + inline ::capnp::List::Reader getVehicleModelAnchor() const; + + inline bool hasLiveAppliedCoordinates() const; + inline ::capnp::List::Reader getLiveAppliedCoordinates() const; + private: ::capnp::_::StructReader _reader; template @@ -10220,6 +10837,22 @@ public: inline ::uint64_t getEffectPrepareMonoTime(); inline void setEffectPrepareMonoTime( ::uint64_t value); + inline bool hasVehicleModelAnchor(); + inline ::capnp::List::Builder getVehicleModelAnchor(); + inline void setVehicleModelAnchor( ::capnp::List::Reader value); + inline void setVehicleModelAnchor(::kj::ArrayPtr value); + inline ::capnp::List::Builder initVehicleModelAnchor(unsigned int size); + inline void adoptVehicleModelAnchor(::capnp::Orphan< ::capnp::List>&& value); + inline ::capnp::Orphan< ::capnp::List> disownVehicleModelAnchor(); + + inline bool hasLiveAppliedCoordinates(); + inline ::capnp::List::Builder getLiveAppliedCoordinates(); + inline void setLiveAppliedCoordinates( ::capnp::List::Reader value); + inline void setLiveAppliedCoordinates(::kj::ArrayPtr value); + inline ::capnp::List::Builder initLiveAppliedCoordinates(unsigned int size); + inline void adoptLiveAppliedCoordinates(::capnp::Orphan< ::capnp::List>&& value); + inline ::capnp::Orphan< ::capnp::List> disownLiveAppliedCoordinates(); + private: ::capnp::_::StructBuilder _builder; template @@ -12823,6 +13456,1137 @@ private: }; #endif // !CAPNP_LITE +class LiveTrackingBinding::Reader { +public: + typedef LiveTrackingBinding Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasIdentity() const; + inline ::capnp::Text::Reader getIdentity() const; + + inline bool hasCpSha256() const; + inline ::capnp::Text::Reader getCpSha256() const; + + inline bool hasControllerRevision() const; + inline ::capnp::Text::Reader getControllerRevision() const; + + inline bool hasBaselineRevision() const; + inline ::capnp::Text::Reader getBaselineRevision() const; + + inline bool hasSourceEpoch() const; + inline ::capnp::Text::Reader getSourceEpoch() const; + + inline bool hasResetGeneration() const; + inline ::capnp::Text::Reader getResetGeneration() const; + + inline bool hasCalibrationGeneration() const; + inline ::capnp::Text::Reader getCalibrationGeneration() const; + + inline bool hasVehicleModelGeneration() const; + inline ::capnp::Text::Reader getVehicleModelGeneration() const; + + inline bool hasNativeIdentity() const; + inline ::capnp::Text::Reader getNativeIdentity() const; + + inline bool hasPhysicalModelRevision() const; + inline ::capnp::Text::Reader getPhysicalModelRevision() const; + + inline bool hasPersistentSourceEpoch() const; + inline ::capnp::Text::Reader getPersistentSourceEpoch() const; + + inline bool getNativeRequired() const; + +private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class LiveTrackingBinding::Builder { +public: + typedef LiveTrackingBinding Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) {} + inline explicit Builder(::capnp::_::StructBuilder base): _builder(base) {} + inline operator Reader() const { return Reader(_builder.asReader()); } + inline Reader asReader() const { return *this; } + + inline ::capnp::MessageSize totalSize() const { return asReader().totalSize(); } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { return asReader().toString(); } +#endif // !CAPNP_LITE + + inline bool hasIdentity(); + inline ::capnp::Text::Builder getIdentity(); + inline void setIdentity( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initIdentity(unsigned int size); + inline void adoptIdentity(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownIdentity(); + + inline bool hasCpSha256(); + inline ::capnp::Text::Builder getCpSha256(); + inline void setCpSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initCpSha256(unsigned int size); + inline void adoptCpSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownCpSha256(); + + inline bool hasControllerRevision(); + inline ::capnp::Text::Builder getControllerRevision(); + inline void setControllerRevision( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initControllerRevision(unsigned int size); + inline void adoptControllerRevision(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownControllerRevision(); + + inline bool hasBaselineRevision(); + inline ::capnp::Text::Builder getBaselineRevision(); + inline void setBaselineRevision( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initBaselineRevision(unsigned int size); + inline void adoptBaselineRevision(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownBaselineRevision(); + + inline bool hasSourceEpoch(); + inline ::capnp::Text::Builder getSourceEpoch(); + inline void setSourceEpoch( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initSourceEpoch(unsigned int size); + inline void adoptSourceEpoch(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownSourceEpoch(); + + inline bool hasResetGeneration(); + inline ::capnp::Text::Builder getResetGeneration(); + inline void setResetGeneration( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initResetGeneration(unsigned int size); + inline void adoptResetGeneration(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownResetGeneration(); + + inline bool hasCalibrationGeneration(); + inline ::capnp::Text::Builder getCalibrationGeneration(); + inline void setCalibrationGeneration( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initCalibrationGeneration(unsigned int size); + inline void adoptCalibrationGeneration(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownCalibrationGeneration(); + + inline bool hasVehicleModelGeneration(); + inline ::capnp::Text::Builder getVehicleModelGeneration(); + inline void setVehicleModelGeneration( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initVehicleModelGeneration(unsigned int size); + inline void adoptVehicleModelGeneration(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownVehicleModelGeneration(); + + inline bool hasNativeIdentity(); + inline ::capnp::Text::Builder getNativeIdentity(); + inline void setNativeIdentity( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initNativeIdentity(unsigned int size); + inline void adoptNativeIdentity(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownNativeIdentity(); + + inline bool hasPhysicalModelRevision(); + inline ::capnp::Text::Builder getPhysicalModelRevision(); + inline void setPhysicalModelRevision( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initPhysicalModelRevision(unsigned int size); + inline void adoptPhysicalModelRevision(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownPhysicalModelRevision(); + + inline bool hasPersistentSourceEpoch(); + inline ::capnp::Text::Builder getPersistentSourceEpoch(); + inline void setPersistentSourceEpoch( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initPersistentSourceEpoch(unsigned int size); + inline void adoptPersistentSourceEpoch(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownPersistentSourceEpoch(); + + inline bool getNativeRequired(); + inline void setNativeRequired(bool value); + +private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class LiveTrackingBinding::Pipeline { +public: + typedef LiveTrackingBinding Pipelines; + + inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) {} + +private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + +class LiveTrackingTrial::Reader { +public: + typedef LiveTrackingTrial Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion() const; + + inline bool hasProtocol() const; + inline ::capnp::Text::Reader getProtocol() const; + + inline bool hasBinding() const; + inline ::cereal::LiveTrackingBinding::Reader getBinding() const; + + inline bool hasTrialId() const; + inline ::capnp::Text::Reader getTrialId() const; + + inline ::uint64_t getSequence() const; + + inline bool hasPhase() const; + inline ::capnp::Text::Reader getPhase() const; + + inline bool hasGroup() const; + inline ::capnp::Text::Reader getGroup() const; + + inline bool hasNames() const; + inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader getNames() const; + + inline bool hasBaseline() const; + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader getBaseline() const; + + inline bool hasTarget() const; + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader getTarget() const; + + inline bool hasBaselineSha256() const; + inline ::capnp::Text::Reader getBaselineSha256() const; + + inline bool hasTargetSha256() const; + inline ::capnp::Text::Reader getTargetSha256() const; + + inline ::uint64_t getStartedMonoTime() const; + + inline ::uint64_t getObservedMonoTime() const; + + inline ::uint64_t getValidUntilMonoTime() const; + + inline bool hasResponseWitness() const; + inline ::cereal::LiveTrackingResponseWitness::Reader getResponseWitness() const; + + inline ::uint64_t getTargetSequence() const; + + inline ::uint64_t getDeadlineMonoTime() const; + +private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class LiveTrackingTrial::Builder { +public: + typedef LiveTrackingTrial Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) {} + inline explicit Builder(::capnp::_::StructBuilder base): _builder(base) {} + inline operator Reader() const { return Reader(_builder.asReader()); } + inline Reader asReader() const { return *this; } + + inline ::capnp::MessageSize totalSize() const { return asReader().totalSize(); } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { return asReader().toString(); } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion(); + inline void setVersion( ::uint16_t value); + + inline bool hasProtocol(); + inline ::capnp::Text::Builder getProtocol(); + inline void setProtocol( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initProtocol(unsigned int size); + inline void adoptProtocol(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownProtocol(); + + inline bool hasBinding(); + inline ::cereal::LiveTrackingBinding::Builder getBinding(); + inline void setBinding( ::cereal::LiveTrackingBinding::Reader value); + inline ::cereal::LiveTrackingBinding::Builder initBinding(); + inline void adoptBinding(::capnp::Orphan< ::cereal::LiveTrackingBinding>&& value); + inline ::capnp::Orphan< ::cereal::LiveTrackingBinding> disownBinding(); + + inline bool hasTrialId(); + inline ::capnp::Text::Builder getTrialId(); + inline void setTrialId( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initTrialId(unsigned int size); + inline void adoptTrialId(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownTrialId(); + + inline ::uint64_t getSequence(); + inline void setSequence( ::uint64_t value); + + inline bool hasPhase(); + inline ::capnp::Text::Builder getPhase(); + inline void setPhase( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initPhase(unsigned int size); + inline void adoptPhase(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownPhase(); + + inline bool hasGroup(); + inline ::capnp::Text::Builder getGroup(); + inline void setGroup( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initGroup(unsigned int size); + inline void adoptGroup(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownGroup(); + + inline bool hasNames(); + inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder getNames(); + inline void setNames( ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setNames(::kj::ArrayPtr value); + inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder initNames(unsigned int size); + inline void adoptNames(::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>&& value); + inline ::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>> disownNames(); + + inline bool hasBaseline(); + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder getBaseline(); + inline void setBaseline( ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder initBaseline(unsigned int size); + inline void adoptBaseline(::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>> disownBaseline(); + + inline bool hasTarget(); + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder getTarget(); + inline void setTarget( ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder initTarget(unsigned int size); + inline void adoptTarget(::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>> disownTarget(); + + inline bool hasBaselineSha256(); + inline ::capnp::Text::Builder getBaselineSha256(); + inline void setBaselineSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initBaselineSha256(unsigned int size); + inline void adoptBaselineSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownBaselineSha256(); + + inline bool hasTargetSha256(); + inline ::capnp::Text::Builder getTargetSha256(); + inline void setTargetSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initTargetSha256(unsigned int size); + inline void adoptTargetSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownTargetSha256(); + + inline ::uint64_t getStartedMonoTime(); + inline void setStartedMonoTime( ::uint64_t value); + + inline ::uint64_t getObservedMonoTime(); + inline void setObservedMonoTime( ::uint64_t value); + + inline ::uint64_t getValidUntilMonoTime(); + inline void setValidUntilMonoTime( ::uint64_t value); + + inline bool hasResponseWitness(); + inline ::cereal::LiveTrackingResponseWitness::Builder getResponseWitness(); + inline void setResponseWitness( ::cereal::LiveTrackingResponseWitness::Reader value); + inline ::cereal::LiveTrackingResponseWitness::Builder initResponseWitness(); + inline void adoptResponseWitness(::capnp::Orphan< ::cereal::LiveTrackingResponseWitness>&& value); + inline ::capnp::Orphan< ::cereal::LiveTrackingResponseWitness> disownResponseWitness(); + + inline ::uint64_t getTargetSequence(); + inline void setTargetSequence( ::uint64_t value); + + inline ::uint64_t getDeadlineMonoTime(); + inline void setDeadlineMonoTime( ::uint64_t value); + +private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class LiveTrackingTrial::Pipeline { +public: + typedef LiveTrackingTrial Pipelines; + + inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) {} + + inline ::cereal::LiveTrackingBinding::Pipeline getBinding(); + inline ::cereal::LiveTrackingResponseWitness::Pipeline getResponseWitness(); +private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + +class LiveTrackingTrial::Coordinate::Reader { +public: + typedef Coordinate Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasName() const; + inline ::capnp::Text::Reader getName() const; + + inline double getValue() const; + +private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class LiveTrackingTrial::Coordinate::Builder { +public: + typedef Coordinate Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) {} + inline explicit Builder(::capnp::_::StructBuilder base): _builder(base) {} + inline operator Reader() const { return Reader(_builder.asReader()); } + inline Reader asReader() const { return *this; } + + inline ::capnp::MessageSize totalSize() const { return asReader().totalSize(); } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { return asReader().toString(); } +#endif // !CAPNP_LITE + + inline bool hasName(); + inline ::capnp::Text::Builder getName(); + inline void setName( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initName(unsigned int size); + inline void adoptName(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownName(); + + inline double getValue(); + inline void setValue(double value); + +private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class LiveTrackingTrial::Coordinate::Pipeline { +public: + typedef Coordinate Pipelines; + + inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) {} + +private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + +class LiveTrackingApplication::Reader { +public: + typedef LiveTrackingApplication Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion() const; + + inline bool hasBinding() const; + inline ::cereal::LiveTrackingBinding::Reader getBinding() const; + + inline bool hasTrialId() const; + inline ::capnp::Text::Reader getTrialId() const; + + inline ::uint64_t getSequence() const; + + inline ::uint64_t getControlMonoTime() const; + + inline bool hasStatus() const; + inline ::capnp::Text::Reader getStatus() const; + + inline bool hasApplied() const; + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader getApplied() const; + + inline bool hasAppliedSha256() const; + inline ::capnp::Text::Reader getAppliedSha256() const; + + inline bool hasConfirmedTargetSha256() const; + inline ::capnp::Text::Reader getConfirmedTargetSha256() const; + + inline bool hasReason() const; + inline ::capnp::Text::Reader getReason() const; + + inline ::uint64_t getTargetSequence() const; + +private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class LiveTrackingApplication::Builder { +public: + typedef LiveTrackingApplication Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) {} + inline explicit Builder(::capnp::_::StructBuilder base): _builder(base) {} + inline operator Reader() const { return Reader(_builder.asReader()); } + inline Reader asReader() const { return *this; } + + inline ::capnp::MessageSize totalSize() const { return asReader().totalSize(); } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { return asReader().toString(); } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion(); + inline void setVersion( ::uint16_t value); + + inline bool hasBinding(); + inline ::cereal::LiveTrackingBinding::Builder getBinding(); + inline void setBinding( ::cereal::LiveTrackingBinding::Reader value); + inline ::cereal::LiveTrackingBinding::Builder initBinding(); + inline void adoptBinding(::capnp::Orphan< ::cereal::LiveTrackingBinding>&& value); + inline ::capnp::Orphan< ::cereal::LiveTrackingBinding> disownBinding(); + + inline bool hasTrialId(); + inline ::capnp::Text::Builder getTrialId(); + inline void setTrialId( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initTrialId(unsigned int size); + inline void adoptTrialId(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownTrialId(); + + inline ::uint64_t getSequence(); + inline void setSequence( ::uint64_t value); + + inline ::uint64_t getControlMonoTime(); + inline void setControlMonoTime( ::uint64_t value); + + inline bool hasStatus(); + inline ::capnp::Text::Builder getStatus(); + inline void setStatus( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initStatus(unsigned int size); + inline void adoptStatus(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownStatus(); + + inline bool hasApplied(); + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder getApplied(); + inline void setApplied( ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder initApplied(unsigned int size); + inline void adoptApplied(::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>> disownApplied(); + + inline bool hasAppliedSha256(); + inline ::capnp::Text::Builder getAppliedSha256(); + inline void setAppliedSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initAppliedSha256(unsigned int size); + inline void adoptAppliedSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownAppliedSha256(); + + inline bool hasConfirmedTargetSha256(); + inline ::capnp::Text::Builder getConfirmedTargetSha256(); + inline void setConfirmedTargetSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initConfirmedTargetSha256(unsigned int size); + inline void adoptConfirmedTargetSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownConfirmedTargetSha256(); + + inline bool hasReason(); + inline ::capnp::Text::Builder getReason(); + inline void setReason( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initReason(unsigned int size); + inline void adoptReason(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownReason(); + + inline ::uint64_t getTargetSequence(); + inline void setTargetSequence( ::uint64_t value); + +private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class LiveTrackingApplication::Pipeline { +public: + typedef LiveTrackingApplication Pipelines; + + inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) {} + + inline ::cereal::LiveTrackingBinding::Pipeline getBinding(); +private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + +class LiveTrackingRestore::Reader { +public: + typedef LiveTrackingRestore Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion() const; + + inline bool hasProtocol() const; + inline ::capnp::Text::Reader getProtocol() const; + + inline bool hasBinding() const; + inline ::cereal::LiveTrackingBinding::Reader getBinding() const; + + inline bool hasHistoryScope() const; + inline ::capnp::Text::Reader getHistoryScope() const; + + inline bool hasHistoryTrialId() const; + inline ::capnp::Text::Reader getHistoryTrialId() const; + + inline bool hasTarget() const; + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader getTarget() const; + + inline bool hasTargetSha256() const; + inline ::capnp::Text::Reader getTargetSha256() const; + + inline ::uint64_t getSequence() const; + + inline ::uint64_t getObservedMonoTime() const; + + inline ::uint64_t getValidUntilMonoTime() const; + + inline bool hasResponseWitness() const; + inline ::cereal::LiveTrackingResponseWitness::Reader getResponseWitness() const; + +private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class LiveTrackingRestore::Builder { +public: + typedef LiveTrackingRestore Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) {} + inline explicit Builder(::capnp::_::StructBuilder base): _builder(base) {} + inline operator Reader() const { return Reader(_builder.asReader()); } + inline Reader asReader() const { return *this; } + + inline ::capnp::MessageSize totalSize() const { return asReader().totalSize(); } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { return asReader().toString(); } +#endif // !CAPNP_LITE + + inline ::uint16_t getVersion(); + inline void setVersion( ::uint16_t value); + + inline bool hasProtocol(); + inline ::capnp::Text::Builder getProtocol(); + inline void setProtocol( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initProtocol(unsigned int size); + inline void adoptProtocol(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownProtocol(); + + inline bool hasBinding(); + inline ::cereal::LiveTrackingBinding::Builder getBinding(); + inline void setBinding( ::cereal::LiveTrackingBinding::Reader value); + inline ::cereal::LiveTrackingBinding::Builder initBinding(); + inline void adoptBinding(::capnp::Orphan< ::cereal::LiveTrackingBinding>&& value); + inline ::capnp::Orphan< ::cereal::LiveTrackingBinding> disownBinding(); + + inline bool hasHistoryScope(); + inline ::capnp::Text::Builder getHistoryScope(); + inline void setHistoryScope( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initHistoryScope(unsigned int size); + inline void adoptHistoryScope(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownHistoryScope(); + + inline bool hasHistoryTrialId(); + inline ::capnp::Text::Builder getHistoryTrialId(); + inline void setHistoryTrialId( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initHistoryTrialId(unsigned int size); + inline void adoptHistoryTrialId(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownHistoryTrialId(); + + inline bool hasTarget(); + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder getTarget(); + inline void setTarget( ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder initTarget(unsigned int size); + inline void adoptTarget(::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>> disownTarget(); + + inline bool hasTargetSha256(); + inline ::capnp::Text::Builder getTargetSha256(); + inline void setTargetSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initTargetSha256(unsigned int size); + inline void adoptTargetSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownTargetSha256(); + + inline ::uint64_t getSequence(); + inline void setSequence( ::uint64_t value); + + inline ::uint64_t getObservedMonoTime(); + inline void setObservedMonoTime( ::uint64_t value); + + inline ::uint64_t getValidUntilMonoTime(); + inline void setValidUntilMonoTime( ::uint64_t value); + + inline bool hasResponseWitness(); + inline ::cereal::LiveTrackingResponseWitness::Builder getResponseWitness(); + inline void setResponseWitness( ::cereal::LiveTrackingResponseWitness::Reader value); + inline ::cereal::LiveTrackingResponseWitness::Builder initResponseWitness(); + inline void adoptResponseWitness(::capnp::Orphan< ::cereal::LiveTrackingResponseWitness>&& value); + inline ::capnp::Orphan< ::cereal::LiveTrackingResponseWitness> disownResponseWitness(); + +private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class LiveTrackingRestore::Pipeline { +public: + typedef LiveTrackingRestore Pipelines; + + inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) {} + + inline ::cereal::LiveTrackingBinding::Pipeline getBinding(); + inline ::cereal::LiveTrackingResponseWitness::Pipeline getResponseWitness(); +private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + +class LiveTrackingResponseCertificate::Reader { +public: + typedef LiveTrackingResponseCertificate Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasVersion() const; + inline ::capnp::Text::Reader getVersion() const; + + inline bool hasModelRevision() const; + inline ::capnp::Text::Reader getModelRevision() const; + + inline ::uint64_t getLagNs() const; + + inline double getSlope() const; + + inline double getIntercept() const; + + inline double getCommandLow() const; + + inline double getCommandHigh() const; + + inline double getHeldoutRmse() const; + + inline bool hasTrainBlockIds() const; + inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader getTrainBlockIds() const; + + inline bool hasHeldoutBlockIds() const; + inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader getHeldoutBlockIds() const; + + inline bool hasSourceInventorySha256() const; + inline ::capnp::Text::Reader getSourceInventorySha256() const; + + inline bool hasIdentificationSha256() const; + inline ::capnp::Text::Reader getIdentificationSha256() const; + + inline ::uint64_t getIdentifiedMonoTime() const; + + inline bool hasProvenance() const; + inline ::capnp::Text::Reader getProvenance() const; + + inline bool hasParentModelRevision() const; + inline ::capnp::Text::Reader getParentModelRevision() const; + + inline bool hasRequalificationSha256() const; + inline ::capnp::Text::Reader getRequalificationSha256() const; + +private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class LiveTrackingResponseCertificate::Builder { +public: + typedef LiveTrackingResponseCertificate Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) {} + inline explicit Builder(::capnp::_::StructBuilder base): _builder(base) {} + inline operator Reader() const { return Reader(_builder.asReader()); } + inline Reader asReader() const { return *this; } + + inline ::capnp::MessageSize totalSize() const { return asReader().totalSize(); } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { return asReader().toString(); } +#endif // !CAPNP_LITE + + inline bool hasVersion(); + inline ::capnp::Text::Builder getVersion(); + inline void setVersion( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initVersion(unsigned int size); + inline void adoptVersion(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownVersion(); + + inline bool hasModelRevision(); + inline ::capnp::Text::Builder getModelRevision(); + inline void setModelRevision( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initModelRevision(unsigned int size); + inline void adoptModelRevision(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownModelRevision(); + + inline ::uint64_t getLagNs(); + inline void setLagNs( ::uint64_t value); + + inline double getSlope(); + inline void setSlope(double value); + + inline double getIntercept(); + inline void setIntercept(double value); + + inline double getCommandLow(); + inline void setCommandLow(double value); + + inline double getCommandHigh(); + inline void setCommandHigh(double value); + + inline double getHeldoutRmse(); + inline void setHeldoutRmse(double value); + + inline bool hasTrainBlockIds(); + inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder getTrainBlockIds(); + inline void setTrainBlockIds( ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setTrainBlockIds(::kj::ArrayPtr value); + inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder initTrainBlockIds(unsigned int size); + inline void adoptTrainBlockIds(::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>&& value); + inline ::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>> disownTrainBlockIds(); + + inline bool hasHeldoutBlockIds(); + inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder getHeldoutBlockIds(); + inline void setHeldoutBlockIds( ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setHeldoutBlockIds(::kj::ArrayPtr value); + inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder initHeldoutBlockIds(unsigned int size); + inline void adoptHeldoutBlockIds(::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>&& value); + inline ::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>> disownHeldoutBlockIds(); + + inline bool hasSourceInventorySha256(); + inline ::capnp::Text::Builder getSourceInventorySha256(); + inline void setSourceInventorySha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initSourceInventorySha256(unsigned int size); + inline void adoptSourceInventorySha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownSourceInventorySha256(); + + inline bool hasIdentificationSha256(); + inline ::capnp::Text::Builder getIdentificationSha256(); + inline void setIdentificationSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initIdentificationSha256(unsigned int size); + inline void adoptIdentificationSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownIdentificationSha256(); + + inline ::uint64_t getIdentifiedMonoTime(); + inline void setIdentifiedMonoTime( ::uint64_t value); + + inline bool hasProvenance(); + inline ::capnp::Text::Builder getProvenance(); + inline void setProvenance( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initProvenance(unsigned int size); + inline void adoptProvenance(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownProvenance(); + + inline bool hasParentModelRevision(); + inline ::capnp::Text::Builder getParentModelRevision(); + inline void setParentModelRevision( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initParentModelRevision(unsigned int size); + inline void adoptParentModelRevision(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownParentModelRevision(); + + inline bool hasRequalificationSha256(); + inline ::capnp::Text::Builder getRequalificationSha256(); + inline void setRequalificationSha256( ::capnp::Text::Reader value); + inline ::capnp::Text::Builder initRequalificationSha256(unsigned int size); + inline void adoptRequalificationSha256(::capnp::Orphan< ::capnp::Text>&& value); + inline ::capnp::Orphan< ::capnp::Text> disownRequalificationSha256(); + +private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class LiveTrackingResponseCertificate::Pipeline { +public: + typedef LiveTrackingResponseCertificate Pipelines; + + inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) {} + +private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + +class LiveTrackingResponseWitness::Reader { +public: + typedef LiveTrackingResponseWitness Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base): _reader(base) {} + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasCertificate() const; + inline ::cereal::LiveTrackingResponseCertificate::Reader getCertificate() const; + + inline ::uint64_t getLastControlMonoTime() const; + + inline ::uint64_t getLastApplyMonoTime() const; + + inline ::uint64_t getLastPoseMonoTime() const; + + inline ::uint64_t getWindowStartMonoTime() const; + + inline ::uint16_t getIndependentPoses() const; + + inline double getResidualRmse() const; + + inline ::uint64_t getValidUntilMonoTime() const; + +private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class LiveTrackingResponseWitness::Builder { +public: + typedef LiveTrackingResponseWitness Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) {} + inline explicit Builder(::capnp::_::StructBuilder base): _builder(base) {} + inline operator Reader() const { return Reader(_builder.asReader()); } + inline Reader asReader() const { return *this; } + + inline ::capnp::MessageSize totalSize() const { return asReader().totalSize(); } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { return asReader().toString(); } +#endif // !CAPNP_LITE + + inline bool hasCertificate(); + inline ::cereal::LiveTrackingResponseCertificate::Builder getCertificate(); + inline void setCertificate( ::cereal::LiveTrackingResponseCertificate::Reader value); + inline ::cereal::LiveTrackingResponseCertificate::Builder initCertificate(); + inline void adoptCertificate(::capnp::Orphan< ::cereal::LiveTrackingResponseCertificate>&& value); + inline ::capnp::Orphan< ::cereal::LiveTrackingResponseCertificate> disownCertificate(); + + inline ::uint64_t getLastControlMonoTime(); + inline void setLastControlMonoTime( ::uint64_t value); + + inline ::uint64_t getLastApplyMonoTime(); + inline void setLastApplyMonoTime( ::uint64_t value); + + inline ::uint64_t getLastPoseMonoTime(); + inline void setLastPoseMonoTime( ::uint64_t value); + + inline ::uint64_t getWindowStartMonoTime(); + inline void setWindowStartMonoTime( ::uint64_t value); + + inline ::uint16_t getIndependentPoses(); + inline void setIndependentPoses( ::uint16_t value); + + inline double getResidualRmse(); + inline void setResidualRmse(double value); + + inline ::uint64_t getValidUntilMonoTime(); + inline void setValidUntilMonoTime( ::uint64_t value); + +private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class LiveTrackingResponseWitness::Pipeline { +public: + typedef LiveTrackingResponseWitness Pipelines; + + inline Pipeline(decltype(nullptr)): _typeless(nullptr) {} + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) {} + + inline ::cereal::LiveTrackingResponseCertificate::Pipeline getCertificate(); +private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + // ======================================================================================= inline bool StarPilotCarControl::Reader::hasHudControl() const { @@ -15544,6 +17308,45 @@ inline ::capnp::Orphan< ::cereal::LearningApplication> StarPilotCarState::Builde ::capnp::bounded<3>() * ::capnp::POINTERS)); } +inline bool StarPilotCarState::Reader::hasFeedbackPublication() const { + return !_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotCarState::Builder::hasFeedbackPublication() { + return !_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::StarPilotCarState::FeedbackPublication::Reader StarPilotCarState::Reader::getFeedbackPublication() const { + return ::capnp::_::PointerHelpers< ::cereal::StarPilotCarState::FeedbackPublication>::get(_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::cereal::StarPilotCarState::FeedbackPublication::Builder StarPilotCarState::Builder::getFeedbackPublication() { + return ::capnp::_::PointerHelpers< ::cereal::StarPilotCarState::FeedbackPublication>::get(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::StarPilotCarState::FeedbackPublication::Pipeline StarPilotCarState::Pipeline::getFeedbackPublication() { + return ::cereal::StarPilotCarState::FeedbackPublication::Pipeline(_typeless.getPointerField(4)); +} +#endif // !CAPNP_LITE +inline void StarPilotCarState::Builder::setFeedbackPublication( ::cereal::StarPilotCarState::FeedbackPublication::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::StarPilotCarState::FeedbackPublication>::set(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), value); +} +inline ::cereal::StarPilotCarState::FeedbackPublication::Builder StarPilotCarState::Builder::initFeedbackPublication() { + return ::capnp::_::PointerHelpers< ::cereal::StarPilotCarState::FeedbackPublication>::init(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void StarPilotCarState::Builder::adoptFeedbackPublication( + ::capnp::Orphan< ::cereal::StarPilotCarState::FeedbackPublication>&& value) { + ::capnp::_::PointerHelpers< ::cereal::StarPilotCarState::FeedbackPublication>::adopt(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::StarPilotCarState::FeedbackPublication> StarPilotCarState::Builder::disownFeedbackPublication() { + return ::capnp::_::PointerHelpers< ::cereal::StarPilotCarState::FeedbackPublication>::disown(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline ::uint64_t StarPilotCarState::Gps::Reader::getSourceMonoTime() const { return _reader.getDataField< ::uint64_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -15750,6 +17553,364 @@ inline ::capnp::Orphan< ::capnp::List> StarPil ::capnp::bounded<0>() * ::capnp::POINTERS)); } +inline ::uint16_t StarPilotCarState::FeedbackPublication::Reader::getVersion() const { + return _reader.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint16_t StarPilotCarState::FeedbackPublication::Builder::getVersion() { + return _builder.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setVersion( ::uint16_t value) { + _builder.setDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotCarState::FeedbackPublication::Reader::hasCardSession() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotCarState::FeedbackPublication::Builder::hasCardSession() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Data::Reader StarPilotCarState::FeedbackPublication::Reader::getCardSession() const { + return ::capnp::_::PointerHelpers< ::capnp::Data>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder StarPilotCarState::FeedbackPublication::Builder::getCardSession() { + return ::capnp::_::PointerHelpers< ::capnp::Data>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setCardSession( ::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Data>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::capnp::Data::Builder StarPilotCarState::FeedbackPublication::Builder::initCardSession(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Data>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), size); +} +inline void StarPilotCarState::FeedbackPublication::Builder::adoptCardSession( + ::capnp::Orphan< ::capnp::Data>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Data>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Data> StarPilotCarState::FeedbackPublication::Builder::disownCardSession() { + return ::capnp::_::PointerHelpers< ::capnp::Data>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Reader::getPublicationSequence() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Builder::getPublicationSequence() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setPublicationSequence( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Reader::getPublicationMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Builder::getPublicationMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setPublicationMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Reader::getSourceCarStateMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Builder::getSourceCarStateMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setSourceCarStateMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotCarState::FeedbackPublication::Reader::getOutputKnown() const { + return _reader.getDataField( + ::capnp::bounded<16>() * ::capnp::ELEMENTS); +} + +inline bool StarPilotCarState::FeedbackPublication::Builder::getOutputKnown() { + return _builder.getDataField( + ::capnp::bounded<16>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setOutputKnown(bool value) { + _builder.setDataField( + ::capnp::bounded<16>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Reader::getOutputControlMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<4>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Builder::getOutputControlMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<4>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setOutputControlMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<4>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Reader::getOutputApplyMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<5>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Builder::getOutputApplyMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<5>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setOutputApplyMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<5>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotCarState::FeedbackPublication::Reader::hasOutputNativeIdentity() const { + return !_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotCarState::FeedbackPublication::Builder::hasOutputNativeIdentity() { + return !_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader StarPilotCarState::FeedbackPublication::Reader::getOutputNativeIdentity() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder StarPilotCarState::FeedbackPublication::Builder::getOutputNativeIdentity() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setOutputNativeIdentity( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder StarPilotCarState::FeedbackPublication::Builder::initOutputNativeIdentity(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), size); +} +inline void StarPilotCarState::FeedbackPublication::Builder::adoptOutputNativeIdentity( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> StarPilotCarState::FeedbackPublication::Builder::disownOutputNativeIdentity() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Reader::getOutputCarrierSequence() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Builder::getOutputCarrierSequence() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setOutputCarrierSequence( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS, value); +} + +inline float StarPilotCarState::FeedbackPublication::Reader::getReportedTorque() const { + return _reader.getDataField( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline float StarPilotCarState::FeedbackPublication::Builder::getReportedTorque() { + return _builder.getDataField( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setReportedTorque(float value) { + _builder.setDataField( + ::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotCarState::FeedbackPublication::Reader::getOutputValid() const { + return _reader.getDataField( + ::capnp::bounded<17>() * ::capnp::ELEMENTS); +} + +inline bool StarPilotCarState::FeedbackPublication::Builder::getOutputValid() { + return _builder.getDataField( + ::capnp::bounded<17>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setOutputValid(bool value) { + _builder.setDataField( + ::capnp::bounded<17>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotCarState::FeedbackPublication::Reader::hasCoBodySha256() const { + return !_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotCarState::FeedbackPublication::Builder::hasCoBodySha256() { + return !_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Data::Reader StarPilotCarState::FeedbackPublication::Reader::getCoBodySha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Data>::get(_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder StarPilotCarState::FeedbackPublication::Builder::getCoBodySha256() { + return ::capnp::_::PointerHelpers< ::capnp::Data>::get(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setCoBodySha256( ::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Data>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline ::capnp::Data::Builder StarPilotCarState::FeedbackPublication::Builder::initCoBodySha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Data>::init(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), size); +} +inline void StarPilotCarState::FeedbackPublication::Builder::adoptCoBodySha256( + ::capnp::Orphan< ::capnp::Data>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Data>::adopt(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Data> StarPilotCarState::FeedbackPublication::Builder::disownCoBodySha256() { + return ::capnp::_::PointerHelpers< ::capnp::Data>::disown(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline bool StarPilotCarState::FeedbackPublication::Reader::hasApplyEvidenceSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotCarState::FeedbackPublication::Builder::hasApplyEvidenceSha256() { + return !_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Data::Reader StarPilotCarState::FeedbackPublication::Reader::getApplyEvidenceSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Data>::get(_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder StarPilotCarState::FeedbackPublication::Builder::getApplyEvidenceSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Data>::get(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setApplyEvidenceSha256( ::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Data>::set(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), value); +} +inline ::capnp::Data::Builder StarPilotCarState::FeedbackPublication::Builder::initApplyEvidenceSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Data>::init(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), size); +} +inline void StarPilotCarState::FeedbackPublication::Builder::adoptApplyEvidenceSha256( + ::capnp::Orphan< ::capnp::Data>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Data>::adopt(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Data> StarPilotCarState::FeedbackPublication::Builder::disownApplyEvidenceSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Data>::disown(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool StarPilotCarState::FeedbackPublication::Reader::hasSourceEpoch() const { + return !_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotCarState::FeedbackPublication::Builder::hasSourceEpoch() { + return !_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader StarPilotCarState::FeedbackPublication::Reader::getSourceEpoch() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder StarPilotCarState::FeedbackPublication::Builder::getSourceEpoch() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setSourceEpoch( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder StarPilotCarState::FeedbackPublication::Builder::initSourceEpoch(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), size); +} +inline void StarPilotCarState::FeedbackPublication::Builder::adoptSourceEpoch( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> StarPilotCarState::FeedbackPublication::Builder::disownSourceEpoch() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} + +inline bool StarPilotCarState::FeedbackPublication::Reader::hasCpSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotCarState::FeedbackPublication::Builder::hasCpSha256() { + return !_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader StarPilotCarState::FeedbackPublication::Reader::getCpSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder StarPilotCarState::FeedbackPublication::Builder::getCpSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setCpSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder StarPilotCarState::FeedbackPublication::Builder::initCpSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), size); +} +inline void StarPilotCarState::FeedbackPublication::Builder::adoptCpSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> StarPilotCarState::FeedbackPublication::Builder::disownCpSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Reader::getCarrierObservedMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<7>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotCarState::FeedbackPublication::Builder::getCarrierObservedMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<7>() * ::capnp::ELEMENTS); +} +inline void StarPilotCarState::FeedbackPublication::Builder::setCarrierObservedMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<7>() * ::capnp::ELEMENTS, value); +} + inline ::uint16_t StarPilotLongitudinalPlan::Reader::getVersion() const { return _reader.getDataField< ::uint16_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -16954,6 +19115,96 @@ inline void SlcState::Builder::setCommandActionSequenceId( ::uint64_t value) { ::capnp::bounded<14>() * ::capnp::ELEMENTS, value); } +inline bool SlcState::Reader::getAutomaticAdoptionPending() const { + return _reader.getDataField( + ::capnp::bounded<140>() * ::capnp::ELEMENTS); +} + +inline bool SlcState::Builder::getAutomaticAdoptionPending() { + return _builder.getDataField( + ::capnp::bounded<140>() * ::capnp::ELEMENTS); +} +inline void SlcState::Builder::setAutomaticAdoptionPending(bool value) { + _builder.setDataField( + ::capnp::bounded<140>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t SlcState::Reader::getPendingSinceMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<15>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t SlcState::Builder::getPendingSinceMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<15>() * ::capnp::ELEMENTS); +} +inline void SlcState::Builder::setPendingSinceMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<15>() * ::capnp::ELEMENTS, value); +} + +inline float SlcState::Reader::getLastReceivedSpeedLimit() const { + return _reader.getDataField( + ::capnp::bounded<27>() * ::capnp::ELEMENTS); +} + +inline float SlcState::Builder::getLastReceivedSpeedLimit() { + return _builder.getDataField( + ::capnp::bounded<27>() * ::capnp::ELEMENTS); +} +inline void SlcState::Builder::setLastReceivedSpeedLimit(float value) { + _builder.setDataField( + ::capnp::bounded<27>() * ::capnp::ELEMENTS, value); +} + +inline bool SlcState::Reader::hasLastReceivedSource() const { + return !_reader.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS).isNull(); +} +inline bool SlcState::Builder::hasLastReceivedSource() { + return !_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader SlcState::Reader::getLastReceivedSource() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder SlcState::Builder::getLastReceivedSource() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS)); +} +inline void SlcState::Builder::setLastReceivedSource( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder SlcState::Builder::initLastReceivedSource(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS), size); +} +inline void SlcState::Builder::adoptLastReceivedSource( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> SlcState::Builder::disownLastReceivedSource() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS)); +} + +inline float SlcState::Reader::getLastReceivedAdjustedLimit() const { + return _reader.getDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS); +} + +inline float SlcState::Builder::getLastReceivedAdjustedLimit() { + return _builder.getDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS); +} +inline void SlcState::Builder::setLastReceivedAdjustedLimit(float value) { + _builder.setDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS, value); +} + inline bool SlcState::SourceReading::Reader::hasSource() const { return !_reader.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); @@ -20248,6 +22499,137 @@ inline void StarPilotModelDataV2::VisionObservation::Builder::setStream( ::cerea ::capnp::bounded<22>() * ::capnp::ELEMENTS, value); } +inline ::uint16_t StarPilotSteeringTracking::Reader::getVersion() const { + return _reader.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint16_t StarPilotSteeringTracking::Builder::getVersion() { + return _builder.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void StarPilotSteeringTracking::Builder::setVersion( ::uint16_t value) { + _builder.setDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotSteeringTracking::Reader::hasSourceReceipt() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotSteeringTracking::Builder::hasSourceReceipt() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::AutomaticLearningSourceReceipt::Reader StarPilotSteeringTracking::Reader::getSourceReceipt() const { + return ::capnp::_::PointerHelpers< ::cereal::AutomaticLearningSourceReceipt>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::cereal::AutomaticLearningSourceReceipt::Builder StarPilotSteeringTracking::Builder::getSourceReceipt() { + return ::capnp::_::PointerHelpers< ::cereal::AutomaticLearningSourceReceipt>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::AutomaticLearningSourceReceipt::Pipeline StarPilotSteeringTracking::Pipeline::getSourceReceipt() { + return ::cereal::AutomaticLearningSourceReceipt::Pipeline(_typeless.getPointerField(0)); +} +#endif // !CAPNP_LITE +inline void StarPilotSteeringTracking::Builder::setSourceReceipt( ::cereal::AutomaticLearningSourceReceipt::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::AutomaticLearningSourceReceipt>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::cereal::AutomaticLearningSourceReceipt::Builder StarPilotSteeringTracking::Builder::initSourceReceipt() { + return ::capnp::_::PointerHelpers< ::cereal::AutomaticLearningSourceReceipt>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void StarPilotSteeringTracking::Builder::adoptSourceReceipt( + ::capnp::Orphan< ::cereal::AutomaticLearningSourceReceipt>&& value) { + ::capnp::_::PointerHelpers< ::cereal::AutomaticLearningSourceReceipt>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::AutomaticLearningSourceReceipt> StarPilotSteeringTracking::Builder::disownSourceReceipt() { + return ::capnp::_::PointerHelpers< ::cereal::AutomaticLearningSourceReceipt>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool StarPilotSteeringTracking::Reader::hasTrial() const { + return !_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotSteeringTracking::Builder::hasTrial() { + return !_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::LiveTrackingTrial::Reader StarPilotSteeringTracking::Reader::getTrial() const { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingTrial>::get(_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::cereal::LiveTrackingTrial::Builder StarPilotSteeringTracking::Builder::getTrial() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingTrial>::get(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::LiveTrackingTrial::Pipeline StarPilotSteeringTracking::Pipeline::getTrial() { + return ::cereal::LiveTrackingTrial::Pipeline(_typeless.getPointerField(1)); +} +#endif // !CAPNP_LITE +inline void StarPilotSteeringTracking::Builder::setTrial( ::cereal::LiveTrackingTrial::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingTrial>::set(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), value); +} +inline ::cereal::LiveTrackingTrial::Builder StarPilotSteeringTracking::Builder::initTrial() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingTrial>::init(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void StarPilotSteeringTracking::Builder::adoptTrial( + ::capnp::Orphan< ::cereal::LiveTrackingTrial>&& value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingTrial>::adopt(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::LiveTrackingTrial> StarPilotSteeringTracking::Builder::disownTrial() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingTrial>::disown(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool StarPilotSteeringTracking::Reader::hasRestore() const { + return !_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotSteeringTracking::Builder::hasRestore() { + return !_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::LiveTrackingRestore::Reader StarPilotSteeringTracking::Reader::getRestore() const { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingRestore>::get(_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::cereal::LiveTrackingRestore::Builder StarPilotSteeringTracking::Builder::getRestore() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingRestore>::get(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::LiveTrackingRestore::Pipeline StarPilotSteeringTracking::Pipeline::getRestore() { + return ::cereal::LiveTrackingRestore::Pipeline(_typeless.getPointerField(2)); +} +#endif // !CAPNP_LITE +inline void StarPilotSteeringTracking::Builder::setRestore( ::cereal::LiveTrackingRestore::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingRestore>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline ::cereal::LiveTrackingRestore::Builder StarPilotSteeringTracking::Builder::initRestore() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingRestore>::init(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void StarPilotSteeringTracking::Builder::adoptRestore( + ::capnp::Orphan< ::cereal::LiveTrackingRestore>&& value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingRestore>::adopt(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::LiveTrackingRestore> StarPilotSteeringTracking::Builder::disownRestore() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingRestore>::disown(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} + inline float CustomReserved6::Reader::getAccelerationJerk() const { return _reader.getDataField( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -22592,6 +24974,84 @@ inline ::capnp::Orphan< ::cereal::LearningApplication> StarPilotLateralState::Bu ::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool StarPilotLateralState::Reader::hasFeedbackConsumption() const { + return !_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotLateralState::Builder::hasFeedbackConsumption() { + return !_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::StarPilotLateralState::FeedbackConsumption::Reader StarPilotLateralState::Reader::getFeedbackConsumption() const { + return ::capnp::_::PointerHelpers< ::cereal::StarPilotLateralState::FeedbackConsumption>::get(_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::cereal::StarPilotLateralState::FeedbackConsumption::Builder StarPilotLateralState::Builder::getFeedbackConsumption() { + return ::capnp::_::PointerHelpers< ::cereal::StarPilotLateralState::FeedbackConsumption>::get(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::StarPilotLateralState::FeedbackConsumption::Pipeline StarPilotLateralState::Pipeline::getFeedbackConsumption() { + return ::cereal::StarPilotLateralState::FeedbackConsumption::Pipeline(_typeless.getPointerField(3)); +} +#endif // !CAPNP_LITE +inline void StarPilotLateralState::Builder::setFeedbackConsumption( ::cereal::StarPilotLateralState::FeedbackConsumption::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::StarPilotLateralState::FeedbackConsumption>::set(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), value); +} +inline ::cereal::StarPilotLateralState::FeedbackConsumption::Builder StarPilotLateralState::Builder::initFeedbackConsumption() { + return ::capnp::_::PointerHelpers< ::cereal::StarPilotLateralState::FeedbackConsumption>::init(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void StarPilotLateralState::Builder::adoptFeedbackConsumption( + ::capnp::Orphan< ::cereal::StarPilotLateralState::FeedbackConsumption>&& value) { + ::capnp::_::PointerHelpers< ::cereal::StarPilotLateralState::FeedbackConsumption>::adopt(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::StarPilotLateralState::FeedbackConsumption> StarPilotLateralState::Builder::disownFeedbackConsumption() { + return ::capnp::_::PointerHelpers< ::cereal::StarPilotLateralState::FeedbackConsumption>::disown(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool StarPilotLateralState::Reader::hasTrackingApplication() const { + return !_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotLateralState::Builder::hasTrackingApplication() { + return !_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::LiveTrackingApplication::Reader StarPilotLateralState::Reader::getTrackingApplication() const { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingApplication>::get(_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::cereal::LiveTrackingApplication::Builder StarPilotLateralState::Builder::getTrackingApplication() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingApplication>::get(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::LiveTrackingApplication::Pipeline StarPilotLateralState::Pipeline::getTrackingApplication() { + return ::cereal::LiveTrackingApplication::Pipeline(_typeless.getPointerField(4)); +} +#endif // !CAPNP_LITE +inline void StarPilotLateralState::Builder::setTrackingApplication( ::cereal::LiveTrackingApplication::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingApplication>::set(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), value); +} +inline ::cereal::LiveTrackingApplication::Builder StarPilotLateralState::Builder::initTrackingApplication() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingApplication>::init(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void StarPilotLateralState::Builder::adoptTrackingApplication( + ::capnp::Orphan< ::cereal::LiveTrackingApplication>&& value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingApplication>::adopt(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::LiveTrackingApplication> StarPilotLateralState::Builder::disownTrackingApplication() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingApplication>::disown(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline ::uint16_t StarPilotLateralState::LaneCentering::Reader::getVersion() const { return _reader.getDataField< ::uint16_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -22710,6 +25170,290 @@ inline ::capnp::Orphan< ::capnp::Text> StarPilotLateralState::LaneCentering::Bui ::capnp::bounded<0>() * ::capnp::POINTERS)); } +inline ::uint16_t StarPilotLateralState::FeedbackConsumption::Reader::getVersion() const { + return _reader.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint16_t StarPilotLateralState::FeedbackConsumption::Builder::getVersion() { + return _builder.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setVersion( ::uint16_t value) { + _builder.setDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Reader::getControlMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Builder::getControlMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setControlMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Reader::getConsumedPublicationMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Builder::getConsumedPublicationMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setConsumedPublicationMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Reader::getConsumedRecvFrame() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Builder::getConsumedRecvFrame() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setConsumedRecvFrame( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Reader::getConsumedReceiptMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<4>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Builder::getConsumedReceiptMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<4>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setConsumedReceiptMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<4>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Reader::getConsumedSeen() const { + return _reader.getDataField( + ::capnp::bounded<16>() * ::capnp::ELEMENTS); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Builder::getConsumedSeen() { + return _builder.getDataField( + ::capnp::bounded<16>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setConsumedSeen(bool value) { + _builder.setDataField( + ::capnp::bounded<16>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Reader::getConsumedValid() const { + return _reader.getDataField( + ::capnp::bounded<17>() * ::capnp::ELEMENTS); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Builder::getConsumedValid() { + return _builder.getDataField( + ::capnp::bounded<17>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setConsumedValid(bool value) { + _builder.setDataField( + ::capnp::bounded<17>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Reader::hasCoBodySha256() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotLateralState::FeedbackConsumption::Builder::hasCoBodySha256() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Data::Reader StarPilotLateralState::FeedbackConsumption::Reader::getCoBodySha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Data>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder StarPilotLateralState::FeedbackConsumption::Builder::getCoBodySha256() { + return ::capnp::_::PointerHelpers< ::capnp::Data>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setCoBodySha256( ::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Data>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::capnp::Data::Builder StarPilotLateralState::FeedbackConsumption::Builder::initCoBodySha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Data>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), size); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::adoptCoBodySha256( + ::capnp::Orphan< ::capnp::Data>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Data>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Data> StarPilotLateralState::FeedbackConsumption::Builder::disownCoBodySha256() { + return ::capnp::_::PointerHelpers< ::capnp::Data>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline float StarPilotLateralState::FeedbackConsumption::Reader::getReportedTorque() const { + return _reader.getDataField( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline float StarPilotLateralState::FeedbackConsumption::Builder::getReportedTorque() { + return _builder.getDataField( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setReportedTorque(float value) { + _builder.setDataField( + ::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + +inline float StarPilotLateralState::FeedbackConsumption::Reader::getRequestTorque() const { + return _reader.getDataField( + ::capnp::bounded<10>() * ::capnp::ELEMENTS); +} + +inline float StarPilotLateralState::FeedbackConsumption::Builder::getRequestTorque() { + return _builder.getDataField( + ::capnp::bounded<10>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setRequestTorque(float value) { + _builder.setDataField( + ::capnp::bounded<10>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Reader::getLatActive() const { + return _reader.getDataField( + ::capnp::bounded<18>() * ::capnp::ELEMENTS); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Builder::getLatActive() { + return _builder.getDataField( + ::capnp::bounded<18>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setLatActive(bool value) { + _builder.setDataField( + ::capnp::bounded<18>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Reader::getInputSafetyLimited() const { + return _reader.getDataField( + ::capnp::bounded<19>() * ::capnp::ELEMENTS); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Builder::getInputSafetyLimited() { + return _builder.getDataField( + ::capnp::bounded<19>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setInputSafetyLimited(bool value) { + _builder.setDataField( + ::capnp::bounded<19>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Reader::getPublishedSafetyLimited() const { + return _reader.getDataField( + ::capnp::bounded<20>() * ::capnp::ELEMENTS); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Builder::getPublishedSafetyLimited() { + return _builder.getDataField( + ::capnp::bounded<20>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setPublishedSafetyLimited(bool value) { + _builder.setDataField( + ::capnp::bounded<20>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Reader::getPriorPublishControlMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t StarPilotLateralState::FeedbackConsumption::Builder::getPriorPublishControlMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setPriorPublishControlMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS, value); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Reader::hasSourceEpoch() const { + return !_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotLateralState::FeedbackConsumption::Builder::hasSourceEpoch() { + return !_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader StarPilotLateralState::FeedbackConsumption::Reader::getSourceEpoch() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder StarPilotLateralState::FeedbackConsumption::Builder::getSourceEpoch() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setSourceEpoch( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder StarPilotLateralState::FeedbackConsumption::Builder::initSourceEpoch(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), size); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::adoptSourceEpoch( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> StarPilotLateralState::FeedbackConsumption::Builder::disownSourceEpoch() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool StarPilotLateralState::FeedbackConsumption::Reader::hasCpSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline bool StarPilotLateralState::FeedbackConsumption::Builder::hasCpSha256() { + return !_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader StarPilotLateralState::FeedbackConsumption::Reader::getCpSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder StarPilotLateralState::FeedbackConsumption::Builder::getCpSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::setCpSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder StarPilotLateralState::FeedbackConsumption::Builder::initCpSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), size); +} +inline void StarPilotLateralState::FeedbackConsumption::Builder::adoptCpSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> StarPilotLateralState::FeedbackConsumption::Builder::disownCpSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} + inline ::cereal::SlcCruiseCommand::Kind SlcCruiseCommand::Reader::getKind() const { return _reader.getDataField< ::cereal::SlcCruiseCommand::Kind>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -22946,6 +25690,40 @@ inline void SlcCruiseCommand::Builder::setTargetMps(float value) { ::capnp::bounded<19>() * ::capnp::ELEMENTS, value); } +inline bool SlcCruiseCommand::Reader::hasSource() const { + return !_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline bool SlcCruiseCommand::Builder::hasSource() { + return !_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader SlcCruiseCommand::Reader::getSource() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder SlcCruiseCommand::Builder::getSource() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void SlcCruiseCommand::Builder::setSource( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder SlcCruiseCommand::Builder::initSource(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), size); +} +inline void SlcCruiseCommand::Builder::adoptSource( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> SlcCruiseCommand::Builder::disownSource() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} + inline bool AolAxisState::Reader::hasSessionId() const { return !_reader.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); @@ -26277,6 +29055,82 @@ inline void LearningControlEvidence::Builder::setEffectPrepareMonoTime( ::uint64 ::capnp::bounded<13>() * ::capnp::ELEMENTS, value); } +inline bool LearningControlEvidence::Reader::hasVehicleModelAnchor() const { + return !_reader.getPointerField( + ::capnp::bounded<19>() * ::capnp::POINTERS).isNull(); +} +inline bool LearningControlEvidence::Builder::hasVehicleModelAnchor() { + return !_builder.getPointerField( + ::capnp::bounded<19>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::List::Reader LearningControlEvidence::Reader::getVehicleModelAnchor() const { + return ::capnp::_::PointerHelpers< ::capnp::List>::get(_reader.getPointerField( + ::capnp::bounded<19>() * ::capnp::POINTERS)); +} +inline ::capnp::List::Builder LearningControlEvidence::Builder::getVehicleModelAnchor() { + return ::capnp::_::PointerHelpers< ::capnp::List>::get(_builder.getPointerField( + ::capnp::bounded<19>() * ::capnp::POINTERS)); +} +inline void LearningControlEvidence::Builder::setVehicleModelAnchor( ::capnp::List::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::List>::set(_builder.getPointerField( + ::capnp::bounded<19>() * ::capnp::POINTERS), value); +} +inline void LearningControlEvidence::Builder::setVehicleModelAnchor(::kj::ArrayPtr value) { + ::capnp::_::PointerHelpers< ::capnp::List>::set(_builder.getPointerField( + ::capnp::bounded<19>() * ::capnp::POINTERS), value); +} +inline ::capnp::List::Builder LearningControlEvidence::Builder::initVehicleModelAnchor(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::List>::init(_builder.getPointerField( + ::capnp::bounded<19>() * ::capnp::POINTERS), size); +} +inline void LearningControlEvidence::Builder::adoptVehicleModelAnchor( + ::capnp::Orphan< ::capnp::List>&& value) { + ::capnp::_::PointerHelpers< ::capnp::List>::adopt(_builder.getPointerField( + ::capnp::bounded<19>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::List> LearningControlEvidence::Builder::disownVehicleModelAnchor() { + return ::capnp::_::PointerHelpers< ::capnp::List>::disown(_builder.getPointerField( + ::capnp::bounded<19>() * ::capnp::POINTERS)); +} + +inline bool LearningControlEvidence::Reader::hasLiveAppliedCoordinates() const { + return !_reader.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS).isNull(); +} +inline bool LearningControlEvidence::Builder::hasLiveAppliedCoordinates() { + return !_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::List::Reader LearningControlEvidence::Reader::getLiveAppliedCoordinates() const { + return ::capnp::_::PointerHelpers< ::capnp::List>::get(_reader.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS)); +} +inline ::capnp::List::Builder LearningControlEvidence::Builder::getLiveAppliedCoordinates() { + return ::capnp::_::PointerHelpers< ::capnp::List>::get(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS)); +} +inline void LearningControlEvidence::Builder::setLiveAppliedCoordinates( ::capnp::List::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::List>::set(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS), value); +} +inline void LearningControlEvidence::Builder::setLiveAppliedCoordinates(::kj::ArrayPtr value) { + ::capnp::_::PointerHelpers< ::capnp::List>::set(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS), value); +} +inline ::capnp::List::Builder LearningControlEvidence::Builder::initLiveAppliedCoordinates(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::List>::init(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS), size); +} +inline void LearningControlEvidence::Builder::adoptLiveAppliedCoordinates( + ::capnp::Orphan< ::capnp::List>&& value) { + ::capnp::_::PointerHelpers< ::capnp::List>::adopt(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::List> LearningControlEvidence::Builder::disownLiveAppliedCoordinates() { + return ::capnp::_::PointerHelpers< ::capnp::List>::disown(_builder.getPointerField( + ::capnp::bounded<20>() * ::capnp::POINTERS)); +} + inline ::uint16_t LearningControlEvidence::FullEffectTrace::Reader::getVersion() const { return _reader.getDataField< ::uint16_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -30910,6 +33764,2080 @@ inline void LearningTorqueProjection::Builder::setUnlimitedRawPresent(bool value ::capnp::bounded<17>() * ::capnp::ELEMENTS, value); } +inline bool LiveTrackingBinding::Reader::hasIdentity() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasIdentity() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getIdentity() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getIdentity() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setIdentity( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initIdentity(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptIdentity( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownIdentity() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasCpSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasCpSha256() { + return !_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getCpSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getCpSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setCpSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initCpSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptCpSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownCpSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasControllerRevision() const { + return !_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasControllerRevision() { + return !_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getControllerRevision() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getControllerRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setControllerRevision( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initControllerRevision(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptControllerRevision( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownControllerRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasBaselineRevision() const { + return !_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasBaselineRevision() { + return !_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getBaselineRevision() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getBaselineRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setBaselineRevision( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initBaselineRevision(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptBaselineRevision( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownBaselineRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasSourceEpoch() const { + return !_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasSourceEpoch() { + return !_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getSourceEpoch() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getSourceEpoch() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setSourceEpoch( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initSourceEpoch(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptSourceEpoch( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownSourceEpoch() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasResetGeneration() const { + return !_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasResetGeneration() { + return !_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getResetGeneration() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getResetGeneration() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setResetGeneration( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initResetGeneration(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptResetGeneration( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownResetGeneration() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasCalibrationGeneration() const { + return !_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasCalibrationGeneration() { + return !_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getCalibrationGeneration() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getCalibrationGeneration() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setCalibrationGeneration( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initCalibrationGeneration(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptCalibrationGeneration( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownCalibrationGeneration() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasVehicleModelGeneration() const { + return !_reader.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasVehicleModelGeneration() { + return !_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getVehicleModelGeneration() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getVehicleModelGeneration() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setVehicleModelGeneration( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initVehicleModelGeneration(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptVehicleModelGeneration( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownVehicleModelGeneration() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasNativeIdentity() const { + return !_reader.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasNativeIdentity() { + return !_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getNativeIdentity() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getNativeIdentity() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setNativeIdentity( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initNativeIdentity(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptNativeIdentity( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownNativeIdentity() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasPhysicalModelRevision() const { + return !_reader.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasPhysicalModelRevision() { + return !_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getPhysicalModelRevision() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getPhysicalModelRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setPhysicalModelRevision( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initPhysicalModelRevision(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptPhysicalModelRevision( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownPhysicalModelRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::hasPersistentSourceEpoch() const { + return !_reader.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingBinding::Builder::hasPersistentSourceEpoch() { + return !_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingBinding::Reader::getPersistentSourceEpoch() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::getPersistentSourceEpoch() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS)); +} +inline void LiveTrackingBinding::Builder::setPersistentSourceEpoch( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingBinding::Builder::initPersistentSourceEpoch(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingBinding::Builder::adoptPersistentSourceEpoch( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingBinding::Builder::disownPersistentSourceEpoch() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingBinding::Reader::getNativeRequired() const { + return _reader.getDataField( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline bool LiveTrackingBinding::Builder::getNativeRequired() { + return _builder.getDataField( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingBinding::Builder::setNativeRequired(bool value) { + _builder.setDataField( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline ::uint16_t LiveTrackingTrial::Reader::getVersion() const { + return _reader.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint16_t LiveTrackingTrial::Builder::getVersion() { + return _builder.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingTrial::Builder::setVersion( ::uint16_t value) { + _builder.setDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingTrial::Reader::hasProtocol() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasProtocol() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingTrial::Reader::getProtocol() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::getProtocol() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::setProtocol( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::initProtocol(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Builder::adoptProtocol( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingTrial::Builder::disownProtocol() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingTrial::Reader::hasBinding() const { + return !_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasBinding() { + return !_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::LiveTrackingBinding::Reader LiveTrackingTrial::Reader::getBinding() const { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::get(_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::cereal::LiveTrackingBinding::Builder LiveTrackingTrial::Builder::getBinding() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::get(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::LiveTrackingBinding::Pipeline LiveTrackingTrial::Pipeline::getBinding() { + return ::cereal::LiveTrackingBinding::Pipeline(_typeless.getPointerField(1)); +} +#endif // !CAPNP_LITE +inline void LiveTrackingTrial::Builder::setBinding( ::cereal::LiveTrackingBinding::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::set(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), value); +} +inline ::cereal::LiveTrackingBinding::Builder LiveTrackingTrial::Builder::initBinding() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::init(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::adoptBinding( + ::capnp::Orphan< ::cereal::LiveTrackingBinding>&& value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::adopt(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::LiveTrackingBinding> LiveTrackingTrial::Builder::disownBinding() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::disown(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingTrial::Reader::hasTrialId() const { + return !_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasTrialId() { + return !_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingTrial::Reader::getTrialId() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::getTrialId() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::setTrialId( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::initTrialId(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Builder::adoptTrialId( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingTrial::Builder::disownTrialId() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline ::uint64_t LiveTrackingTrial::Reader::getSequence() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingTrial::Builder::getSequence() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingTrial::Builder::setSequence( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingTrial::Reader::hasPhase() const { + return !_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasPhase() { + return !_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingTrial::Reader::getPhase() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::getPhase() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::setPhase( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::initPhase(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Builder::adoptPhase( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingTrial::Builder::disownPhase() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingTrial::Reader::hasGroup() const { + return !_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasGroup() { + return !_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingTrial::Reader::getGroup() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::getGroup() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::setGroup( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::initGroup(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Builder::adoptGroup( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingTrial::Builder::disownGroup() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingTrial::Reader::hasNames() const { + return !_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasNames() { + return !_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader LiveTrackingTrial::Reader::getNames() const { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::get(_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder LiveTrackingTrial::Builder::getNames() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::get(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::setNames( ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::set(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), value); +} +inline void LiveTrackingTrial::Builder::setNames(::kj::ArrayPtr value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::set(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), value); +} +inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder LiveTrackingTrial::Builder::initNames(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::init(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Builder::adoptNames( + ::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>&& value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::adopt(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>> LiveTrackingTrial::Builder::disownNames() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::disown(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingTrial::Reader::hasBaseline() const { + return !_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasBaseline() { + return !_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader LiveTrackingTrial::Reader::getBaseline() const { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::get(_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder LiveTrackingTrial::Builder::getBaseline() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::get(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::setBaseline( ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::set(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), value); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder LiveTrackingTrial::Builder::initBaseline(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::init(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Builder::adoptBaseline( + ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::adopt(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>> LiveTrackingTrial::Builder::disownBaseline() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::disown(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingTrial::Reader::hasTarget() const { + return !_reader.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasTarget() { + return !_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader LiveTrackingTrial::Reader::getTarget() const { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::get(_reader.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS)); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder LiveTrackingTrial::Builder::getTarget() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::get(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::setTarget( ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::set(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS), value); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder LiveTrackingTrial::Builder::initTarget(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::init(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Builder::adoptTarget( + ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::adopt(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>> LiveTrackingTrial::Builder::disownTarget() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::disown(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingTrial::Reader::hasBaselineSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasBaselineSha256() { + return !_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingTrial::Reader::getBaselineSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::getBaselineSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::setBaselineSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::initBaselineSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Builder::adoptBaselineSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingTrial::Builder::disownBaselineSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingTrial::Reader::hasTargetSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasTargetSha256() { + return !_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingTrial::Reader::getTargetSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::getTargetSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::setTargetSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Builder::initTargetSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Builder::adoptTargetSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingTrial::Builder::disownTargetSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<9>() * ::capnp::POINTERS)); +} + +inline ::uint64_t LiveTrackingTrial::Reader::getStartedMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingTrial::Builder::getStartedMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingTrial::Builder::setStartedMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingTrial::Reader::getObservedMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingTrial::Builder::getObservedMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingTrial::Builder::setObservedMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingTrial::Reader::getValidUntilMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<4>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingTrial::Builder::getValidUntilMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<4>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingTrial::Builder::setValidUntilMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<4>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingTrial::Reader::hasResponseWitness() const { + return !_reader.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Builder::hasResponseWitness() { + return !_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::LiveTrackingResponseWitness::Reader LiveTrackingTrial::Reader::getResponseWitness() const { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::get(_reader.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS)); +} +inline ::cereal::LiveTrackingResponseWitness::Builder LiveTrackingTrial::Builder::getResponseWitness() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::get(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::LiveTrackingResponseWitness::Pipeline LiveTrackingTrial::Pipeline::getResponseWitness() { + return ::cereal::LiveTrackingResponseWitness::Pipeline(_typeless.getPointerField(10)); +} +#endif // !CAPNP_LITE +inline void LiveTrackingTrial::Builder::setResponseWitness( ::cereal::LiveTrackingResponseWitness::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::set(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS), value); +} +inline ::cereal::LiveTrackingResponseWitness::Builder LiveTrackingTrial::Builder::initResponseWitness() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::init(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Builder::adoptResponseWitness( + ::capnp::Orphan< ::cereal::LiveTrackingResponseWitness>&& value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::adopt(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::LiveTrackingResponseWitness> LiveTrackingTrial::Builder::disownResponseWitness() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::disown(_builder.getPointerField( + ::capnp::bounded<10>() * ::capnp::POINTERS)); +} + +inline ::uint64_t LiveTrackingTrial::Reader::getTargetSequence() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<5>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingTrial::Builder::getTargetSequence() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<5>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingTrial::Builder::setTargetSequence( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<5>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingTrial::Reader::getDeadlineMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingTrial::Builder::getDeadlineMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingTrial::Builder::setDeadlineMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingTrial::Coordinate::Reader::hasName() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingTrial::Coordinate::Builder::hasName() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingTrial::Coordinate::Reader::getName() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Coordinate::Builder::getName() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void LiveTrackingTrial::Coordinate::Builder::setName( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingTrial::Coordinate::Builder::initName(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingTrial::Coordinate::Builder::adoptName( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingTrial::Coordinate::Builder::disownName() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline double LiveTrackingTrial::Coordinate::Reader::getValue() const { + return _reader.getDataField( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline double LiveTrackingTrial::Coordinate::Builder::getValue() { + return _builder.getDataField( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingTrial::Coordinate::Builder::setValue(double value) { + _builder.setDataField( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline ::uint16_t LiveTrackingApplication::Reader::getVersion() const { + return _reader.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint16_t LiveTrackingApplication::Builder::getVersion() { + return _builder.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingApplication::Builder::setVersion( ::uint16_t value) { + _builder.setDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingApplication::Reader::hasBinding() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingApplication::Builder::hasBinding() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::LiveTrackingBinding::Reader LiveTrackingApplication::Reader::getBinding() const { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::cereal::LiveTrackingBinding::Builder LiveTrackingApplication::Builder::getBinding() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::LiveTrackingBinding::Pipeline LiveTrackingApplication::Pipeline::getBinding() { + return ::cereal::LiveTrackingBinding::Pipeline(_typeless.getPointerField(0)); +} +#endif // !CAPNP_LITE +inline void LiveTrackingApplication::Builder::setBinding( ::cereal::LiveTrackingBinding::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::cereal::LiveTrackingBinding::Builder LiveTrackingApplication::Builder::initBinding() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void LiveTrackingApplication::Builder::adoptBinding( + ::capnp::Orphan< ::cereal::LiveTrackingBinding>&& value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::LiveTrackingBinding> LiveTrackingApplication::Builder::disownBinding() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingApplication::Reader::hasTrialId() const { + return !_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingApplication::Builder::hasTrialId() { + return !_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingApplication::Reader::getTrialId() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::getTrialId() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void LiveTrackingApplication::Builder::setTrialId( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::initTrialId(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingApplication::Builder::adoptTrialId( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingApplication::Builder::disownTrialId() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline ::uint64_t LiveTrackingApplication::Reader::getSequence() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingApplication::Builder::getSequence() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingApplication::Builder::setSequence( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingApplication::Reader::getControlMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingApplication::Builder::getControlMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingApplication::Builder::setControlMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingApplication::Reader::hasStatus() const { + return !_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingApplication::Builder::hasStatus() { + return !_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingApplication::Reader::getStatus() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::getStatus() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void LiveTrackingApplication::Builder::setStatus( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::initStatus(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingApplication::Builder::adoptStatus( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingApplication::Builder::disownStatus() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingApplication::Reader::hasApplied() const { + return !_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingApplication::Builder::hasApplied() { + return !_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader LiveTrackingApplication::Reader::getApplied() const { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::get(_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder LiveTrackingApplication::Builder::getApplied() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::get(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void LiveTrackingApplication::Builder::setApplied( ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::set(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), value); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder LiveTrackingApplication::Builder::initApplied(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::init(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingApplication::Builder::adoptApplied( + ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::adopt(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>> LiveTrackingApplication::Builder::disownApplied() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::disown(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingApplication::Reader::hasAppliedSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingApplication::Builder::hasAppliedSha256() { + return !_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingApplication::Reader::getAppliedSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::getAppliedSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void LiveTrackingApplication::Builder::setAppliedSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::initAppliedSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingApplication::Builder::adoptAppliedSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingApplication::Builder::disownAppliedSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingApplication::Reader::hasConfirmedTargetSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingApplication::Builder::hasConfirmedTargetSha256() { + return !_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingApplication::Reader::getConfirmedTargetSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::getConfirmedTargetSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void LiveTrackingApplication::Builder::setConfirmedTargetSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::initConfirmedTargetSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingApplication::Builder::adoptConfirmedTargetSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingApplication::Builder::disownConfirmedTargetSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingApplication::Reader::hasReason() const { + return !_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingApplication::Builder::hasReason() { + return !_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingApplication::Reader::getReason() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::getReason() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline void LiveTrackingApplication::Builder::setReason( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingApplication::Builder::initReason(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingApplication::Builder::adoptReason( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingApplication::Builder::disownReason() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} + +inline ::uint64_t LiveTrackingApplication::Reader::getTargetSequence() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingApplication::Builder::getTargetSequence() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingApplication::Builder::setTargetSequence( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS, value); +} + +inline ::uint16_t LiveTrackingRestore::Reader::getVersion() const { + return _reader.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint16_t LiveTrackingRestore::Builder::getVersion() { + return _builder.getDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingRestore::Builder::setVersion( ::uint16_t value) { + _builder.setDataField< ::uint16_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingRestore::Reader::hasProtocol() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingRestore::Builder::hasProtocol() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingRestore::Reader::getProtocol() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingRestore::Builder::getProtocol() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void LiveTrackingRestore::Builder::setProtocol( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingRestore::Builder::initProtocol(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingRestore::Builder::adoptProtocol( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingRestore::Builder::disownProtocol() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingRestore::Reader::hasBinding() const { + return !_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingRestore::Builder::hasBinding() { + return !_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::LiveTrackingBinding::Reader LiveTrackingRestore::Reader::getBinding() const { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::get(_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::cereal::LiveTrackingBinding::Builder LiveTrackingRestore::Builder::getBinding() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::get(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::LiveTrackingBinding::Pipeline LiveTrackingRestore::Pipeline::getBinding() { + return ::cereal::LiveTrackingBinding::Pipeline(_typeless.getPointerField(1)); +} +#endif // !CAPNP_LITE +inline void LiveTrackingRestore::Builder::setBinding( ::cereal::LiveTrackingBinding::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::set(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), value); +} +inline ::cereal::LiveTrackingBinding::Builder LiveTrackingRestore::Builder::initBinding() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::init(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void LiveTrackingRestore::Builder::adoptBinding( + ::capnp::Orphan< ::cereal::LiveTrackingBinding>&& value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::adopt(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::LiveTrackingBinding> LiveTrackingRestore::Builder::disownBinding() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingBinding>::disown(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingRestore::Reader::hasHistoryScope() const { + return !_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingRestore::Builder::hasHistoryScope() { + return !_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingRestore::Reader::getHistoryScope() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingRestore::Builder::getHistoryScope() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void LiveTrackingRestore::Builder::setHistoryScope( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingRestore::Builder::initHistoryScope(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingRestore::Builder::adoptHistoryScope( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingRestore::Builder::disownHistoryScope() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingRestore::Reader::hasHistoryTrialId() const { + return !_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingRestore::Builder::hasHistoryTrialId() { + return !_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingRestore::Reader::getHistoryTrialId() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingRestore::Builder::getHistoryTrialId() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void LiveTrackingRestore::Builder::setHistoryTrialId( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingRestore::Builder::initHistoryTrialId(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingRestore::Builder::adoptHistoryTrialId( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingRestore::Builder::disownHistoryTrialId() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingRestore::Reader::hasTarget() const { + return !_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingRestore::Builder::hasTarget() { + return !_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader LiveTrackingRestore::Reader::getTarget() const { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::get(_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder LiveTrackingRestore::Builder::getTarget() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::get(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void LiveTrackingRestore::Builder::setTarget( ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::set(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), value); +} +inline ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>::Builder LiveTrackingRestore::Builder::initTarget(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::init(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingRestore::Builder::adoptTarget( + ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::adopt(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>> LiveTrackingRestore::Builder::disownTarget() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::cereal::LiveTrackingTrial::Coordinate, ::capnp::Kind::STRUCT>>::disown(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingRestore::Reader::hasTargetSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingRestore::Builder::hasTargetSha256() { + return !_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingRestore::Reader::getTargetSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingRestore::Builder::getTargetSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void LiveTrackingRestore::Builder::setTargetSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingRestore::Builder::initTargetSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingRestore::Builder::adoptTargetSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingRestore::Builder::disownTargetSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline ::uint64_t LiveTrackingRestore::Reader::getSequence() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingRestore::Builder::getSequence() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingRestore::Builder::setSequence( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingRestore::Reader::getObservedMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingRestore::Builder::getObservedMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingRestore::Builder::setObservedMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingRestore::Reader::getValidUntilMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingRestore::Builder::getValidUntilMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingRestore::Builder::setValidUntilMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingRestore::Reader::hasResponseWitness() const { + return !_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingRestore::Builder::hasResponseWitness() { + return !_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::LiveTrackingResponseWitness::Reader LiveTrackingRestore::Reader::getResponseWitness() const { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::get(_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline ::cereal::LiveTrackingResponseWitness::Builder LiveTrackingRestore::Builder::getResponseWitness() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::get(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::LiveTrackingResponseWitness::Pipeline LiveTrackingRestore::Pipeline::getResponseWitness() { + return ::cereal::LiveTrackingResponseWitness::Pipeline(_typeless.getPointerField(6)); +} +#endif // !CAPNP_LITE +inline void LiveTrackingRestore::Builder::setResponseWitness( ::cereal::LiveTrackingResponseWitness::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::set(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), value); +} +inline ::cereal::LiveTrackingResponseWitness::Builder LiveTrackingRestore::Builder::initResponseWitness() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::init(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline void LiveTrackingRestore::Builder::adoptResponseWitness( + ::capnp::Orphan< ::cereal::LiveTrackingResponseWitness>&& value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::adopt(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::LiveTrackingResponseWitness> LiveTrackingRestore::Builder::disownResponseWitness() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseWitness>::disown(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingResponseCertificate::Reader::hasVersion() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseCertificate::Builder::hasVersion() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingResponseCertificate::Reader::getVersion() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::getVersion() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseCertificate::Builder::setVersion( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::initVersion(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingResponseCertificate::Builder::adoptVersion( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingResponseCertificate::Builder::disownVersion() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingResponseCertificate::Reader::hasModelRevision() const { + return !_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseCertificate::Builder::hasModelRevision() { + return !_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingResponseCertificate::Reader::getModelRevision() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::getModelRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseCertificate::Builder::setModelRevision( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::initModelRevision(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingResponseCertificate::Builder::adoptModelRevision( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingResponseCertificate::Builder::disownModelRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline ::uint64_t LiveTrackingResponseCertificate::Reader::getLagNs() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingResponseCertificate::Builder::getLagNs() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseCertificate::Builder::setLagNs( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline double LiveTrackingResponseCertificate::Reader::getSlope() const { + return _reader.getDataField( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline double LiveTrackingResponseCertificate::Builder::getSlope() { + return _builder.getDataField( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseCertificate::Builder::setSlope(double value) { + _builder.setDataField( + ::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + +inline double LiveTrackingResponseCertificate::Reader::getIntercept() const { + return _reader.getDataField( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} + +inline double LiveTrackingResponseCertificate::Builder::getIntercept() { + return _builder.getDataField( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseCertificate::Builder::setIntercept(double value) { + _builder.setDataField( + ::capnp::bounded<2>() * ::capnp::ELEMENTS, value); +} + +inline double LiveTrackingResponseCertificate::Reader::getCommandLow() const { + return _reader.getDataField( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} + +inline double LiveTrackingResponseCertificate::Builder::getCommandLow() { + return _builder.getDataField( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseCertificate::Builder::setCommandLow(double value) { + _builder.setDataField( + ::capnp::bounded<3>() * ::capnp::ELEMENTS, value); +} + +inline double LiveTrackingResponseCertificate::Reader::getCommandHigh() const { + return _reader.getDataField( + ::capnp::bounded<4>() * ::capnp::ELEMENTS); +} + +inline double LiveTrackingResponseCertificate::Builder::getCommandHigh() { + return _builder.getDataField( + ::capnp::bounded<4>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseCertificate::Builder::setCommandHigh(double value) { + _builder.setDataField( + ::capnp::bounded<4>() * ::capnp::ELEMENTS, value); +} + +inline double LiveTrackingResponseCertificate::Reader::getHeldoutRmse() const { + return _reader.getDataField( + ::capnp::bounded<5>() * ::capnp::ELEMENTS); +} + +inline double LiveTrackingResponseCertificate::Builder::getHeldoutRmse() { + return _builder.getDataField( + ::capnp::bounded<5>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseCertificate::Builder::setHeldoutRmse(double value) { + _builder.setDataField( + ::capnp::bounded<5>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingResponseCertificate::Reader::hasTrainBlockIds() const { + return !_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseCertificate::Builder::hasTrainBlockIds() { + return !_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader LiveTrackingResponseCertificate::Reader::getTrainBlockIds() const { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::get(_reader.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder LiveTrackingResponseCertificate::Builder::getTrainBlockIds() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::get(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseCertificate::Builder::setTrainBlockIds( ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline void LiveTrackingResponseCertificate::Builder::setTrainBlockIds(::kj::ArrayPtr value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::set(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), value); +} +inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder LiveTrackingResponseCertificate::Builder::initTrainBlockIds(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::init(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingResponseCertificate::Builder::adoptTrainBlockIds( + ::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>&& value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::adopt(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>> LiveTrackingResponseCertificate::Builder::disownTrainBlockIds() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::disown(_builder.getPointerField( + ::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingResponseCertificate::Reader::hasHeldoutBlockIds() const { + return !_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseCertificate::Builder::hasHeldoutBlockIds() { + return !_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader LiveTrackingResponseCertificate::Reader::getHeldoutBlockIds() const { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::get(_reader.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder LiveTrackingResponseCertificate::Builder::getHeldoutBlockIds() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::get(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseCertificate::Builder::setHeldoutBlockIds( ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::set(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), value); +} +inline void LiveTrackingResponseCertificate::Builder::setHeldoutBlockIds(::kj::ArrayPtr value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::set(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), value); +} +inline ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>::Builder LiveTrackingResponseCertificate::Builder::initHeldoutBlockIds(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::init(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingResponseCertificate::Builder::adoptHeldoutBlockIds( + ::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>&& value) { + ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::adopt(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>> LiveTrackingResponseCertificate::Builder::disownHeldoutBlockIds() { + return ::capnp::_::PointerHelpers< ::capnp::List< ::capnp::Text, ::capnp::Kind::BLOB>>::disown(_builder.getPointerField( + ::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingResponseCertificate::Reader::hasSourceInventorySha256() const { + return !_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseCertificate::Builder::hasSourceInventorySha256() { + return !_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingResponseCertificate::Reader::getSourceInventorySha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::getSourceInventorySha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseCertificate::Builder::setSourceInventorySha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::initSourceInventorySha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingResponseCertificate::Builder::adoptSourceInventorySha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingResponseCertificate::Builder::disownSourceInventorySha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingResponseCertificate::Reader::hasIdentificationSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseCertificate::Builder::hasIdentificationSha256() { + return !_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingResponseCertificate::Reader::getIdentificationSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::getIdentificationSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseCertificate::Builder::setIdentificationSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::initIdentificationSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingResponseCertificate::Builder::adoptIdentificationSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingResponseCertificate::Builder::disownIdentificationSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline ::uint64_t LiveTrackingResponseCertificate::Reader::getIdentifiedMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingResponseCertificate::Builder::getIdentifiedMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseCertificate::Builder::setIdentifiedMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS, value); +} + +inline bool LiveTrackingResponseCertificate::Reader::hasProvenance() const { + return !_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseCertificate::Builder::hasProvenance() { + return !_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingResponseCertificate::Reader::getProvenance() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::getProvenance() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseCertificate::Builder::setProvenance( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::initProvenance(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingResponseCertificate::Builder::adoptProvenance( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingResponseCertificate::Builder::disownProvenance() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<6>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingResponseCertificate::Reader::hasParentModelRevision() const { + return !_reader.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseCertificate::Builder::hasParentModelRevision() { + return !_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingResponseCertificate::Reader::getParentModelRevision() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::getParentModelRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseCertificate::Builder::setParentModelRevision( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::initParentModelRevision(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingResponseCertificate::Builder::adoptParentModelRevision( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingResponseCertificate::Builder::disownParentModelRevision() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<7>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingResponseCertificate::Reader::hasRequalificationSha256() const { + return !_reader.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseCertificate::Builder::hasRequalificationSha256() { + return !_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS).isNull(); +} +inline ::capnp::Text::Reader LiveTrackingResponseCertificate::Reader::getRequalificationSha256() const { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_reader.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::getRequalificationSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::get(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseCertificate::Builder::setRequalificationSha256( ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::set(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS), value); +} +inline ::capnp::Text::Builder LiveTrackingResponseCertificate::Builder::initRequalificationSha256(unsigned int size) { + return ::capnp::_::PointerHelpers< ::capnp::Text>::init(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS), size); +} +inline void LiveTrackingResponseCertificate::Builder::adoptRequalificationSha256( + ::capnp::Orphan< ::capnp::Text>&& value) { + ::capnp::_::PointerHelpers< ::capnp::Text>::adopt(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::capnp::Text> LiveTrackingResponseCertificate::Builder::disownRequalificationSha256() { + return ::capnp::_::PointerHelpers< ::capnp::Text>::disown(_builder.getPointerField( + ::capnp::bounded<8>() * ::capnp::POINTERS)); +} + +inline bool LiveTrackingResponseWitness::Reader::hasCertificate() const { + return !_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline bool LiveTrackingResponseWitness::Builder::hasCertificate() { + return !_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); +} +inline ::cereal::LiveTrackingResponseCertificate::Reader LiveTrackingResponseWitness::Reader::getCertificate() const { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseCertificate>::get(_reader.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::cereal::LiveTrackingResponseCertificate::Builder LiveTrackingResponseWitness::Builder::getCertificate() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseCertificate>::get(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::cereal::LiveTrackingResponseCertificate::Pipeline LiveTrackingResponseWitness::Pipeline::getCertificate() { + return ::cereal::LiveTrackingResponseCertificate::Pipeline(_typeless.getPointerField(0)); +} +#endif // !CAPNP_LITE +inline void LiveTrackingResponseWitness::Builder::setCertificate( ::cereal::LiveTrackingResponseCertificate::Reader value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseCertificate>::set(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), value); +} +inline ::cereal::LiveTrackingResponseCertificate::Builder LiveTrackingResponseWitness::Builder::initCertificate() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseCertificate>::init(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void LiveTrackingResponseWitness::Builder::adoptCertificate( + ::capnp::Orphan< ::cereal::LiveTrackingResponseCertificate>&& value) { + ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseCertificate>::adopt(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); +} +inline ::capnp::Orphan< ::cereal::LiveTrackingResponseCertificate> LiveTrackingResponseWitness::Builder::disownCertificate() { + return ::capnp::_::PointerHelpers< ::cereal::LiveTrackingResponseCertificate>::disown(_builder.getPointerField( + ::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline ::uint64_t LiveTrackingResponseWitness::Reader::getLastControlMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingResponseWitness::Builder::getLastControlMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseWitness::Builder::setLastControlMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingResponseWitness::Reader::getLastApplyMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingResponseWitness::Builder::getLastApplyMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseWitness::Builder::setLastApplyMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingResponseWitness::Reader::getLastPoseMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingResponseWitness::Builder::getLastPoseMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseWitness::Builder::setLastPoseMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<2>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingResponseWitness::Reader::getWindowStartMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingResponseWitness::Builder::getWindowStartMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseWitness::Builder::setWindowStartMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<3>() * ::capnp::ELEMENTS, value); +} + +inline ::uint16_t LiveTrackingResponseWitness::Reader::getIndependentPoses() const { + return _reader.getDataField< ::uint16_t>( + ::capnp::bounded<16>() * ::capnp::ELEMENTS); +} + +inline ::uint16_t LiveTrackingResponseWitness::Builder::getIndependentPoses() { + return _builder.getDataField< ::uint16_t>( + ::capnp::bounded<16>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseWitness::Builder::setIndependentPoses( ::uint16_t value) { + _builder.setDataField< ::uint16_t>( + ::capnp::bounded<16>() * ::capnp::ELEMENTS, value); +} + +inline double LiveTrackingResponseWitness::Reader::getResidualRmse() const { + return _reader.getDataField( + ::capnp::bounded<5>() * ::capnp::ELEMENTS); +} + +inline double LiveTrackingResponseWitness::Builder::getResidualRmse() { + return _builder.getDataField( + ::capnp::bounded<5>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseWitness::Builder::setResidualRmse(double value) { + _builder.setDataField( + ::capnp::bounded<5>() * ::capnp::ELEMENTS, value); +} + +inline ::uint64_t LiveTrackingResponseWitness::Reader::getValidUntilMonoTime() const { + return _reader.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t LiveTrackingResponseWitness::Builder::getValidUntilMonoTime() { + return _builder.getDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS); +} +inline void LiveTrackingResponseWitness::Builder::setValidUntilMonoTime( ::uint64_t value) { + _builder.setDataField< ::uint64_t>( + ::capnp::bounded<6>() * ::capnp::ELEMENTS, value); +} + } // namespace CAPNP_END_HEADER diff --git a/openpilot/cereal/gen/cpp/log.capnp.c++ b/openpilot/cereal/gen/cpp/log.capnp.c++ index 5ebd329be3..2f567514a9 100644 --- a/openpilot/cereal/gen/cpp/log.capnp.c++ +++ b/openpilot/cereal/gen/cpp/log.capnp.c++ @@ -30434,7 +30434,7 @@ const ::capnp::_::RawSchema s_8795cd2944faeab6 = { 0, 5, i_8795cd2944faeab6, nullptr, nullptr, { &s_8795cd2944faeab6, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<2641> b_d314cfd957229c11 = { +static const ::capnp::_::AlignedData<2643> b_d314cfd957229c11 = { { 0, 0, 0, 0, 5, 0, 6, 0, 17, 156, 34, 87, 217, 207, 20, 211, 10, 0, 0, 0, 1, 0, 2, 0, @@ -31238,297 +31238,297 @@ static const ::capnp::_::AlignedData<2641> b_d314cfd957229c11 = { 84, 0, 145, 255, 0, 0, 0, 0, 0, 0, 1, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 77, 22, 0, 0, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 76, 22, 0, 0, 3, 0, 1, 0, - 88, 22, 0, 0, 2, 0, 1, 0, - 85, 0, 144, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 113, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 85, 22, 0, 0, 130, 0, 0, 0, + 77, 22, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 22, 0, 0, 3, 0, 1, 0, 96, 22, 0, 0, 2, 0, 1, 0, + 85, 0, 144, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 113, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 93, 22, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 22, 0, 0, 3, 0, 1, 0, + 104, 22, 0, 0, 2, 0, 1, 0, 86, 0, 143, 255, 0, 0, 0, 0, 0, 0, 1, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 93, 22, 0, 0, 162, 0, 0, 0, + 101, 22, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 22, 0, 0, 3, 0, 1, 0, - 108, 22, 0, 0, 2, 0, 1, 0, + 104, 22, 0, 0, 3, 0, 1, 0, + 116, 22, 0, 0, 2, 0, 1, 0, 87, 0, 142, 255, 0, 0, 0, 0, 0, 0, 1, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 105, 22, 0, 0, 194, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 108, 22, 0, 0, 3, 0, 1, 0, - 120, 22, 0, 0, 2, 0, 1, 0, - 88, 0, 141, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 116, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 117, 22, 0, 0, 74, 0, 0, 0, + 113, 22, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 22, 0, 0, 3, 0, 1, 0, 128, 22, 0, 0, 2, 0, 1, 0, + 88, 0, 141, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 116, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 22, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 22, 0, 0, 3, 0, 1, 0, + 136, 22, 0, 0, 2, 0, 1, 0, 48, 0, 140, 255, 0, 0, 0, 0, 0, 0, 1, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 125, 22, 0, 0, 242, 0, 0, 0, + 133, 22, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 132, 22, 0, 0, 3, 0, 1, 0, - 144, 22, 0, 0, 2, 0, 1, 0, + 140, 22, 0, 0, 3, 0, 1, 0, + 152, 22, 0, 0, 2, 0, 1, 0, 49, 0, 139, 255, 0, 0, 0, 0, 0, 0, 1, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 141, 22, 0, 0, 226, 0, 0, 0, + 149, 22, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 22, 0, 0, 3, 0, 1, 0, - 160, 22, 0, 0, 2, 0, 1, 0, + 156, 22, 0, 0, 3, 0, 1, 0, + 168, 22, 0, 0, 2, 0, 1, 0, 50, 0, 138, 255, 0, 0, 0, 0, 0, 0, 1, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 22, 0, 0, 202, 0, 0, 0, + 165, 22, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 22, 0, 0, 3, 0, 1, 0, - 176, 22, 0, 0, 2, 0, 1, 0, + 172, 22, 0, 0, 3, 0, 1, 0, + 184, 22, 0, 0, 2, 0, 1, 0, 73, 0, 137, 255, 0, 0, 0, 0, 0, 0, 1, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 22, 0, 0, 250, 0, 0, 0, + 181, 22, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 22, 0, 0, 3, 0, 1, 0, - 192, 22, 0, 0, 2, 0, 1, 0, + 188, 22, 0, 0, 3, 0, 1, 0, + 200, 22, 0, 0, 2, 0, 1, 0, 74, 0, 136, 255, 0, 0, 0, 0, 0, 0, 1, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 189, 22, 0, 0, 234, 0, 0, 0, + 197, 22, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 196, 22, 0, 0, 3, 0, 1, 0, - 208, 22, 0, 0, 2, 0, 1, 0, + 204, 22, 0, 0, 3, 0, 1, 0, + 216, 22, 0, 0, 2, 0, 1, 0, 75, 0, 135, 255, 0, 0, 0, 0, 0, 0, 1, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 205, 22, 0, 0, 210, 0, 0, 0, + 213, 22, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 22, 0, 0, 3, 0, 1, 0, - 224, 22, 0, 0, 2, 0, 1, 0, + 220, 22, 0, 0, 3, 0, 1, 0, + 232, 22, 0, 0, 2, 0, 1, 0, 145, 0, 134, 255, 0, 0, 0, 0, 0, 0, 1, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 22, 0, 0, 234, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 228, 22, 0, 0, 3, 0, 1, 0, - 240, 22, 0, 0, 2, 0, 1, 0, - 76, 0, 133, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 124, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 237, 22, 0, 0, 114, 0, 0, 0, + 229, 22, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 22, 0, 0, 3, 0, 1, 0, 248, 22, 0, 0, 2, 0, 1, 0, - 77, 0, 132, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 125, 0, 0, 0, + 76, 0, 133, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 22, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 22, 0, 0, 3, 0, 1, 0, 0, 23, 0, 0, 2, 0, 1, 0, + 77, 0, 132, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 125, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 22, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 252, 22, 0, 0, 3, 0, 1, 0, + 8, 23, 0, 0, 2, 0, 1, 0, 78, 0, 131, 255, 0, 0, 0, 0, 0, 0, 1, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 22, 0, 0, 170, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 23, 0, 0, 3, 0, 1, 0, - 12, 23, 0, 0, 2, 0, 1, 0, - 21, 0, 130, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 9, 23, 0, 0, 82, 0, 0, 0, + 5, 23, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 23, 0, 0, 3, 0, 1, 0, 20, 23, 0, 0, 2, 0, 1, 0, + 21, 0, 130, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 127, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 23, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 23, 0, 0, 3, 0, 1, 0, + 28, 23, 0, 0, 2, 0, 1, 0, 39, 0, 129, 255, 0, 0, 0, 0, 0, 0, 1, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 23, 0, 0, 138, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 20, 23, 0, 0, 3, 0, 1, 0, - 32, 23, 0, 0, 2, 0, 1, 0, - 37, 0, 128, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 129, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 29, 23, 0, 0, 106, 0, 0, 0, + 25, 23, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 23, 0, 0, 3, 0, 1, 0, 40, 23, 0, 0, 2, 0, 1, 0, - 7, 0, 127, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 130, 0, 0, 0, + 37, 0, 128, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 23, 0, 0, 122, 0, 0, 0, + 37, 23, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 23, 0, 0, 3, 0, 1, 0, 48, 23, 0, 0, 2, 0, 1, 0, - 16, 0, 126, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 131, 0, 0, 0, + 7, 0, 127, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 23, 0, 0, 98, 0, 0, 0, + 45, 23, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 23, 0, 0, 3, 0, 1, 0, 56, 23, 0, 0, 2, 0, 1, 0, + 16, 0, 126, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 131, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 53, 23, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 52, 23, 0, 0, 3, 0, 1, 0, + 64, 23, 0, 0, 2, 0, 1, 0, 23, 0, 125, 255, 0, 0, 0, 0, 0, 0, 1, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 23, 0, 0, 138, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 56, 23, 0, 0, 3, 0, 1, 0, - 68, 23, 0, 0, 2, 0, 1, 0, - 72, 0, 124, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 133, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 65, 23, 0, 0, 90, 0, 0, 0, + 61, 23, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 23, 0, 0, 3, 0, 1, 0, 76, 23, 0, 0, 2, 0, 1, 0, + 72, 0, 124, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 133, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 73, 23, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 23, 0, 0, 3, 0, 1, 0, + 84, 23, 0, 0, 2, 0, 1, 0, 34, 0, 123, 255, 0, 0, 0, 0, 0, 0, 1, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 73, 23, 0, 0, 106, 0, 0, 0, + 81, 23, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 23, 0, 0, 3, 0, 1, 0, - 100, 23, 0, 0, 2, 0, 1, 0, + 80, 23, 0, 0, 3, 0, 1, 0, + 108, 23, 0, 0, 2, 0, 1, 0, 62, 0, 122, 255, 0, 0, 0, 0, 0, 0, 1, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 23, 0, 0, 50, 0, 0, 0, + 105, 23, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 92, 23, 0, 0, 3, 0, 1, 0, - 120, 23, 0, 0, 2, 0, 1, 0, + 100, 23, 0, 0, 3, 0, 1, 0, + 128, 23, 0, 0, 2, 0, 1, 0, 89, 0, 121, 255, 0, 0, 0, 0, 0, 0, 1, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 23, 0, 0, 138, 0, 0, 0, + 125, 23, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 23, 0, 0, 3, 0, 1, 0, - 132, 23, 0, 0, 2, 0, 1, 0, + 128, 23, 0, 0, 3, 0, 1, 0, + 140, 23, 0, 0, 2, 0, 1, 0, 90, 0, 120, 255, 0, 0, 0, 0, 0, 0, 1, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 129, 23, 0, 0, 178, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 132, 23, 0, 0, 3, 0, 1, 0, - 144, 23, 0, 0, 2, 0, 1, 0, - 91, 0, 119, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 138, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 141, 23, 0, 0, 82, 0, 0, 0, + 137, 23, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 23, 0, 0, 3, 0, 1, 0, 152, 23, 0, 0, 2, 0, 1, 0, - 92, 0, 118, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 139, 0, 0, 0, + 91, 0, 119, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 149, 23, 0, 0, 122, 0, 0, 0, + 149, 23, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 23, 0, 0, 3, 0, 1, 0, 160, 23, 0, 0, 2, 0, 1, 0, + 92, 0, 118, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 139, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 23, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 156, 23, 0, 0, 3, 0, 1, 0, + 168, 23, 0, 0, 2, 0, 1, 0, 93, 0, 117, 255, 0, 0, 0, 0, 0, 0, 1, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 23, 0, 0, 194, 0, 0, 0, + 165, 23, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 23, 0, 0, 3, 0, 1, 0, - 172, 23, 0, 0, 2, 0, 1, 0, + 168, 23, 0, 0, 3, 0, 1, 0, + 180, 23, 0, 0, 2, 0, 1, 0, 94, 0, 116, 255, 0, 0, 0, 0, 0, 0, 1, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 23, 0, 0, 138, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 172, 23, 0, 0, 3, 0, 1, 0, - 184, 23, 0, 0, 2, 0, 1, 0, - 95, 0, 115, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 142, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 181, 23, 0, 0, 106, 0, 0, 0, + 177, 23, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 23, 0, 0, 3, 0, 1, 0, 192, 23, 0, 0, 2, 0, 1, 0, - 96, 0, 114, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 143, 0, 0, 0, + 95, 0, 115, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 189, 23, 0, 0, 130, 0, 0, 0, + 189, 23, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 23, 0, 0, 3, 0, 1, 0, 200, 23, 0, 0, 2, 0, 1, 0, - 97, 0, 113, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 144, 0, 0, 0, + 96, 0, 114, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 23, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 192, 23, 0, 0, 3, 0, 1, 0, - 204, 23, 0, 0, 2, 0, 1, 0, - 98, 0, 112, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 145, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 201, 23, 0, 0, 66, 0, 0, 0, + 197, 23, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 23, 0, 0, 3, 0, 1, 0, 208, 23, 0, 0, 2, 0, 1, 0, - 31, 0, 111, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 146, 0, 0, 0, + 97, 0, 113, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 205, 23, 0, 0, 106, 0, 0, 0, + 205, 23, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 23, 0, 0, 3, 0, 1, 0, + 212, 23, 0, 0, 2, 0, 1, 0, + 98, 0, 112, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 145, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 23, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 23, 0, 0, 3, 0, 1, 0, 216, 23, 0, 0, 2, 0, 1, 0, - 52, 0, 110, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 147, 0, 0, 0, + 31, 0, 111, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 23, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 23, 0, 0, 3, 0, 1, 0, 224, 23, 0, 0, 2, 0, 1, 0, - 65, 0, 109, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 148, 0, 0, 0, + 52, 0, 110, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 23, 0, 0, 122, 0, 0, 0, + 221, 23, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 23, 0, 0, 3, 0, 1, 0, 232, 23, 0, 0, 2, 0, 1, 0, + 65, 0, 109, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 148, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 23, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 228, 23, 0, 0, 3, 0, 1, 0, + 240, 23, 0, 0, 2, 0, 1, 0, 100, 0, 108, 255, 0, 0, 0, 0, 0, 0, 1, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 229, 23, 0, 0, 194, 0, 0, 0, + 237, 23, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 23, 0, 0, 3, 0, 1, 0, - 244, 23, 0, 0, 2, 0, 1, 0, + 240, 23, 0, 0, 3, 0, 1, 0, + 252, 23, 0, 0, 2, 0, 1, 0, 66, 0, 107, 255, 0, 0, 0, 0, 0, 0, 1, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 23, 0, 0, 162, 0, 0, 0, + 249, 23, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 244, 23, 0, 0, 3, 0, 1, 0, - 0, 24, 0, 0, 2, 0, 1, 0, + 252, 23, 0, 0, 3, 0, 1, 0, + 8, 24, 0, 0, 2, 0, 1, 0, 36, 0, 106, 255, 0, 0, 0, 0, 0, 0, 1, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 23, 0, 0, 178, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 24, 0, 0, 3, 0, 1, 0, - 12, 24, 0, 0, 2, 0, 1, 0, - 58, 0, 105, 255, 0, 0, 0, 0, - 0, 0, 1, 0, 152, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 9, 24, 0, 0, 114, 0, 0, 0, + 5, 24, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 24, 0, 0, 3, 0, 1, 0, 20, 24, 0, 0, 2, 0, 1, 0, + 58, 0, 105, 255, 0, 0, 0, 0, + 0, 0, 1, 0, 152, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 24, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 24, 0, 0, 3, 0, 1, 0, + 28, 24, 0, 0, 2, 0, 1, 0, 59, 0, 104, 255, 0, 0, 0, 0, 0, 0, 1, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 24, 0, 0, 138, 0, 0, 0, + 25, 24, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 24, 0, 0, 3, 0, 1, 0, - 32, 24, 0, 0, 2, 0, 1, 0, + 28, 24, 0, 0, 3, 0, 1, 0, + 40, 24, 0, 0, 2, 0, 1, 0, 108, 111, 103, 77, 111, 110, 111, 84, 105, 109, 101, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, @@ -32666,8 +32666,10 @@ static const ::capnp::_::AlignedData<2641> b_d314cfd957229c11 = { 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 99, 117, 115, 116, 111, 109, 82, 101, - 115, 101, 114, 118, 101, 100, 53, 0, + 115, 116, 97, 114, 112, 105, 108, 111, + 116, 83, 116, 101, 101, 114, 105, 110, + 103, 84, 114, 97, 99, 107, 105, 110, + 103, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 85, 164, 81, 217, 44, 118, 205, 165, 0, 0, 0, 0, 0, 0, 0, 0, @@ -33192,10 +33194,10 @@ static const ::capnp::_::RawSchema* const d_d314cfd957229c11[] = { &s_fe346a9de48d9b50, &s_fe35ad896ffaeacf, }; -static const uint16_t m_d314cfd957229c11[] = {98, 101, 133, 30, 142, 125, 124, 55, 42, 149, 108, 148, 60, 70, 87, 76, 63, 5, 23, 127, 69, 22, 28, 153, 152, 35, 7, 112, 113, 129, 6, 132, 151, 71, 59, 92, 128, 85, 26, 19, 10, 91, 21, 48, 3, 41, 40, 99, 100, 1, 65, 126, 146, 150, 64, 94, 32, 96, 8, 46, 25, 72, 51, 44, 37, 62, 36, 16, 14, 122, 119, 120, 117, 121, 118, 49, 18, 0, 24, 95, 78, 105, 143, 144, 145, 9, 75, 2, 86, 15, 82, 104, 83, 38, 84, 27, 134, 68, 20, 54, 58, 56, 47, 53, 45, 12, 81, 80, 33, 89, 90, 31, 13, 131, 147, 130, 17, 4, 11, 73, 138, 141, 139, 140, 116, 111, 103, 136, 109, 137, 110, 107, 114, 115, 97, 123, 52, 66, 135, 43, 34, 39, 102, 57, 50, 106, 79, 93, 67, 61, 74, 88, 77, 29}; +static const uint16_t m_d314cfd957229c11[] = {98, 101, 133, 30, 142, 125, 124, 55, 42, 149, 108, 148, 60, 70, 87, 76, 63, 5, 23, 127, 69, 22, 28, 153, 152, 35, 7, 113, 129, 6, 132, 151, 71, 59, 92, 128, 85, 26, 19, 10, 91, 21, 48, 3, 41, 40, 99, 100, 1, 65, 126, 146, 150, 64, 94, 32, 96, 8, 46, 25, 72, 51, 44, 37, 62, 36, 16, 14, 122, 119, 120, 117, 121, 118, 49, 18, 0, 24, 95, 78, 105, 143, 144, 145, 9, 75, 2, 86, 15, 82, 104, 83, 38, 84, 27, 134, 68, 20, 54, 58, 56, 47, 53, 45, 12, 81, 80, 33, 89, 90, 31, 13, 131, 147, 130, 17, 4, 11, 73, 138, 141, 139, 140, 116, 111, 103, 136, 109, 137, 110, 107, 114, 115, 112, 97, 123, 52, 66, 135, 43, 34, 39, 102, 57, 50, 106, 79, 93, 67, 61, 74, 88, 77, 29}; static const uint16_t i_d314cfd957229c11[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 0, 67}; const ::capnp::_::RawSchema s_d314cfd957229c11 = { - 0xd314cfd957229c11, b_d314cfd957229c11.words, 2641, d_d314cfd957229c11, m_d314cfd957229c11, + 0xd314cfd957229c11, b_d314cfd957229c11.words, 2643, d_d314cfd957229c11, m_d314cfd957229c11, 111, 154, i_d314cfd957229c11, nullptr, nullptr, { &s_d314cfd957229c11, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE diff --git a/openpilot/cereal/gen/cpp/log.capnp.h b/openpilot/cereal/gen/cpp/log.capnp.h index 3fc7b03191..99a06499bf 100644 --- a/openpilot/cereal/gen/cpp/log.capnp.h +++ b/openpilot/cereal/gen/cpp/log.capnp.h @@ -3296,7 +3296,7 @@ struct Event { STARPILOT_CAR_STATE, STARPILOT_LONGITUDINAL_PLAN, SLC_VISION_OBSERVATION, - CUSTOM_RESERVED5, + STARPILOT_STEERING_TRACKING, CUSTOM_RESERVED6, STARPILOT_RADAR_STATE, STARPILOT_SELFDRIVE_STATE, @@ -23221,9 +23221,9 @@ public: inline bool hasSlcVisionObservation() const; inline ::cereal::StarPilotModelDataV2::Reader getSlcVisionObservation() const; - inline bool isCustomReserved5() const; - inline bool hasCustomReserved5() const; - inline ::cereal::CustomReserved5::Reader getCustomReserved5() const; + inline bool isStarpilotSteeringTracking() const; + inline bool hasStarpilotSteeringTracking() const; + inline ::cereal::StarPilotSteeringTracking::Reader getStarpilotSteeringTracking() const; inline bool isCustomReserved6() const; inline bool hasCustomReserved6() const; @@ -24304,13 +24304,13 @@ public: inline void adoptSlcVisionObservation(::capnp::Orphan< ::cereal::StarPilotModelDataV2>&& value); inline ::capnp::Orphan< ::cereal::StarPilotModelDataV2> disownSlcVisionObservation(); - inline bool isCustomReserved5(); - inline bool hasCustomReserved5(); - inline ::cereal::CustomReserved5::Builder getCustomReserved5(); - inline void setCustomReserved5( ::cereal::CustomReserved5::Reader value); - inline ::cereal::CustomReserved5::Builder initCustomReserved5(); - inline void adoptCustomReserved5(::capnp::Orphan< ::cereal::CustomReserved5>&& value); - inline ::capnp::Orphan< ::cereal::CustomReserved5> disownCustomReserved5(); + inline bool isStarpilotSteeringTracking(); + inline bool hasStarpilotSteeringTracking(); + inline ::cereal::StarPilotSteeringTracking::Builder getStarpilotSteeringTracking(); + inline void setStarpilotSteeringTracking( ::cereal::StarPilotSteeringTracking::Reader value); + inline ::cereal::StarPilotSteeringTracking::Builder initStarpilotSteeringTracking(); + inline void adoptStarpilotSteeringTracking(::capnp::Orphan< ::cereal::StarPilotSteeringTracking>&& value); + inline ::capnp::Orphan< ::cereal::StarPilotSteeringTracking> disownStarpilotSteeringTracking(); inline bool isCustomReserved6(); inline bool hasCustomReserved6(); @@ -57727,57 +57727,57 @@ inline ::capnp::Orphan< ::cereal::StarPilotModelDataV2> Event::Builder::disownSl ::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline bool Event::Reader::isCustomReserved5() const { - return which() == Event::CUSTOM_RESERVED5; +inline bool Event::Reader::isStarpilotSteeringTracking() const { + return which() == Event::STARPILOT_STEERING_TRACKING; } -inline bool Event::Builder::isCustomReserved5() { - return which() == Event::CUSTOM_RESERVED5; +inline bool Event::Builder::isStarpilotSteeringTracking() { + return which() == Event::STARPILOT_STEERING_TRACKING; } -inline bool Event::Reader::hasCustomReserved5() const { - if (which() != Event::CUSTOM_RESERVED5) return false; +inline bool Event::Reader::hasStarpilotSteeringTracking() const { + if (which() != Event::STARPILOT_STEERING_TRACKING) return false; return !_reader.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); } -inline bool Event::Builder::hasCustomReserved5() { - if (which() != Event::CUSTOM_RESERVED5) return false; +inline bool Event::Builder::hasStarpilotSteeringTracking() { + if (which() != Event::STARPILOT_STEERING_TRACKING) return false; return !_builder.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS).isNull(); } -inline ::cereal::CustomReserved5::Reader Event::Reader::getCustomReserved5() const { - KJ_IREQUIRE((which() == Event::CUSTOM_RESERVED5), +inline ::cereal::StarPilotSteeringTracking::Reader Event::Reader::getStarpilotSteeringTracking() const { + KJ_IREQUIRE((which() == Event::STARPILOT_STEERING_TRACKING), "Must check which() before get()ing a union member."); - return ::capnp::_::PointerHelpers< ::cereal::CustomReserved5>::get(_reader.getPointerField( + return ::capnp::_::PointerHelpers< ::cereal::StarPilotSteeringTracking>::get(_reader.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline ::cereal::CustomReserved5::Builder Event::Builder::getCustomReserved5() { - KJ_IREQUIRE((which() == Event::CUSTOM_RESERVED5), +inline ::cereal::StarPilotSteeringTracking::Builder Event::Builder::getStarpilotSteeringTracking() { + KJ_IREQUIRE((which() == Event::STARPILOT_STEERING_TRACKING), "Must check which() before get()ing a union member."); - return ::capnp::_::PointerHelpers< ::cereal::CustomReserved5>::get(_builder.getPointerField( + return ::capnp::_::PointerHelpers< ::cereal::StarPilotSteeringTracking>::get(_builder.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline void Event::Builder::setCustomReserved5( ::cereal::CustomReserved5::Reader value) { +inline void Event::Builder::setStarpilotSteeringTracking( ::cereal::StarPilotSteeringTracking::Reader value) { _builder.setDataField( - ::capnp::bounded<4>() * ::capnp::ELEMENTS, Event::CUSTOM_RESERVED5); - ::capnp::_::PointerHelpers< ::cereal::CustomReserved5>::set(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::ELEMENTS, Event::STARPILOT_STEERING_TRACKING); + ::capnp::_::PointerHelpers< ::cereal::StarPilotSteeringTracking>::set(_builder.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS), value); } -inline ::cereal::CustomReserved5::Builder Event::Builder::initCustomReserved5() { +inline ::cereal::StarPilotSteeringTracking::Builder Event::Builder::initStarpilotSteeringTracking() { _builder.setDataField( - ::capnp::bounded<4>() * ::capnp::ELEMENTS, Event::CUSTOM_RESERVED5); - return ::capnp::_::PointerHelpers< ::cereal::CustomReserved5>::init(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::ELEMENTS, Event::STARPILOT_STEERING_TRACKING); + return ::capnp::_::PointerHelpers< ::cereal::StarPilotSteeringTracking>::init(_builder.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline void Event::Builder::adoptCustomReserved5( - ::capnp::Orphan< ::cereal::CustomReserved5>&& value) { +inline void Event::Builder::adoptStarpilotSteeringTracking( + ::capnp::Orphan< ::cereal::StarPilotSteeringTracking>&& value) { _builder.setDataField( - ::capnp::bounded<4>() * ::capnp::ELEMENTS, Event::CUSTOM_RESERVED5); - ::capnp::_::PointerHelpers< ::cereal::CustomReserved5>::adopt(_builder.getPointerField( + ::capnp::bounded<4>() * ::capnp::ELEMENTS, Event::STARPILOT_STEERING_TRACKING); + ::capnp::_::PointerHelpers< ::cereal::StarPilotSteeringTracking>::adopt(_builder.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); } -inline ::capnp::Orphan< ::cereal::CustomReserved5> Event::Builder::disownCustomReserved5() { - KJ_IREQUIRE((which() == Event::CUSTOM_RESERVED5), +inline ::capnp::Orphan< ::cereal::StarPilotSteeringTracking> Event::Builder::disownStarpilotSteeringTracking() { + KJ_IREQUIRE((which() == Event::STARPILOT_STEERING_TRACKING), "Must check which() before get()ing a union member."); - return ::capnp::_::PointerHelpers< ::cereal::CustomReserved5>::disown(_builder.getPointerField( + return ::capnp::_::PointerHelpers< ::cereal::StarPilotSteeringTracking>::disown(_builder.getPointerField( ::capnp::bounded<0>() * ::capnp::POINTERS)); } diff --git a/openpilot/cereal/libcereal.a b/openpilot/cereal/libcereal.a index 692164d1ab..f96e540929 100644 Binary files a/openpilot/cereal/libcereal.a and b/openpilot/cereal/libcereal.a differ diff --git a/openpilot/cereal/log.capnp b/openpilot/cereal/log.capnp index e2c90bfaf9..c2f669ab5c 100644 --- a/openpilot/cereal/log.capnp +++ b/openpilot/cereal/log.capnp @@ -2643,7 +2643,7 @@ struct Event { starpilotCarState @109 :Custom.StarPilotCarState; starpilotLongitudinalPlan @110 :Custom.StarPilotLongitudinalPlan; slcVisionObservation @111 :Custom.StarPilotModelDataV2; - customReserved5 @112 :Custom.CustomReserved5; + starpilotSteeringTracking @112 :Custom.StarPilotSteeringTracking; # Temporary historical starpilotPlan replay transport. # When old-route replay support is removed, retain reserved slot @113. # Adapter: openpilot/starpilot/ui/old_starpilot_route_shim.py. diff --git a/openpilot/cereal/services.h b/openpilot/cereal/services.h index 7fc59e18d9..8bb5ebdc9c 100644 --- a/openpilot/cereal/services.h +++ b/openpilot/cereal/services.h @@ -32,6 +32,7 @@ static std::map services = { { "extrinsicsCalibration", {"extrinsicsCalibration", true, 4.000000, 4, 256000}}, { "lateralTorqueParameters", {"lateralTorqueParameters", true, 4.000000, 1, 256000}}, { "autoSteeringLearning", {"autoSteeringLearning", true, 4.000000, 1, 256000}}, + { "starpilotSteeringTracking", {"starpilotSteeringTracking", true, 20.000000, 5, 256000}}, { "lateralDelay", {"lateralDelay", true, 4.000000, 1, 256000}}, { "operatingSystemLog", {"operatingSystemLog", true, 0.000000, -1, 256000}}, { "carState", {"carState", true, 100.000000, 10, 256000}}, diff --git a/openpilot/cereal/services.py b/openpilot/cereal/services.py index 831f303d3c..566449cb3a 100755 --- a/openpilot/cereal/services.py +++ b/openpilot/cereal/services.py @@ -51,6 +51,7 @@ _services: dict[str, tuple] = { "extrinsicsCalibration": (True, 4., 4), "lateralTorqueParameters": (True, 4., 1), "autoSteeringLearning": (True, 4., 1), + "starpilotSteeringTracking": (True, 20., 5), "lateralDelay": (True, 4., 1), "operatingSystemLog": (True, 0.), "carState": (True, 100., 10), diff --git a/openpilot/common/libcommon.a b/openpilot/common/libcommon.a index ab52133caf..284b24be12 100644 Binary files a/openpilot/common/libcommon.a and b/openpilot/common/libcommon.a differ diff --git a/openpilot/common/libparams_c.so b/openpilot/common/libparams_c.so index 9196e851d5..a2dcdf53f4 100755 Binary files a/openpilot/common/libparams_c.so and b/openpilot/common/libparams_c.so differ diff --git a/openpilot/common/params_keys.h b/openpilot/common/params_keys.h index 944a76108a..dc6dfe61ba 100644 --- a/openpilot/common/params_keys.h +++ b/openpilot/common/params_keys.h @@ -107,6 +107,7 @@ inline static std::unordered_map keys = { {"SteeringLearningPreferences", {PERSISTENT, BYTES}}, {"AutoSteeringLearningReset", {PERSISTENT, STRING}}, {"AutomaticSteeringProfiles", {PERSISTENT | DONT_LOG, BYTES}}, + {"AutomaticSteeringLiveTrials", {PERSISTENT | DONT_LOG, BYTES}}, {"ForceAutoTuneOff", {PERSISTENT, BOOL, "0"}}, {"SteerLatAccel", {PERSISTENT, FLOAT}}, {"SteerLatAccelStock", {PERSISTENT, FLOAT}}, diff --git a/openpilot/selfdrive/car/card.py b/openpilot/selfdrive/car/card.py index 9aeafa8471..27a7f33266 100644 --- a/openpilot/selfdrive/car/card.py +++ b/openpilot/selfdrive/car/card.py @@ -162,12 +162,13 @@ class Car: self.curve_replay = os.getenv('CURVE_REPLAY_RUNTIME') == '1' or feature_requested(self.params, 'curve') self.aol_replay = os.getenv('AOL_REPLAY_RUNTIME') == '1' or feature_requested(self.params, 'aol') self.conditional_replay = os.getenv('CONDITIONAL_MODE_REPLAY_RUNTIME') == '1' or feature_requested(self.params, 'conditional') - subscribed_services = ['pandaStates', 'carControl', 'onroadEvents', 'deviceState'] + subscribed_services = ['pandaStates', 'carControl', 'onroadEvents', 'deviceState', 'autoSteeringLearning'] if self.aol_replay: subscribed_services.extend(('aolSafetyWire', 'managerState', 'aolAxisState', 'selfdriveState')) if self.slc_replay: subscribed_services.append('slcState') - self.sm = messaging.SubMaster(subscribed_services) + self.sm = messaging.SubMaster(subscribed_services, ignore_alive=['autoSteeringLearning'], + ignore_valid=['autoSteeringLearning'], ignore_avg_freq=['autoSteeringLearning']) self.slc_cruise_event_id = 0 self.slc_producer_session = uuid.uuid4().hex self.slc_command_sock = messaging.sub_sock('slcCruiseCommand', conflate=False) if self.slc_replay else None @@ -941,6 +942,13 @@ class Car: co_send = messaging.new_message('carOutput') co_send.valid = self.sm.all_checks(['carControl']) co_send.carOutput.actuatorsOutput = self.last_actuators_output + self.learning_feedback_publication = None + capture = self._feedback_capture(int(co_send.logMonoTime)) + if capture is not None: + try: + self.learning_feedback_publication = capture.publication(co_send) + except Exception: + capture.invalidate() # Optional observation cannot affect the original CO. self.pm.send('carOutput', co_send) # kick off controlsd step while we actuate the latest carControl packet @@ -1012,6 +1020,16 @@ class Car: companion = learning_companion except Exception: self._clear_learning_evidence() + publication = getattr(self, 'learning_feedback_publication', None) + if publication is not None: + try: + from openpilot.starpilot.flm.feedback_wire import write_publication + feedback_companion = (companion.as_reader().as_builder() if companion is not None else + messaging.new_message('starpilotCarState', valid=bool(cs_send.valid), logMonoTime=int(cs_send.logMonoTime))) + write_publication(feedback_companion.starpilotCarState, publication) + companion = feedback_companion + except Exception: + self.learning_feedback_publication = None self.car_gps_publisher.update(self.CI.CS, self.pm, companion) self.pm.send('carState', cs_send) if getattr(self, 'learning_stamp_observer', None) is not None: @@ -1267,7 +1285,33 @@ class Car: (not ps.safetyRxChecksInvalid or inactive_keepalive and self.volt_startup_keepalive()) for ps, cfg in zip(pandas, self.CP.safetyConfigs, strict=True))) + def _feedback_capture(self, now_ns): + # Default OFF performs no feedback-module import, digest or witness allocation. + try: + source = self.sm['autoSteeringLearning'].learning.sourceReceipt + if not source.enabled or source.manual: + self.learning_feedback_capture = None + return None + from openpilot.starpilot.flm.feedback_observation import card_capture_for + return card_capture_for(self, now_ns) + except Exception: + self.learning_feedback_capture = None + return None + + def _invalidate_feedback_output(self, *, assignment=False): + capture = getattr(self, 'learning_feedback_capture', None) + if capture is not None: + try: + if assignment: + capture.begin_assignment() + else: + capture.invalidate() + except Exception: + self.learning_feedback_capture = None + def _clear_learning_evidence(self): + self._invalidate_feedback_output() + self.learning_feedback_publication = None self.learning_stamp_observer = None self.learning_applied_evidence = self.learning_previous_applied_evidence = None self.learning_send_mono_ns = self.learning_send_committed_mono_ns = 0 @@ -1499,6 +1543,7 @@ class Car: if (not owner.pending_target and self.startup_panda_configured() and owner.sources_current(self.CI, now_boot_ns)): inactive = car.CarControl.new_message().as_reader() + self._invalidate_feedback_output(assignment=True) self.last_actuators_output, can_sends = self.CI.apply(inactive, now_boot_ns) self.publish_sendcan(can_sends, valid=CS.canValid) return @@ -1515,8 +1560,10 @@ class Car: if not self.vehicle_startup.before_control(configured=True, ci=self.CI, now_ns=now_boot_ns, control_current=current): self.publish_sendcan([], valid=False) return + self._invalidate_feedback_output(assignment=True) self.last_actuators_output, can_sends = self.CI.apply(inactive, now_boot_ns) else: + self._invalidate_feedback_output(assignment=True) self.last_actuators_output, can_sends = cast(GMInterface, self.CI).apply(inactive, now_boot_ns, startup_keepalive=True) self.publish_sendcan(can_sends, valid=CS.canValid) return @@ -1572,12 +1619,19 @@ class Car: learning_snapshot = applied_snapshot except Exception: self._clear_learning_evidence() + self._invalidate_feedback_output(assignment=True) self.last_actuators_output, can_sends = self.CI.apply(CC, now_nanos) self.publish_sendcan(can_sends, valid=CS.canValid) if learning_snapshot is not None and self.learning_send_committed_mono_ns: try: self.learning_applied_evidence = learning_snapshot(self, CC, CS, self.last_actuators_output, apply_ns=self.learning_send_committed_mono_ns, state_ns=self.learning_car_state_source_ns) + capture = getattr(self, 'learning_feedback_capture', None) + if capture is not None: + try: + capture.committed(self.last_actuators_output, self.learning_applied_evidence) + except Exception: + self.learning_feedback_capture = None # Keep the original applied evidence. except Exception: self._clear_learning_evidence() diff --git a/openpilot/selfdrive/controls/controlsd.py b/openpilot/selfdrive/controls/controlsd.py index dd722182f4..f892a55c61 100755 --- a/openpilot/selfdrive/controls/controlsd.py +++ b/openpilot/selfdrive/controls/controlsd.py @@ -75,7 +75,9 @@ class Controls: 'extrinsicsCalibration', 'deviceMotion', 'longitudinalPlan', 'lateralManeuverPlan', 'carState', 'carOutput', 'driverMonitoringState', 'onroadEvents', 'driverAssistance'] + (['aolAxisState', 'aolSafetyWire'] if self.aol_replay or self.ordinary_axis_ack_required else [])) - optional = list(dict.fromkeys(self.longitudinal_inputs.optional_services + ['deviceState', 'autoSteeringLearning'])) + optional = list(dict.fromkeys( + self.longitudinal_inputs.optional_services + ['deviceState', 'autoSteeringLearning', 'starpilotSteeringTracking', 'pandaStates'] + )) if optional: ignore = set(optional) self.sm = messaging.SubMaster(services + optional, poll='selfdriveState', @@ -122,7 +124,8 @@ class Controls: "startupPidGain": [list(row) for row in self.LaC.pid._k_p], "startupPidLimits": [self.LaC.pid.neg_limit, self.LaC.pid.pos_limit]}) - self.global_learning = GlobalRuntime(self.CP, self.CI, self.LaC) + from openpilot.starpilot.flm.live_trial_store import load_confirmed_history + self.global_learning = GlobalRuntime(self.CP, self.CI, self.LaC, confirmed_live_history=load_confirmed_history(self.params)) from openpilot.starpilot.lateral.controller_selection import model_turn_assist_supported assist_enabled = self.vehicle_startup_preferences.turn_assist and model_turn_assist_supported(self.CP) @@ -438,6 +441,7 @@ class Controls: hudControl.leftLaneDepart = self.sm['driverAssistance'].leftLaneDeparture hudControl.rightLaneDepart = self.sm['driverAssistance'].rightLaneDeparture + feedback_input_limited = self.steer_limited_by_safety if CC.latActive: CO = self.sm['carOutput'] if self.CP.steerControlType == car.CarParams.SteerControlType.angle: @@ -505,6 +509,21 @@ class Controls: feedback = learning_feedback except Exception: self._discard_global_learning() + if (automatic is not None and automatic.eligible and automatic.source is not None and + automatic.source.enabled and not automatic.source.manual and + self.CP.steerControlType == car.CarParams.SteerControlType.torque): + try: + from openpilot.starpilot.flm.feedback_observation import capture_controls + from openpilot.starpilot.flm.feedback_wire import write_consumption + observation = capture_controls(self, CC, control_ns=int(cc_send.logMonoTime), input_limited=feedback_input_limited) + if observation is not None: + feedback_copy = feedback.as_reader().as_builder() + write_consumption(feedback_copy.starpilotLateralState, observation) + feedback = feedback_copy + except Exception: + self.learning_feedback_prior_publish_ns = 0 + else: + self.learning_feedback_prior_publish_ns = 0 self.pm.send('starpilotLateralState', feedback) def run(self): diff --git a/openpilot/selfdrive/controls/lib/latcontrol_torque.py b/openpilot/selfdrive/controls/lib/latcontrol_torque.py index e97ab6a148..60eb1b9c54 100644 --- a/openpilot/selfdrive/controls/lib/latcontrol_torque.py +++ b/openpilot/selfdrive/controls/lib/latcontrol_torque.py @@ -100,7 +100,7 @@ class LatControlTorque(LatControl): result = self.update_standard(active, CS, VM, params, steer_limited_by_safety, desired_curvature, curvature_limited, lat_delay) return auto_stages.apply_selected_output(self, CS, VM, params, desired_curvature, result, - active=active, limited=steer_limited_by_safety or curvature_limited) + active=active, limited=steer_limited_by_safety or curvature_limited, curvature_limited=curvature_limited) def update_standard(self, active, CS, VM, params, steer_limited_by_safety, desired_curvature, curvature_limited, lat_delay, *, friction_policy=None): @@ -115,7 +115,7 @@ class LatControlTorque(LatControl): curvature_deadzone = abs(VM.calc_curvature(math.radians(self.steering_angle_deadzone_deg + flm.deadband(self.flm_surface, CS.vEgo)), CS.vEgo, 0.0)) lateral_accel_deadzone = curvature_deadzone * CS.vEgo ** 2 lateral_accel_deadzone = auto_stages.apply_selected_deadzone(self, CS, VM, float(lateral_accel_deadzone), desired_curvature, - active=active, limited=steer_limited_by_safety or curvature_limited) + active=active, limited=steer_limited_by_safety or curvature_limited, curvature_limited=curvature_limited) delay_frames = int(np.clip(lat_delay / self.dt + 1, 1, self.lat_accel_request_buffer_len)) expected_lateral_accel = self.lat_accel_request_buffer[-delay_frames] @@ -134,10 +134,10 @@ class LatControlTorque(LatControl): if friction_policy is not None: threshold = friction_policy.friction_threshold(CS.vEgo, setpoint, desired_lateral_jerk) ff, threshold = auto_stages.apply_stages(self, CS, setpoint, desired_lateral_jerk, ff, threshold, - active=active, limited=steer_limited_by_safety or curvature_limited) + active=active, limited=steer_limited_by_safety or curvature_limited, curvature_limited=curvature_limited) selected_friction = self.friction(error + JERK_GAIN * desired_lateral_jerk, lateral_accel_deadzone, threshold, CS, setpoint) ff += auto_stages.apply_selected_friction(self, CS, setpoint, float(selected_friction), - active=active, limited=steer_limited_by_safety or curvature_limited) + active=active, limited=steer_limited_by_safety or curvature_limited, curvature_limited=curvature_limited) if not active: output_torque = 0.0 diff --git a/openpilot/starpilot/flm/SConscript b/openpilot/starpilot/flm/SConscript new file mode 100644 index 0000000000..d7c3d25815 --- /dev/null +++ b/openpilot/starpilot/flm/SConscript @@ -0,0 +1,28 @@ +Import('envCython') +import hashlib +import json +import platform +import struct +import sys +import sysconfig + +native = envCython.Clone() +native.Append(CCFLAGS=['-O2', '-fno-fast-math', '-fno-finite-math-only']) +extension = native.Program('_primitive_walk.so', '_primitive_walk.pyx', LIBS=[]) + + +def write_binding(target, source, env): + files = {'_primitive_walk.pyx': source[1], 'native_primitives.py': source[2], + 'SConscript': source[3], 'SConstruct': source[4]} + value = {'schema': 1, 'revision': 'flm-exact-primitives-v1', + 'sources': {name: hashlib.sha256(open(str(path), 'rb').read()).hexdigest() for name, path in files.items()}, + 'artifact_sha256': hashlib.sha256(open(str(source[0]), 'rb').read()).hexdigest(), + 'abi': {'implementation': sys.implementation.name, 'cache_tag': sys.implementation.cache_tag, + 'python': list(sys.version_info[:2]), 'SOABI': sysconfig.get_config_var('SOABI'), + 'machine': platform.machine(), 'pointer_bits': struct.calcsize('P') * 8, 'platform': sys.platform}} + with open(str(target[0]), 'w') as output: + output.write(json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False) + '\n') + + +native.Command('_primitive_walk.binding.json', + [extension[0], '_primitive_walk.pyx', 'native_primitives.py', 'SConscript', '#SConstruct'], write_binding) diff --git a/openpilot/starpilot/flm/_primitive_walk.binding.json b/openpilot/starpilot/flm/_primitive_walk.binding.json new file mode 100644 index 0000000000..fb9958a2de --- /dev/null +++ b/openpilot/starpilot/flm/_primitive_walk.binding.json @@ -0,0 +1 @@ +{"abi":{"SOABI":"cpython-312-aarch64-linux-gnu","cache_tag":"cpython-312","implementation":"cpython","machine":"aarch64","platform":"linux","pointer_bits":64,"python":[3,12]},"artifact_sha256":"86cc336b284b1ed0744a83e23f04a5e32f51ea3284bdfe493a69c9b38b6cad3b","revision":"flm-exact-primitives-v1","schema":1,"sources":{"SConscript":"8e7589098349bc9400c9b99578fcdf661db204578762b2121ce9d2a1d34b90bd","SConstruct":"b921b56f52db299276b1a14e470685cae1cdcdd1b7865a714cb2027a354d51ca","_primitive_walk.pyx":"7750f6486cc0cbc129cd922c5c1537aa5af8e0606bd8b23b7eb9726b134e8cae","native_primitives.py":"af1dbeb0d91ccde641e32a3a77aa26c84c11132ace6dc0d9696073976d814904"}} diff --git a/openpilot/starpilot/flm/_primitive_walk.pyx b/openpilot/starpilot/flm/_primitive_walk.pyx new file mode 100644 index 0000000000..7469b59cbd --- /dev/null +++ b/openpilot/starpilot/flm/_primitive_walk.pyx @@ -0,0 +1,322 @@ +# distutils: language = c++ +# cython: language_level=3 +import math +import json +import hashlib +import sys +from libcpp.string cimport string +from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING, PyBytes_GET_SIZE +from cpython.unicode cimport PyUnicode_GET_LENGTH, PyUnicode_READ_CHAR + +FALLBACK = object() +cdef Py_ssize_t LIMIT = 128 * 1024 * 1024 +cdef int DEPTH = 32 + +cdef bint budget(): + cdef int frames = 0 + cdef object frame + if sys.getrecursionlimit() < 512: + return False + frame = sys._getframe() + while frame is not None and frames <= 64: + frames += 1 + frame = frame.f_back + return frames <= 64 + +cdef object limits(object item, int depth, dict memo, set active): + cdef object kind = type(item), ident = id(item), cached, child, key, row, value + cdef Py_ssize_t cost = 0, child_cost + cdef int height = 0, maximum = -1 + if depth > DEPTH or ident in active: + return FALLBACK + cached = memo.get(ident) + if cached is not None: + return cached if depth + cached[1] <= DEPTH else FALLBACK + if item is None: cost = 1 + elif kind is bool: cost = 2 + elif kind is int: + value = 9 + (abs(item).bit_length() + 7) // 8 + if value > LIMIT: return FALLBACK + cost = value + elif kind is float: + if not math.isfinite(item): return FALLBACK + cost = 9 + elif kind is str or kind is bytes: + value = 9 + len(item.encode('utf8', 'surrogatepass') if kind is str else item) + if value > LIMIT: return FALLBACK + cost = value + elif kind is tuple or kind is list or kind is dict: + if kind is dict: + for key in item: + if type(key) is not str: return FALLBACK + active.add(ident) + cost = 9 + try: + if kind is dict: + for key, child in item.items(): + for value in (key, child): + row = limits(value, depth + 1, memo, active) + if row is FALLBACK: return FALLBACK + child_cost = row[0] + if child_cost > LIMIT - cost: return FALLBACK + cost += child_cost + maximum = max(maximum, row[1]) + else: + for child in item: + row = limits(child, depth + 1, memo, active) + if row is FALLBACK: return FALLBACK + child_cost = row[0] + if child_cost > LIMIT - cost: return FALLBACK + cost += child_cost + maximum = max(maximum, row[1]) + finally: + active.remove(ident) + height = 1 + maximum + else: return FALLBACK + if depth + height > DEPTH or cost > LIMIT: return FALLBACK + row = (cost, height) + memo[ident] = row + return row + +cdef object clone(object item): + cdef object kind = type(item), child, key + cdef dict mapping + cdef list rows + if kind is dict: + mapping = {} + for key, child in item.items(): mapping[key] = clone(child) + return mapping + if kind is tuple or kind is list: + rows = [] + for child in item: rows.append(clone(child)) + return tuple(rows) if kind is tuple else rows + return item + +cdef object typed(object item): + cdef object kind = type(item), child, key + cdef list rows = [] + if item is None or kind is bool or kind is int or kind is str: + return [kind.__name__, item] + if kind is float: return ['float64', item.hex()] + if kind is bytes: return ['bytes', item.hex()] + if kind is tuple or kind is list: + for child in item: rows.append(typed(child)) + return [kind.__name__, rows] + for key in sorted(item): rows.append([key, typed(item[key])]) + return ['dict', rows] + +def independent_primitive_graph(object value): + if not budget() or limits(value, 0, {}, set()) is FALLBACK: return FALLBACK + return clone(value) + +cdef object shape(object item, int depth, dict memo, set active): + cdef object kind = type(item), ident, cached, child, key, height + cdef int maximum = -1 + if depth > DEPTH: return FALLBACK + if item is None or kind is bool or kind is int or kind is str or kind is bytes: return 0 + if kind is float: return 0 if math.isfinite(item) else FALLBACK + if kind is not tuple and kind is not list and kind is not dict: return FALLBACK + ident = id(item) + if ident in active: return FALLBACK + cached = memo.get(ident) + if cached is not None: return cached if depth + cached <= DEPTH else FALLBACK + if kind is dict: + for key in item: + if type(key) is not str: return FALLBACK + active.add(ident) + try: + for child in (item.values() if kind is dict else item): + height = shape(child, depth + 1, memo, active) + if height is FALLBACK: return FALLBACK + maximum = max(maximum, height) + finally: + active.remove(ident) + height = 1 + maximum + if depth + height > DEPTH: return FALLBACK + memo[ident] = height + return height + +cdef bint large(object value, int depth): + cdef object kind = type(value), child + if kind is tuple or kind is list: return len(value) > 256 + if kind is dict and depth < 3: + for child in value.values(): + if large(child, depth + 1): return True + return False + +cdef void update_digest(object digest, object value): + cdef object kind = type(value), child, key, text + cdef list batch + cdef Py_ssize_t start, index = 0 + if (kind is tuple or kind is list) and len(value) > 256: + digest.update(b'["tuple",[' if kind is tuple else b'["list",[') + for start in range(0, len(value), 64): + if start: digest.update(b',') + batch = [] + for child in value[start:start + 64]: batch.append(typed(child)) + text = json.dumps(batch, separators=(',', ':'), allow_nan=False) + digest.update(text[1:-1].encode()) + digest.update(b']]') + elif kind is dict and large(value, 0): + digest.update(b'["dict",[') + for key in sorted(value): + if index: digest.update(b',') + digest.update(('[' + json.dumps(key, separators=(',', ':'), allow_nan=False) + ',').encode()) + update_digest(digest, value[key]) + digest.update(b']') + index += 1 + digest.update(b']]') + else: + digest.update(json.dumps(typed(value), separators=(',', ':'), allow_nan=False).encode()) + +# One fixed-capacity native buffer, no Python token lists or typed-tree copy. +# Every byte is ASCII JSON. Admission remains the original full shape prepass. +cdef class JsonBuffer: + cdef string data + cdef object digest + + def __cinit__(self, object digest): + self.digest = digest + self.data.reserve(65536) + + cdef void flush(self) except *: + if self.data.size(): + self.digest.update(PyBytes_FromStringAndSize(self.data.data(), self.data.size())) + self.data.clear() + + cdef void put(self, char value) except *: + self.data.push_back(value) + if self.data.size() == 65536: self.flush() + + cdef void literal(self, const char* value, Py_ssize_t size) except *: + cdef Py_ssize_t offset = 0, available, take + while offset < size: + available = 65536 - self.data.size() + take = min(available, size - offset) + self.data.append(value + offset, take) + offset += take + if self.data.size() == 65536: self.flush() + +cdef void unicode_json(JsonBuffer output, object value) except *: + cdef Py_ssize_t index, length + cdef bytes encoded + cdef unsigned int code, high, low + cdef const char* digits = "0123456789abcdef" + # A digest flush can release the GIL or reenter Python after shape(). A later + # dict key may therefore no longer be an exact str. The old typed-tree JSON + # path encodes such a raw key as a JSON value; preserve that safe behavior. + if type(value) is not str: + encoded = json.dumps(value, separators=(',', ':'), allow_nan=False).encode() + output.literal(PyBytes_AS_STRING(encoded), PyBytes_GET_SIZE(encoded)) + return + length = PyUnicode_GET_LENGTH(value) + output.put(34) + for index in range(length): + code = PyUnicode_READ_CHAR(value, index) + if code == 34 or code == 92: + output.put(92) + output.put(code) + elif code == 8: output.literal('\\b', 2) + elif code == 9: output.literal('\\t', 2) + elif code == 10: output.literal('\\n', 2) + elif code == 12: output.literal('\\f', 2) + elif code == 13: output.literal('\\r', 2) + elif 32 <= code <= 126: + output.put(code) + else: + if code > 65535: + high = 55296 + ((code - 65536) >> 10) + low = 56320 + ((code - 65536) & 1023) + else: + high = code + low = 0 + output.literal('\\u', 2) + output.put(digits[(high >> 12) & 15]) + output.put(digits[(high >> 8) & 15]) + output.put(digits[(high >> 4) & 15]) + output.put(digits[high & 15]) + if code > 65535: + output.literal('\\u', 2) + output.put(digits[(low >> 12) & 15]) + output.put(digits[(low >> 8) & 15]) + output.put(digits[(low >> 4) & 15]) + output.put(digits[low & 15]) + output.put(34) + +cdef void buffer_typed(JsonBuffer output, object value) except *: + cdef object kind = type(value), child, key + cdef bytes encoded + cdef const unsigned char* raw + cdef const char* digits = "0123456789abcdef" + cdef Py_ssize_t index = 0, length, offset, take, byte_index + cdef char hex_chunk[8192] + if value is None: output.literal('["NoneType",null]', 17) + elif kind is bool: + if value: output.literal('["bool",true]', 13) + else: output.literal('["bool",false]', 14) + elif kind is int: + output.literal('["int",', 7) + # Exact int repr is also the C JSON encoder's decimal spelling and keeps + # CPython's configured digit-limit exceptions. No subclass method can run. + encoded = str(value).encode('ascii') + output.literal(PyBytes_AS_STRING(encoded), PyBytes_GET_SIZE(encoded)) + output.put(93) + elif kind is float: + output.literal('["float64",', 11) + unicode_json(output, value.hex()) + output.put(93) + elif kind is str: + output.literal('["str",', 7) + unicode_json(output, value) + output.put(93) + elif kind is bytes: + output.literal('["bytes","', 10) + raw = PyBytes_AS_STRING(value) + length = PyBytes_GET_SIZE(value) + for offset in range(0, length, 4096): + take = min(4096, length - offset) + for byte_index in range(take): + hex_chunk[2 * byte_index] = digits[raw[offset + byte_index] >> 4] + hex_chunk[2 * byte_index + 1] = digits[raw[offset + byte_index] & 15] + output.literal(hex_chunk, 2 * take) + output.literal('"]', 2) + elif kind is tuple or kind is list: + if kind is tuple: output.literal('["tuple",[', 10) + else: output.literal('["list",[', 9) + for child in value: + if index: output.put(44) + buffer_typed(output, child) + index += 1 + output.literal(']]', 2) + else: + output.literal('["dict",[', 9) + for key in sorted(value): + if index: output.put(44) + output.put(91) + unicode_json(output, key) + output.put(44) + buffer_typed(output, value[key]) + output.put(93) + index += 1 + output.literal(']]', 2) + +def whole_body_digest(object value): + cdef JsonBuffer output + if not budget() or shape(value, 0, {}, set()) is FALLBACK: return FALLBACK + digest = hashlib.sha256() + if large(value, 0): + output = JsonBuffer(digest) + buffer_typed(output, value) + output.flush() + else: + update_digest(digest, value) + return digest.hexdigest() + + + +cdef extern from "Python.h": + int PY_MAJOR_VERSION + int PY_MINOR_VERSION + +def compiled_abi(): + return (PY_MAJOR_VERSION, PY_MINOR_VERSION, sizeof(void*) * 8, 'flm-exact-primitives-v1') diff --git a/openpilot/starpilot/flm/_primitive_walk.so b/openpilot/starpilot/flm/_primitive_walk.so new file mode 100755 index 0000000000..18ae5988e9 Binary files /dev/null and b/openpilot/starpilot/flm/_primitive_walk.so differ diff --git a/openpilot/starpilot/flm/adaptive_profile.py b/openpilot/starpilot/flm/adaptive_profile.py index d2f0711917..3ba6891ae6 100644 --- a/openpilot/starpilot/flm/adaptive_profile.py +++ b/openpilot/starpilot/flm/adaptive_profile.py @@ -95,6 +95,25 @@ def loss(samples, coefficients): def evaluate(identity, parameters, blocks, *, now_ns, required_cohorts, response_revision, sequence=1, ridge=.01, prior=None): + return _evaluate(identity, parameters, blocks, now_ns=now_ns, required_cohorts=required_cohorts, + response_revision=response_revision, sequence=sequence, ridge=ridge, prior=prior) + + +def evaluate_event_proposal(identity, parameters, blocks, *, trial_step, event_mean_model=None, **kwargs): + """One bounded TRAIN ridge proposal; all ordinary evidence/heldout gates remain.""" + from openpilot.starpilot.flm.event_bounded_ridge import bounded_ridge + if not 1 <= len(parameters) <= 4 or not finite(trial_step) or trial_step <= 0: + return Result('unavailable', 'event_proposal_contract') + if event_mean_model is not None: + from openpilot.starpilot.flm.event_mean_evidence import validate_model + validate_model(event_mean_model,blocks) + return _evaluate(identity, parameters, blocks, _bounded_solver=lambda matrix, rhs, centre: + bounded_ridge(matrix, rhs, parameters, centre, trial_step), + **({'_event_mean_model':event_mean_model} if event_mean_model is not None else {}), **kwargs) + + +def _evaluate(identity, parameters, blocks, *, now_ns, required_cohorts, response_revision, + sequence=1, ridge=.01, prior=None, _bounded_solver=None, _event_mean_model=None): """Fit a logged stage response residual; never generates exploration commands.""" if (type(identity) is not str or len(identity) != 64 or type(now_ns) is not int or now_ns <= 0 or not required_cohorts or not response_revision or type(sequence) is not int or sequence <= 0 or @@ -142,8 +161,9 @@ def evaluate(identity, parameters, blocks, *, now_ns, required_cohorts, response if max(abs(value) for value in inverse_column) > 100: raise ValueError("confounded_features") regularized = [[value + (ridge * len(rows) if i == j else 0) for j, value in enumerate(row)] for i, row in enumerate(gram)] - coefficients = solve(regularized, [sum(row.weight * row.features[i] * row.residual_target for row in rows) + - ridge * len(rows) * centre[i] for i in range(count)]) + rhs = [sum(row.weight * row.features[i] * row.residual_target for row in rows) + + ridge * len(rows) * centre[i] for i in range(count)] + coefficients = solve(regularized, rhs) if _bounded_solver is None else _bounded_solver(regularized, rhs, centre) except ValueError: return Result('collecting', 'confounded_stage') if any(not p.low <= value <= p.high or abs(value) > p.max_step for p, value in zip(parameters, coefficients, strict=True)): @@ -155,6 +175,9 @@ def evaluate(identity, parameters, blocks, *, now_ns, required_cohorts, response mean = sum(independent) / len(independent) variance = sum((x - mean) ** 2 for x in independent) / (len(independent) - 1) lower = mean - 12.707 * math.sqrt(variance / len(independent)) + if _event_mean_model is not None: + from openpilot.starpilot.flm.event_mean_evidence import lower as event_lower + lower = event_lower(_event_mean_model,tuple(b for b in heldout if b.cohort==cohort),tuple(independent),cohort=cohort) if lower <= 0: return Result('rejected', 'heldout_cohort_regression_or_noise') improvements.append((cohort, lower)) diff --git a/openpilot/starpilot/flm/application_wire.py b/openpilot/starpilot/flm/application_wire.py index 89fb7ac204..cb42e24d4a 100644 --- a/openpilot/starpilot/flm/application_wire.py +++ b/openpilot/starpilot/flm/application_wire.py @@ -7,6 +7,11 @@ MAX_AGE_NS = 500_000_000 def write_application(envelope, outcome): + from openpilot.starpilot.flm.live_trial_owner import LiveTrialApplication + if type(outcome) is LiveTrialApplication: + from openpilot.starpilot.flm.live_trial_wire import write_application as write_live_application + write_live_application(envelope, outcome) + return if not isinstance(outcome, ApplicationOutcome): return envelope.learningApplication = { diff --git a/openpilot/starpilot/flm/candidate_feedback.py b/openpilot/starpilot/flm/candidate_feedback.py new file mode 100644 index 0000000000..def92abd25 --- /dev/null +++ b/openpilot/starpilot/flm/candidate_feedback.py @@ -0,0 +1,241 @@ +"""Versioned causal candidate feedback; never original/current admission.""" +from dataclasses import dataclass +import hashlib +import math +import pickle +from typing import cast + +LEGACY_POLICY = 'original-consumed-feedback-regime-v1' +CLOSED_POLICY = 'candidate-native-consumed-co-closed-loop-v1' +PROOF_VERSION = 'candidate-native-feedback-chronology-v1' +_SEAL = object() + + +def policy(value): + if type(value) is not str or value not in (LEGACY_POLICY,CLOSED_POLICY): + raise ValueError('explicit candidate feedback policy') + return value + + +def uses_closed_loop(engine): + # Opting OUT of a shortcut grants no source/native/health authority. Unknown + # shells/proxies retain the old path without attribute callbacks or recursion. + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.feedback_replay import FeedbackReplayEvidence,_SEAL as feedback_seal + if type(engine) is not EventControllerReplay: + return False + owned = object.__getattribute__(engine,'__dict__') + evidence = owned.get('verified_feedback_tape') + if type(evidence) is not FeedbackReplayEvidence: + return False + values = object.__getattribute__(evidence,'__dict__') + value = values.get('candidate_policy') + return values.get('_seal') is feedback_seal and type(value) is str and value == CLOSED_POLICY + + +@dataclass(frozen=True) +class CandidateFeedbackProof: + version: str + source_digest: str + original_digest: str + coordinates_digest: str + native_policy: str + chronology_digest: str + row_count: int + _seal: object + + +def strict_reference_proof_key(proof, engine, coordinates, request_count): + """Fresh exact strict chronology certificate; numerical FD reuse only. + + The returned immutable primitive bytes detect later mutation of a retained + proof object. They never replace the fresh engine/source key at the caller. + """ + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + if (type(proof) is not CandidateFeedbackProof or proof._seal is not _SEAL or + type(coordinates) is not tuple or type(request_count) is not int or + any(type(value) not in (int,float) or not math.isfinite(value) for value in coordinates)): + return None + evidence = require_feedback_engine(engine) + fields = (proof.version,proof.source_digest,proof.original_digest,proof.coordinates_digest, + proof.native_policy,proof.chronology_digest,proof.row_count) + if (any(type(value) is not str for value in fields[:6]) or type(proof.row_count) is not int or + proof.version != PROOF_VERSION or evidence.candidate_policy != CLOSED_POLICY or + proof.source_digest != evidence.engine_input_digest or proof.original_digest != evidence.original.digest or + proof.coordinates_digest != hashlib.sha256(pickle.dumps(coordinates,protocol=5)).hexdigest() or + proof.native_policy != 'strict' or not 1 <= proof.row_count == request_count <= 12000 or + len(proof.chronology_digest) != 64 or any(c not in '0123456789abcdef' for c in proof.chronology_digest)): + return None + return pickle.dumps(fields,protocol=5) + +def _strict_reference_fields(proof, evidence, coordinates, request_count): + """Numerical fields only; caller MUST own fresh full pre/post source checks. + + Public callers retain strict_reference_proof_key and its own fresh admission. + This helper creates no seal, proof, cache entry or current/native permission. + """ + from openpilot.starpilot.flm.feedback_replay import FeedbackReplayEvidence, _SEAL as feedback_seal + if (type(proof) is not CandidateFeedbackProof or proof._seal is not _SEAL or + type(evidence) is not FeedbackReplayEvidence or evidence._seal is not feedback_seal or + type(coordinates) is not tuple or type(request_count) is not int or + any(type(value) not in (int,float) or not math.isfinite(value) for value in coordinates)): + return None + fields = (proof.version,proof.source_digest,proof.original_digest,proof.coordinates_digest, + proof.native_policy,proof.chronology_digest,proof.row_count) + if (any(type(value) is not str for value in fields[:6]) or type(proof.row_count) is not int or + proof.version != PROOF_VERSION or evidence.candidate_policy != CLOSED_POLICY or + proof.source_digest != evidence.engine_input_digest or proof.original_digest != evidence.original.digest or + proof.coordinates_digest != hashlib.sha256(pickle.dumps(coordinates,protocol=5)).hexdigest() or + proof.native_policy != 'strict' or not 1 <= proof.row_count == request_count <= 12000 or + len(proof.chronology_digest) != 64 or any(c not in '0123456789abcdef' for c in proof.chronology_digest)): + return None + return pickle.dumps(fields,protocol=5) + + +class CandidateFeedbackLoop: + """One rollout, constant working state; native reports remain their real map. + + Immutable original observations are never rewritten. Only the next update's + safety_limited input is reconstructed. The consumed CO coordinate is exact, + not nearest-time lookup, physical delivery or an invented receiver contract. + """ + def __init__(self, engine, reported, coordinates, *, native_policy, original_baseline=False): + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + from openpilot.starpilot.flm.torque_emission_replay import EmissionHistory + evidence = require_feedback_engine(engine) + if (evidence.candidate_policy != CLOSED_POLICY or type(reported) is not dict or reported or + type(engine.emissions) is not EmissionHistory or type(coordinates) is not tuple or + any(type(value) not in (int,float) or not math.isfinite(value) for value in coordinates) or + type(original_baseline) is not bool or type(native_policy) is not str or + native_policy not in ('strict','native-shadow-identification')): + raise ValueError('registered candidate native feedback scope') + self.engine,self.evidence,self.reported = engine,evidence,reported + self.coordinates_digest = hashlib.sha256(pickle.dumps(coordinates,protocol=5)).hexdigest() + self.native_policy,self.original_baseline = native_policy,original_baseline + self._iterator = iter(self._rows()) + self._pending = None + self._previous_window = self._previous_ns = self._previous_flag = None + self._stream,self._count = hashlib.sha256(),0 + + def _rows(self): + for bounds,subset,activation in self.evidence.windows: + context = tuple(row for row in self.evidence.original.context_only if bounds[0] <= row.control_ns <= bounds[1]) + if len(context) > 1 or context and context[0].control_ns != bounds[0]: + raise ValueError('candidate feedback only actual leading context') + for row in context: + if row.control_ns >= activation: + raise ValueError('unknown leading CO cannot be a candidate treatment') + yield bounds,activation,row,None + for joined in subset.rows: + yield bounds,activation,joined.consumption,joined.publication + + @staticmethod + def _record(bounds,activation,row,pub,input_flag,request,reported,flag): + return (PROOF_VERSION,bounds,activation,row.control_ns,row.prior_publish_ns, + row.publication_ns,row.recv_frame,row.receipt_ns,row.co_body_sha256, + None if pub is None else (pub.output_control_ns,pub.output_apply_ns,pub.output_carrier_sequence, + pub.native_identity,pub.applied_sha256),input_flag,request,reported,flag) + + @staticmethod + def _add(stream,record): + body = pickle.dumps(record,protocol=5) + stream.update(len(body).to_bytes(8,'big')) + stream.update(body) + + def before(self, frame): + if self._pending is not None: + raise ValueError('candidate feedback previous control unfinished') + try: + bounds,activation,row,pub = next(self._iterator) + except StopIteration as error: + raise ValueError('candidate feedback extra controller frame') from error + if type(frame.control_ns) is not int or frame.control_ns != row.control_ns: + raise ValueError('candidate feedback exact controller chronology') + same_window = bounds == self._previous_window + if same_window and row.prior_publish_ns != self._previous_ns: + raise ValueError('candidate feedback previous publication omitted') + previous = cast(bool, self._previous_flag if same_window else row.input_limited) + if row.control_ns < activation and previous is not row.input_limited: + raise ValueError('candidate changed original preactivation input') + values = tuple(pair for pair in frame.inputs if pair[0] == 'safety_limited') + if (len(values) != 1 or type(values[0][1]) not in (int,float) or + values[0][1] != float(row.input_limited)): + raise ValueError('original actual safety_limited input changed') + self._pending = bounds,activation,row,pub,previous + if previous is row.input_limited: + return frame.inputs + return tuple((key,float(previous)) if key == 'safety_limited' else (key,value) for key,value in frame.inputs) + + def _equation(self,bounds,activation,row,pub,input_flag,request,reported_map): + from openpilot.starpilot.flm.feedback_observation import f32 + request = f32(request) + original = self.original_baseline or row.control_ns < activation or pub is None + if pub is None or pub.output_control_ns < activation: + reported = row.reported_torque if pub is None else pub.reported_torque + else: + if (not pub.output_control_ns <= pub.output_apply_ns <= pub.publication_ns <= row.control_ns or + pub.output_apply_ns not in reported_map): + raise ValueError('candidate consumed exact native CO not yet produced') + reported = f32(reported_map[pub.output_apply_ns]) + flag = bool(row.lat_active and abs(request-reported) > .01) + if original and (request != row.requested_torque or reported != row.reported_torque or + input_flag is not row.input_limited or flag is not row.published_limited): + raise ValueError('candidate changed original baseline or preactivation feedback') + return request,reported,flag + + def after(self,control_ns,request): + if self._pending is None or control_ns != self._pending[2].control_ns: + raise ValueError('candidate feedback controller pair missing') + bounds,activation,row,pub,input_flag = self._pending + request,reported,flag = self._equation(bounds,activation,row,pub,input_flag,request,self.reported) + self._add(self._stream,self._record(bounds,activation,row,pub,input_flag,request,reported,flag)) + self._count += 1 + self._previous_window,self._previous_ns,self._previous_flag = bounds,control_ns,flag + self._pending = None + + def finish(self,requests,reported): + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + if self._pending is not None or reported is not self.reported or type(requests) is not dict: + raise ValueError('candidate feedback complete native/controller maps required') + try: + next(self._iterator) + except StopIteration: + pass + else: + raise ValueError('candidate feedback original control omitted') + check = hashlib.sha256() + prior_window = prior_ns = prior_flag = None + count = 0 + for bounds,activation,row,pub in self._rows(): + if row.control_ns not in requests: + raise ValueError('candidate feedback complete request map required') + same_window = prior_window == bounds + if same_window and row.prior_publish_ns != prior_ns: + raise ValueError('candidate feedback prior candidate omitted') + input_flag = prior_flag if same_window else row.input_limited + request,torque,flag = self._equation(bounds,activation,row,pub,input_flag,requests[row.control_ns],reported) + self._add(check,self._record(bounds,activation,row,pub,input_flag,request,torque,flag)) + prior_window,prior_ns,prior_flag = bounds,row.control_ns,flag + count += 1 + evidence = require_feedback_engine(self.engine) + if (evidence.candidate_policy != CLOSED_POLICY or evidence.engine_input_digest != self.evidence.engine_input_digest or + evidence.original.digest != self.evidence.original.digest or count != self._count or not 1 <= count <= 12000 or + len(requests) != count or check.digest() != self._stream.digest()): + raise ValueError('candidate feedback owned chronology/native results changed') + return CandidateFeedbackProof(PROOF_VERSION,evidence.engine_input_digest,evidence.original.digest, + self.coordinates_digest,self.native_policy,check.hexdigest(),count,_SEAL) + + +def report_policy_matches(submitted,offered): + """Declaration matching only; physical/native/current guards remain callers'.""" + if type(submitted) is not tuple or len(submitted) not in (3,4) or type(offered) is not dict: + return False + expected = {'protocol','binding_digest','archive_digest','verified_digest','known_rows','context_rows'} + if len(submitted) == 4: + if type(submitted[3]) is not str or submitted[3] != CLOSED_POLICY: + return False + expected |= {'candidate_feedback_policy','candidate_feedback_proof_version'} + if (type(offered.get('candidate_feedback_policy')) is not str or offered['candidate_feedback_policy'] != CLOSED_POLICY or + type(offered.get('candidate_feedback_proof_version')) is not str or offered['candidate_feedback_proof_version'] != PROOF_VERSION): + return False + return set(offered) == expected diff --git a/openpilot/starpilot/flm/canonical_surface_adapter.py b/openpilot/starpilot/flm/canonical_surface_adapter.py index b4d6b08c5a..34f501328e 100644 --- a/openpilot/starpilot/flm/canonical_surface_adapter.py +++ b/openpilot/starpilot/flm/canonical_surface_adapter.py @@ -61,6 +61,9 @@ GROUPS = { 'deadband.low': ('center_deadband_crawl_deg', 'center_deadband_low_deg', 'center_deadband_mid_deg'), 'deadband.high': ('center_deadband_fast_deg', 'center_deadband_highway_deg'), } +# Numerical probe groups are separate from one-coordinate trial ownership. +PROBE_GROUPS = dict(GROUPS, all30=NAMES) + # Exact existing Dom universal raw domains, frozen without controller imports. # This prevents adaptive_profile -> canonical -> torque_surface -> policy -> # torque_stages -> selected_effects import cycles. Tests compare every row to @@ -114,6 +117,15 @@ def canonical_names(names): return tuple(n for n in NAMES if n in names) +def probe_names(names): + """Numerical measurements only; actual trial's four-knob cap is unchanged.""" + if (type(names) is not tuple or not 1 <= len(names) <= len(NAMES) or + any(type(name) is not str for name in names) or len(set(names)) != len(names) or + any(name not in NAMES for name in names)): + raise ValueError('one to thirty exact canonical numerical probe names') + return tuple(name for name in NAMES if name in names) + + _RAW_BY_NAME = MappingProxyType({name: (value, limits) for name, value, limits in RAW_DOMAINS}) @@ -229,9 +241,12 @@ class CanonicalSurfaceEffects: No Params, fitting, JSON, cloned controller or native writes belong here. """ - def __init__(self, coordinates, source_baseline_revision, *, accumulated=None): + def __init__(self, coordinates, source_baseline_revision, *, accumulated=None, allow_limited_profile=False): from openpilot.starpilot.flm import torque_surface as surface_math + if type(allow_limited_profile) is not bool: + raise ValueError('explicit bounded profile policy') + self.allow_limited_profile = allow_limited_profile trial = validate_trial(coordinates) values = dict.fromkeys(NAMES, 0.0) if accumulated is not None: @@ -293,7 +308,7 @@ class CanonicalSurfaceEffects: self.stage_calls += 1 if self.stage_calls != 1: raise ValueError('duplicate actual FLM stage') - if not frame.valid() or not frame.active or frame.driver_override or frame.limited: + if not frame.valid() or not frame.active or frame.driver_override or (frame.limited and (frame.curvature_limited or not self.allow_limited_profile)): return frame.baseline_ff, frame.baseline_threshold m, s = self.math, self.surface # Exact existing Dom universal controller order; notably no invented reciprocal diff --git a/openpilot/starpilot/flm/clipping_quality.py b/openpilot/starpilot/flm/clipping_quality.py index 238d1d1f31..a2a8050dab 100644 --- a/openpilot/starpilot/flm/clipping_quality.py +++ b/openpilot/starpilot/flm/clipping_quality.py @@ -13,6 +13,8 @@ FIXED_MAP_IDENTIFICATION = 'fixed-map-no-identification-v1' RESTORE_ACTIVATION_VERSION = 'fixed-eight-whole-map-restore-v1' RESTORE_AGGREGATE_ACTIVATION_VERSION = 'fixed-eight-whole-map-restore-support-v2' RESTORE_ZERO_REFERENCE_VERSION = 'fixed-eight-whole-map-restore-zero-reference-v3' +RESTORE_FEEDBACK_VERSION = 'fixed-eight-whole-map-feedback-restore-v4' +RESTORE_FEEDBACK_STATISTICS = 'fixed-two-sided-t95-12.707-v1' def clipping_quality(value): @@ -22,7 +24,23 @@ def clipping_quality(value): def event_clipping_quality(payload): - if type(payload.get('event_version')) is int and payload['event_version'] in (4, 5, 6): + if 'event_restore_statistics' in payload and (type(payload.get('event_version')) is not int or payload['event_version'] != 8): + raise ValueError('feedback restore statistics require explicit version8') + if type(payload.get('event_version')) is int and payload['event_version'] == 7: + from openpilot.starpilot.flm.feedback_replay import PROTOCOL + from openpilot.starpilot.flm.event_mean_evidence import VERSION + if (payload.get('plant_protocol') != PROTOCOL or payload.get('event_mean_version') != VERSION or + payload.get('event_operation') != 'trial'): + raise ValueError('explicit original feedback mean trial policy') + # Exactly the version3 quality/identification predicates, with no altered + # debt, clipping, native, or shadow admission policy for the new evidence. + return event_clipping_quality({**payload,'event_version':3}) + if type(payload.get('event_version')) is int and payload['event_version'] == 8: + from openpilot.starpilot.flm.feedback_replay import PROTOCOL + if (payload.get('plant_protocol') != PROTOCOL or + payload.get('event_restore_statistics') != RESTORE_FEEDBACK_STATISTICS or 'event_mean_version' in payload): + raise ValueError('explicit original feedback fixed-map restore policy') + if type(payload.get('event_version')) is int and payload['event_version'] in (4, 5, 6, 8): if (payload.get('event_operation') != 'restore' or payload.get('event_restore_policy') != restore_policy(payload['event_version']) or payload.get('event_clipping_quality') != restore_quality(payload['event_version']) or @@ -94,14 +112,14 @@ def identification_for_quality(contract): def restore_policy(version): - if type(version) is not int or version not in (4, 5, 6): + if type(version) is not int or version not in (4, 5, 6, 8): raise ValueError('explicit whole-map restore version') - return {4: RESTORE_ACTIVATION_VERSION, 5: RESTORE_AGGREGATE_ACTIVATION_VERSION, 6: RESTORE_ZERO_REFERENCE_VERSION}[version] + return {4: RESTORE_ACTIVATION_VERSION, 5: RESTORE_AGGREGATE_ACTIVATION_VERSION, 6: RESTORE_ZERO_REFERENCE_VERSION, 8: RESTORE_FEEDBACK_VERSION}[version] def restore_quality(version): restore_policy(version) - return ZERO_REFERENCE_QUALITY if version == 6 else QUALIFIED_EVENT_CLIPPING_QUALITY + return ZERO_REFERENCE_QUALITY if version in (6, 8) else QUALIFIED_EVENT_CLIPPING_QUALITY def zero_reference_relief_is_admitted(recorded_request, reference_request, candidate_request, diff --git a/openpilot/starpilot/flm/controller_replay.py b/openpilot/starpilot/flm/controller_replay.py index 099e344ac8..bc4ceb2e62 100644 --- a/openpilot/starpilot/flm/controller_replay.py +++ b/openpilot/starpilot/flm/controller_replay.py @@ -1,3 +1,4 @@ +from openpilot.starpilot.flm.normalized_trace_archive import guarded_archive """Exact selected torque controller replay, exclusively in the learner process. Only capture_snapshot/frame_input belong in controls. They copy bounded numeric @@ -7,8 +8,10 @@ invalidates evidence. Conversion tables belong to the registered actual CI/polic """ from openpilot.starpilot.flm.torque_eligibility import replay_context +from openpilot.starpilot.flm.native_refinement_wire import valid_inputs from collections import deque, OrderedDict +from openpilot.starpilot.flm.normalized_trace_archive import trace_for,archive_key,stored_record,TraceArchive,read_ref from dataclasses import dataclass import math import struct @@ -158,7 +161,10 @@ def _decode(values, path, old=None): seq = [_decode(values, path + '.' + str(i)) for i in range(length)] return deque(seq, maxlen=int(values[path + '.maxlen']) or None) if kind == 3 else tuple(seq) if kind == 2 else seq value = values[path] - return bool(value) if isinstance(old, bool) else int(value) if isinstance(old, int) else value + # Constructor zeros can be ints even for continuous PID/filter state. Never + # truncate an original fractional checkpoint to that initial Python type. + # Exact integral counters retain their integer representation. + return bool(value) if isinstance(old, bool) else int(value) if isinstance(old, int) and value.is_integer() else value def restore_snapshot(controller, vm, snapshot): @@ -496,7 +502,7 @@ def _run(controller, vm, inputs, objective, coordinates, torque_values=(), centr if values['active'] and not cs.steeringPressed and not values['safety_limited'] and source.calls != 1: raise ValueError('actual selected stage hook absent or duplicated') if (effect_source is not None and values['active'] and not cs.steeringPressed and - not values['safety_limited'] and not values['curvature_limited']): + not values['curvature_limited']): if effect_source.stage_calls != 1 or effect_source.friction_calls != 1: raise ValueError('actual selected effect hook absent or duplicated') return float(output), log @@ -505,7 +511,7 @@ def _run(controller, vm, inputs, objective, coordinates, torque_values=(), centr @dataclass(frozen=True) class ReplayFrame: control_ns: int - inputs: tuple[tuple[str, float], ...] + inputs: tuple[tuple[str, float | str], ...] selected_command: float applied_coordinates: tuple[float, ...] block_id: str = '' @@ -524,6 +530,8 @@ class ReplayFrame: effect_prepare_ns: int = 0 controller_domain: str = 'torque' desired_kinematic_accel: float | None = None + vehicle_model_anchor: tuple[float, ...] = () + live_applied_coordinates: tuple[float, ...] = () @dataclass(frozen=True) @@ -674,10 +682,17 @@ class ControllerReplay: self.__dict__['_ControllerReplay__recorded_cp_digest_cache'] = (raw, value, car_params_digest) return value + @guarded_archive() def normalize(self, frame, *, epsilon=0.0001, _verify_only=False): + return self._normalize_frame(frame,epsilon=epsilon,_verify_only=_verify_only) + + def _normalize_frame(self, frame, *, epsilon=0.0001, _verify_only=False): + # Called only by whole-archive guarded public/snapshot operations. Every + # consumed row still resolves freshly; no per-frame success cache exists. if ( self.failed or not isinstance(frame, ReplayFrame) + or not valid_inputs(frame.inputs) or frame.controller_domain != ('torque_pid' if self.factory.cp.lateralTuning.which() == 'pid' else 'torque') or not math.isfinite(epsilon) or not 0 < epsilon <= 0.03 @@ -699,13 +714,15 @@ class ControllerReplay: before = capture_snapshot(self.actual, self.vm) centre = _FixedCentreSource(self.baseline_revision, frame.centre_values) recorded_effect = None - if frame.effect_trace is not None: + trace = trace_for(frame,self) + if trace is not None: from openpilot.starpilot.flm.selected_effects import EffectReplayRecordedSource - if (frame.effect_trace.control_ns != frame.effect_prepare_ns or - frame.effect_trace.envelope.selected_baseline_revision != self.baseline_revision or - frame.effect_trace.envelope.cp_sha256 != self._recorded_cp_digest()): + if (trace.control_ns != frame.effect_prepare_ns or + trace.envelope.selected_baseline_revision != self.baseline_revision or + trace.envelope.cp_sha256 != self._recorded_cp_digest()): raise ValueError('recorded full30 trace source/prepare clock mismatch') - recorded_effect = EffectReplayRecordedSource(frame.effect_trace) + recorded_effect = EffectReplayRecordedSource(trace) + trace = None _verify_only = True actual, _ = _run(self.actual, self.vm, frame.inputs, self.objective, frame.applied_coordinates, frame.torque_values, centre, frame.actual_stage_residual, effect_trial=recorded_effect) @@ -839,7 +856,7 @@ class ControllerReplay: or self.last_ns and not 0 < frame.control_ns - self.last_ns <= MAX_FRAME_GAP_NS or not math.isfinite(frame.selected_command) - or len(frame.inputs) > MAX_FRAME_VALUES + or not valid_inputs(frame.inputs) ): self.failed = True return False @@ -1302,6 +1319,7 @@ class ControllerReplay: except (ValueError, TypeError, KeyError, AttributeError, OverflowError): return False + @guarded_archive() def _current_raw_state(self, control_ns): """At4Hz, at most100 actual steps from the latest genuine1Hz checkpoint.""" from itertools import islice @@ -1319,6 +1337,8 @@ class ControllerReplay: before = None previous = 0 for frame in frames: + if any(frame.live_applied_coordinates): + return None # New live history uses its explicit raw-evidence prefix verifier. if previous and frame.frame_sequence != previous + 1: return None previous = frame.frame_sequence @@ -1328,7 +1348,7 @@ class ControllerReplay: recorded_effect = None if frame.effect_trace is not None: from openpilot.starpilot.flm.selected_effects import EffectReplayRecordedSource - recorded_effect = EffectReplayRecordedSource(frame.effect_trace) + recorded_effect = EffectReplayRecordedSource(trace_for(frame,self)) actual, _ = _run( self.probe, self.probe_vm, frame.inputs, self.objective, frame.applied_coordinates, frame.torque_values, centre, frame.actual_stage_residual, effect_trial=recorded_effect @@ -1422,11 +1442,42 @@ class ControllerReplay: 'model_revision': self.model_revision, 'windows': tuple(windows), 'checkpoints': tuple((stamp, checkpoint.revision, checkpoint.values) for stamp, checkpoint in checkpoints), - 'frames': tuple(asdict(frame) for frame in frames), + 'frames': tuple(stored_record(frame) for frame in frames), 'emissions': self.emissions.snapshot(windows), } + if archive_key(self) is not None: + data['version'] = 2 + data['trace_archive'] = self.trace_archive.literal + from openpilot.starpilot.flm.primitive_source_memo import independent_primitive_graph + from openpilot.starpilot.flm.normalized_trace_archive import RESERVED_BYTES + data = independent_primitive_graph(data,reserved_bytes=RESERVED_BYTES) return pack_primitive_snapshot(data) + @guarded_archive() + def _load_snapshot_frames(self, frames, checkpoints, *, normalized): + # A ring may retain a partial oldest checkpoint. Discard only that prefix; + # missing heldout rows are detected by validate's exact sample join. + started = False + for row in frames: + row = dict(row) + for name in ('vehicle_model_anchor', 'live_applied_coordinates'): + if name in row: + row[name] = tuple(row[name]) + if row.get('effect_trace') is not None: + from openpilot.starpilot.flm.effect_trace import trace_from_dict + row = dict(row, effect_trace=read_ref(row['effect_trace'],self.trace_archive) if normalized else trace_from_dict(row['effect_trace'])) + frame = ReplayFrame(**row) + if frame.control_ns in checkpoints: + self.checkpoint(frame.control_ns, checkpoints[frame.control_ns]) + started = True + if not started: + continue + normalize = self._normalize_frame if normalized and type(self) is ControllerReplay else self.normalize + if normalize(frame, epsilon=0.01, _verify_only=True) is None: + raise ValueError('child actual selected replay mismatch') + if not started or self.failed: + raise ValueError('child checkpoint unavailable') + @classmethod def from_snapshot(cls, factory, objective, data, *, rebuild=True): """Recompute actual neutral verification/features; never trust saved flags.""" @@ -1435,8 +1486,12 @@ class ControllerReplay: if type(rebuild) is not bool: raise ValueError('explicit replay rebuild policy') values = unpack_primitive_snapshot(data) + if values.get('version') == 2: + from openpilot.starpilot.flm.primitive_source_memo import independent_primitive_graph + from openpilot.starpilot.flm.normalized_trace_archive import RESERVED_BYTES + values = independent_primitive_graph(values,reserved_bytes=RESERVED_BYTES) if ( - values.get('version') != 1 + values.get('version') not in (1,2) or values.get('objective') != asdict(objective) or values.get('mode') != str(factory.mode) or values.get('turn_assist') != factory.turn_assist @@ -1446,6 +1501,10 @@ class ControllerReplay: ): raise ValueError('fit job selected controller identity') engine = cls(factory, objective, identity=values['identity'], baseline_revision=values['baseline_revision'], model_revision=values['model_revision']) + if values.get('version') == 2: + engine.trace_archive = TraceArchive(values['trace_archive']) + elif 'trace_archive' in values: + raise ValueError('legacy replay cannot adopt normalized traces') engine.emissions.load(values['emissions'], tuple(values.get('windows', ()))) if type(values.get('raw_mode', False)) is not bool: raise ValueError('raw replay mode') @@ -1454,23 +1513,7 @@ class ControllerReplay: frames = values['frames'] if not frames or len(frames) > MAX_REPLAY_FRAMES or len(checkpoints) > 600: raise ValueError('fit job history bound') - # A ring may retain a partial oldest checkpoint. Discard only that prefix; - # missing heldout rows are detected by validate's exact sample join. - started = False - for row in frames: - if row.get('effect_trace') is not None: - from openpilot.starpilot.flm.effect_trace import trace_from_dict - row = dict(row, effect_trace=trace_from_dict(row['effect_trace'])) - frame = ReplayFrame(**row) - if frame.control_ns in checkpoints: - engine.checkpoint(frame.control_ns, checkpoints[frame.control_ns]) - started = True - if not started: - continue - if engine.normalize(frame, epsilon=0.01, _verify_only=True) is None: - raise ValueError('child actual selected replay mismatch') - if not started or engine.failed: - raise ValueError('child checkpoint unavailable') + engine._load_snapshot_frames(frames,checkpoints,normalized=values.get('version') == 2) engine.replay_windows = tuple(values.get('windows', ())) if rebuild: engine.rebuild_causal_features(epsilon=0.01) diff --git a/openpilot/starpilot/flm/effect_current_probe.py b/openpilot/starpilot/flm/effect_current_probe.py index 8c0d04c192..4a7e91bbdd 100644 --- a/openpilot/starpilot/flm/effect_current_probe.py +++ b/openpilot/starpilot/flm/effect_current_probe.py @@ -1,3 +1,5 @@ +from openpilot.starpilot.flm.normalized_trace_archive import guarded_archive +from openpilot.starpilot.flm.normalized_trace_archive import trace_for """Fresh original/CAN probe of the exact FIRST full30 application transition. Heldout whole-trajectory fitting is separate. This never manufactures future @@ -13,6 +15,7 @@ from openpilot.starpilot.flm.selected_effects import EffectReplaySource, EffectR from openpilot.starpilot.flm.canonical_surface_adapter import CanonicalSurfaceEffects +@guarded_archive() def current_effect_probe(engine, profile, control, output, model): try: envelope = profile.effect_envelope.validate(profile) @@ -26,7 +29,7 @@ def current_effect_probe(engine, profile, control, output, model): frame=next((row for row in islice(reversed(engine.records),60) if row.control_ns==control.control_ns),None) if frame is None or frame.effect_prepare_ns != control.effect_prepare_ns or not 0 <= engine.last_ns-frame.control_ns <= 500_000_000: return False - trace=frame.effect_trace + trace=trace_for(frame,engine) before=dict.fromkeys(NAMES,0.) if trace is None else dict(trace.applied) prior=dict(envelope.prior_coordinates) # A proposal is based on the actual CURRENT accepted target, not a saved diff --git a/openpilot/starpilot/flm/event_bounded_ridge.py b/openpilot/starpilot/flm/event_bounded_ridge.py new file mode 100644 index 0000000000..1bb3580c51 --- /dev/null +++ b/openpilot/starpilot/flm/event_bounded_ridge.py @@ -0,0 +1,48 @@ +"""Small deterministic box-constrained TRAIN ridge solve, without loss search.""" +from itertools import product + +from openpilot.starpilot.flm.adaptive_profile import finite, solve + + +def bounded_ridge(matrix, rhs, parameters, centre, trial_step): + """Enumerate at most 3**4 active faces of the strictly convex ridge quadratic. + + A face fixes each active coordinate to a bound and re-solves its free + coordinates. This is not componentwise clamping of an unconstrained fit. + No heldout values or actuator/native permission enter this calculation. + """ + count = len(parameters) + if (not 1 <= count <= 4 or len(centre) != count or len(rhs) != count or len(matrix) != count or + not finite(trial_step) or trial_step <= 0 or + any(len(row) != count or not all(finite(value) for value in row) for row in matrix) or + not all(finite(value) for value in (*rhs, *centre))): + raise ValueError('bounded_ridge_contract') + bounds = tuple((max(p.low, -p.max_step, old - trial_step), min(p.high, p.max_step, old + trial_step)) + for p, old in zip(parameters, centre, strict=True)) + if any(not all(finite(value) for value in pair) or pair[0] > pair[1] for pair in bounds): + raise ValueError('bounded_ridge_bounds') + best = None + for face in product((0, -1, 1), repeat=count): # free, lower, upper; deterministic ties + free = tuple(i for i, status in enumerate(face) if status == 0) + candidate = [bounds[i][0 if status == -1 else 1] if status else 0. for i, status in enumerate(face)] + if free: + reduced = [[matrix[i][j] for j in free] for i in free] + residual = [rhs[i] - sum(matrix[i][j] * candidate[j] for j in range(count) if face[j]) for i in free] + try: + fitted = solve(reduced, residual) + except ValueError: + continue + for i, value in zip(free, fitted, strict=True): + candidate[i] = value + if any(not finite(value) or not low <= value <= high + for value, (low, high) in zip(candidate, bounds, strict=True)): + continue + objective = sum(candidate[i] * (sum(matrix[i][j] * candidate[j] for j in range(count)) - 2 * rhs[i]) + for i in range(count)) + if not finite(objective): + continue + if best is None or objective < best[0]: + best = objective, tuple(candidate) + if best is None: + raise ValueError('bounded_ridge_unavailable') + return best[1] diff --git a/openpilot/starpilot/flm/event_collection.py b/openpilot/starpilot/flm/event_collection.py index d18f05c5de..46ac8b927d 100644 --- a/openpilot/starpilot/flm/event_collection.py +++ b/openpilot/starpilot/flm/event_collection.py @@ -48,7 +48,7 @@ class EventCollectionAdapter(TorqueCollectionAdapter): if replay_version != 3 and self.clipping_quality != RECORDED_DEBT: raise ValueError('state-aligned quality requires explicit event version3') from openpilot.starpilot.flm.clipping_quality import restore_quality - if type(restore_version) is not int or restore_version not in (2, 4, 5, 6) or (restore_version in (4, 5, 6) and + if type(restore_version) is not int or restore_version not in (2, 4, 5, 6, 8) or (restore_version in (4, 5, 6, 8) and (replay_version != 3 or self.clipping_quality != restore_quality(restore_version))): raise ValueError('explicit whole-map restore producer contract') self.restore_version = restore_version @@ -70,9 +70,15 @@ class EventCollectionAdapter(TorqueCollectionAdapter): self.model_speed_bounds = None self.target_changed = False self.actual_target_sha256 = None + self.original_context_receipts = None if source is not None: self.bind_source(source) + def original_source_packet(self, collector, windows, now_ns): + """Separate explicit export; never a fit_job_snapshot or live grant.""" + from openpilot.starpilot.flm.original_source_capture import export_packet + return export_packet(self, collector, windows, now_ns) + def select_objective(self, key): if not getattr(self, '_event_initialized', False): return super().select_objective(key) @@ -136,6 +142,26 @@ class EventCollectionAdapter(TorqueCollectionAdapter): return self.objective.required_cohorts return () if self.event_objective is None else self.event_objective.required_cohorts + def enable_original_context_capture(self): + if self.original_context_receipts is None: + from openpilot.starpilot.flm.original_context_receipts import OriginalContextReceipts + self.original_context_receipts = OriginalContextReceipts() + + def observe_original_applied(self, output): + if self.original_context_receipts is not None: + self.original_context_receipts.record_output(output) + if self.original_context_receipts.failed: + return + # Replay needs every original Card carrier, including outputs received + # too late to become fitting samples. This records history only; the + # collector's current freshness/eligibility checks still run unchanged. + self.record_actual_carrier(output) + control = self.original_context_receipts.controls.get(output.control_ns) + if control is not None: + # Keep the existing exact source, native lease and CAN checks. Missing + # or invalid emissions remain absent and make replay fail closed. + self.record_actual_emission(control, output) + def prepare_control(self, evidence): trace = getattr(evidence, 'effect_trace', None) target = None if trace is None else trace.envelope @@ -156,7 +182,10 @@ class EventCollectionAdapter(TorqueCollectionAdapter): prior_full_profile_sha256=target.full_profile_sha256).for_trial(GROUPS[self.key[1]]) self.event_objective = EffectObjective(self.key[1], self.envelope.trial_names, self.envelope.trial_anchor, self.source_epoch) - return super().prepare_control(evidence) + prepared = super().prepare_control(evidence) + if prepared is not None and self.original_context_receipts is not None: + self.original_context_receipts.record_control(evidence) + return prepared def physical_observable(self, value): # Raw storage is shared by all scheduled groups, including crawl FF. @@ -236,7 +265,7 @@ class EventCollectionAdapter(TorqueCollectionAdapter): if self.replay_version == 3 and operation == 'trial' else {}), **({'event_clipping_quality': self.clipping_quality, 'event_identification_policy': FIXED_MAP_IDENTIFICATION, 'event_restore_policy': restore_policy(self.restore_version)} - if operation == 'restore' and self.restore_version in (4, 5, 6) else {}), + if operation == 'restore' and self.restore_version in (4, 5, 6, 8) else {}), 'event_objective': asdict(self.event_objective), 'effect_envelope': asdict(envelope), 'objective': asdict(self.objective), 'now_ns': now_ns, 'sequence': sequence, 'prior': tuple(dict(envelope.prior_coordinates)[name] for name in self.event_objective.names), @@ -252,6 +281,13 @@ class EventCollectionAdapter(TorqueCollectionAdapter): from openpilot.starpilot.flm.full_surface_profile import FullSurfaceEnvelope from openpilot.starpilot.flm.event_activation import activation_from_dict, activation_matches_model, activation_model_from_dict try: + plant_mode = getattr(self,'source_plant_protocol',None) is not None + if plant_mode or 'plant_protocol' in report: + from openpilot.starpilot.flm.source_plant_runtime import install as install_source_plant + offered_context = tuple(report.get('physical_context',())) + if not plant_mode or not install_source_plant(self,report,offered_context,commit=False): + return False + physical_context = offered_context envelope = result.profile.effect_envelope if (not isinstance(envelope, FullSurfaceEnvelope) or self.envelope is None or envelope.source_binding() != self.envelope.source_binding() or @@ -271,11 +307,13 @@ class EventCollectionAdapter(TorqueCollectionAdapter): return False model = activation_model_from_dict(report['model']) report_version = report.get('event_version', 2) - if type(report_version) is not int or report_version == 3 and 'restore_activation_plan' in report: + if type(report_version) is not int or report_version in (3,7) and 'restore_activation_plan' in report: return False if envelope.operation == 'restore' and report_version != self.restore_version: return False - if report_version == 3: + if report_version in (3,7): + if report_version == 7 and (not plant_mode or report.get('plant_protocol') != 'chronological-source-feedback-mean-fit-v1'): + return False plan = activation_from_dict(report['activation_plan']) if (self.replay_version != 3 or envelope.operation != 'trial' or plan.identity != self.identity or plan.baseline_revision != self.baseline_revision or @@ -284,13 +322,18 @@ class EventCollectionAdapter(TorqueCollectionAdapter): not set(result.profile.train_ids + result.profile.heldout_ids).issubset(plan.stage_blocks) or not activation_matches_model(plan, model)): return False - elif report_version in (4, 5, 6): + elif report_version in (4, 5, 6, 8): from openpilot.starpilot.flm.restore_activation import restore_plan_from_dict, restore_plan_matches from openpilot.starpilot.flm.clipping_quality import restore_policy plan = restore_plan_from_dict(report['restore_activation_plan']) expected_policy = restore_policy(report_version) + if report_version == 8: + from openpilot.starpilot.flm.clipping_quality import RESTORE_FEEDBACK_STATISTICS + if (not plant_mode or report.get('plant_protocol') != 'chronological-source-feedback-mean-fit-v1' or + report.get('event_restore_statistics') != RESTORE_FEEDBACK_STATISTICS or 'event_mean_basis' in report): + return False if (self.restore_version != report_version or plan.version != expected_policy or - report_version == 6 and plan.clipping_quality != self.clipping_quality or + report_version in (6, 8) and plan.clipping_quality != self.clipping_quality or envelope.operation != 'restore' or 'activation_plan' in report or plan.identity != self.identity or plan.baseline_revision != self.baseline_revision or plan.source_epoch != self.source_epoch or plan.cutoff_ns != report['model_cutoff_ns'] or @@ -309,12 +352,14 @@ class EventCollectionAdapter(TorqueCollectionAdapter): physical_model_revision(model) != validation.physical_model_revision or validation.baseline_revision != self.baseline_revision): return False self.event_objective = EffectObjective(self.key[1], envelope.trial_names, envelope.trial_anchor, self.source_epoch) - if report_version in (3, 4, 5, 6) and self.model != model: + if report_version in (3, 4, 5, 6, 7, 8) and self.model != model: self.current_physical_valid = False self.last_physical_ns = 0 self.model, self.model_cutoff_ns = model, report['model_cutoff_ns'] self.model_speed_bounds = bounds self.fit_profile = result.profile + if plant_mode: + install_source_plant(self,report,physical_context,commit=True) return True except (AttributeError, KeyError, TypeError, ValueError): return False diff --git a/openpilot/starpilot/flm/event_collector.py b/openpilot/starpilot/flm/event_collector.py index 44c5de6c23..226b7ee4d5 100644 --- a/openpilot/starpilot/flm/event_collector.py +++ b/openpilot/starpilot/flm/event_collector.py @@ -144,6 +144,7 @@ class EventFullrateCollector(FullrateCollector): self.recent_controls.clear() self.recent_applied.clear() self.ready.clear() + self.probe_ready.clear() self.points.clear() self.raw_points.clear() self._pending_storage = () @@ -179,6 +180,9 @@ class EventFullrateCollector(FullrateCollector): return accepted def job_snapshot(self, now_ns): + if getattr(self.adapter,'source_plant_protocol',None) is not None: + from openpilot.starpilot.flm.source_plant_runtime import snapshot + return snapshot(self,now_ns) # Guards are cheap. Whole-event reconstruction stays at the original60s # cadence, including unsuccessful selections; reports only reuse its result. if self.snapshot_readiness_context != self._snapshot_context(): @@ -213,3 +217,9 @@ class EventFullrateCollector(FullrateCollector): self.snapshot_readiness_context = self._snapshot_context() self.snapshot_readiness.update(observedMonoTime=now_ns, objective=str(getattr(self.adapter, 'key', ''))[:128]) return snapshot + + def mark_job_submitted(self, now_ns): + super().mark_job_submitted(now_ns) + if getattr(self.adapter,'source_plant_protocol',None) is not None: + from openpilot.starpilot.flm.source_plant_runtime import submitted + submitted(self) diff --git a/openpilot/starpilot/flm/event_controller_replay.py b/openpilot/starpilot/flm/event_controller_replay.py index 7882d12eaf..b504ac8e2c 100644 --- a/openpilot/starpilot/flm/event_controller_replay.py +++ b/openpilot/starpilot/flm/event_controller_replay.py @@ -1,6 +1,8 @@ +from openpilot.starpilot.flm.normalized_trace_archive import guarded_archive +from openpilot.starpilot.flm.normalized_trace_archive import trace_for,archive_key """Child-only canonical effect trials through complete selected controller/CAN history.""" -from typing import cast +from typing import cast, NamedTuple from collections.abc import Mapping import math @@ -61,8 +63,21 @@ class _LazyControllerRequests(Mapping[int, float]): self.finished = True +class _EquivalentBaselineView(NamedTuple): + owner: object + cache: tuple + entry: tuple + original_key: object + selected_key: object + proof_fingerprint: bytes + + class EventControllerReplay: - """Reuse original source/native/encoder proofs; replace only trial coordinates.""" + """Original source proof plus explicitly versioned candidate controller replay. + + Legacy trials retain frozen feedback. Declared closed-loop candidates rebuild + only causal safety_limited inputs from their actual encoded native reports. + """ def __init__(self, engine, objective, original_anchor, *, restore_envelope=None, activation_plan=None): if (engine.failed or not engine.raw_mode or not 1 <= len(objective.names) <= 4 or len(set(objective.names)) != len(objective.names) or len(objective.anchor_digest) != 64): @@ -72,6 +87,7 @@ class EventControllerReplay: self.activation_plan = activation_plan self.__identification_token = object() self.__strict_rollouts = None + self.__subset_baseline_view = None self.__causal_features = None self.__prepared_car_states = None self.__controller_prefix = None @@ -100,7 +116,7 @@ class EventControllerReplay: return getattr(self.engine, name) def _prior_matches(self, frame): - trace = getattr(frame, 'effect_trace', None) + trace = trace_for(frame,self) if trace is None: return not any(self.prior_full.values()) and not any(frame.applied_coordinates) try: @@ -152,6 +168,9 @@ class EventControllerReplay: self.__strict_rollouts = None self.__controller_prefix = None return self._rollout_uncached(coordinates) + equivalent = self._equivalent_subset_baseline(coordinates, source_key) + if equivalent is not None: + return equivalent if self.__strict_rollouts is None or self.__strict_rollouts[0] != source_key: self.__strict_rollouts = source_key, {} slots = self.__strict_rollouts[1] @@ -167,6 +186,8 @@ class EventControllerReplay: type(self.emissions) is EmissionHistory) build_prefix = slot == 'baseline' and prefix_scoped self.__pending_controller_prefix = None + # A cache entry may carry only the proof completed by THIS strict execution. + self.__candidate_feedback_proof = None if prefix_scoped: result = self._rollout_uncached(coordinates, _prefix_context=(source_key, build_prefix)) else: @@ -179,9 +200,18 @@ class EventControllerReplay: self.__controller_prefix = self.__pending_controller_prefix self.__pending_controller_prefix = None frozen = tuple(MappingProxyType(dict(part)) for part in result) - slots[slot] = key, frozen + from openpilot.starpilot.flm.candidate_feedback import uses_closed_loop, strict_reference_proof_key + if uses_closed_loop(self): + proof = self.__candidate_feedback_proof + proof_key = strict_reference_proof_key(proof,self,coordinates,len(frozen[0])) + if proof_key is not None: + slots[slot] = key, frozen, (proof,proof_key) + # No complete strict chronology: do not create a reusable result. + else: + slots[slot] = key, frozen return tuple(dict(part) for part in frozen) + @guarded_archive() def _rollout_uncached(self, coordinates, *, restore_target=False, restore_omit_group=None, _identification_token=None, _prefix_context=None): from openpilot.starpilot.flm.selected_effects import (EffectTrial, EffectReplayTransition, EffectReplayRecordedSource, @@ -255,6 +285,8 @@ class EventControllerReplay: certificate = None prefix_requests, prefix_saturated, prefix_states = {}, {}, {} + candidate_feedback = None + def controller_frames(): nonlocal build_prefix prior_ns = prior_sequence = index = 0 @@ -278,7 +310,7 @@ class EventControllerReplay: prior_ns and not 0 < frame.control_ns - prior_ns <= MAX_FRAME_GAP_NS or prior_sequence and frame.frame_sequence != prior_sequence + 1): raise ValueError('event source/history mismatch') - trace = getattr(frame, 'effect_trace', None) + trace = trace_for(frame,self) prepare_ns = getattr(frame, 'effect_prepare_ns', 0) activation = None if self.activation_plan is None else self.activation_plan.activation_for(*windows[index]) if self.activation_plan is not None and activation is None: @@ -321,6 +353,8 @@ class EventControllerReplay: source = trial else: raise ValueError('event exact owner transition clock unavailable') + trace = None + run_inputs = frame.inputs if candidate_feedback is None else candidate_feedback.before(frame) if certificate is not None and reusable_prefix and windows[index] in certificate[3]: # Retain every source/history/clock/prepare/recorded-source check above. # Later state is restored ONLY at the exact declared activation tick. @@ -329,7 +363,7 @@ class EventControllerReplay: else: prepared = None if prepared_states is None else prepared_states.get(frame.inputs) kwargs = {} if prepared is None else {'_prepared_car_state': prepared} - output, log = _run(controller, vm, frame.inputs, self.engine.objective, (0.0, 0.0), frame.torque_values, + output, log = _run(controller, vm, run_inputs, self.engine.objective, (0.0, 0.0), frame.torque_values, actual_stage_residual=frame.actual_stage_residual, effect_trial=source, **kwargs) is_saturated = bool(log.saturated) if (neutral or original_prefix) and (not same_final_command(output, frame.selected_command) or @@ -340,6 +374,8 @@ class EventControllerReplay: if not math.isfinite(output) or abs(output) > 1.000001: raise ValueError('event selected command envelope') requests[frame.control_ns], saturated[frame.control_ns] = output, is_saturated + if candidate_feedback is not None: + candidate_feedback.after(frame.control_ns,output) if build_prefix and reusable_prefix: prefix_requests[frame.control_ns], prefix_saturated[frame.control_ns] = raw_output, is_saturated prior_ns, prior_sequence = frame.control_ns, frame.frame_sequence @@ -358,10 +394,30 @@ class EventControllerReplay: quality = {} if self.activation_plan is None else {'quality_contract': self.activation_plan.clipping_quality} if shadow: quality['identification_policy'] = NATIVE_SHADOW_IDENTIFICATION + feedback = getattr(self,'verified_feedback_tape',None) + reported = None + if feedback is not None: + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + evidence = require_feedback_engine(self) + reported = {} + quality['feedback_reported'] = reported + from openpilot.starpilot.flm.candidate_feedback import CLOSED_POLICY,CandidateFeedbackLoop + if evidence.candidate_policy == CLOSED_POLICY: + if not supported or type(self.emissions) is not EmissionHistory: + raise ValueError('closed-loop candidate requires actual registered controller/native replay') + candidate_feedback = CandidateFeedbackLoop(self,reported,tuple(coordinates), + native_policy='native-shadow-identification' if shadow else 'strict', + original_baseline=not restore_target and tuple(coordinates) == self.reference) encoded = self.emissions.rollout(native_requests, windows[0][0], windows[-1][1], windows, **quality) # Native success is insufficient: late controller-only frames and the exact # original final-window/tick guard must ALSO finish before any result exists. lazy_requests.finish() + if reported is not None: + if candidate_feedback is not None: + self.__candidate_feedback_proof = candidate_feedback.finish(requests,reported) + else: + from openpilot.starpilot.flm.feedback_replay import verify_candidate_engine + verify_candidate_engine(self,requests,reported) if build_prefix and prefix_requests and prefix_states: self.__pending_controller_prefix = (cast(tuple[object, bool], _prefix_context)[0], MappingProxyType(prefix_requests), MappingProxyType(prefix_saturated), MappingProxyType(prefix_states)) @@ -401,6 +457,11 @@ class EventControllerReplay: self.objective, self.reference, self.prior_full, self.original_anchor, self.activation_plan, self.restore_envelope, tuple(self.records), tuple(self.checkpoints), tuple(self.replay_windows), native) + if archive_key(self) is not None: + data = (*data,archive_key(self)) + if getattr(self,'verified_feedback_tape',None) is not None: + from openpilot.starpilot.flm.feedback_replay import owned_feedback_key + data += (owned_feedback_key(self),) return hashlib.sha256(pickle.dumps(data, protocol=5)).digest() except (AttributeError, TypeError, ValueError, OverflowError, pickle.PicklingError): return None # Unsupported cache key means full replay, never readiness. @@ -411,6 +472,10 @@ class EventControllerReplay: This transfers numeric columns, never a native verdict, shadow token, profile, validation, lease, or optimizer choice. Any unknown case rebuilds. """ + from openpilot.starpilot.flm.candidate_feedback import uses_closed_loop + if uses_closed_loop(self) != uses_closed_loop(original): + return False + import pickle import struct from types import MappingProxyType @@ -524,10 +589,152 @@ class EventControllerReplay: all(type(value) is bool for value in prefix[2].values()) and all(window in self.replay_windows and type(value) is Snapshot for window, value in prefix[3].items())): self.__controller_prefix = (new_key, *(MappingProxyType(dict(part)) for part in prefix[1:])) + # Separate numerical view only. No borrowed strict proof or native verdict + # is installed in this selected engine's strict cache or proof fields. + cache = original.__dict__.get('_EventControllerReplay__strict_rollouts') + entry = cache[1].get('baseline') if type(cache) is tuple and len(cache) == 2 and type(cache[1]) is dict else None + if (epsilon == .01 and uses_closed_loop(self) and type(cache) is tuple and len(cache) == 2 and cache[0] == old_key and + type(entry) is tuple and len(entry) == 3 and type(entry[2]) is tuple and len(entry[2]) == 2 and + type(entry[2][1]) is bytes): + self.__subset_baseline_view = _EquivalentBaselineView(original,cache,entry,old_key,new_key,entry[2][1]) return True except (AttributeError, TypeError, ValueError, KeyError, OverflowError, pickle.PicklingError): return False + def _equivalent_subset_baseline(self, coordinates, source_key): + """Only an identical baseline's numeric maps; never a new execution proof. + + The full owner's strict proof stays bound to its original objective. Both + baselines take the neutral recorded-source branch on EVERY frame; shared + records/checkpoints/inputs/native state therefore give identical updates. + Full30 target/source checks conservatively preserve the existing trial guards. + Every use freshly binds both owners before and after callback-capable work. + """ + import pickle + import struct + from types import MappingProxyType + from openpilot.starpilot.flm.candidate_feedback import uses_closed_loop,strict_reference_proof_key,CandidateFeedbackProof,_SEAL + from openpilot.starpilot.flm.canonical_surface_adapter import NAMES,parameters_for + from openpilot.starpilot.flm.controller_replay import ControllerReplay,SelectedControllerFactory + from openpilot.starpilot.flm.event_objective import EffectObjective + from openpilot.starpilot.flm.event_activation import EventActivationPlan,ZERO_REFERENCE_ACTIVATION_VERSION + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + from openpilot.starpilot.flm.pid_controller_replay import PidControllerFactory + from openpilot.starpilot.flm.selected_effects import EffectTrial,EffectReplayTransition + from openpilot.starpilot.flm.torque_emission_replay import EmissionHistory + from openpilot.starpilot.lateral.controller_selection import ControllerMode + from capnp.lib.capnp import _DynamicStructReader + from opendbc.car.car_helpers import interfaces + from abc import ABCMeta + from collections import OrderedDict + + try: + view = self.__dict__.get('_EventControllerReplay__subset_baseline_view') + if (type(view) is not _EquivalentBaselineView or type(self) is not EventControllerReplay or + type(source_key) is not bytes or type(view.original_key) is not bytes or type(view.selected_key) is not bytes or + type(coordinates) is not tuple or any(type(value) not in (int,float) or not math.isfinite(value) for value in coordinates) or + type(view.cache) is not tuple or len(view.cache) != 2 or type(view.cache[0]) is not bytes or + type(view.cache[1]) is not dict or any(type(key) is not str for key in view.cache[1]) or + type(view.entry) is not tuple or len(view.entry) != 3 or type(view.proof_fingerprint) is not bytes): + return None + original = view.owner + if (type(original) is not EventControllerReplay or original is self or + type(self.engine) is not ControllerReplay or self.engine is not original.engine or + type(self.emissions) is not EmissionHistory or + self.restore_envelope is not None or original.restore_envelope is not None or + type(self.activation_plan) is not EventActivationPlan or self.activation_plan.version != ZERO_REFERENCE_ACTIVATION_VERSION or + self.failed or original.failed or not self.raw_mode or not original.raw_mode or + any(getattr(self,name) is not getattr(original,name) for name in + ('factory','records','checkpoints','replay_windows','emissions','activation_plan')) or + not uses_closed_loop(self) or not uses_closed_loop(original) or source_key != view.selected_key or + pickle.dumps(coordinates,protocol=5) != pickle.dumps(self.reference,protocol=5)): + return None + # Unknown callback-bearing factory/collection shapes retain real replay. + factory = self.factory + if type(factory) not in (SelectedControllerFactory,PidControllerFactory): + return None + interface_metaclass = type(factory.interface_class) + if (type(factory.cp) is not _DynamicStructReader or type(self.emissions.cp) is not _DynamicStructReader or + type(factory.mode) not in (str,ControllerMode) or + (interface_metaclass is not type and interface_metaclass is not ABCMeta) or + factory.interface_class is not interfaces.get(str(factory.cp.carFingerprint)) or + type(self.emissions.events) is not OrderedDict or type(self.emissions.carriers) is not OrderedDict or + type(factory) is SelectedControllerFactory and type(factory.policy) not in (str,type(None))): + return None + if (original._causal_input_key(.01) != view.original_key or + original.__dict__.get('_EventControllerReplay__strict_rollouts') is not view.cache or + view.cache[0] != view.original_key or view.cache[1].get('baseline') is not view.entry or + type(self.objective) is not EffectObjective or type(original.objective) is not EffectObjective or + not set(self.objective.names) < set(original.objective.names) or + self.objective.group != original.objective.group or self.objective.context_key != original.objective.context_key or + self.objective.parameters != parameters_for(self.objective.names) or + original.objective.parameters != parameters_for(original.objective.names)): + return None + old_evidence,new_evidence = require_feedback_engine(original),require_feedback_engine(self) + if (old_evidence.original is not new_evidence.original or old_evidence.binding != new_evidence.binding or + tuple((bounds,subset.digest,activation) for bounds,subset,activation in old_evidence.windows) != + tuple((bounds,subset.digest,activation) for bounds,subset,activation in new_evidence.windows)): + return None + expected = self.activation_plan.version,'strict-trial',pickle.dumps(original.reference,protocol=5) + if (view.entry[0] != expected or type(view.entry[1]) is not tuple or len(view.entry[1]) != 3 or + any(type(part) is not MappingProxyType for part in view.entry[1]) or + type(view.entry[2]) is not tuple or len(view.entry[2]) != 2 or view.entry[2][1] != view.proof_fingerprint or + strict_reference_proof_key(view.entry[2][0],original,original.reference,len(view.entry[1][0])) != view.proof_fingerprint): + return None + def bits(values): + if type(values) is not dict or tuple(values) != NAMES or any(type(values[name]) is not float for name in NAMES): + raise ValueError('exact full30 baseline values') + return struct.pack('!30d',*(values[name] for name in NAMES)) + left = EffectTrial(original.objective,original.reference,original.baseline_revision,original.original_anchor) + right = EffectTrial(self.objective,self.reference,self.baseline_revision,self.original_anchor) + left_transition,right_transition = EffectReplayTransition(left),EffectReplayTransition(right) + left_source,right_source = left.source(),right.source() + prior_bits = bits(original.prior_full) + if (bits(self.prior_full) != prior_bits or bits(left_transition.original_target) != prior_bits or + bits(right_transition.original_target) != prior_bits or bits(left_transition.target) != prior_bits or + bits(right_transition.target) != prior_bits or bits(left_source.values) != bits(right_source.values) or + left_source.full_profile_sha256 != right_source.full_profile_sha256 or + left_source.original_effect_anchor != original.objective.anchor_digest or + right_source.original_effect_anchor != self.objective.anchor_digest): + return None + for owner in (original,self): + reference = tuple(owner.prior_full[name] for name in owner.objective.names) + if (any(type(value) is not float for value in owner.reference) or + pickle.dumps(reference,protocol=5) != pickle.dumps(owner.reference,protocol=5)): + return None + old_anchor,new_anchor = dict(original.original_anchor),dict(self.original_anchor) + old_anchor.pop('anchor_digest') + new_anchor.pop('anchor_digest') + if pickle.dumps(old_anchor,protocol=5) != pickle.dumps(new_anchor,protocol=5): + return None + result = tuple(dict(part) for part in view.entry[1]) + # This repeat may execute original source callbacks; both source keys are + # recomputed AFTER it. The final proof snapshot below is primitive-only. + if strict_reference_proof_key(view.entry[2][0],original,original.reference,len(result[0])) != view.proof_fingerprint: + return None + def proof_snapshot(proof): + if type(proof) is not CandidateFeedbackProof: + return None + owned = object.__getattribute__(proof,'__dict__') + if type(owned) is not dict or any(type(key) is not str for key in owned) or owned.get('_seal') is not _SEAL: + return None + values = tuple(owned.get(name) for name in ('version','source_digest','original_digest', + 'coordinates_digest','native_policy','chronology_digest','row_count')) + if any(type(value) is not str for value in values[:6]) or type(values[6]) is not int: + return None + return pickle.dumps(values,protocol=5) + if (original._causal_input_key(.01) != view.original_key or self._causal_input_key(.01) != view.selected_key or + self.__dict__.get('_EventControllerReplay__subset_baseline_view') is not view or + original.__dict__.get('verified_feedback_tape') is not old_evidence or + self.__dict__.get('verified_feedback_tape') is not new_evidence or + original.__dict__.get('_EventControllerReplay__strict_rollouts') is not view.cache or + view.cache[1].get('baseline') is not view.entry or view.entry[2][1] != view.proof_fingerprint or + proof_snapshot(view.entry[2][0]) != view.proof_fingerprint): + return None + return result + except (AttributeError,TypeError,ValueError,KeyError,OverflowError,pickle.PicklingError): + return None + def ensure_causal_features(self, *, epsilon=.01): """Reuse only this engine's successful original full rebuild; never IPC flags.""" if self.restore_envelope is not None: @@ -539,6 +746,135 @@ class EventControllerReplay: self.projected = dict(self.__causal_features[1]) return len(self.projected) + def _strict_fd_reference(self, coordinates, certificate): + """Reuse one proven strict baseline only for an identical shadow FD side. + + Exact full input bytes are rebound on every hit. This is a native-limited + numerical result, never a profile/current admission or a saved shadow proof. + """ + if certificate is None: + return None + import pickle + from types import MappingProxyType + from openpilot.starpilot.flm.controller_replay import ControllerReplay + from openpilot.starpilot.flm.torque_emission_replay import EmissionHistory + from openpilot.starpilot.flm.event_activation import EventActivationPlan, ZERO_REFERENCE_ACTIVATION_VERSION + try: + plan = self.__dict__.get('activation_plan') + if (type(self) is not EventControllerReplay or type(self.__dict__.get('engine')) is not ControllerReplay or + type(self.emissions) is not EmissionHistory or self.restore_envelope is not None or + type(plan) is not EventActivationPlan or plan.version != ZERO_REFERENCE_ACTIVATION_VERSION or + plan.identification_policy != NATIVE_SHADOW_IDENTIFICATION): + return None + from openpilot.starpilot.flm.candidate_feedback import uses_closed_loop, _strict_reference_fields + closed = uses_closed_loop(self) + saved, entry, reference_bits = certificate + current = self.__dict__.get('_EventControllerReplay__strict_rollouts') + if (current is not saved or type(saved) is not tuple or len(saved) != 2 or type(saved[1]) is not dict or + saved[1].get('baseline') is not entry or type(entry) is not tuple or len(entry) != (3 if closed else 2) or + entry[0] != (plan.version, 'strict-trial', reference_bits) or + type(entry[1]) is not tuple or len(entry[1]) != 3 or + any(type(part) is not MappingProxyType for part in entry[1]) or + pickle.dumps(tuple(coordinates), protocol=5) != reference_bits): + return None + key = self._causal_input_key(.01) # Exact same owned-key contract as strict rollout cache. + if key is None or key != saved[0]: + return None + if closed: + retained = entry[2] + if type(retained) is not tuple or len(retained) != 2 or type(retained[1]) is not bytes: + return None + # The complete pre-key above has ALREADY validated this owned + # evidence's source digest and every subset/activation seal. Only the + # primitive proof projection is needed here; the complete post-key + # below remains mandatory after ALL callback-capable field reads. + evidence = object.__getattribute__(self,'__dict__').get('verified_feedback_tape') + fresh = _strict_reference_fields(retained[0],evidence,self.reference,len(entry[1][0])) + if fresh is None or fresh != retained[1]: + return None + # Proof validation itself must not open a mutable-input gap. + if self._causal_input_key(.01) != key: + return None + return dict(entry[1][1]) # Copy native outputs; no mutable cache alias. + except (AttributeError, KeyError, TypeError, ValueError, OverflowError, pickle.PicklingError): + return None # Unknown/stale proof keeps the original shadow execution. + + def _zero_gain_fd_reference(self, coordinates, certificate): + """Exact strict baseline for a provably input-neutral torque FD probe only. + + Turn-in/unwind FF modifiers multiply ff_gain in the canonical torque + formula. With full neutral original/reference state and no effect traces, + changing only one such modifier leaves ALL selected controller hook outputs + identical at every frame, including the gradual owner transition. This + never covers crawl, PID, restore, candidate feasibility or failed probes. + """ + from openpilot.starpilot.flm.controller_replay import ControllerReplay, SelectedControllerFactory + from openpilot.selfdrive.controls.lib.latcontrol_torque import LatControlTorque + from openpilot.starpilot.flm.selected_effects import EffectTrial, EffectReplayTransition + from openpilot.starpilot.flm.canonical_surface_adapter import CanonicalSurfaceEffects, NAMES + from openpilot.starpilot.flm import torque_surface + try: + if (type(self) is not EventControllerReplay or type(self.engine) is not ControllerReplay or + type(self.factory) is not SelectedControllerFactory or type(self.actual) is not LatControlTorque or + self.restore_envelope is not None or type(coordinates) is not tuple or + len(coordinates) != len(self.reference) or not 1 <= len(coordinates) <= 4 or + any(type(value) not in (int,float) or not math.isfinite(value) or abs(value) > .25 for value in coordinates) or + any(value != 0. for value in self.reference) or set(self.prior_full) != set(NAMES) or + any(value != 0. for value in self.prior_full.values()) or + any(getattr(frame,'effect_trace',None) is not None for frame in self.records)): + return None + changed = tuple(name for name,value,base in zip(self.objective.names,coordinates,self.reference,strict=True) if value != base) + if len(changed) != 1 or changed[0] not in ( + 'turn_in_boost_left','turn_in_boost_right','unwind_taper_left','unwind_taper_right'): + return None + trial = EffectTrial(self.objective,coordinates,self.baseline_revision,self.original_anchor) + transition = EffectReplayTransition(trial) # Validate the exact original full30 target/anchor. + source = trial.source() + if (type(source) is not CanonicalSurfaceEffects or source.math is not torque_surface or + tuple(source.values) != NAMES or source.values['ff_gain_left'] != 0. or source.values['ff_gain_right'] != 0. or + any(value != 0. for value in transition.original_target.values()) or + any(value != 0. for name,value in transition.target.items() if name != changed[0])): + return None + # Neutral strict replay does not execute a candidate owner transition. + # Reproduce that candidate-only lifecycle over EVERY original frame; + # source equivalence cannot waive missing/regressing prepare clocks. + windows = tuple(self.replay_windows) + if not windows: + return None + index, previous_prepare = 0, 0 + for frame in self.records: + while index < len(windows) and frame.control_ns > windows[index][1]: + index += 1 + if index >= len(windows) or not windows[index][0] <= frame.control_ns <= windows[index][1]: + return None + if frame.control_ns == windows[index][0]: + transition.reset() + previous_prepare = 0 + activation = self.activation_plan.activation_for(*windows[index]) + if activation is None: + return None + original_prefix = activation[0] == 0 or frame.control_ns < activation[0] + activation_tick = frame.control_ns == activation[0] + prepare_ns = getattr(frame,'effect_prepare_ns',0) + if activation_tick and prepare_ns != activation[1]: + return None + if original_prefix: + continue + if not 0 < prepare_ns <= frame.control_ns or previous_prepare and prepare_ns <= previous_prepare: + return None # Unknown clock patterns keep the original full probe. + actual_source = transition.source_for_frame(prepare_ns,None,new_owner=activation_tick).source + if (type(actual_source) is not CanonicalSurfaceEffects or actual_source.math is not torque_surface or + tuple(actual_source.values) != NAMES or + any(value != 0. for name,value in actual_source.values.items() if name != changed[0])): + return None + previous_prepare = prepare_ns + # Existing proof checks exact plan, strict successful baseline, owned + # factory/CP/checkpoints/frames/native packets and signed-bit input key. + # It returns a copy only after this source equivalence proof succeeds. + return self._strict_fd_reference(self.reference,certificate) + except (AttributeError,TypeError,ValueError,KeyError,OverflowError): + return None # Unknown source/transition retains the original actual FD. + def rebuild_causal_features(self, *, epsilon=.01): self.__causal_features = None if self.restore_envelope is not None: @@ -549,6 +885,14 @@ class EventControllerReplay: plan = self.__dict__.get('activation_plan') identify = {} if plan is None or plan.identification_policy != NATIVE_SHADOW_IDENTIFICATION else { '_identification_token': self.__identification_token} + reference_certificate = None + if identify: + import pickle + saved = self.__dict__.get('_EventControllerReplay__strict_rollouts') + if type(saved) is tuple and len(saved) == 2 and type(saved[1]) is dict: + entry = saved[1].get('baseline') + if entry is not None: + reference_certificate = (saved, entry, pickle.dumps(self.reference, protocol=5)) sensitivities = [] for index, parameter in enumerate(self.objective.parameters): plus, minus = list(self.reference), list(self.reference) @@ -562,11 +906,19 @@ class EventControllerReplay: # Legacy experiments retain their original final-quality probe gates. high = low = None try: - _, high, _ = self._rollout(tuple(plus), **identify) + high = self._strict_fd_reference(tuple(plus), reference_certificate) + if high is None and identify: + high = self._zero_gain_fd_reference(tuple(plus), reference_certificate) + if high is None: + _, high, _ = self._rollout(tuple(plus), **identify) except ValueError: pass try: - _, low, _ = self._rollout(tuple(minus), **identify) + low = self._strict_fd_reference(tuple(minus), reference_certificate) + if low is None and identify: + low = self._zero_gain_fd_reference(tuple(minus), reference_certificate) + if low is None: + _, low, _ = self._rollout(tuple(minus), **identify) except ValueError: pass if high is None and low is None: @@ -619,6 +971,15 @@ class EventControllerReplay: return False def validate(self, profile, control, output, physical_context, blocks, model): + from openpilot.starpilot.flm.clipping_quality import RESTORE_FEEDBACK_VERSION + if self.activation_plan is not None and self.activation_plan.version == RESTORE_FEEDBACK_VERSION: + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + try: + if getattr(self,'event_mean_model',None) is not None: + return None + require_feedback_engine(self) + except (ValueError,TypeError,KeyError,AttributeError): + return None if (self.failed or profile.policy != (RESTORE_POLICY if self.restore_envelope is not None else POLICY) or self.restore_envelope is not None and profile.effect_envelope != self.restore_envelope or profile.identity != self.identity or profile.parameters != self.objective.parameters or len(profile.coefficients) != len(self.objective.names) or @@ -670,6 +1031,11 @@ class EventControllerReplay: mean = sum(values) / len(values) variance = sum((value - mean) ** 2 for value in values) / (len(values) - 1) lower = mean - 12.707 * math.sqrt(variance / len(values)) + event_mean_model = getattr(self,'event_mean_model',None) + if event_mean_model is not None: + from openpilot.starpilot.flm.event_mean_evidence import lower as event_lower + cohort = next(name for name,items in improvements.items() if items is values) + lower = event_lower(event_mean_model,tuple(block for block in selected if block.cohort==cohort),tuple(values),cohort=cohort) if not math.isfinite(lower) or lower <= 0: return None lower_bounds.append(lower) diff --git a/openpilot/starpilot/flm/event_episodes.py b/openpilot/starpilot/flm/event_episodes.py index c10f7dc591..99c51051bb 100644 --- a/openpilot/starpilot/flm/event_episodes.py +++ b/openpilot/starpilot/flm/event_episodes.py @@ -7,6 +7,23 @@ from openpilot.starpilot.flm.physical_response import kinematic_accel from openpilot.starpilot.flm.controller_replay import MAX_FRAME_GAP_NS +def _episode_excited(engine, apply_ns, objective): + """Every original counted-episode endpoint needs selected encoded support. + + This is the same per-parameter RMS floor as objective_excited, evaluated on + the actual cycle instead of a pooled storage bundle. No targets/loss select + points; zero-effect/opposite-direction cycles cannot satisfy the quota. + """ + count = len(objective.parameters) + if any(stamp not in engine.projected for stamp in apply_ns): + return False # Reject the entire cycle; never drop an unsupported endpoint. + rows = [engine.projected[stamp][1] for stamp in apply_ns] + if not rows or any(len(row) != count or any(not math.isfinite(value) for value in row) for row in rows): + raise ValueError('counted episode actual encoded objective basis') + return all(sum(row[index] ** 2 for row in rows) / len(rows) >= parameter.excitation_floor ** 2 + for index, parameter in enumerate(objective.parameters)) + + def tracking_episodes(engine, raw_blocks, model, objective): """Caller supplies independently verified stage endpoints after model cutoff. @@ -25,7 +42,7 @@ def tracking_episodes(engine, raw_blocks, model, objective): frame = frames.get(stamp) if frame is None or frame.desired_physical_accel is None or not math.isfinite(frame.desired_physical_accel): raise ValueError('event exact control-frame physical target absent') - points[stamp] = (block['block_id'], abs(frame.desired_physical_accel - point['physical']['lateral_accel'])) + points[stamp] = (block['block_id'], abs(frame.desired_physical_accel - point['physical']['lateral_accel']), point['apply_ns']) windows = tuple(engine.replay_windows) episodes = [] start = last = neutral_start = 0 @@ -42,9 +59,7 @@ def tracking_episodes(engine, raw_blocks, model, objective): floor = max(.10, 6 * model.heldout_rmse) if kind == 'large' else max(.015, 3 * model.heldout_rmse) if error < floor or not 200_000_000 <= end - start <= 10_000_000_000: return - if kind == 'subtle' and objective.group.startswith(('ff.', 'turn.', 'unwind.', 'angle.')): - return - if kind == 'large' and objective.group.startswith(('center', 'deadband.')): + if not _episode_excited(engine, tuple(value[2] for _,value in selected), objective): return ids = tuple(dict.fromkeys(value[0] for _, value in sorted(selected))) digest = hashlib.sha256(repr((engine.identity, engine.baseline_revision, objective.group, diff --git a/openpilot/starpilot/flm/event_evaluate.py b/openpilot/starpilot/flm/event_evaluate.py index 7aba6e86b6..0a88113938 100644 --- a/openpilot/starpilot/flm/event_evaluate.py +++ b/openpilot/starpilot/flm/event_evaluate.py @@ -2,7 +2,7 @@ from dataclasses import replace import math -from openpilot.starpilot.flm.adaptive_profile import Result, evaluate, loss +from openpilot.starpilot.flm.adaptive_profile import Result, evaluate_event_proposal, loss from openpilot.starpilot.flm.event_controller_replay import POLICY TRIAL_STEP = .005 @@ -18,7 +18,7 @@ def _lower(values): def evaluate_event(identity, parameters, blocks, *, episodes, now_ns, required_cohorts, response_revision, - sequence=1, ridge=.01, prior=None, effect_envelope=None, native_feasible=None): + sequence=1, ridge=.01, prior=None, effect_envelope=None, native_feasible=None, event_mean_model=None): """Actual fit boundary; caller reconstructs episodes from exact child trace. An episode crossing train/heldout boundaries is rejected, not counted twice. @@ -60,13 +60,14 @@ def evaluate_event(identity, parameters, blocks, *, episodes, now_ns, required_c counts[episode['kind']] += 1 if counts['large'] < 3 and counts['subtle'] < 9: return Result('collecting', 'comparable_tracking_episodes') - fitted = evaluate(identity, parameters, blocks, now_ns=now_ns, required_cohorts=required_cohorts, - response_revision=response_revision, sequence=sequence, ridge=ridge, prior=prior) + fitted = evaluate_event_proposal(identity, parameters, blocks, trial_step=TRIAL_STEP, now_ns=now_ns, + required_cohorts=required_cohorts, + response_revision=response_revision, sequence=sequence, ridge=ridge, prior=prior, + **({'event_mean_model':event_mean_model} if event_mean_model is not None else {})) if fitted.profile is None: return fitted reference = (0.0,) * len(parameters) if prior is None else prior - proposal = tuple(min(max(target, old - TRIAL_STEP), old + TRIAL_STEP) - for old, target in zip(reference, fitted.profile.coefficients, strict=True)) + proposal = fitted.profile.coefficients if proposal == reference: return Result('collecting', 'no_identifiable_event_trial') if native_feasible is not None: @@ -94,10 +95,14 @@ def evaluate_event(identity, parameters, blocks, *, episodes, now_ns, required_c for cohort in sorted(required_cohorts): selected = [block for block in heldout if block.cohort == cohort] # Recompute BOTH before/after and original-baseline heldout evidence for the - # actual tiny proposal. The unconstrained ridge proposal is never applied. + # actual tiny proposal. The bounded training proposal is never admitted without these checks. gains = [loss(block.samples, reference) - loss(block.samples, proposal) for block in selected] original_gains = [loss(block.samples, (0.0,) * len(parameters)) - loss(block.samples, proposal) for block in selected] lower = min(_lower(gains), _lower(original_gains)) + if event_mean_model is not None: + from openpilot.starpilot.flm.event_mean_evidence import lower as event_lower + lower = min(event_lower(event_mean_model,tuple(selected),tuple(gains),cohort=cohort), + event_lower(event_mean_model,tuple(selected),tuple(original_gains),cohort=cohort)) if not math.isfinite(lower) or lower <= 0: return Result('rejected', 'event_tiny_trial_heldout_regression_or_noise') improvements.append((cohort, lower)) diff --git a/openpilot/starpilot/flm/event_mean_evidence.py b/openpilot/starpilot/flm/event_mean_evidence.py new file mode 100644 index 0000000000..559daaef61 --- /dev/null +++ b/openpilot/starpilot/flm/event_mean_evidence.py @@ -0,0 +1,167 @@ +"""Explicit event-only paired-mean evidence model; never native authority. + +Coverage is conditional on independent, approximately normal ORIGINAL whole- +block gains. Source disjointness is checked; it does not establish independence. +""" +from dataclasses import asdict, dataclass +import hashlib +import math +from collections.abc import Sequence +from typing import TYPE_CHECKING, TypedDict, cast + +if TYPE_CHECKING: + from openpilot.starpilot.flm.adaptive_profile import Block + + +class _EventPoint(TypedDict): + control_ns: int + apply_ns: int + mono_ns: int + + +class _EventPacket(TypedDict, total=False): + event_windows: tuple[tuple[int, int, str], ...] + samples: Sequence[_EventPoint] + grouping_revision: str + identity: str + block_id: str + cohort: str + +VERSION='event-paired-whole-block-t95-v1' +ASSUMPTION='independent-approximately-normal-paired-whole-block-gains' +_SEAL=object() + + +def _digest(value): + from openpilot.starpilot.flm.physical_input_observation import body_digest + return body_digest(value) + + +def arithmetic_lower(values, ids): + """Numerical primitive ONLY; this function cannot qualify an event or model.""" + from openpilot.starpilot.flm.automatic import T95 + if (type(values) is not tuple or type(ids) is not tuple or len(ids)!=len(values) or + not 2<=len(values)<=len(T95)+1 or + any(type(identity) is not str or not identity for identity in ids) or len(set(ids))!=len(ids) or + any(type(value) not in (int,float) or not math.isfinite(value) for value in values)): + raise ValueError('event paired-mean finite unique bounded units') + mean=sum(values)/len(values) + variance=sum((value-mean)**2 for value in values)/(len(values)-1) + result=mean-T95[len(values)-2]*math.sqrt(variance/len(values)) + if not math.isfinite(result): + raise ValueError('event paired-mean finite bound') + return result + + +@dataclass(frozen=True) +class WholeBlockUnit: + block_id: str + cohort: str + first_ns: int + last_ns: int + windows: tuple[tuple[int, int, str], ...] + block_sha256: str + + +@dataclass(frozen=True) +class EventMeanEvidence: + version: str + assumption: str + identity: str + source_geometry_sha256: str + units: tuple[WholeBlockUnit,...] + split: int + digest: str + _seal: object + + +def _model_body(model): + return {'version':model.version,'assumption':model.assumption,'identity':model.identity, + 'source_geometry_sha256':model.source_geometry_sha256,'units':tuple(asdict(unit) for unit in model.units), + 'split':model.split} + + +def build_model(engine, events, blocks): + """Bind child-reconstructed whole-source lineage and exact numeric blocks. + + Caller still must run every controller/native/physical/heldout/current guard. + No parent-declared block count, renamed boundary, or arbitrary packet is proof. + """ + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.adaptive_profile import Block + from openpilot.starpilot.flm.event_windows import whole_event_windows,GROUPING_REVISION + if (type(engine) is not EventControllerReplay or type(events) is not tuple or type(blocks) is not tuple or + not 8<=len(blocks)<=24 or len(events)!=len(blocks)): + raise ValueError('actual whole-block event mean model source') + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + require_feedback_engine(engine) + actual=tuple(whole_event_windows(engine)) + allowed={(first,last,kind) for _,first,last,kind in actual} + units=[] + used=set() + previous=0 + for event,block in zip(events,blocks,strict=True): + if type(block) is not Block or type(event) is not dict: + raise ValueError('typed original event mean model blocks') + event = cast(_EventPacket, event) + windows=event.get('event_windows') + if type(windows) is not tuple or not windows or any(type(row) is not tuple or len(row)!=3 or row not in allowed for row in windows): + raise ValueError('event mean model complete original geometry') + windows = cast(tuple[tuple[int, int, str], ...], windows) + if any(first<=previous_end for (_,previous_end,_),(first,_,_) in zip(windows,windows[1:], strict=False)): + raise ValueError('event mean model complete original geometry') + expected='event-v3:'+hashlib.sha256(repr((GROUPING_REVISION,engine.identity,engine.baseline_revision, + engine.objective.context_key,windows)).encode()).hexdigest() + if (event.get('grouping_revision')!=GROUPING_REVISION or event.get('identity')!=engine.identity or + event.get('block_id')!=expected or block.block_id!=expected or block.identity!=engine.identity or + block.block_id in used or block.cohort!=event.get('cohort') or windows[0][0]<=previous or + len(block.samples)!=len(event.get('samples',()))): + raise ValueError('event mean model disjoint original block identity') + for sample,point in zip(block.samples,event['samples'],strict=True): + if (sample.control_ns,sample.applied_ns,sample.mono_ns)!=(point['control_ns'],point['apply_ns'],point['mono_ns']): + raise ValueError('event mean model exact original endpoints') + if not any(first<=sample.control_ns<=last for first,last,_ in windows): + raise ValueError('event mean model endpoint outside complete source event') + used.add(block.block_id) + previous=windows[-1][1] + units.append(WholeBlockUnit(block.block_id,block.cohort,windows[0][0],previous,windows,_digest(asdict(block)))) + geometry=_digest({'actual_windows':actual,'identity':engine.identity,'baseline':engine.baseline_revision, + 'context':engine.objective.context_key,'cp_bytes':engine.factory.cp.as_builder().to_bytes()}) + model=EventMeanEvidence(VERSION,ASSUMPTION,engine.identity,geometry,tuple(units),len(units)//2,'',_SEAL) + from dataclasses import replace + return replace(model,digest=_digest(_model_body(model))) + + +def validate_model(model, blocks): + from openpilot.starpilot.flm.adaptive_profile import Block + if (type(model) is not EventMeanEvidence or model._seal is not _SEAL or model.version!=VERSION or + model.assumption!=ASSUMPTION or model.digest!=_digest(_model_body(model)) or + type(blocks) is not tuple or len(blocks)!=len(model.units)): + raise ValueError('owned event paired-mean model') + if any(type(block) is not Block or (block.identity,block.block_id,block.cohort)!=(model.identity,unit.block_id,unit.cohort) or + _digest(asdict(block))!=unit.block_sha256 for block,unit in zip(blocks,model.units,strict=True)): + raise ValueError('event paired-mean model changed numeric evidence') + return model + + +def lower(model, blocks, values, *, cohort): + """Only ALL exact heldout units in one original cohort determine n and df.""" + if (type(model) is not EventMeanEvidence or model._seal is not _SEAL or model.version!=VERSION or + model.assumption!=ASSUMPTION or model.digest!=_digest(_model_body(model))): + raise ValueError('owned event paired-mean model') + expected=tuple(unit for unit in model.units[model.split:] if unit.cohort==cohort) + if type(blocks) is not tuple or len(blocks)!=len(expected): + raise ValueError('all original heldout cohort units required') + for block,unit in zip(blocks,expected,strict=True): + if (block.identity,block.block_id,block.cohort)!=(model.identity,unit.block_id,cohort) or _digest(asdict(cast('Block', block)))!=unit.block_sha256: + raise ValueError('event paired-mean heldout identity/body changed') + return arithmetic_lower(values,tuple(unit.block_id for unit in expected)) + + +def report_basis(model): + if type(model) is not EventMeanEvidence or model._seal is not _SEAL or model.digest!=_digest(_model_body(model)): + raise ValueError('owned event mean report model') + return {**_model_body(model),'digest':model.digest,'upper_cdf':.975,'two_sided_confidence':.95, + 'disjoint_source_does_not_prove_independence':True,'table':'automatic.T95-conservative-upper-rounded-df1..23', + 'heldout_units_by_cohort':{cohort:sum(unit.cohort==cohort for unit in model.units[model.split:]) + for cohort in sorted({unit.cohort for unit in model.units})}} diff --git a/openpilot/starpilot/flm/event_raw_fit.py b/openpilot/starpilot/flm/event_raw_fit.py index 8a14853aa0..9d0aa408b2 100644 --- a/openpilot/starpilot/flm/event_raw_fit.py +++ b/openpilot/starpilot/flm/event_raw_fit.py @@ -1,3 +1,4 @@ +from openpilot.starpilot.flm.normalized_trace_archive import archive_of """Actual child path: whole events -> measured plant -> tiny canonical effect trial.""" from openpilot.starpilot.flm.clipping_quality import event_clipping_quality from dataclasses import asdict @@ -55,7 +56,7 @@ def original_event_evidence(payload, engine, objective): if packet['identity'] != engine.identity or not 40 <= len(packet['samples']) <= 240: raise ValueError('event packet provenance/bound') for point in packet['samples']: - command, output = control_from_dict(point['control']), output_from_dict(point['output']) + command, output = control_from_dict(point['control'],trace_archive=archive_of(engine)), output_from_dict(point['output']) key = point['control_ns'], point['apply_ns'], point['mono_ns'] if (key in seen or point['mono_ns'] <= previous or command.identity != engine.identity or command.baseline_revision != engine.baseline_revision or not command.enabled or command.manual or @@ -107,43 +108,71 @@ def _control_frame_index(records): return frames -def fit_event_torque(payload, engine, objective): +def fit_event_torque(payload, engine, objective, *, source_plant=None): frames = _control_frame_index(engine.records) - envelope, context, lag, packets, events = original_event_evidence(payload, engine, objective) - if len(events) < 16: - return _unavailable('whole_physical_event_support', storagePackets=len(packets), wholeBlocks=len(events), requiredWholeBlocks=16) - if payload.get('event_version') == 3: - from openpilot.starpilot.flm.event_activation import bind_activation_model, build_activation_plan - prepared = prepare_fixed_event_calibration(payload, engine, objective) - if type(prepared) is dict: - return prepared - original_model, cutoff, events, declared_stage = prepared - plan = build_activation_plan(engine, model=original_model, cutoff_ns=cutoff, stage_events=declared_stage, physical_context=context, - quality_contract=event_clipping_quality(payload)) - if engine.activation_plan != plan: - raise ValueError('event candidate changed frozen activation experiment') - model, split = bind_activation_model(original_model, plan), 8 + if source_plant is not None: + from openpilot.starpilot.flm.source_plant_fit import PROTOCOL, PreparedPlant + from openpilot.starpilot.flm.full_surface_profile import envelope_from_dict + from openpilot.starpilot.flm.event_activation import bind_activation_model + plant, events = source_plant + declared_protocol = PROTOCOL + if payload.get('event_version') == 7: + from openpilot.starpilot.flm.feedback_replay import PROTOCOL as FEEDBACK_PROTOCOL + declared_protocol = FEEDBACK_PROTOCOL + if (not isinstance(plant, PreparedPlant) or payload.get('plant_protocol') != declared_protocol or + tuple(payload['physical_context']) != plant.context or engine.activation_plan is None or + payload.get('event_version') not in (3,7) or payload.get('event_operation') != 'trial'): + raise ValueError('explicit child-owned independent source plant') + envelope = envelope_from_dict(payload['effect_envelope']) + context, lag, packets = plant.model.context, plant.model.lag_ns, events + cutoff, split = plant.identification.cutoff_ns, 0 + model = bind_activation_model(plant.model, engine.activation_plan) else: - # Chronologically earlier measured episodes calibrate the response. At least - # eight later whole-event bundles remain for disjoint stage train/heldout. - model = None - rows = [] - cutoff = split = 0 - for index, event in enumerate(events[:-8]): - for point in event['samples']: - output = output_from_dict(point['output']) - command = -delivered(output) - rows.append(PhysicalRow(event['block_id'], output.apply_ns, point['mono_ns'], command, - point['physical']['lateral_accel'], context, - 'left' if command > 0 else 'right', True, False, False)) - if index >= 7: - model = fit_event_physical(engine.identity, context, tuple(rows), lag_ns=lag, now_ns=event['samples'][-1]['mono_ns']) - if model is not None: - cutoff = max(row.response_ns for row in rows) - split = index + 1 - break - if model is None: - return _unavailable('event_independent_physical_model', wholeBlocks=len(events), modelPoints=len(rows)) + if payload.get('event_version') == 7: + raise ValueError('new event mean requires explicit original feedback source plant') + envelope, context, lag, packets, events = original_event_evidence(payload, engine, objective) + if len(events) < 16: + return _unavailable('whole_physical_event_support', storagePackets=len(packets), wholeBlocks=len(events), requiredWholeBlocks=16) + if payload.get('event_version') == 3: + from openpilot.starpilot.flm.event_activation import bind_activation_model, build_activation_plan + prepared = prepare_fixed_event_calibration(payload, engine, objective) + if type(prepared) is dict: + return prepared + original_model, cutoff, events, declared_stage = prepared + plan = build_activation_plan(engine, model=original_model, cutoff_ns=cutoff, stage_events=declared_stage, physical_context=context, + quality_contract=event_clipping_quality(payload)) + if engine.activation_plan != plan: + raise ValueError('event candidate changed frozen activation experiment') + model, split = bind_activation_model(original_model, plan), 8 + else: + # Chronologically earlier measured episodes calibrate the response. At least + # eight later whole-event bundles remain for disjoint stage train/heldout. + model = None + rows = [] + cutoff = split = 0 + for index, event in enumerate(events[:-8]): + for point in event['samples']: + output = output_from_dict(point['output']) + command = -delivered(output) + rows.append(PhysicalRow(event['block_id'], output.apply_ns, point['mono_ns'], command, + point['physical']['lateral_accel'], context, + 'left' if command > 0 else 'right', True, False, False)) + if index >= 7: + model = fit_event_physical(engine.identity, context, tuple(rows), lag_ns=lag, now_ns=event['samples'][-1]['mono_ns']) + if model is not None: + cutoff = max(row.response_ns for row in rows) + split = index + 1 + break + if model is None: + return _unavailable('event_independent_physical_model', wholeBlocks=len(events), modelPoints=len(rows)) + if payload.get('event_version') == 7: + from openpilot.starpilot.flm.event_mean_evidence import VERSION + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + if payload.get('event_mean_version') != VERSION: + raise ValueError('explicit new event mean version') + require_feedback_engine(engine) + elif 'event_mean_version' in payload: + raise ValueError('legacy event cannot adopt a new mean-evidence policy') engine.ensure_causal_features() blocks, stage_events = [], [] unexcited_stage_blocks = 0 @@ -156,7 +185,7 @@ def fit_event_torque(payload, engine, objective): continue samples = [] for point in event['samples']: - command, output = control_from_dict(point['control']), output_from_dict(point['output']) + command, output = control_from_dict(point['control'],trace_archive=archive_of(engine)), output_from_dict(point['output']) if point['mono_ns'] <= previous: raise ValueError('event stage chronology') previous = point['mono_ns'] @@ -175,6 +204,11 @@ def fit_event_torque(payload, engine, objective): stage_events.append(event) if len(blocks) < 8: return _unavailable('event_disjoint_stage_support', wholeBlocks=len(events), stageBlocks=len(blocks), unexcitedStageBlocks=unexcited_stage_blocks) + event_mean_model = None + if payload.get('event_version') == 7: + from openpilot.starpilot.flm.event_mean_evidence import build_model + event_mean_model = build_model(engine,tuple(stage_events),tuple(blocks)) + engine.event_mean_model = event_mean_model episodes = tracking_episodes(engine, tuple(stage_events), model, objective) prior = None if payload.get('prior') is None else tuple(payload['prior']) from openpilot.starpilot.flm.event_activation import ZERO_REFERENCE_ACTIVATION_VERSION @@ -183,18 +217,25 @@ def fit_event_torque(payload, engine, objective): result = evaluate_event(engine.identity, objective.parameters, tuple(blocks), episodes=episodes, now_ns=payload['now_ns'], required_cohorts=objective.required_cohorts, response_revision='event-independent-physical-v2', sequence=payload['sequence'], prior=prior, - effect_envelope=envelope, native_feasible=native_feasible) + effect_envelope=envelope, native_feasible=native_feasible, + **({'event_mean_model':event_mean_model} if event_mean_model is not None else {})) + mean_basis = {} + if event_mean_model is not None: + from openpilot.starpilot.flm.event_mean_evidence import report_basis + mean_basis = {'event_mean_basis':report_basis(event_mean_model)} validation = None if result.profile is not None: - current, applied = control_from_dict(payload['control']), output_from_dict(payload['output']) + current, applied = control_from_dict(payload['control'],trace_archive=archive_of(engine)), output_from_dict(payload['output']) validation = engine.validate(result.profile, current, applied, tuple(payload['physical_context']), tuple(blocks), model) if validation is None or validation.physical_model_revision != physical_model_revision(model): - return {'status': 'rejected', 'reason': 'event_actual_controller_replay', 'profile': None, 'validation': None} + return {'status':'rejected','reason':'event_actual_controller_replay','profile':None,'validation':None, + **mean_basis} return {'status': result.status, 'reason': result.reason, + **mean_basis, 'profile': None if result.profile is None else asdict(result.profile), 'validation': None if validation is None else asdict(validation), 'model': asdict(model), 'model_cutoff_ns': cutoff, 'objective_key': payload.get('objective_key'), - **({'event_version': 3, 'activation_plan': asdict(engine.activation_plan)} if payload.get('event_version') == 3 else {}), + **({'event_version': payload['event_version'], 'activation_plan': asdict(engine.activation_plan)} if payload.get('event_version') in (3,7) else {}), 'event_counts': {'large': sum(event['kind'] == 'large' for event in episodes), 'subtle': sum(event['kind'] == 'subtle' for event in episodes)}, 'whole_blocks': len(events), 'stage_blocks': len(blocks), 'unexcited_stage_blocks': unexcited_stage_blocks, diff --git a/openpilot/starpilot/flm/event_restore_fit.py b/openpilot/starpilot/flm/event_restore_fit.py index 3e792d32a3..ee0d1efc33 100644 --- a/openpilot/starpilot/flm/event_restore_fit.py +++ b/openpilot/starpilot/flm/event_restore_fit.py @@ -1,3 +1,4 @@ +from openpilot.starpilot.flm.normalized_trace_archive import archive_of """Fresh whole-map measured/controller proof for an explicitly retained prior. Restoration does not estimate coefficients or pass a thirty-dimensional map as @@ -39,7 +40,7 @@ def _restore_original_evidence(payload, engine, objective): if packet['identity'] != engine.identity or not 40 <= len(packet['samples']) <= 240: raise ValueError('restore endpoint packet identity') for point in packet['samples']: - command, output = control_from_dict(point['control']), output_from_dict(point['output']) + command, output = control_from_dict(point['control'],trace_archive=archive_of(engine)), output_from_dict(point['output']) key = point['control_ns'], point['apply_ns'], point['mono_ns'] if (key in seen or point['mono_ns'] <= previous or command.identity != engine.identity or command.baseline_revision != engine.baseline_revision or command.source_epoch != envelope.source_epoch or @@ -94,47 +95,70 @@ def _fixed_restore_rollout(payload, engine, coordinates, *, restore_target=False try: return engine._rollout(coordinates, restore_target=restore_target, restore_omit_group=restore_omit_group) except ValueError as error: - if payload.get('event_version') not in (4, 5, 6): + if payload.get('event_version') not in (4, 5, 6, 8): raise return unavailable('restore_actual_full_map_controller_replay', status='rejected', event_version=payload['event_version'], replay_phase='group-off' if restore_omit_group is not None else 'full-map' if restore_target else 'baseline', replay_reason=str(error), restore_activation_plan=asdict(engine.activation_plan)) -def fit_event_restore(payload, engine, objective): - envelope, context, packets, events = _restore_original_evidence(payload, engine, objective) - frames = _control_frame_index(engine.records) - if len(events) < 16: - return unavailable('restore_whole_physical_event_support', wholeBlocks=len(events), requiredWholeBlocks=16) - declared = payload.get('event_version') in (4, 5, 6) - if declared: +def fit_event_restore(payload, engine, objective, *, source_plant=None): + feedback_restore = payload.get('event_version') == 8 + if feedback_restore: + from openpilot.starpilot.flm.clipping_quality import event_clipping_quality + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + event_clipping_quality(payload) + if source_plant is None or getattr(engine,'event_mean_model',None) is not None: + raise ValueError('feedback restore requires fresh source plant and fixed conservative statistics') + require_feedback_engine(engine) + if source_plant is not None: + from openpilot.starpilot.flm.source_plant_fit import PROTOCOL, PreparedPlant + from openpilot.starpilot.flm.feedback_replay import PROTOCOL as FEEDBACK_PROTOCOL from openpilot.starpilot.flm.event_activation import bind_activation_model - from openpilot.starpilot.flm.restore_activation import build_restore_plan - prepared = prepare_fixed_restore_calibration(payload, engine, objective) - if type(prepared) is dict: - return prepared - original_model, cutoff, events, declared_stage = prepared - plan = build_restore_plan(engine, model=original_model, cutoff_ns=cutoff, stage_events=declared_stage, - physical_context=context, envelope=envelope, restore_version=payload['event_version']) - if engine.activation_plan != plan: - raise ValueError('restore changed frozen full-map activation declaration') - model, split = bind_activation_model(original_model, plan), 8 + plant, events = source_plant + if (not isinstance(plant, PreparedPlant) or payload.get('plant_protocol') != (FEEDBACK_PROTOCOL if feedback_restore else PROTOCOL) or + tuple(payload['physical_context']) != plant.context or engine.activation_plan is None or + payload.get('event_version') not in ((8,) if feedback_restore else (4,5,6)) or payload.get('event_operation') != 'restore'): + raise ValueError('explicit child-owned independent restore plant') + envelope = envelope_from_dict(payload['effect_envelope']) + envelope.validate() + context, packets, frames = plant.model.context, events, _control_frame_index(engine.records) + cutoff, split, declared = plant.identification.cutoff_ns, 0, True + model = bind_activation_model(plant.model, engine.activation_plan) else: - model, rows, cutoff, split = None, [], 0, 0 - for index, event in enumerate(events[:-8]): - for point in event['samples']: - output = output_from_dict(point['output']) - command = -delivered(output) - rows.append(PhysicalRow(event['block_id'], output.apply_ns, point['mono_ns'], command, point['physical']['lateral_accel'], - context, 'left' if command > 0 else 'right', True, False, False)) - if index >= 7: - model = fit_event_physical(engine.identity, context, tuple(rows), lag_ns=payload['physical_context'][-1], - now_ns=event['samples'][-1]['mono_ns']) - if model is not None: - cutoff, split = max(row.response_ns for row in rows), index + 1 - break - if model is None: - return unavailable('restore_independent_physical_model', wholeBlocks=len(events), modelPoints=len(rows)) + envelope, context, packets, events = _restore_original_evidence(payload, engine, objective) + frames = _control_frame_index(engine.records) + if len(events) < 16: + return unavailable('restore_whole_physical_event_support', wholeBlocks=len(events), requiredWholeBlocks=16) + declared = payload.get('event_version') in (4, 5, 6) + if declared: + from openpilot.starpilot.flm.event_activation import bind_activation_model + from openpilot.starpilot.flm.restore_activation import build_restore_plan + prepared = prepare_fixed_restore_calibration(payload, engine, objective) + if type(prepared) is dict: + return prepared + original_model, cutoff, events, declared_stage = prepared + plan = build_restore_plan(engine, model=original_model, cutoff_ns=cutoff, stage_events=declared_stage, + physical_context=context, envelope=envelope, restore_version=payload['event_version']) + if engine.activation_plan != plan: + raise ValueError('restore changed frozen full-map activation declaration') + model, split = bind_activation_model(original_model, plan), 8 + else: + model, rows, cutoff, split = None, [], 0, 0 + for index, event in enumerate(events[:-8]): + for point in event['samples']: + output = output_from_dict(point['output']) + command = -delivered(output) + rows.append(PhysicalRow(event['block_id'], output.apply_ns, point['mono_ns'], command, point['physical']['lateral_accel'], + context, 'left' if command > 0 else 'right', True, False, False)) + if index >= 7: + model = fit_event_physical(engine.identity, context, tuple(rows), lag_ns=payload['physical_context'][-1], + now_ns=event['samples'][-1]['mono_ns']) + if model is not None: + cutoff, split = max(row.response_ns for row in rows), index + 1 + break + if model is None: + return unavailable('restore_independent_physical_model', wholeBlocks=len(events), modelPoints=len(rows)) stage = tuple(event for event in events[split:] if all(min(point['control_ns'], point['apply_ns'], point['mono_ns']) > cutoff for point in event['samples'])) if len(stage) < 8: @@ -195,14 +219,14 @@ def fit_event_restore(payload, engine, objective): for group in effects: # Joint group impact must be present in this actual supported trajectory. # No unobservable saved low-speed/window coordinate restores on faith. - if payload.get('event_version') in (5, 6): + if payload.get('event_version') in (5, 6, 8): group_gains[group].append(sum(group_errors[group]) / len(samples) - after_loss) elif sum(value > 1e-8 for value in signals[group]) < 40: group_gains[group].append(None) else: group_gains[group].append(sum(group_errors[group]) / len(samples) - after_loss) middle = len(blocks) // 2 - if payload.get('event_version') in (5, 6): + if payload.get('event_version') in (5, 6, 8): from openpilot.starpilot.flm.restore_support import capped_group_credits, independent_half_supported, all_event_lower for group, trace in effects.items(): evidence = tuple((event['block_id'], tuple((point['control_ns'], point['apply_ns'], @@ -217,7 +241,7 @@ def fit_event_restore(payload, engine, objective): support_credits=half, required_credits=80, required_effectful_events=2) for group, gains in group_gains.items(): for cohort in (gains[:middle], gains[middle:]): - if payload.get('event_version') in (5, 6): + if payload.get('event_version') in (5, 6, 8): if all_event_lower(cohort, _lower) is None: return unavailable('restore_group_independent_heldout_support', group=group) else: @@ -233,13 +257,14 @@ def fit_event_restore(payload, engine, objective): objective.parameters, tuple(dict(envelope.coordinates)[name] for name in objective.names), tuple(block.block_id for block in blocks[:middle]), tuple(block.block_id for block in blocks[middle:]), baseline_loss, candidate_loss, ((objective.required_cohorts[0], lower),), RESTORE_POLICY, envelope) - validation = engine.validate(profile, control_from_dict(payload['control']), output_from_dict(payload['output']), + validation = engine.validate(profile, control_from_dict(payload['control'],trace_archive=archive_of(engine)), output_from_dict(payload['output']), tuple(payload['physical_context']), tuple(blocks), model) if validation is None or validation.physical_model_revision != physical_model_revision(model): return unavailable('restore_actual_full_map_controller_replay') return {'status': 'qualified', 'reason': 'restored_history_fresh_whole_map_pending_current_probe', 'profile': asdict(profile), 'validation': asdict(validation), 'model': asdict(model), **({'event_version': payload['event_version'], 'restore_activation_plan': asdict(engine.activation_plan)} if declared else {}), + **({'event_restore_statistics': payload['event_restore_statistics']} if feedback_restore else {}), 'model_cutoff_ns': cutoff, 'whole_blocks': len(events), 'stage_blocks': len(stage), 'restoration_qualified_groups': tuple(effects), 'objective_key': payload.get('objective_key'), 'speed_mps_bounds': (min(dict(point['control']['stage_values'])['speed'] for event in events for point in event['samples']), diff --git a/openpilot/starpilot/flm/feedback_archive.py b/openpilot/starpilot/flm/feedback_archive.py new file mode 100644 index 0000000000..f8f716cd69 --- /dev/null +++ b/openpilot/starpilot/flm/feedback_archive.py @@ -0,0 +1,125 @@ +"""Bounded ORIGINAL companion observations; no native/model authority.""" +from collections import OrderedDict, deque +from dataclasses import astuple +import pickle + +VERSION='original-feedback-archive-v1' +MAX_ROWS=12000 + + +def archive_binding(packet): + """No feedback field in its own binding; original context remains bit exact.""" + from openpilot.starpilot.flm.physical_input_observation import body_digest + from opendbc.car.steering_projection import TorqueProjection,projection_to_dict + context=packet['context_receipts'] + if type(context) is not dict or type(context.get('outputs')) is not tuple: + raise ValueError('original feedback context output tuple required') + outputs=[] + for row in context['outputs']: + projection=row.get('projection') + if isinstance(projection,TorqueProjection): + projection=projection_to_dict(projection) + if projection is None: + raise ValueError('original feedback projection binding schema') + row={**row,'projection':projection} + outputs.append(row) + # export_packet precedes its existing classless projection normalization. + # Bind exactly that known codec on BOTH sides; no full-frame copy/repack. + context={**context,'outputs':tuple(outputs)} + from openpilot.starpilot.flm.normalized_trace_archive import normalized_packet,reserve_for_packet + normalized = normalized_packet(packet) + reserve_for_packet(packet) + body = {'version':VERSION + '-normalized-v2' if normalized else VERSION,'context_receipts':context, + 'replay_prefix':packet['replay_prefix'],'original_replay_source':packet['original_replay_source'], + 'capture_epoch':packet['capture_epoch'],'capture_epoch_start_ns':packet['capture_epoch_start_ns'], + 'capture_epoch_cutoff_ns':packet['capture_epoch_cutoff_ns']} + if normalized: + body['trace_archive'] = packet['trace_archive'] + return body_digest(body) + + +class OriginalFeedbackArchive: + def __init__(self): + self.publications=OrderedDict() + self.consumptions=OrderedDict() + self.failed=False + self.failures=deque() + self.poison_through_ns=0 + + def reject(self, received_ns): + if type(received_ns) is not int or received_ns<=0: + self.failed=True + else: + if len(self.failures)>=MAX_ROWS: + self.poison_through_ns=max(self.poison_through_ns,self.failures.popleft()) + self.failures.append(received_ns) + return False + + def _record(self, destination, stamp, row, received_ns): + if type(received_ns) is not int or not 0MAX_ROWS: + destination.popitem(last=False) + return True + + def publication(self, row, received_ns): + from openpilot.starpilot.flm.feedback_observation import validate_publication + validate_publication(row) + return self._record(self.publications,row.publication_ns,row,received_ns) + + def consumption(self, row, control, received_ns): + from openpilot.starpilot.flm.feedback_observation import validate_consumption,f32 + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + validate_consumption(row) + if (type(control) is not ControlEvidence or row.control_ns!=control.control_ns or + row.source_epoch!=control.source_epoch or row.requested_torque!=f32(control.requested) or + row.lat_active is not control.active or row.published_limited is not control.limited or + dict(control.replay_inputs).get('safety_limited')!=float(row.input_limited)): + return self.reject(row.control_ns) + return self._record(self.consumptions,row.control_ns,row,received_ns) + + def export(self, packet): + from openpilot.starpilot.flm.feedback_archive_codec import compact_observations + if self.failed: + raise ValueError('original feedback archive failed') + stamps=tuple(row['control_ns'] for row in packet['context_receipts']['controls']) + if (not stamps or any(stamp not in self.consumptions for stamp in stamps) or + self.poison_through_ns and stamps[0]<=self.poison_through_ns+100_000_000 or + any(stamps[0]-100_000_000<=stamp<=stamps[-1]+100_000_000 for stamp in self.failures)): + raise ValueError('complete original feedback consumption unavailable') + consumptions=tuple(self.consumptions[stamp] for stamp in stamps) + # Original metadata interval, independent of quality, response or fit outcome. + # Keep the preceding consumed CO if retained, plus all publications through + # the last original controller frame; no heldout/flag-based row filtering. + first=min(row.publication_ns for row in consumptions) + publications=tuple(row for stamp,row in self.publications.items() if first<=stamp<=stamps[-1]) + return compact_observations(publications,consumptions,binding_digest=archive_binding(packet)) + + +def record_companion(collector,event,received_ns,*,control=None): + """New-version tap only. Missing/malformed siblings poison this prefix.""" + archive=getattr(collector.adapter,'original_feedback_archive',None) + if archive is None: + return + from openpilot.starpilot.flm.feedback_wire import read_publication,read_consumption + try: + if control is not None: + if event.which() != 'starpilotLateralState': + raise ValueError('original feedback consumption carrier') + row=read_consumption(event.starpilotLateralState) + if row is None or not archive.consumption(row,control,received_ns): + raise ValueError('complete original feedback consumption required') + else: + if event.which() != 'starpilotCarState': + raise ValueError('original feedback publication carrier') + row=read_publication(event.starpilotCarState) + if row is None or not archive.publication(row,received_ns): + raise ValueError('complete original feedback publication required') + except (AttributeError,KeyError,TypeError,ValueError,OverflowError): + archive.reject(received_ns) diff --git a/openpilot/starpilot/flm/feedback_archive_codec.py b/openpilot/starpilot/flm/feedback_archive_codec.py new file mode 100644 index 0000000000..f39b090d62 --- /dev/null +++ b/openpilot/starpilot/flm/feedback_archive_codec.py @@ -0,0 +1,146 @@ +"""Lossless bounded column wire for untrusted original feedback observations. + +No object decoder, numeric equality deduplication or admission token. Integer +columns use exact signed delta varints; floats retain all binary64 bits. This +reduces repeated field names and hex strings before the existing level1 IPC. +""" +import struct +from typing import cast + +VERSION = 'original-feedback-columns-v1' +MAX_ROWS = 12000 +MAX_BYTES = 16*1024*1024 +_PUB = (('version','i'),('session','h16'),('sequence','i'),('publication_ns','i'), + ('source_epoch','h32'),('cp_sha256','h32'),('output_known','b'),('output_control_ns','i'), + ('output_state_ns','i'),('output_apply_ns','i'),('output_carrier_sequence','i'), + ('reported_torque','f'),('valid','b'),('co_body_sha256','h32'),('applied_sha256','o32'), + ('native_identity','o32'),('carrier_observed_ns','i')) +_CON = (('version','i'),('control_ns','i'),('publication_ns','i'),('recv_frame','i'), + ('receipt_ns','i'),('seen','b'),('valid','b'),('co_body_sha256','h32'), + ('reported_torque','f'),('requested_torque','f'),('lat_active','b'),('input_limited','b'), + ('published_limited','b'),('prior_publish_ns','i'),('source_epoch','h32'),('cp_sha256','h32')) + + +def _column(values, kind): + wire = bytearray() + previous = 0 + for value in values: + if kind == 'i': + if type(value) is not int or not 0 <= value < 2**64: + raise ValueError('feedback integer column') + delta, previous = value-previous, value + number = delta*2 if delta >= 0 else -delta*2-1 + while number >= 128: + wire.append((number & 127)|128) + number >>= 7 + wire.append(number) + elif kind == 'b': + if type(value) is not bool: + raise ValueError('feedback Boolean column') + wire.append(int(value)) + elif kind == 'f': + if type(value) is not float: + raise ValueError('feedback exact float column') + wire.extend(struct.pack('!d',value)) + else: + size = int(kind[1:]) + if type(value) is not str or len(value) not in ((0,size*2) if kind[0]=='o' else (size*2,)): + raise ValueError('feedback digest column') + raw = bytes.fromhex(value) + if raw.hex() != value: + raise ValueError('feedback canonical lowercase digest') + if kind[0] == 'o': + wire.append(bool(raw)) + wire.extend(raw) + return bytes(wire) + + +def _values(wire, kind, count): + if type(wire) is not bytes or len(wire) > MAX_BYTES: + raise ValueError('bounded primitive feedback column') + result, offset, previous = [], 0, 0 + for _ in range(count): + if kind == 'i': + number = shift = 0 + while True: + if offset >= len(wire) or shift > 63: + raise ValueError('bounded feedback integer encoding') + byte = wire[offset] + offset += 1 + number |= (byte&127)<= len(wire) or wire[offset] not in (0,1): + raise ValueError('feedback Boolean encoding') + value = bool(wire[offset]) + offset += 1 + elif kind == 'f': + if offset+8 > len(wire): + raise ValueError('feedback float encoding') + value = struct.unpack_from('!d',wire,offset)[0] + offset += 8 + else: + size = int(kind[1:]) + if kind[0] == 'o': + if offset >= len(wire) or wire[offset] not in (0,1): + raise ValueError('feedback optional digest encoding') + if not wire[offset]: + size = 0 + offset += 1 + if offset+size > len(wire): + raise ValueError('feedback digest encoding') + value = wire[offset:offset+size].hex() + offset += size + result.append(value) + if offset != len(wire): + raise ValueError('feedback column trailing data') + return result + + +def compact_observations(publications,consumptions,*,binding_digest): + from openpilot.starpilot.flm.feedback_observation import export_observations + body = export_observations(publications,consumptions,binding_digest=binding_digest) + groups = [] + for name,schema in (('publications',_PUB),('consumptions',_CON)): + rows = body[name] + groups.append((len(rows),tuple(_column((row[field] for row in rows),kind) for field,kind in schema))) + wire = {'version':VERSION,'protocol':body['protocol'],'binding':bytes.fromhex(binding_digest), + 'digest':bytes.fromhex(body['digest']),'groups':tuple(groups)} + if sum(len(column) for _,columns in groups for column in columns) > MAX_BYTES: + raise ValueError('feedback aggregate column bound') + return wire + + +def expand_observations(wire,*,binding_digest): + from openpilot.starpilot.flm.feedback_observation import PROTOCOL,decode_observations + if (type(wire) is not dict or set(wire) != {'version','protocol','binding','digest','groups'} or + wire['version'] != VERSION or wire['protocol'] != PROTOCOL or + type(wire['binding']) is not bytes or len(wire['binding']) != 32 or wire['binding'].hex() != binding_digest or + type(wire['digest']) is not bytes or len(wire['digest']) != 32 or + type(wire['groups']) is not tuple or len(wire['groups']) != 2): + raise ValueError('explicit bounded feedback column protocol') + body = {'protocol':PROTOCOL,'binding_digest':binding_digest,'digest':wire['digest'].hex()} + total = 0 + for name,schema,group in zip(('publications','consumptions'),(_PUB,_CON),wire['groups'], strict=False): + if (type(group) is not tuple or len(group) != 2 or type(group[0]) is not int or not 1 <= group[0] <= MAX_ROWS or + type(group[1]) is not tuple or len(group[1]) != len(schema)): + raise ValueError('feedback column shape/count') + count, columns = group[0], cast(tuple[bytes, ...], group[1]) + if any(type(column) is not bytes for column in columns): + raise ValueError('primitive feedback columns only') + total += sum(len(column) for column in columns) + if total > MAX_BYTES: + raise ValueError('feedback aggregate column bound') + decoded = tuple(_values(column,kind,count) for column,(_,kind) in zip(columns,schema, strict=False)) + body[name] = [dict(zip((field for field,_ in schema),values, strict=False)) for values in zip(*decoded, strict=False)] + # Returns untrusted records only. Authentic original replay is a separate gate. + return decode_observations(body,binding_digest=binding_digest) diff --git a/openpilot/starpilot/flm/feedback_observation.py b/openpilot/starpilot/flm/feedback_observation.py new file mode 100644 index 0000000000..e1b6ba218d --- /dev/null +++ b/openpilot/starpilot/flm/feedback_observation.py @@ -0,0 +1,643 @@ +"""Versioned observations of Controls' real consumed CO feedback. + +Capture only; no native/current/profile authority. Original source and native +replay must independently verify every apply before creating a verified tape. +""" +from dataclasses import asdict, dataclass, replace, Field, _FIELD +import math +import re +import struct +import uuid +from types import GetSetDescriptorType + +PROTOCOL = 'original-consumed-co-feedback-v1' +MAX_ROWS = 12000 +MAX_AGE_NS = 100_000_000 +_VERIFIED = object() +_LOWER_HEX = re.compile('[0-9a-f]*') + + +def f32(value): + if type(value) not in (float, int) or not math.isfinite(value): + raise ValueError('finite feedback operand') + return struct.unpack(' row.output_apply_ns+MAX_AGE_NS or + not row.output_control_ns <= row.output_apply_ns or not _hex(row.applied_sha256)): + raise ValueError('feedback known actual apply') + elif (row.output_control_ns,row.output_state_ns,row.output_apply_ns,row.output_carrier_sequence,row.applied_sha256,row.carrier_observed_ns) != (0,0,0,0,'',0): + raise ValueError('unknown output cannot claim an apply') + return row + + +def validate_consumption(row): + if type(row) is not CoConsumption or type(row.version) is not int or row.version != 1: + raise ValueError('typed feedback consumption version') + if (not _positive(row.control_ns) or + any(type(v) is not int or not 0 <= v < 2**64 for v in (row.publication_ns,row.recv_frame,row.receipt_ns)) or + row.seen and (not all(_positive(v) for v in (row.publication_ns,row.recv_frame,row.receipt_ns)) or + not row.publication_ns <= row.receipt_ns <= row.control_ns <= row.publication_ns+MAX_AGE_NS) or + type(row.prior_publish_ns) is not int or not 0 <= row.prior_publish_ns < row.control_ns or + any(type(v) is not bool for v in (row.seen,row.valid,row.lat_active,row.input_limited,row.published_limited)) or + not all(_hex(v) for v in (row.co_body_sha256,row.source_epoch,row.cp_sha256)) or + any(f32(v) != v for v in (row.reported_torque,row.requested_torque))): + raise ValueError('feedback consumption fields/clock') + expected = bool(row.lat_active and abs(row.requested_torque-row.reported_torque) > .01) + if row.published_limited is not expected: + raise ValueError('original published feedback equation') + return row + + +class CardFeedbackCapture: + """Owner-local association, invalidated BEFORE every output setter.""" + def __init__(self, source_epoch, cp_sha256): + if not _hex(source_epoch) or not _hex(cp_sha256): + raise ValueError('feedback source context') + self.source_epoch,self.cp_sha256 = source_epoch,cp_sha256 + self.session = uuid.uuid4().hex + self.sequence = self.last_header = 0 + self.output = self.applied = None + self.assignment_serial = self.associated_serial = 0 + + def begin_assignment(self): + self.assignment_serial += 1 + self.invalidate() + + def invalidate(self): + self.output = self.applied = None + self.associated_serial = 0 + + def committed(self, actual, applied): + from openpilot.starpilot.flm.fullrate_collector import AppliedEvidence + from opendbc.car.steering_projection import TorqueProjection + # This is an observation association, never an applied/native verifier. + if (self.assignment_serial <= 0 or type(applied) is not AppliedEvidence or applied.domain != 'normalized_torque' or + type(applied.projection) is not TorqueProjection or + f32(float(actual.torque)) != f32(applied.applied) or f32(applied.projection.reported_torque) != f32(applied.applied)): + self.invalidate() + return + self.output,self.applied = actual,applied + self.associated_serial = self.assignment_serial + + def publication(self, event): + header = int(event.logMonoTime) + if header <= self.last_header: + self.invalidate() + raise ValueError('nonincreasing CO owner publication') + self.last_header = header + self.sequence += 1 + actual = event.carOutput.actuatorsOutput + # Object identity at setter establishes ownership; body equality only checks + # it survived assignment, never discovers which apply supplied this output. + applied = self.applied + known = self.output is not None and applied is not None and self.associated_serial == self.assignment_serial + if known and actual.to_dict() != self.output.to_dict(): + self.invalidate() + raise ValueError('held owner output changed before CO publication') + row = CoPublication(1,self.session,self.sequence,header,self.source_epoch,self.cp_sha256,known, + 0 if not known else applied.control_ns,0 if not known else applied.state_ns, + 0 if not known else applied.apply_ns,0 if not known else applied.carrier_frame, + f32(float(actual.torque)),bool(event.valid),co_body(header,bool(event.valid),actual.to_dict()), + '' if not known else applied_digest(applied),'',0 if not known else applied.carrier_observed_ns) + return validate_publication(row) + + +def card_capture_for(owner, now_ns): + """Only actual enabled/current settings receipt enables observational work.""" + from types import SimpleNamespace + from openpilot.starpilot.flm.learning_wire import read_source_receipt + from openpilot.starpilot.flm.runtime_adapter_registry import car_params_digest + sm = owner.sm + try: + # This service is optional for native authority; all_checks deliberately + # ignores it there. Observational capture must check its REAL bits directly. + service = 'autoSteeringLearning' + stamp = int(sm.logMonoTime[service]) + receipt = int(sm.recv_time[service]*1e9) + if (type(now_ns) is not int or not now_ns > 0 or + not all((sm.seen[service],sm.valid[service],sm.alive[service],sm.freq_ok[service])) or + not 0 < stamp <= now_ns <= stamp+500_000_000 or + not 0 < receipt <= now_ns <= receipt+500_000_000): + raise ValueError('feedback settings source/receipt invalid or stale') + digest = getattr(owner,'_feedback_cp_sha256',None) + if digest is None: + digest = owner._feedback_cp_sha256 = car_params_digest(owner.CP) + source = read_source_receipt(SimpleNamespace(valid=bool(sm.valid[service]),autoSteeringLearning=sm[service]), + now_ns=now_ns,cp_sha256=digest) + if source is None or not source.enabled or source.manual: + raise ValueError('feedback settings off/manual') + capture = getattr(owner,'learning_feedback_capture',None) + if type(capture) is not CardFeedbackCapture or (capture.source_epoch,capture.cp_sha256) != (source.source_epoch,digest): + capture = owner.learning_feedback_capture = CardFeedbackCapture(source.source_epoch,digest) + return capture + except (AttributeError,KeyError,TypeError,ValueError,OverflowError): + owner.learning_feedback_capture = None + return None + + +def capture_controls(owner, cc, *, control_ns, input_limited): + """Freeze actual consumption only; no association guessed by torque equality.""" + automatic = getattr(owner,'global_learning',None) + source = None if automatic is None else automatic.source + if (automatic is None or not automatic.eligible or source is None or not source.enabled or source.manual or + not source.observed_ns <= control_ns <= source.expires_ns or str(owner.CP.steerControlType) != 'torque'): + owner.learning_feedback_prior_publish_ns = 0 + return None + context = (source.source_epoch,source.cp_sha256) + if getattr(owner,'learning_feedback_context',None) != context: + owner.learning_feedback_prior_publish_ns = 0 + owner.learning_feedback_context = context + sm = owner.sm + header = int(sm.logMonoTime['carOutput']) + output = sm['carOutput'].actuatorsOutput + row = CoConsumption(1,control_ns,header,int(sm.recv_frame['carOutput']),int(sm.recv_time['carOutput']*1e9), + bool(sm.seen['carOutput']),bool(sm.valid['carOutput']),co_body(header,bool(sm.valid['carOutput']),output.to_dict()), + f32(float(output.torque)),f32(float(cc.actuators.torque)),bool(cc.latActive),bool(input_limited), + bool(owner.steer_limited_by_safety),int(getattr(owner,'learning_feedback_prior_publish_ns',0)), + source.source_epoch,source.cp_sha256) + owner.learning_feedback_prior_publish_ns = control_ns + return validate_consumption(row) + + +@dataclass(frozen=True) +class FeedbackJoin: + consumption: CoConsumption + publication: CoPublication + + +@dataclass(frozen=True) +class VerifiedFeedbackTape: + """Child-local result. Never decode this type from an untrusted wire.""" + protocol: str + rows: tuple[FeedbackJoin, ...] + binding_digest: str + digest: str + context_only: tuple[CoConsumption, ...] + parent_digest: str + _seal: object + + +def join_original(publications, consumptions): + """Pure bounded exact join. NOT independent native/source execution proof.""" + if (type(publications) is not tuple or type(consumptions) is not tuple or + not 1 <= len(publications) <= MAX_ROWS or not 1 <= len(consumptions) <= MAX_ROWS): + raise ValueError('bounded complete feedback tape') + by_header = {} + prior = None + for pub in publications: + validate_publication(pub) + if pub.publication_ns in by_header or prior is not None and ( + pub.session != prior.session or pub.sequence <= prior.sequence or pub.publication_ns <= prior.publication_ns or + (pub.source_epoch,pub.cp_sha256) != (prior.source_epoch,prior.cp_sha256)): + raise ValueError('ambiguous/reordered/reset CO publication') + if prior is not None and pub.output_known and prior.output_known and pub.output_apply_ns == prior.output_apply_ns and ( + pub.output_carrier_sequence < prior.output_carrier_sequence or pub.carrier_observed_ns < prior.carrier_observed_ns): + raise ValueError('held publication carrier chronology') + by_header[pub.publication_ns] = pub + prior = pub + rows = [] + previous = None + for row in consumptions: + validate_consumption(row) + if not row.seen or not row.valid: + raise ValueError('unseen/invalid consumed output is observation only') + pub = by_header.get(row.publication_ns) + if pub is None or not pub.output_known or ( + pub.co_body_sha256,pub.reported_torque,pub.valid,pub.source_epoch,pub.cp_sha256) != ( + row.co_body_sha256,row.reported_torque,row.valid,row.source_epoch,row.cp_sha256): + raise ValueError('exact consumed CO/apply provenance unavailable') + if previous is not None and (row.control_ns <= previous.control_ns or + row.recv_frame < previous.recv_frame or row.publication_ns < previous.publication_ns or + row.prior_publish_ns != previous.control_ns or row.input_limited is not previous.published_limited): + raise ValueError('original publish to next-controller feedback linkage') + rows.append(FeedbackJoin(row,pub)) + previous = row + return tuple(rows) + + +def _plain_record_values(item, kind): + """Read only a plain exact class's complete owned, callback-free dictionary.""" + if type(item) is not kind or type(kind) is not type or kind.__bases__ != (object,): + return None + namespace = kind.__dict__ + descriptor = namespace.get('__dict__') + metadata = namespace.get('__dataclass_fields__') + if ('__getattribute__' in namespace or '__getattr__' in namespace or + type(descriptor) is not GetSetDescriptorType or descriptor.__objclass__ is not kind or + type(metadata) is not dict): + return None + declared = [] + for name, field in metadata.items(): + if (type(name) is not str or type(field) is not Field or type(field.name) is not str or field.name != name or + field._field_type is not _FIELD or name in namespace): + return None + declared.append(name) + owned = object.__getattribute__(item, '__dict__') + if (type(owned) is not dict or any(type(key) is not str for key in owned) or + any(name not in owned for name in declared)): + return None + return tuple((name, owned[name]) for name in declared) + + +def _feedback_asdict(record): + """Fresh complete record values; unknown shapes retain whole asdict order.""" + def flat(item, kind): + values = _plain_record_values(item, kind) + if values is None: + return None + body = {} + for name, value in values: + if value is not None and type(value) not in (bool, int, float, str, bytes): + return None + body[name] = value + return body + + kind = type(record) + if kind is CoPublication or kind is CoConsumption: + body = flat(record, kind) + if body is not None: + return body + elif kind is FeedbackJoin: + joined = _plain_record_values(record, FeedbackJoin) + if joined is not None and tuple(name for name, _ in joined) == ('consumption', 'publication'): + consumption = flat(joined[0][1], CoConsumption) + if consumption is not None: + publication = flat(joined[1][1], CoPublication) + if publication is not None: + return {'consumption': consumption, 'publication': publication} + return asdict(record) + + +def _tape_digest_uncached(rows, context_only, binding_digest, parent_digest): + return _digest({'binding':binding_digest,'parent':parent_digest, + 'rows':[_feedback_asdict(row) for row in rows],'context_only':[_feedback_asdict(row) for row in context_only]}) + + +# Child-local cost cache only. Fresh exact full bytes are required on EVERY hit. +from collections import OrderedDict +_TAPE_DIGEST_CACHE=OrderedDict() +_TAPE_DIGEST_CACHE_BYTES=0 +_TAPE_DIGEST_MAX_ENTRIES=4 +_TAPE_DIGEST_MAX_KEY=4*1024*1024 +_TAPE_DIGEST_MAX_BYTES=8*1024*1024 + + +def _tape_digest_key(rows,context_only,binding_digest,parent_digest): + if (type(rows) is not tuple or len(rows)>MAX_ROWS or type(context_only) is not tuple or len(context_only)>MAX_ROWS or + type(binding_digest) is not str or type(parent_digest) is not str or len(binding_digest)>128 or len(parent_digest)>128): + return None + def primitive_record(record,kind): + if type(record) is not kind: + return None + values=tuple(getattr(record,name) for name in kind.__dataclass_fields__) + for value in values: + leaf=type(value) + if leaf is str: + if len(value)>128: + return None + elif leaf is int: + if abs(value).bit_length()>64: + return None + elif leaf is float: + if not math.isfinite(value): + return None + elif leaf is not bool: + return None + return values + pairs=[] + for joined in rows: + if type(joined) is not FeedbackJoin: + return None + c=primitive_record(joined.consumption,CoConsumption) + p=primitive_record(joined.publication,CoPublication) + if c is None or p is None: + return None + pairs.append((c,p)) + context=[] + for record in context_only: + values=primitive_record(record,CoConsumption) + if values is None: + return None + context.append(values) + import pickle + # ONLY plain native scalar tuples cross pickle. Exact known record types and + # every declared field were checked above; extra attrs/customreduce never run. + key=pickle.dumps((tuple(pairs),tuple(context),binding_digest,parent_digest),protocol=5) + return key if len(key)<=min(_TAPE_DIGEST_MAX_KEY,_TAPE_DIGEST_MAX_BYTES) else None + + +def _tape_digest_status(rows,context_only,binding_digest,parent_digest): + global _TAPE_DIGEST_CACHE_BYTES + key=_tape_digest_key(rows,context_only,binding_digest,parent_digest) + if key is not None and key in _TAPE_DIGEST_CACHE: + _TAPE_DIGEST_CACHE.move_to_end(key) + return _TAPE_DIGEST_CACHE[key], True + result=_tape_digest_uncached(rows,context_only,binding_digest,parent_digest) + if key is not None: + while _TAPE_DIGEST_CACHE and (len(_TAPE_DIGEST_CACHE)>=_TAPE_DIGEST_MAX_ENTRIES or + _TAPE_DIGEST_CACHE_BYTES+len(key)>_TAPE_DIGEST_MAX_BYTES): + old,_=_TAPE_DIGEST_CACHE.popitem(last=False) + _TAPE_DIGEST_CACHE_BYTES-=len(old) + _TAPE_DIGEST_CACHE[key]=result + _TAPE_DIGEST_CACHE_BYTES+=len(key) + return result, key is not None + + +def _tape_digest(rows,context_only,binding_digest,parent_digest): + return _tape_digest_status(rows,context_only,binding_digest,parent_digest)[0] + + +def _sealed_status(tape,binding_digest): + if not (type(tape) is VerifiedFeedbackTape and tape._seal is _VERIFIED and tape.protocol == PROTOCOL and + tape.binding_digest == binding_digest and _hex(binding_digest) and bool(tape.rows)): + return False, False + expected_digest = tape.digest + digest, closed = _tape_digest_status(tape.rows,tape.context_only,binding_digest,tape.parent_digest) + matched = expected_digest == digest + return matched, closed and type(tape.protocol) is str and type(expected_digest) is str + + +def _sealed(tape,binding_digest): + return _sealed_status(tape,binding_digest)[0] + + +def _control_matches(control,row,cp_digest): + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + if (type(control) is not ControlEvidence or row.cp_sha256 != cp_digest or control.source_epoch != row.source_epoch or + f32(control.requested) != row.requested_torque or bool(control.active) != row.lat_active or + bool(control.limited) != row.published_limited or dict(control.replay_inputs).get('safety_limited') != float(row.input_limited) or + not control.enabled or control.manual or not control.valid or not control.active or control.override or control.fault or + not 0 < control.source_receipt_ns <= control.control_ns <= control.source_receipt_ns+500_000_000): + raise ValueError('actual original control/source feedback mismatch') + if control.native_required: + if (not control.native_allowed or not _hex(control.native_identity) or + not 0 < control.native_observed_ns <= control.control_ns <= control.native_expires_ns <= control.native_observed_ns+200_000_000): + raise ValueError('original feedback native control lease/identity') + elif control.native_allowed or control.native_identity or (control.native_observed_ns,control.native_expires_ns) != (0,0): + raise ValueError('ordinary feedback cannot fabricate native ACK') + + +def verify_original(publications, consumptions, controls, outputs, engine, binding_digest, *, + expected_control_ids, context_packet, frames, allow_initial_context_only=False): + """Complete owned original proof; optional FIRST q is explicit context only.""" + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.fullrate_collector import AppliedEvidence + from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current + from openpilot.starpilot.flm.original_context_receipts import reuse_verified_context + from openpilot.starpilot.flm.torque_raw_fit import output_from_dict + from openpilot.starpilot.flm.runtime_adapter_registry import car_params_digest + if type(engine) is not EventControllerReplay or not _hex(binding_digest): + raise ValueError('actual original replay and owned context binding required') + verified_controls = reuse_verified_context(context_packet,engine,frames) + if verified_controls is None or type(controls) is not dict or controls != verified_controls: + raise ValueError('completed child-local exact original source/native replay proof required') + owned_outputs = {v.apply_ns:v for v in (output_from_dict(body) for body in context_packet['outputs'])} + if type(outputs) is not dict or outputs != owned_outputs: + raise ValueError('exact original applied context receipt bodies required') + _archive_records(publications,consumptions) + frame_ids = tuple(frame.control_ns for frame in frames) + if (type(expected_control_ids) is not tuple or expected_control_ids != frame_ids or + expected_control_ids != tuple(row.control_ns for row in consumptions) or type(allow_initial_context_only) is not bool): + raise ValueError('all original controller feedback membership required') + cp_digest = car_params_digest(engine.factory.cp) + for row in consumptions: + _control_matches(controls.get(row.control_ns),row,cp_digest) + by_header = {pub.publication_ns:pub for pub in publications} + first = consumptions[0] + first_pub = by_header.get(first.publication_ns) + first_unclosed = (not first.seen or not first.valid or first_pub is None or not first_pub.output_known or + first_pub.output_control_ns not in controls or first_pub.output_apply_ns not in outputs) + context_only = () + known = consumptions + if first_unclosed: + if not allow_initial_context_only or len(consumptions) < 2 or first.control_ns != frame_ids[0]: + raise ValueError('initial original CO context is unclosed') + context_only,known = (first,),consumptions[1:] + # Retain raw archive digest, including unclosed first q and unconsumed COs. + # Only known consumed headers are joined; ALL controller rows remain required. + needed = {row.publication_ns for row in known} + used_publications = tuple(pub for pub in publications if pub.publication_ns in needed) + rows = join_original(used_publications,known) + if context_only and (known[0].prior_publish_ns != first.control_ns or known[0].input_limited is not first.published_limited): + raise ValueError('initial context to first known feedback linkage') + for joined in rows: + row,pub = joined.consumption,joined.publication + producing = controls.get(pub.output_control_ns) + output = outputs.get(pub.output_apply_ns) + if producing is None or type(output) is not AppliedEvidence: + raise ValueError('actual original producing apply unavailable') + if (pub.output_carrier_sequence < output.carrier_frame or pub.carrier_observed_ns < output.carrier_observed_ns): + raise ValueError('held original carrier receipt regressed') + held = replace(output,carrier_frame=pub.output_carrier_sequence,carrier_observed_ns=pub.carrier_observed_ns) + if (output.control_ns != pub.output_control_ns or output.state_ns != pub.output_state_ns or + output.apply_ns != pub.output_apply_ns or applied_digest(held) != pub.applied_sha256 or + not consumed_sources_current(producing,output) or not engine.applied_matches(producing,output) or + producing.source_epoch != row.source_epoch or f32(output.applied) != pub.reported_torque or + pub.native_identity and pub.native_identity != producing.native_identity or + not pub.output_apply_ns <= row.control_ns <= pub.output_apply_ns+MAX_AGE_NS or + not output.valid or not output.active or output.override or output.fault or not output.native_allowed or + producing.native_required and output.apply_ns > producing.native_expires_ns): + raise ValueError('original feedback source/native/actual CAN mismatch') + parent = _digest({'publications':[_feedback_asdict(v) for v in publications],'consumptions':[_feedback_asdict(v) for v in consumptions]}) + digest = _tape_digest(rows,context_only,binding_digest,parent) + return VerifiedFeedbackTape(PROTOCOL,rows,binding_digest,digest,context_only,parent,_VERIFIED) + + +def _checked_subset(tape, exact_control_ids, binding_digest): + sealed, closed = _sealed_status(tape,binding_digest) + if not sealed or type(exact_control_ids) is not tuple or not exact_control_ids: + raise ValueError('owned verified feedback tape/exact window required') + stamps = tuple(row.consumption.control_ns for row in tape.rows) + try: + start = stamps.index(exact_control_ids[0]) + except ValueError as error: + raise ValueError('subset cannot include unknown/context-only feedback') from error + selected = tape.rows[start:start+len(exact_control_ids)] + if tuple(row.consumption.control_ns for row in selected) != exact_control_ids: + raise ValueError('complete contiguous original feedback window required') + return selected, closed and all(type(stamp) is int for stamp in exact_control_ids) + + +def _mint_subset(tape, selected, binding_digest): + digest = _tape_digest(selected,(),binding_digest,tape.digest) + return VerifiedFeedbackTape(PROTOCOL,selected,binding_digest,digest,(),tape.digest,_VERIFIED) + + +def subset_verified(tape, exact_control_ids, *, binding_digest): + """Owner-minted contiguous original window; no wire-created subset seal.""" + selected, _ = _checked_subset(tape,exact_control_ids,binding_digest) + return _mint_subset(tape,selected,binding_digest) + + +def validate_subset_verified(tape, exact_control_ids, *, binding_digest): + """Fresh validation; discard mint only for the just-checked closed domain.""" + selected, closed = _checked_subset(tape,exact_control_ids,binding_digest) + if not closed: + _mint_subset(tape,selected,binding_digest) + + +def verify_candidate_flags(tape, candidate_requests, candidate_reported, *, activation_ns, binding_digest): + """Narrow regime equality only. No modified inputs or native permission.""" + if (not _sealed(tape,binding_digest) or not _positive(activation_ns) or + tape.rows[0].consumption.control_ns > activation_ns): + raise ValueError('owned verified feedback tape/prefix required') + return _flag_equations(tape.rows,candidate_requests,candidate_reported,activation_ns) + + +def _flag_equations(rows,candidate_requests,candidate_reported,activation_ns): + """Pure arithmetic for differential tests; NEVER native/admission proof.""" + if type(candidate_requests) is not dict or type(candidate_reported) is not dict: + raise ValueError('explicit candidate feedback maps') + if set(candidate_requests) != {r.consumption.control_ns for r in rows}: + raise ValueError('all candidate feedback frames required') + for joined in rows: + row,pub = joined.consumption,joined.publication + request = f32(candidate_requests[row.control_ns]) + if row.control_ns < activation_ns: + if request != row.requested_torque: + raise ValueError('candidate changed original preactivation prefix') + continue + if pub.output_control_ns < activation_ns: + reported = pub.reported_torque + else: + if pub.output_apply_ns not in candidate_reported: + raise ValueError('candidate reported CO coordinate missing') + reported = f32(candidate_reported[pub.output_apply_ns]) + hint = bool(row.lat_active and abs(request-reported) > .01) + if hint is not row.published_limited: + raise ValueError('candidate changed original consumed-feedback regime') + return True + + +def _archive_records(publications,consumptions): + if (type(publications) is not tuple or type(consumptions) is not tuple or + not 1 <= len(publications) <= MAX_ROWS or not 1 <= len(consumptions) <= MAX_ROWS): + raise ValueError('bounded complete feedback archive') + for pub in publications: + validate_publication(pub) + for row in consumptions: + validate_consumption(row) + if (len({v.publication_ns for v in publications}) != len(publications) or + any(b.publication_ns <= a.publication_ns for a,b in zip(publications,publications[1:], strict=False)) or + any(b.control_ns <= a.control_ns for a,b in zip(consumptions,consumptions[1:], strict=False))): + raise ValueError('duplicate/reordered original feedback archive') + + +def export_observations(publications, consumptions, *, binding_digest): + """Separate bounded archive body; hashes bind content, never execute proof.""" + if not _hex(binding_digest): + raise ValueError('explicit feedback archive binding') + # Preserve unknown/context observations too; this is NEVER a verified type. + _archive_records(publications,consumptions) + body = {'protocol':PROTOCOL,'binding_digest':binding_digest, + 'publications':[_feedback_asdict(v) for v in publications], + 'consumptions':[_feedback_asdict(v) for v in consumptions]} + return {**body,'digest':_digest(body)} + + +def decode_observations(packet, *, binding_digest): + """Untrusted separate records only. Never returns VerifiedFeedbackTape.""" + from dataclasses import fields + if type(packet) is not dict or set(packet) != {'protocol','binding_digest','publications','consumptions','digest'}: + raise ValueError('explicit feedback archive version required; old packets ineligible') + body = {k:v for k,v in packet.items() if k != 'digest'} + if (packet['protocol'] != PROTOCOL or not _hex(binding_digest) or packet['binding_digest'] != binding_digest or + not _hex(packet['digest'])): + raise ValueError('feedback archive exact binding/body') + values = [] + for name,kind,validator in (('publications',CoPublication,validate_publication),('consumptions',CoConsumption,validate_consumption)): + raw = packet[name] + if type(raw) is not list or not 1 <= len(raw) <= MAX_ROWS: + raise ValueError('feedback archive row budget') + keys = {v.name for v in fields(kind)} + if any(type(v) is not dict or set(v) != keys for v in raw): + raise ValueError('feedback archive exact typed fields') + values.append(tuple(validator(kind(**v)) for v in raw)) + if packet['digest'] != _digest(body): + raise ValueError('feedback archive exact binding/body') + _archive_records(*values) + return tuple(values) diff --git a/openpilot/starpilot/flm/feedback_replay.py b/openpilot/starpilot/flm/feedback_replay.py new file mode 100644 index 0000000000..35452280b3 --- /dev/null +++ b/openpilot/starpilot/flm/feedback_replay.py @@ -0,0 +1,136 @@ +"""NEW event source feedback proof, separate from legacy event admission.""" +from dataclasses import dataclass +from openpilot.starpilot.flm.candidate_feedback import LEGACY_POLICY,CLOSED_POLICY,PROOF_VERSION,policy as validate_candidate_policy + +PROTOCOL='chronological-source-feedback-mean-fit-v1' +EVENT_VERSION=7 +_SEAL=object() + + +@dataclass(frozen=True) +class FeedbackReplayEvidence: + binding: str + original: object + windows: tuple + engine_input_digest: str + _seal: object + candidate_policy: str = LEGACY_POLICY + + +def _inputs(engine, *, candidate_policy=LEGACY_POLICY): + # Hash trusted in-process objects ONLY; no pickle data is ever decoded here. + # Keep the same original CP/factory/controller/native/checkpoint/body coverage + # as the existing causal owned-input key, without recursively hashing this seal. + import hashlib + import pickle + from openpilot.starpilot.flm.controller_replay import SelectedControllerFactory + from openpilot.starpilot.flm.pid_controller_replay import PidControllerFactory + factory=engine.factory + if type(factory) not in (SelectedControllerFactory,PidControllerFactory): + raise ValueError('registered original feedback factory required') + policy=('torque',str(factory.policy)) if type(factory) is SelectedControllerFactory else ('stock-pid',None) + selected=(type(factory).__module__,type(factory).__qualname__,factory.cp.as_builder().to_bytes(), + str(factory.mode),policy,factory.turn_assist,factory.manual_baseline,factory.manual_source_epoch, + factory.interface_class.__module__,factory.interface_class.__qualname__) + native=(engine.emissions.cp.as_builder().to_bytes(),engine.emissions.identity,engine.emissions.baseline_revision, + tuple(engine.emissions.events.items()),tuple(engine.emissions.carriers.items())) + body=(engine.identity,engine.baseline_revision,engine.engine.model_revision,engine.engine.failed, + engine.engine.raw_mode,selected,engine.engine.objective,engine.objective,engine.reference, + engine.prior_full,engine.original_anchor,engine.activation_plan,engine.restore_envelope, + tuple(engine.records),tuple(engine.checkpoints),tuple(engine.replay_windows),native) + from openpilot.starpilot.flm.normalized_trace_archive import archive_key + if archive_key(engine) is not None: + body = (*body,archive_key(engine)) + validate_candidate_policy(candidate_policy) + if candidate_policy != LEGACY_POLICY: + body = (*body,candidate_policy,PROOF_VERSION) + return hashlib.sha256(pickle.dumps(body,protocol=5)).hexdigest() + + +def verify_archive(packet, original): + from openpilot.starpilot.flm.feedback_archive import archive_binding + from openpilot.starpilot.flm.feedback_observation import verify_original + from openpilot.starpilot.flm.feedback_archive_codec import expand_observations + from openpilot.starpilot.flm.torque_raw_fit import output_from_dict + from openpilot.starpilot.flm.normalized_trace_archive import normalized_packet,archive_of + if normalized_packet(packet): + import pickle + owner_archive = archive_of(original) + if owner_archive is None or pickle.dumps(owner_archive.literal,protocol=5) != pickle.dumps(packet['trace_archive'],protocol=5): + raise ValueError('normalized feedback original literal source differs') + binding=archive_binding(packet) + publications,consumptions=expand_observations(packet['feedback_observations'],binding_digest=binding) + frames=tuple(original.records) + outputs={value.apply_ns:value for value in (output_from_dict(row) for row in packet['context_receipts']['outputs'])} + return verify_original(publications,consumptions,original.original_context_controls,outputs,original,binding, + expected_control_ids=tuple(frame.control_ns for frame in frames),context_packet=packet['context_receipts'], + frames=frames,allow_initial_context_only=True) + + +def attach_feedback_engine(engine, tape, *, candidate_policy=LEGACY_POLICY): + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.feedback_observation import subset_verified + if type(engine) is not EventControllerReplay or engine.activation_plan is None: + raise ValueError('new feedback requires actual declared stage engine') + windows=[] + for bounds in engine.replay_windows: + controls=tuple(frame.control_ns for frame in engine.records if bounds[0]<=frame.control_ns<=bounds[1]) + if not controls or controls[0]!=bounds[0] or controls[-1]!=bounds[1]: + raise ValueError('complete feedback engine window required') + context=tuple(row.control_ns for row in tape.context_only) + if context: + # The sealed global first q may precede a later stage-only replay. + if len(context) != 1 or context[0] > engine.records[0].control_ns: + raise ValueError('only original leading feedback context allowed') + controls=tuple(stamp for stamp in controls if stamp not in context) + subset=subset_verified(tape,controls,binding_digest=tape.binding_digest) + activation=engine.activation_plan.activation_for(*bounds) + if activation is None: + raise ValueError('original feedback activation clock required') + # An original context-only window has no treatment. Requiring every request + # to equal original through its end proves the unchanged prefix instead. + flag_activation=activation[0] or bounds[1]+1 + windows.append((bounds,subset,flag_activation)) + validate_candidate_policy(candidate_policy) + evidence=FeedbackReplayEvidence(tape.binding_digest,tape,tuple(windows), + _inputs(engine,candidate_policy=candidate_policy),_SEAL,candidate_policy) + engine.verified_feedback_tape=evidence + return evidence + + +def require_feedback_engine(engine): + from openpilot.starpilot.flm.feedback_observation import validate_subset_verified + evidence=getattr(engine,'verified_feedback_tape',None) + if (type(evidence) is not FeedbackReplayEvidence or evidence._seal is not _SEAL or + type(evidence.candidate_policy) is not str or evidence.candidate_policy not in (LEGACY_POLICY,CLOSED_POLICY) or + evidence.engine_input_digest!=_inputs(engine,candidate_policy=evidence.candidate_policy) or + tuple(bounds for bounds,_,_ in evidence.windows)!=tuple(engine.replay_windows)): + raise ValueError('owned original feedback engine evidence required') + for bounds,subset,activation in evidence.windows: + controls=tuple(frame.control_ns for frame in engine.records if bounds[0]<=frame.control_ns<=bounds[1] and + frame.control_ns not in tuple(row.control_ns for row in evidence.original.context_only)) + # Validate the owner seal/digest and exact full window again, never trust an + # instance flag or wire-created VerifiedFeedbackTape-shaped object. + validate_subset_verified(subset,controls,binding_digest=evidence.binding) + if (engine.activation_plan.activation_for(*bounds)[0] or bounds[1]+1)!=activation: + raise ValueError('feedback activation changed') + return evidence + + +def verify_candidate_engine(engine,requests,reported): + from openpilot.starpilot.flm.feedback_observation import verify_candidate_flags + evidence=require_feedback_engine(engine) + if evidence.candidate_policy != LEGACY_POLICY: + raise ValueError('closed-loop candidate requires its own native chronology proof') + for _bounds,subset,activation in evidence.windows: + selected={row.consumption.control_ns:requests[row.consumption.control_ns] for row in subset.rows} + verify_candidate_flags(subset,selected,reported,activation_ns=activation,binding_digest=evidence.binding) + return True + + +def owned_feedback_key(engine): + """New feedback evidence is part of every strict/FD/prefix owned-input key.""" + evidence=require_feedback_engine(engine) + key = (PROTOCOL,evidence.binding,evidence.original.digest, + tuple((bounds,subset.digest,activation) for bounds,subset,activation in evidence.windows)) + return key if evidence.candidate_policy == LEGACY_POLICY else (*key,evidence.candidate_policy,PROOF_VERSION) diff --git a/openpilot/starpilot/flm/feedback_wire.py b/openpilot/starpilot/flm/feedback_wire.py new file mode 100644 index 0000000000..0f5cbfe117 --- /dev/null +++ b/openpilot/starpilot/flm/feedback_wire.py @@ -0,0 +1,89 @@ +"""Sibling fork-carrier codec; never edits legacy learning evidence helpers.""" +from dataclasses import asdict +from capnp.lib.capnp import _DynamicStructReader, _DynamicStructBuilder +from openpilot.cereal import custom +from openpilot.starpilot.flm.feedback_observation import ( + CoPublication, CoConsumption, validate_publication, validate_consumption, +) + +_PUB = ( + ('version','version'),('cardSession','session'),('publicationSequence','sequence'), + ('publicationMonoTime','publication_ns'),('sourceCarStateMonoTime','output_state_ns'), + ('outputKnown','output_known'),('outputControlMonoTime','output_control_ns'), + ('outputApplyMonoTime','output_apply_ns'),('outputNativeIdentity','native_identity'), + ('outputCarrierSequence','output_carrier_sequence'),('reportedTorque','reported_torque'), + ('outputValid','valid'),('coBodySha256','co_body_sha256'),('applyEvidenceSha256','applied_sha256'), + ('sourceEpoch','source_epoch'),('cpSha256','cp_sha256'),('carrierObservedMonoTime','carrier_observed_ns'), +) +_CON = ( + ('version','version'),('controlMonoTime','control_ns'),('consumedPublicationMonoTime','publication_ns'), + ('consumedRecvFrame','recv_frame'),('consumedReceiptMonoTime','receipt_ns'),('consumedSeen','seen'), + ('consumedValid','valid'),('coBodySha256','co_body_sha256'),('reportedTorque','reported_torque'), + ('requestTorque','requested_torque'),('latActive','lat_active'),('inputSafetyLimited','input_limited'), + ('publishedSafetyLimited','published_limited'),('priorPublishControlMonoTime','prior_publish_ns'), + ('sourceEpoch','source_epoch'),('cpSha256','cp_sha256'), +) +_DATA = {'session':16,'co_body_sha256':32,'applied_sha256':32} +_BOOL = {'output_known','valid','seen','lat_active','input_limited','published_limited'} +_FLOAT = {'reported_torque','requested_torque'} +_TEXT = {'source_epoch','cp_sha256','native_identity'} + + +def _carrier(parent, schema): + if type(parent) not in (_DynamicStructReader,_DynamicStructBuilder) or parent.schema != schema: + raise ValueError('exact reserved feedback carrier schema') + + +def _write(row, fields): + values = asdict(row) + return {wire:bytes.fromhex(values[name]) if name in _DATA else values[name] for wire,name in fields} + + +def _read(row, fields): + values = {} + for wire,name in fields: + raw = getattr(row,wire) + if name in _DATA: + value = bytes(raw) + if len(value) != _DATA[name] and not (name == 'applied_sha256' and len(value) == 0): + raise ValueError('exact feedback Data length') + values[name] = value.hex() + elif name in _BOOL: + values[name] = bool(raw) + elif name in _FLOAT: + values[name] = float(raw) + elif name in _TEXT: + if len(str(raw)) > 64: + raise ValueError('feedback Text bound') + values[name] = str(raw) + else: + values[name] = int(raw) + return values + + +def write_publication(parent, row): + _carrier(parent,custom.StarPilotCarState.schema) + validate_publication(row) + parent.feedbackPublication = _write(row,_PUB) + + +def read_publication(parent): + _carrier(parent,custom.StarPilotCarState.schema) + row = parent.feedbackPublication + if row.version == 0: + return None + return validate_publication(CoPublication(**_read(row,_PUB))) + + +def write_consumption(parent, row): + _carrier(parent,custom.StarPilotLateralState.schema) + validate_consumption(row) + parent.feedbackConsumption = _write(row,_CON) + + +def read_consumption(parent): + _carrier(parent,custom.StarPilotLateralState.schema) + row = parent.feedbackConsumption + if row.version == 0: + return None + return validate_consumption(CoConsumption(**_read(row,_CON))) diff --git a/openpilot/starpilot/flm/fit_process.py b/openpilot/starpilot/flm/fit_process.py index 9265d8b40a..c3b238c87d 100644 --- a/openpilot/starpilot/flm/fit_process.py +++ b/openpilot/starpilot/flm/fit_process.py @@ -14,6 +14,10 @@ INTERVAL_NS = 60_000_000_000 class FitProcess: + worker_module = 'openpilot.starpilot.flm.fit_worker' + wall_ns = WALL_NS + interval_ns = INTERVAL_NS + def __init__(self): self.child = None self.pending = b'' @@ -25,7 +29,7 @@ class FitProcess: self.reason = 'idle' def submit(self, snapshot, *, token, now_ns): - if self.child is not None or self.last_start_ns and now_ns - self.last_start_ns < INTERVAL_NS: + if self.child is not None or self.last_start_ns and now_ns - self.last_start_ns < self.interval_ns: return False try: payload = json.dumps(snapshot, separators=(',', ':'), allow_nan=False).encode() @@ -33,7 +37,13 @@ class FitProcess: self.reason = 'fit_snapshot_invalid' self.last_start_ns = now_ns return False - if len(payload) > MAX_INPUT: + return self.submit_bytes(payload, token=token, now_ns=now_ns) + + def submit_bytes(self, payload, *, token, now_ns): + """Bounded prepared IPC; the owned worker still validates its full input.""" + if self.child is not None or self.last_start_ns and now_ns - self.last_start_ns < self.interval_ns: + return False + if type(payload) is not bytes or not 0 < len(payload) <= MAX_INPUT: self.reason = 'fit_snapshot_bound' self.last_start_ns = now_ns return False @@ -44,7 +54,7 @@ class FitProcess: environment['STARPILOT_FLM_PARENT_PID'] = str(os.getpid()) try: self.child = subprocess.Popen( - [sys.executable, '-B', '-m', 'openpilot.starpilot.flm.fit_worker'], + [sys.executable, '-B', '-m', self.worker_module], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, @@ -94,7 +104,7 @@ class FitProcess: child = self.child if child is None: return None - if token != self.token or now_ns - self.started_ns > WALL_NS: + if token != self.token or now_ns - self.started_ns > self.wall_ns: self.cancel('fit_context_changed' if token != self.token else 'fit_wall_limit') if self.reaping: if child.poll() is not None: diff --git a/openpilot/starpilot/flm/fit_worker.py b/openpilot/starpilot/flm/fit_worker.py index 04e1a148d8..80e54f7c22 100644 --- a/openpilot/starpilot/flm/fit_worker.py +++ b/openpilot/starpilot/flm/fit_worker.py @@ -32,6 +32,8 @@ def main(): if len(raw) > MAX_INPUT: raise ValueError('fit input bound') snapshot = json.loads(raw) + from openpilot.starpilot.flm.native_primitives import enable_fit_worker + enable_fit_worker() from openpilot.starpilot.flm.learner_factory import fit_snapshot # One isolated, address-space/CPU-bounded job. Reference counting remains diff --git a/openpilot/starpilot/flm/fullrate_collector.py b/openpilot/starpilot/flm/fullrate_collector.py index 3b76bdc7f4..5cc52458df 100644 --- a/openpilot/starpilot/flm/fullrate_collector.py +++ b/openpilot/starpilot/flm/fullrate_collector.py @@ -9,6 +9,7 @@ from typing import TypeGuard, cast from collections import Counter, OrderedDict, deque from openpilot.starpilot.flm.torque_eligibility import asynchronous_hint, bounded_context_output, exact_unclipped_pair, consumed_sources_current +from openpilot.starpilot.flm.native_refinement_wire import valid_inputs from dataclasses import dataclass, replace import math @@ -96,6 +97,8 @@ class ControlEvidence: applied_profile_expires_ns: int = 0 effect_trace: object | None = None effect_prepare_ns: int = 0 + vehicle_model_anchor: tuple[float, ...] = () + live_applied_coordinates: tuple[float, ...] = () @dataclass(frozen=True) @@ -162,6 +165,9 @@ class RawBlock: class FullrateCollector: def __init__(self, adapter): self.adapter = adapter + self.live_tracking = None + self.live_response_pair = None + self._last_live_ingest_ns = 0 self.session = uuid.uuid4().hex self.sequence = self.block_sequence = 0 self.controls = OrderedDict() @@ -169,6 +175,9 @@ class FullrateCollector: self.recent_controls = OrderedDict() self.recent_applied = OrderedDict() self.ready = deque(maxlen=MAX_PENDING) + # Historical NUMERICAL inputs only. Pending/live joins keep MAX_PENDING; + # the actor still enforces original2s age, source/native identity and expiry. + self.probe_ready = deque(maxlen=256) self.last_carrier_frame = 0 self.blocks = deque(maxlen=MAX_BLOCKS) self.points = [] @@ -190,7 +199,65 @@ class FullrateCollector: self.model_block_sequence = self.model_points = 0 self.model_sealed = False + def enable_original_source_capture(self, *, clock_contract=None): + """Explicit private experiment; never called by the production parent.""" + from openpilot.starpilot.flm.original_source_capture import OriginalSourceCapture, OriginalSourceClockContract + if getattr(self, 'original_source_capture', None) is not None: + raise ValueError('original source capture already enabled') + if clock_contract is None: + from openpilot.starpilot.flm.producer_clock_equivalence import pinned_default_contract + clock_contract = pinned_default_contract() + if not isinstance(clock_contract, OriginalSourceClockContract): + raise ValueError('independently pinned original producer-clock contract required') + clock_contract.validate() + enable_context = getattr(self.adapter, 'enable_original_context_capture', None) + if enable_context is None: + raise ValueError('actual original context-only archive required') + enable_context() + self.original_source_clock_contract = clock_contract + self.original_source_capture = OriginalSourceCapture() + + def begin_original_source_experiment(self, receipt): + capture = getattr(self, 'original_source_capture', None) + if capture is None: + raise ValueError('original source capture not enabled') + capture.begin_epoch(receipt) + + def reset_original_source_capture(self, reason, *, hard=False): + capture = getattr(self, 'original_source_capture', None) + if capture is None: + return + pending = getattr(self, '_original_source_pose_queue', ()) + if hard: + # Revoke every previously seen/pending source publication. Clearing seen + # alone would recapture the same old KF publication after OFF/ON. + floor = max((getattr(self, '_original_source_revoked_before_ns', 0), + getattr(self, '_original_source_seen_pose', 0), + *(row.mono_ns for row in pending), *(row.mono_ns for row in capture.rows[-1:]))) + self._original_source_revoked_before_ns = floor + for name in ('_original_source_pose_context', '_original_source_clock', + '_original_source_calibrator', '_original_source_vm_anchor', '_original_source_equivalence'): + self.__dict__.pop(name, None) + self._original_source_seen_pose = 0 + self._original_source_clock_generation = getattr(self, '_original_source_clock_generation', 0) + 1 + self._original_source_pose_queue = () + capture.reset(reason, hard=hard) + + def original_source_pose(self, value, now_ns): + capture = getattr(self, 'original_source_capture', None) + if capture is None: + return False + # The 5120 raw-row cap includes legacy retained rows and the <=4 original + # pending poses. No second 5120-row allowance is created by this opt-in. + retained = len(self.raw_points) + sum(len(block.samples) for block in self.blocks) + pending = len(getattr(self, '_original_source_pose_queue', ())) + return capture.add(value, now_ns, retained_raw_rows=retained + pending) + def split(self, reason, *, reset=False): + self.reset_original_source_capture(reason, hard=reset) + if self.live_tracking is not None: + self.live_tracking.interrupt(reason, context_changed=reset) + self.live_response_pair = None if reason in self.training_interruptions or len(self.training_interruptions) < 24: self.training_interruptions[reason] += 1 self.controls.clear() @@ -198,6 +265,7 @@ class FullrateCollector: self.recent_controls.clear() self.recent_applied.clear() self.ready.clear() + self.probe_ready.clear() self.last_carrier_frame = 0 self.points.clear() self.raw_points.clear() @@ -215,6 +283,13 @@ class FullrateCollector: self.physical_context = None self.fit_block_sequence = 0 + def _retain_original_pair(self, command, output): + # Called only AFTER existing source/receiver/native admission above. + # Keep the exact same objects and clocks; this supplies no new proof. + pair=(command,output) + self.ready.append(pair) + self.probe_ready.append(pair) + def overrun(self): self.dropped += 1 self.split('fullrate_overrun') @@ -224,6 +299,8 @@ class FullrateCollector: return False def control(self, value, now_ns): + if isinstance(value, ControlEvidence) and (value.manual or not value.enabled): + self.reset_original_source_capture('original_source_disabled_or_manual', hard=True) if isinstance(value, ControlEvidence) and value.effect_trace is not None: try: from openpilot.starpilot.flm.effect_trace import FullEffectTrace @@ -268,7 +345,7 @@ class FullrateCollector: or not value.policy or not value.controller_revision or not replay_values(value.replay_snapshot, 1024) - or not replay_values(value.replay_inputs, 32) + or not valid_inputs(value.replay_inputs) or type(value.snapshot_sequence) is not int or value.snapshot_sequence < 0 or type(value.turn_assist) is not bool @@ -343,6 +420,10 @@ class FullrateCollector: if self.controls and value.control_ns <= next(reversed(self.controls)): self.split('reordered_control') return False + self._last_live_ingest_ns = now_ns + if self.live_tracking is not None: + self.live_tracking.observe_source_control(value, now_ns=now_ns, + current_native=getattr(self, '_learning_current_native_allowed', False)) self.controls[value.control_ns] = value self.recent_controls[value.control_ns] = value while len(self.recent_controls) > MAX_PENDING: @@ -353,6 +434,9 @@ class FullrateCollector: return True def output(self, value, now_ns): + context_only = getattr(self.adapter, 'observe_original_applied', None) + if context_only is not None: + context_only(value) # Original owner receipt, never ready/plant permission. if ( not isinstance(value, AppliedEvidence) or not finite(value.applied) @@ -394,6 +478,7 @@ class FullrateCollector: self.split('changed_duplicate_output', reset=True) return False return True + self._last_live_ingest_ns = now_ns self.applied[key] = value self.recent_applied[key] = value while len(self.recent_applied) > MAX_PENDING: @@ -457,7 +542,14 @@ class FullrateCollector: if record is None or not record(command, output): self.split('actual_emission_recorder_unavailable', reset=True) return - self.ready.append((command, output)) + self._retain_original_pair(command, output) + if self.live_tracking is not None: + try: + profile = self.live_tracking.profile(command) + self.live_tracking.observe_control(command, output, profile_sha256=profile, + now_ns=self._last_live_ingest_ns, current_native=getattr(self, '_learning_current_native_allowed', False)) + except (AttributeError, KeyError, TypeError, ValueError): + self.live_tracking.interrupt('actual_tracking_native_pair_unqualified') def physical(self, value, now_ns): if ( diff --git a/openpilot/starpilot/flm/global_runtime.py b/openpilot/starpilot/flm/global_runtime.py index 9aeb53c429..1a18ccfe14 100644 --- a/openpilot/starpilot/flm/global_runtime.py +++ b/openpilot/starpilot/flm/global_runtime.py @@ -3,6 +3,7 @@ import math from collections import OrderedDict from types import SimpleNamespace +from typing import cast from openpilot.starpilot.flm.adaptive_profile import ApplicationOutcome, Parameter, ProfileOwner from openpilot.starpilot.flm.runtime_adapter_registry import ( selected_descriptor, @@ -30,6 +31,7 @@ def native_witness_current(native, *, now_ns, active): SERVICE = 'autoSteeringLearning' +TRACKING_SERVICE = 'starpilotSteeringTracking' NON_TORQUE_SCHEMAS = { 'pid': 'pid-ff-pre-pid-v1', 'angle': 'angle-no-offset-v1', @@ -39,11 +41,12 @@ NON_TORQUE_SCHEMAS = { class GlobalRuntime: - def __init__(self, cp, ci, controller): + def __init__(self, cp, ci, controller, *, confirmed_live_history=()): # Construct AFTER actual LaC selection, startup native/manual configuration. self.cp, self.ci, self.controller = cp, ci, controller from openpilot.starpilot.flm.calibration_lifecycle import ConsumerCalibrationLifecycle self._calibration_lifecycle = ConsumerCalibrationLifecycle() + self._calibration_pending: tuple[str, int] | None = None self.health_services = ('carState', 'modelV2', 'vehicleParameters', 'lateralDelay', 'driverMonitoringState', 'extrinsicsCalibration') try: self.descriptor = selected_descriptor(cp, ci, controller) @@ -68,6 +71,11 @@ class GlobalRuntime: self.session = '' self.outer_sequence = -1 self._runtime_admission_cache = None + self.live_trial_owner = None + self._live_trial_delivery_cache = None + self._confirmed_live_history = tuple(confirmed_live_history) + self.live_vehicle_model_generation = '' + self._live_vm_anchor = None self.last_now = 0 self.used = False self.eligible = False @@ -112,6 +120,8 @@ class GlobalRuntime: def _neutral(self, reason): self.controller.automatic_flm_effects = None + if self.live_trial_owner is not None: + self.live_trial_owner.neutral(reason) if self.effect_owner is not None: self.effect_owner.source = None self.effect_owner.trace = None @@ -128,30 +138,73 @@ class GlobalRuntime: def prepare(self, sm, *, now_ns, active, healthy, controller=None, manual_current=False, native_required=False, native_witness=None): """Bounded current source/admission checks; invoked immediately before LaC.""" self.used, self.stage, self.frame, self.eligible = False, None, None, False + self.outcome = None self.controller.automatic_flm_effects = None self.source = None # Capture a fresh V2 generation and calibration interruption before ANY # unrelated adapter/native/vehicle health gate can return. + receipt_service = SERVICE + if TRACKING_SERVICE in sm.seen and sm.seen[TRACKING_SERVICE] and int(sm[TRACKING_SERVICE].version)!=0: + receipt_service = TRACKING_SERVICE + live_delivery = receipt_service == TRACKING_SERVICE tracker = self._calibration_lifecycle early_source = None try: if (type(now_ns) is int and now_ns > 0 and self.descriptor is not None and - sm.seen[SERVICE] and sm.alive[SERVICE] and sm.valid[SERVICE] and sm.freq_ok[SERVICE] and - 0 <= now_ns - int(sm.recv_time[SERVICE] * 1e9) <= SOURCE_LEASE_NS): - envelope = sm[SERVICE] - if envelope.learning.sourceReceipt.version in (2, 3): - early_source = read_source_receipt(SimpleNamespace(valid=True, autoSteeringLearning=envelope), - now_ns=now_ns, cp_sha256=self.descriptor.cp_sha256) - if early_source is not None and not tracker.generation: - tracker.capture(early_source.calibration_generation) + sm.seen[receipt_service] and sm.alive[receipt_service] and sm.valid[receipt_service] and sm.freq_ok[receipt_service] and + 0 <= now_ns - int(sm.recv_time[receipt_service] * 1e9) <= SOURCE_LEASE_NS): + envelope = sm[receipt_service] + receipt = envelope.sourceReceipt if live_delivery else envelope.learning.sourceReceipt + if receipt.version in (2, 3): + if live_delivery: + from openpilot.starpilot.flm.live_trial_wire import read_tracking_source + early_source = read_tracking_source(envelope,now_ns=now_ns,cp_sha256=self.descriptor.cp_sha256) + else: + early_source = read_source_receipt(SimpleNamespace(valid=True, autoSteeringLearning=envelope), + now_ns=now_ns, cp_sha256=self.descriptor.cp_sha256) stamp = int(sm.logMonoTime['extrinsicsCalibration']) receipt = int(sm.recv_time['extrinsicsCalibration'] * 1e9) current_calibration = bool(type(now_ns) is int and now_ns > 0 and sm.all_checks(['extrinsicsCalibration']) and 0 < stamp <= now_ns <= stamp + 500_000_000 and 0 < receipt <= now_ns <= receipt + 500_000_000) + if not tracker.generation: + from openpilot.starpilot.flm.calibration_lifecycle import calibrated_basis + pending = self._calibration_pending + generation = early_source.calibration_generation if early_source is not None else None + if pending is not None and generation is not None and generation != pending[0]: + tracker.retire(pending[0]) + pending = self._calibration_pending = None + calibration = calibrated_basis(sm['extrinsicsCalibration']) + frequency_only = bool(not current_calibration and type(now_ns) is int and now_ns > 0 and + sm.seen['extrinsicsCalibration'] and sm.alive['extrinsicsCalibration'] and sm.valid['extrinsicsCalibration'] and + not sm.freq_ok['extrinsicsCalibration'] and + 0 < stamp <= now_ns <= stamp + 500_000_000 and 0 < receipt <= now_ns <= receipt + 500_000_000 and + calibration is not None) + defer_binding = bool(frequency_only and generation is not None and not tracker.exhausted and + generation not in tracker.rejected and + (pending is None or calibration is not None and calibration[1] >= pending[1])) + if defer_binding: + assert generation is not None and calibration is not None + self._calibration_pending = (generation, calibration[1]) + else: + # Invalid/stale first calibration and pending regressions retain the + # original capture-before-observe rejection. No pending row admits. + if generation is None and pending is not None and not current_calibration: + generation = pending[0] + if generation is not None: + tracker.capture(generation) + if pending is not None and generation == pending[0]: + tracker.blocks = pending[1] + self._calibration_pending = None tracker.observe(sm['extrinsicsCalibration'], current=current_calibration, now_ns=now_ns) except (AttributeError, KeyError, TypeError, ValueError, OverflowError): + if not tracker.generation: + if early_source is not None: + tracker.capture(early_source.calibration_generation) + elif self._calibration_pending is not None: + tracker.capture(self._calibration_pending[0]) + self._calibration_pending = None tracker.reject(now_ns=now_ns) self.outcome = None self.centre_frame = None @@ -170,6 +223,7 @@ class GlobalRuntime: self.native_identity_cache = (key, native_identity_digest(key)) self.native_identity = self.native_identity_cache[1] self.active, self.healthy, self.now_ns = bool(active), bool(healthy), now_ns + self._live_sm = sm if self.descriptor is None or controller is not None and controller is not self.controller: self._neutral('actual_adapter_unavailable') return @@ -218,14 +272,18 @@ class GlobalRuntime: raise ValueError('stale_original_source') if self.controller.sat_time > 0: raise ValueError('controller_saturation') - if not (sm.seen[SERVICE] and sm.alive[SERVICE] and sm.valid[SERVICE] and sm.freq_ok[SERVICE]): + if not (sm.seen[receipt_service] and sm.alive[receipt_service] and sm.valid[receipt_service] and sm.freq_ok[receipt_service]): raise ValueError('source_unavailable') - received = int(sm.recv_time[SERVICE] * 1e9) + received = int(sm.recv_time[receipt_service] * 1e9) if not 0 <= now_ns - received <= SOURCE_LEASE_NS: raise ValueError('stale_receipt') - envelope = sm[SERVICE] + envelope = sm[receipt_service] event = SimpleNamespace(valid=True, autoSteeringLearning=envelope) - source = early_source if early_source is not None else read_source_receipt(event, now_ns=now_ns, cp_sha256=self.descriptor.cp_sha256) + if live_delivery: + from openpilot.starpilot.flm.live_trial_wire import read_tracking_source + source = early_source if early_source is not None else read_tracking_source(envelope,now_ns=now_ns,cp_sha256=self.descriptor.cp_sha256) + else: + source = early_source if early_source is not None else read_source_receipt(event, now_ns=now_ns, cp_sha256=self.descriptor.cp_sha256) if ( source is None or source.cp_sha256 != self.descriptor.cp_sha256 @@ -248,6 +306,7 @@ class GlobalRuntime: self.generation += 1 self.owner = self.torque_owner = self.centre_source = None self.effect_owner = self.effect_envelope = None + self.live_trial_owner = None self.effect_full_hash = '' self.profile = self.validation = None self.snapshot_ns = self.snapshot_sequence = 0 @@ -257,6 +316,18 @@ class GlobalRuntime: self.eligible = bool(source.enabled and not source.manual and not self.live_manual and active and healthy) if not source.enabled or source.manual or self.live_manual or not active: raise ValueError('manual_or_off_or_inactive') + live_offer, live_restore = self._current_live_trial(sm,envelope) if live_delivery else (None,None) + if live_offer is not None and live_restore is not None: + raise ValueError('simultaneous trial and history restore') + if live_offer is not None or live_restore is not None: + self._prepare_live_trial(live_offer or live_restore, source, now_ns=now_ns, active=active, healthy=healthy, restoring=live_restore is not None) + self.reason = 'qualified' + return + if live_delivery: + raise ValueError('live_tracking_collecting') + if self.effect_owner is self.live_trial_owner and self.live_trial_owner is not None: + self.live_trial_owner.neutral('live_offer_withdrawn') + self.effect_owner = None admission = self._current_runtime_admission(sm,event) if admission is None: raise ValueError('collecting') @@ -332,6 +403,71 @@ class GlobalRuntime: except (AttributeError, KeyError, TypeError, ValueError, OverflowError): self._neutral('unavailable') + def _current_live_trial(self, sm, envelope): + from capnp.lib.capnp import _DynamicStructReader + from openpilot.starpilot.flm.live_trial_wire import read_trial,read_restore + if int(envelope.version) != 1: + raise ValueError('unknown live tracking delivery version') + frame = sm.recv_frame[TRACKING_SERVICE] + if type(frame) is not int or not isinstance(envelope,_DynamicStructReader): + self._live_trial_delivery_cache = None + return read_trial(envelope),read_restore(envelope) + key = (frame,int(sm.logMonoTime[TRACKING_SERVICE]),float(sm.recv_time[TRACKING_SERVICE])) + cached = self._live_trial_delivery_cache + if cached is not None and cached[0] == key and cached[1] is envelope: + return cached[2] + value = (read_trial(envelope),read_restore(envelope)) + self._live_trial_delivery_cache = (key,envelope,value) + return value + + def _prepare_live_trial(self, offer, source, *, now_ns, active, healthy, restoring=False): + from openpilot.starpilot.flm.live_trial_owner import TrackingTrialOwner + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + # Live experiments always require the actual current native permission, + # ordinary Panda permission and typed axis grants remain distinct domains. + if self.native_required: + if not native_witness_current(self.native_witness, now_ns=now_ns, active=active): + raise ValueError('live_native_permission_unavailable') + else: + from openpilot.starpilot.flm.runtime_adapter_registry import current_authority + if not current_authority(self._live_sm,self.cp,now_ns=now_ns,required=False)[0]: + raise ValueError('live_ordinary_native_permission_unavailable') + if not self.live_vehicle_model_generation: + raise ValueError('live_vehicle_model_unobserved') + expected = LiveTrialBinding(self.identity,self.descriptor.cp_sha256,self.descriptor.controller_revision, + self.baseline_revision,source.source_epoch,source.reset,source.calibration_generation, + self.live_vehicle_model_generation,self.native_identity if self.native_required else '',offer.binding.physical_model_revision, + source.persistent_source_epoch,self.native_required).validate() + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + validate_response_witness(offer.response_witness,expected,now_ns) + if offer.binding != expected: + raise ValueError('live_current_source_binding_changed') + if self.descriptor.domain == 'torque_pid': + from openpilot.starpilot.flm.pid_stages import SUPPORTED, coordinates_supported + if (not restoring and any(name not in SUPPORTED for name in offer.names)) or not coordinates_supported(dict(offer.target)): + raise ValueError('PID live trial requires one of26 supported hooks') + owner = self.live_trial_owner + if owner is None or owner.binding != expected: + if owner is not None: + owner.neutral('live_binding_changed') + owner = TrackingTrialOwner(expected,domain=self.descriptor.domain) + self.live_trial_owner = owner + permission = bool(active and healthy and self.eligible) + if restoring: + from openpilot.starpilot.flm.live_trial_store import LiveConfirmedPrior,scope,stable_binding + priors = tuple(prior for prior in self._confirmed_live_history if type(prior) is LiveConfirmedPrior and + prior.historical_only is True and scope(prior.binding)==offer.history_scope and stable_binding(prior.binding)==stable_binding(expected)) + if len(priors)!=1 or not owner.prepare_restore(offer,priors[0],now_ns=now_ns,permission=permission,binding=expected): + raise ValueError('confirmed_history_not_currently_restorable') + elif not owner.prepare(offer, now_ns=now_ns, permission=permission, binding=expected): + raise ValueError('live_current_permission_unavailable') + self.owner = self.torque_owner = self.centre_source = None + self.profile = self.validation = None + self.effect_envelope = None + self.effect_owner = owner + self.effect_full_hash = offer.target_sha256 + self.controller.automatic_flm_effects = owner + def _current_runtime_admission(self, sm, event): # SubMaster alone owns recv_frame: a new delivery ALWAYS changes this key. # Keep the original immutable reader alive so Python ID reuse cannot hit it. @@ -382,7 +518,7 @@ class GlobalRuntime: 'context_key': source.source_epoch, 'persistent_source_epoch': source.persistent_source_epoch, 'current_full_profile_sha256': envelope.prior_full_profile_sha256 if renewing else current_hash} admit_context(envelope,profile=profile,**context) - previous = self.effect_owner + previous = cast(EffectOwner | None, self.effect_owner) if (previous is None or previous.objective != objective or previous.physical_model_revision != validation.physical_model_revision or not renewing): owner = EffectOwner(self.identity, objective, selected_baseline_revision=self.baseline_revision, @@ -439,6 +575,14 @@ class GlobalRuntime: ('model_source_ns.high', float(int(model_source_ns) >> 32)), ('model_source_ns.low', float(int(model_source_ns) & 0xFFFFFFFF)), ) + # Original receipt metadata only. No controller update, model, new native + # permission or full checkpoint is produced here. + provenance = getattr(self.controller, 'native_learning_provenance', ()) + if provenance: + from openpilot.starpilot.flm.native_refinement_wire import FIELD, encode + compact = encode(tuple(provenance)) + if compact is not None: + self.replay_inputs += ((FIELD, compact),) regime = vm_regime(dict(self.replay_inputs), self.vm_anchor) if regime is None or self.vm_anchor is not None and regime != self.vm_anchor: if self.validation is not None: @@ -446,6 +590,10 @@ class GlobalRuntime: self.eligible = False self._neutral('vehicle_model_material_change') self.vm_anchor = regime + if regime != self._live_vm_anchor: + from openpilot.starpilot.flm.runtime_adapter_registry import digest + self.live_vehicle_model_generation = '' if regime is None else digest(regime) + self._live_vm_anchor = regime if self.observation is not None: self.observation.begin_frame() policy = getattr(getattr(self.controller, 'starpilot_extension', None), 'policy', None) @@ -571,6 +719,13 @@ class GlobalRuntime: trace = effect.trace if effect is not None else None has_effect = any(effect.applied_full.values()) if effect is not None else any(coordinates) applied = bool(self.used and self.stage_effect and self.reason == 'qualified' and has_effect and self.active and self.healthy) + if effect is self.live_trial_owner and effect is not None: + application = effect.application + if application is not None: + actual = bool(effect.used and effect.frame_complete(self.descriptor.domain) and + self.active and self.healthy and self.eligible and self.reason == 'qualified') + self.outcome = effect.acknowledge_frame(actual) + return self.outcome = ApplicationOutcome( self.identity, self.profile.sequence if self.profile is not None else 0, @@ -738,6 +893,34 @@ class GlobalRuntime: self.centre_history[control_ns] = (key, min(self.profile.expires_ns, self.source.expires_ns), tuple(p.name for p in self.profile.parameters)) while len(self.centre_history) > 64: self.centre_history.popitem(last=False) + # This is the actual hook-consumed map, never the offered target. The + # preparation stamp and publication stamp are distinct but same-frame. + from openpilot.starpilot.flm.live_trial_owner import LiveTrialApplication + live_values = () + application = self.outcome + live_owner = self.live_trial_owner + current_offer = None if live_owner is None else (live_owner.restore_offer or live_owner.offer) + if ( + type(application) is LiveTrialApplication + and live_owner is not None + and current_offer is not None + and self.effect_owner is live_owner + and self.controller.automatic_flm_effects is live_owner + and live_owner.frame_complete(self.descriptor.domain) + and application.status in ('applied', 'restored') + and application.binding == live_owner.binding + and application.control_ns == self.now_ns + and 0 <= control_ns - application.control_ns <= 30_000_000 + and current_offer.observed_ns <= control_ns <= current_offer.expires_ns + and self.source.observed_ns <= control_ns <= self.source.expires_ns + and active and self.healthy and self.eligible and self.reason == 'qualified' + and self.source.enabled and not (self.source.manual or self.live_manual) + and (not self.native_required or native_allowed) + ): + live_values = tuple(value for _, value in application.applied) + if live_owner is not None and self.controller.automatic_flm_effects is live_owner and not live_values: + # A consumed but unqualified live map is unknown, never legacy neutral. + return None return ControlEvidence( self.identity, self.generation, @@ -784,4 +967,6 @@ class GlobalRuntime: applied_profile_expires_ns=self.profile.expires_ns if current_profile else 0, effect_trace=self.effect_owner.trace if self.controller.automatic_flm_effects is self.effect_owner and self.effect_owner is not None else None, effect_prepare_ns=self.now_ns, + vehicle_model_anchor=() if self.vm_anchor is None else tuple(self.vm_anchor), + live_applied_coordinates=live_values, ) diff --git a/openpilot/starpilot/flm/learner_factory.py b/openpilot/starpilot/flm/learner_factory.py index 2862996e65..b473333e35 100644 --- a/openpilot/starpilot/flm/learner_factory.py +++ b/openpilot/starpilot/flm/learner_factory.py @@ -89,6 +89,8 @@ def _frame(evidence): applied_parameter_names=tuple(evidence.applied_parameter_names), effect_trace=evidence.effect_trace, effect_prepare_ns=evidence.effect_prepare_ns, + vehicle_model_anchor=evidence.vehicle_model_anchor, + live_applied_coordinates=evidence.live_applied_coordinates, ) @@ -162,6 +164,7 @@ class TorqueCollectionAdapter: baseline_revision=evidence.baseline_revision, model_revision='raw-child-v1', ) + self.replay.source_controller_revision = descriptor.controller_revision self.snapshot_sequence = 0 self.model = None self.model_cutoff_ns = 0 @@ -459,6 +462,11 @@ def fit_snapshot(snapshot): payload = unpack_primitive_snapshot(base64.b64decode(snapshot['snapshot_b64'], validate=True)) if payload.get('version') != 1 or payload.get('family') != snapshot.get('family'): raise ValueError('fit family') + if 'plant_protocol' in payload: + if tuple(snapshot.get('objective_key',())) != tuple(payload.get('objective_key',())): + raise ValueError('source experiment wrapper objective key') + from openpilot.starpilot.flm.source_plant_pipeline import fit_snapshot as fit_source_plant + return fit_source_plant(payload) event_version = payload.get('event_version') event_job = type(event_version) is int and event_version in (2, 3, 4, 5, 6) if 'event_version' in payload and not event_job: diff --git a/openpilot/starpilot/flm/learning_wire.py b/openpilot/starpilot/flm/learning_wire.py index 02083193f1..d9861da068 100644 --- a/openpilot/starpilot/flm/learning_wire.py +++ b/openpilot/starpilot/flm/learning_wire.py @@ -10,6 +10,7 @@ from capnp.lib.capnp import _DynamicStructReader from openpilot.cereal import custom, messaging from openpilot.starpilot.flm.adaptive_profile import Parameter, Profile, ReplayValidation, replay_accepts from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence, identity, finite +from openpilot.starpilot.flm.native_refinement_wire import valid_inputs, EXTENDED_INPUTS class _FullSurfaceFields(TypedDict): @@ -69,6 +70,13 @@ def _read_named_values(values, bound): return tuple(result) +def _read_original_inputs(values): + result = _read_named_values(values, EXTENDED_INPUTS) + if not valid_inputs(result): + raise ValueError('versioned original replay input bounds') + return result + + def _write_effect_trace(trace): from openpilot.starpilot.flm.effect_trace import FullEffectTrace if not isinstance(trace, FullEffectTrace): @@ -124,6 +132,11 @@ def write_control(parent, evidence): or not identity(evidence.identity) or not 1 <= len(evidence.features) <= MAX_FIELDS or not all(finite(v) for v in (*evidence.features, evidence.requested, evidence.baseline_prediction)) + or not valid_inputs(evidence.replay_inputs) + or len(evidence.live_applied_coordinates) not in (0, 30) + or not all(finite(v) for v in evidence.live_applied_coordinates) + or len(evidence.vehicle_model_anchor) not in (0, 3) + or not all(finite(v) and v != 0.0 for v in evidence.vehicle_model_anchor) ): raise ValueError('control evidence') parent.learningEvidence = { @@ -151,6 +164,8 @@ def write_control(parent, evidence): 'replaySnapshot': _named_values(evidence.replay_snapshot), 'replayInputs': _named_values(evidence.replay_inputs), 'snapshotSequence': evidence.snapshot_sequence, + 'vehicleModelAnchor': evidence.vehicle_model_anchor, + 'liveAppliedCoordinates': evidence.live_applied_coordinates, 'turnAssist': evidence.turn_assist, 'appliedParameterNames': evidence.applied_parameter_names, 'nativeRequired': evidence.native_required, @@ -194,6 +209,10 @@ def read_control(event): or len(row.stageValues) > 40 or len(row.baselineValues) > 32 or len(row.appliedParameterNames) > 8 + or len(row.liveAppliedCoordinates) not in (0, 30) + or not all(finite(float(v)) for v in row.liveAppliedCoordinates) + or len(row.vehicleModelAnchor) not in (0, 3) + or not all(finite(float(v)) and float(v) != 0.0 for v in row.vehicleModelAnchor) ): raise ValueError('control version or bound') return ControlEvidence( @@ -228,7 +247,7 @@ def read_control(event): tuple(float(v) for v in row.baselineValues), str(row.controllerRevision), _read_named_values(row.replaySnapshot, 1024), - _read_named_values(row.replayInputs, 32), + _read_original_inputs(row.replayInputs), int(row.snapshotSequence), bool(row.turnAssist), tuple(_string(str(name)) for name in row.appliedParameterNames), @@ -242,6 +261,8 @@ def read_control(event): int(row.appliedProfileValidUntilMonoTime), _read_effect_trace(row.effectTrace), int(row.effectPrepareMonoTime), + tuple(float(v) for v in row.vehicleModelAnchor), + tuple(float(v) for v in row.liveAppliedCoordinates), ) diff --git a/openpilot/starpilot/flm/live_tracking.py b/openpilot/starpilot/flm/live_tracking.py new file mode 100644 index 0000000000..cb56d8a73b --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking.py @@ -0,0 +1,1340 @@ +"""Optional learner's observation-triggered trials, never Controls permission. + +Inputs are completed whole episodes from the genuine fullrate/physical join. +This module neither replays a route nor changes controller/native limits. +Application is an explicit separate lease, not a ReplayValidation token. +""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Sequence +from dataclasses import dataclass +from functools import lru_cache +from typing import TYPE_CHECKING, Literal, NotRequired, TypedDict +import hashlib +import math +import statistics +from openpilot.starpilot.flm.automatic import T95 + +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, GROUPS, coordinate_bounds, full_profile_sha256 +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding, LiveTrialOffer, VERSION, MAX_STEP, MAX_LEASE_NS + +if TYPE_CHECKING: + from openpilot.starpilot.flm.live_tracking_response import ResponseCertificate, ValidatedResponseWitness + +RETRY_NS = 20_000_000_000 +TRIAL_TIMEOUT_NS = 120_000_000_000 +MAX_EPISODES = 24 +MAX_DISCOVERY_EPISODES = 128 + + +def _beta_fraction(a: float, b: float, x: float) -> float: + """Bounded continued fraction for the Student survival integral.""" + tiny = 1e-300 + c, d = 1.0, 1.0 - (a + b) * x / (a + 1.0) + d = 1.0 / (d if abs(d) > tiny else tiny) + value = d + for m in range(1, 129): + for numerator in (m * (b - m) * x / ((a + 2 * m - 1) * (a + 2 * m)), -(a + m) * (a + b + m) * x / ((a + 2 * m) * (a + 2 * m + 1))): + d = 1.0 + numerator * d + c = 1.0 + numerator / c + d = 1.0 / (d if abs(d) > tiny else tiny) + c = c if abs(c) > tiny else tiny + delta = c * d + value *= delta + if abs(delta - 1.0) < 1e-13: + return value + raise ValueError('Student confidence integration did not converge') + + +def _student_survival(t: float, df: int) -> float: + x, a, b = df / (df + t * t), df / 2.0, 0.5 + if x == 1.0: + return 0.5 + scale = math.exp(math.lgamma(a + b) - math.lgamma(a) - math.lgamma(b) + a * math.log(x) + b * math.log1p(-x)) + beta = scale * _beta_fraction(a, b, x) / a if x < (a + 1.0) / (a + b + 2.0) else 1.0 - scale * _beta_fraction(b, a, 1.0 - x) / b + return 0.5 * beta + + +@lru_cache(maxsize=256, typed=True) +def _critical(df: int, alpha: float) -> float: + if type(df) is not int or df < 1 or type(alpha) is not float or not 0 < alpha < 0.5: + raise ValueError('bounded Student confidence arguments') + low, high = 0.0, 1.0 + while _student_survival(high, df) > alpha and high < 1e8: + high *= 2.0 + if _student_survival(high, df) > alpha: + raise ValueError('Student confidence outside numerical range') + log_scale = math.lgamma((df + 1) / 2.0) - math.lgamma(df / 2.0) - 0.5 * math.log(df * math.pi) + value = (low + high) / 2.0 + for _ in range(32): + tail = _student_survival(value, df) + if tail > alpha: + low = value + else: + high = value + # Verify a conservative upper root with the SAME survival function. No + # approximate normal tail, rounded df or lower confidence is substituted. + if abs(tail - alpha) <= alpha * 1e-13: + upper = math.nextafter(value + 2e-12 * max(1.0, abs(value)), math.inf) + if _student_survival(upper, df) <= alpha: + return upper + pdf = math.exp(log_scale - (df + 1) / 2.0 * math.log1p(value * value / df)) + candidate = value + (tail - alpha) / pdf if pdf > 0 else math.nan + value = candidate if low < candidate < high else (low + high) / 2.0 + for _ in range(64): + middle = (low + high) / 2.0 + if _student_survival(middle, df) > alpha: + low = middle + else: + high = middle + return high + + +@dataclass(frozen=True) +class HookSensitivity: + name: str + error_feature_sum: float + feature_energy: float + native_separated: int + native_endpoints: int + max_step_physical_rms: float # RMS physical acceleration DELTA at .005, after native encoding and retained slope. + native_gradient_complete: bool = False + + def validate(self): + if ( + type(self.name) is not str + or self.name not in NAMES + or type(self.native_gradient_complete) is not bool + or any(type(value) not in (int, float) or not math.isfinite(value) for value in (self.error_feature_sum, self.feature_energy, self.max_step_physical_rms)) + or self.feature_energy < 0 + or self.max_step_physical_rms < 0 + or type(self.native_separated) is not int + or type(self.native_endpoints) is not int + or not 0 <= self.native_separated <= self.native_endpoints <= 240 + or self.native_gradient_complete + and self.native_endpoints < 4 + ): + raise ValueError('actual finite hook/native probe support required') + return self + + +@dataclass(frozen=True) +class TrackingEpisode: + """Numerical evidence only; input adapters must verify actual source joins. + + sensitivity is an algorithmic actual-controller probe, NOT a plant fit. + The response model/lag and future trial score are independently measured. + """ + + binding: LiveTrialBinding + profile_sha256: str + episode_id: str + start_ns: int + end_ns: int + kind: str + direction: int + speed_low: float + speed_high: float + goal_rms: float + error_rms: float + reversal_hz: float + endpoints: int + control_max_gap_ns: int + sensitivity: tuple[HookSensitivity, ...] + goal_shape: tuple[float, ...] = () + native_gradient_complete: bool = False + normalization_accel: float | None = None + + def validate(self): + if self.normalization_accel is not None and ( + type(self.normalization_accel) not in (int, float) or not math.isfinite(self.normalization_accel) or self.normalization_accel != max(0.1, self.goal_rms) + ): + raise ValueError('explicit kinematic tracking normalization required') + if ( + type(self.binding) is not LiveTrialBinding + or type(self.kind) is not str + or self.kind not in ('large', 'subtle') + or type(self.direction) is not int + or self.direction not in ((-1, 0, 1) if self.normalization_accel is not None else (-1, 1)) + or type(self.native_gradient_complete) is not bool + or any(type(v) is not int for v in (self.start_ns, self.end_ns, self.endpoints, self.control_max_gap_ns)) + or not 0 < self.start_ns < self.end_ns + or not 200_000_000 <= self.end_ns - self.start_ns <= 10_000_000_000 + or not 4 <= self.endpoints <= 240 + or not 0 < self.control_max_gap_ns <= 50_000_000 + or any(type(v) not in (int, float) or not math.isfinite(v) for v in (self.speed_low, self.speed_high, self.goal_rms, self.error_rms, self.reversal_hz)) + or not 0 < self.speed_low <= self.speed_high <= 1.15 * self.speed_low + or min(self.goal_rms, self.error_rms, self.reversal_hz) < 0 + or type(self.sensitivity) is not tuple + or len(self.sensitivity) > 30 + or type(self.goal_shape) is not tuple + or len(self.goal_shape) != 8 + or any(type(value) not in (int, float) or not math.isfinite(value) for value in self.goal_shape) + ): + raise ValueError('complete real tracking episode required') + self.binding.validate() + if any(type(value) is not str or len(value) != 64 or any(c not in '0123456789abcdef' for c in value) for value in (self.profile_sha256, self.episode_id)): + raise ValueError('exact actual episode/profile identities required') + names = set() + for row in self.sensitivity: + if type(row) is not HookSensitivity: + raise ValueError('actual independent hook probe required') + row.validate() + if row.name in names: + raise ValueError('distinct actual hook probes required') + names.add(row.name) + return self + + +def _map(values: tuple[tuple[str, float], ...]) -> dict[str, float]: + if type(values) is not tuple or tuple(name for name, _ in values) != NAMES: + raise ValueError('canonical complete thirty-coordinate target required') + result = {} + for name, value in values: + low, high = coordinate_bounds(name) + if type(value) not in (int, float) or not math.isfinite(value) or not low <= value <= high: + raise ValueError('original own-car coordinate domain') + result[name] = float(value) + return result + + +def _required(kind: str) -> int: + return 3 if kind == 'large' else 9 + + +def _look_quotas(required: int, feasible: int) -> tuple[int, ...]: + later = min(2 * required, feasible, MAX_EPISODES) + return (required, later) if required >= 9 and later >= required + 3 else (required,) + + +def _discovery_support(rows, kind, model_rmse): + # Maneuver geometry and miss severity are separate. A modest tracking miss + # in a substantial curve needs the longer small-error evidence path, rather + # than being excluded by the large-error floor forever. + large = tuple(rows[-3:]) + if kind == 'large' and len(large) == 3 and all(row.error_rms >= (0.10 if model_rmse is None else max(0.10, 6 * model_rmse)) for row in large): + return large, 3 + small = tuple(rows[-9:]) + if len(small) == 9 and all(row.error_rms >= (0.015 if model_rmse is None else max(0.015, 3 * model_rmse)) for row in small): + return small, 9 + return (), None + + +def _kind_for_group(group: str, kind: str) -> bool: + # Actual native probes determine which hook affects this maneuver. A small + # tracking oscillation can need feedforward/friction changes too; the label + # alone must not exclude a measured controller effect. Confidence, native + # visibility and independent future comparisons remain mandatory. + return group in GROUPS and kind in ('large', 'subtle') + + +def _group_direction_matches(group: str, kind: str, direction: int) -> bool: + return kind != 'large' or not (group.endswith('.left') and direction < 0 or group.endswith('.right') and direction > 0) + + +@dataclass(frozen=True) +class DemandSignature: + kind: str + direction: int + duration_ns: int + speed_low: float + speed_high: float + goal_rms: float + goal_shape: tuple[float, ...] + normalization_accel: float | None = None + + +def _demand(episode: TrackingEpisode) -> DemandSignature: + return DemandSignature( + episode.kind, + episode.direction, + episode.end_ns - episode.start_ns, + episode.speed_low, + episode.speed_high, + episode.goal_rms, + episode.goal_shape, + episode.normalization_accel, + ) + + +@lru_cache(maxsize=256) +def _input_shape_descriptor(shape): + return sum(shape) / 8, math.sqrt(sum((a - b) ** 2 for a, b in zip(shape, shape[1:], strict=False)) / 7) + + +def _matches_demand(left: DemandSignature, right: TrackingEpisode) -> bool: + if left.normalization_accel is not None and right.normalization_accel is not None: + # Fixed input windows need not replay an identical waveform phase. Match + # steering demand magnitude, direction/mean and rate content instead. No + # measured error, native result or future score chooses these descriptors. + left_mean, left_rate = _input_shape_descriptor(left.goal_shape) + right_mean, right_rate = _input_shape_descriptor(right.goal_shape) + return ( + left.kind == right.kind + and (left.kind != 'large' or left.direction == right.direction) + and max(left.duration_ns, right.end_ns - right.start_ns) <= 1.15 * min(left.duration_ns, right.end_ns - right.start_ns) + and max(left.speed_high, right.speed_high) <= 1.15 * min(left.speed_low, right.speed_low) + and min(_score_scale(left), _score_scale(right)) > 0 + and max(_score_scale(left), _score_scale(right)) <= 1.25 * min(_score_scale(left), _score_scale(right)) + and abs(left_mean - right_mean) <= 0.25 + and max(0.15, left_rate, right_rate) <= 1.5 * max(0.15, min(left_rate, right_rate)) + ) + return ( + left.kind == right.kind + and left.direction == right.direction + and max(left.duration_ns, right.end_ns - right.start_ns) <= 1.15 * min(left.duration_ns, right.end_ns - right.start_ns) + and max(abs(a - b) for a, b in zip(left.goal_shape, right.goal_shape, strict=True)) <= 0.15 + and max(left.speed_high, right.speed_high) <= 1.15 * min(left.speed_low, right.speed_low) + and min(_score_scale(left), _score_scale(right)) > 0 + and max(_score_scale(left), _score_scale(right)) <= 1.25 * min(_score_scale(left), _score_scale(right)) + ) + + +def _matches(left: TrackingEpisode, right: TrackingEpisode) -> bool: + return _matches_demand(_demand(left), right) + + +def _score_scale(episode) -> float: + return episode.goal_rms if episode.normalization_accel is None else episode.normalization_accel + + +def _score(episode: TrackingEpisode) -> float: + # Raw error falls when the road demands less acceleration even with an + # unchanged proportional tracking error. Compare dimensionless tracking. + return episode.error_rms / _score_scale(episode) if _score_scale(episode) > 0 else math.inf + + +def _noise(rows: Sequence[TrackingEpisode], model_rmse: float | None) -> tuple[float, float, int]: + """Whole-episode uncertainty, with a measured-model noise floor. + + Positive short-range serial covariance inflates the mean variance. This + bounded HAC estimate is an empirical road-noise defense, not a guarantee + under arbitrary nonstationarity; source/demand matching remains mandatory. + """ + values = [_score(row) for row in rows] + center = statistics.mean(values) + variance = statistics.variance(values) + if model_rmse is not None: + variance = max(variance, (model_rmse / min(row.goal_rms for row in rows)) ** 2) + inflation = 1.0 + lags = min(3, len(rows) - 1) + if variance > 0: + for lag in range(1, lags + 1): + covariance = sum((values[i] - center) * (values[i - lag] - center) for i in range(lag, len(values))) / len(values) + inflation += 2 * (1.0 - lag / (lags + 1)) * max(0.0, covariance) / variance + return variance, inflation, max(1, math.floor(len(rows) / inflation) - 1) + + +def _feedback_result(before, future): + """Small-step empirical feedback, NOT an inferential confidence test. + + Both fixed batches are unselected actual tracking windows. Require a mean + AND median improvement of at least0.5%, with at least80% of later windows + staying within the observed baseline error envelope. Existing reversal, gross-regression and + source/native guards are independently mandatory. Road drift can confound + this rule; it does not establish physical causation or a false-positive rate. + """ + if ( + len(before) not in (3, 9) + or len(future) != len(before) + or any(type(value) not in (int, float) or not math.isfinite(value) or value < 0 for value in (*before, *future)) + ): + raise ValueError('fixed finite empirical tracking batches required') + old_mean, new_mean = statistics.mean(before), statistics.mean(future) + old_median, new_median = statistics.median(before), statistics.median(future) + margin = max(0.0001, 0.005 * old_mean) + required = math.ceil(0.8 * len(future)) + if old_mean - new_mean > margin and old_median - new_median > margin and sum(value <= max(before) + margin for value in future) >= required: + return 1 + if new_mean - old_mean > margin and new_median - old_median > margin and sum(value >= min(before) - margin for value in future) >= required: + return -1 + return 0 + + +def _welch_df(left_variance: float, left_df: int, right_variance: float, right_df: int) -> int: + denominator = left_variance**2 / left_df + right_variance**2 / right_df + return max(1, math.floor((left_variance + right_variance) ** 2 / denominator)) if denominator else left_df + right_df + + +class TrialState(TypedDict): + trial_id: str + phase: Literal['trial', 'confirmed', 'rollback'] + group: str + names: tuple[str, ...] + baseline: dict[str, float] + target: dict[str, float] + started_ns: int + deadline_ns: int + required_episodes: int + critical: float + alpha: float + accepted: bool + look_quotas: NotRequired[tuple[int, ...]] + target_sequence: NotRequired[int] + demand: NotRequired[DemandSignature] + published: NotRequired[bool] + prior_accepted: NotRequired[TrialState | None] + + +class DiscoveryState(TypedDict): + group: str + name: str + target: dict[str, float] + expected: float + anchor: TrackingEpisode + selected_end_ns: int + quota: int + expires_ns: int + recovery: bool + minimum_episodes: NotRequired[int] + + +class SelectorHint(TypedDict): + group: str + name: str + delta: float + demand: DemandSignature + binding: LiveTrialBinding + profile_sha256: str + model_revision: str + controller_domain: str + selected_ns: int + expires_ns: int + source_episode_ids: tuple[str, ...] + quota: int + minimum_episodes: NotRequired[int] + + +TrialPlan = tuple[int, float, float] +TrialChoice = tuple[float, str, str, dict[str, float], tuple[TrackingEpisode, ...], float, float, tuple[int, int], TrackingEpisode] + + +def discovery_needs_probes(discovery) -> bool: + # A recovery selector must re-establish its effect with fresh actual native + # endpoints. Its new comparison episodes cannot be measurement-only until + # that existing final revalidation has completed. + return discovery is not None and discovery['recovery'] is True + + +@lru_cache(maxsize=32, typed=True) +def _comparison_thresholds(variance, inflation, feasible, minimum, alpha): + """Numerical work only; never cache evidence, current health or permission. + + Same original operation order and Student/alpha/df calculations. Hooks with + the same measured planning context share this immutable arithmetic grid; + each hook's expected effect still selects its own original allocation. + """ + rows = [] + for before_count in range(minimum, MAX_EPISODES + 1): + left = variance * inflation / before_count + before_df = max(1, math.floor(before_count / inflation) - 1) + futures = [] + for future_count in range(minimum, feasible + 1): + right = variance * inflation / future_count + future_df = max(1, math.floor(future_count / inflation) - 1) + look_alpha = alpha / len(_look_quotas(future_count, feasible)) + critical = _critical(_welch_df(left, before_df, right, future_df), look_alpha) + futures.append((future_count, (critical + 0.8416212335729143) * math.sqrt(left + right))) + rows.append((before_count, tuple(futures))) + return tuple(rows) + + +class LiveTrackingOptimizer: + def __init__( + self, binding: LiveTrialBinding, accepted_full30: tuple[tuple[str, float], ...], *, controller_domain: str, response_certificate: ResponseCertificate + ): + from openpilot.starpilot.flm.live_tracking_response import validate_certificate + + certificate = validate_certificate(response_certificate) + if type(binding) is not LiveTrialBinding or certificate.binding != binding or controller_domain not in ('torque', 'torque_pid'): + raise ValueError('genuinely identified current response binding required') + binding.validate() + self.binding = binding + self.accepted = _map(accepted_full30) + from openpilot.starpilot.flm.live_tracking_response import TrackingSourceContext + + self.domain, self.lag_ns = controller_domain, certificate.lag_ns + self.rmse = None if type(certificate) is TrackingSourceContext else certificate.heldout_rmse + self.certificate = certificate + self.response_witness: ValidatedResponseWitness | None = None + self.ready_since: dict[str, int] = {} + # Completed negative trials guide selection only. They never contribute + # scores, confidence or authority to a subsequent independent experiment. + self.rejected_directions: deque[tuple[str, int, str, DemandSignature]] = deque(maxlen=128) + self.closed_loop_worse: deque[tuple[str, int, str, DemandSignature, int]] = deque(maxlen=64) + self.inconclusive_trials: deque[tuple[str, int, str, DemandSignature, int]] = deque(maxlen=64) + self.baseline: deque[TrackingEpisode] = deque(maxlen=MAX_DISCOVERY_EPISODES) + self.future: list[TrackingEpisode] = [] + self.seen: deque[str] = deque(maxlen=256) + self.planning_history: deque[TrackingEpisode] = deque(maxlen=MAX_DISCOVERY_EPISODES if self.rmse is None else MAX_EPISODES) + self.sequence = self.publication_sequence = self.last_attempt = self.last_correction = self.last_ack_ns = 0 + self.offer_state: TrialState | None = None + self.last_ack: tuple[str, int, str, str, int] | None = None + self.before: tuple[TrackingEpisode, ...] = () + self.discovery: DiscoveryState | None = None + self.selector_hint: SelectorHint | None = None + self.selector_pending = False + self.selector_after_ns = self.last_episode_end_ns = 0 + self.comparison: list[TrackingEpisode] = [] + self.settled_ns: int | None = None + self.status, self.reason = 'collecting', 'qualified_tracking_episodes_required' + self.model_status = 'unavailable' + + def _hash(self, values: dict[str, float] | tuple[tuple[str, float], ...]) -> str: + return full_profile_sha256(self.binding.baseline_revision, values) + + def interrupt(self, reason: str, *, context_changed: bool = False) -> None: + # Already completed clean geometry survives an ordinary brake/override. + # The input builder discards interrupted partial episodes independently. + retain_comparison = not context_changed and self.discovery is not None and (self.offer_state is None or self.offer_state.get('accepted') is True) + if not retain_comparison: + self.discovery = None + self.comparison.clear() + self.selector_pending = self.selector_hint is not None + self.selector_after_ns = max(self.selector_after_ns, self.last_episode_end_ns, self.last_ack_ns) + # Only completed baseline observations survive. No old witness, command, + # trial outcome or deadline becomes current. A pending experiment still + # follows the unchanged rollback path below. + if context_changed: + self.baseline.clear() + self.planning_history.clear() + self.inconclusive_trials.clear() + self._discard_selector() + self.future.clear() + self.before = () + if self.offer_state is not None and not self.offer_state.get('accepted', False): + state = self.offer_state.copy() + if state['phase'] == 'trial' and state.get('published', True) is False: + # No transmission was attempted, so the consumer has no such pending + # trial to roll back. Keep any earlier accepted owner with its OWN + # sequence; the abandoned attempt still spends its confidence budget. + self.offer_state = state.get('prior_accepted') + else: + state['phase'], state['target'] = 'rollback', dict(state['baseline']) + self.offer_state = state + self.status, self.reason = 'paused', reason + self.settled_ns = None + + def _discard_selector(self) -> None: + self.selector_hint = None + self.selector_pending = False + self.discovery = None + self.comparison.clear() + + def _current_selector(self, now_ns: int) -> SelectorHint | None: + hint = self.selector_hint + changed = hint is not None and ( + hint['binding'] != self.binding + or hint['profile_sha256'] != self._hash(self.accepted) + or hint['model_revision'] != self.certificate.model_revision + or hint['controller_domain'] != self.domain + or len(hint['source_episode_ids']) < hint.get('minimum_episodes', _required(hint['demand'].kind)) + ) + if hint is not None and (changed or not hint['selected_ns'] <= now_ns <= hint['expires_ns']): + self._discard_selector() + if changed: + self.baseline.clear() + self.planning_history.clear() + return None + return hint + + def preferred_probe_group(self, now_ns: int) -> str | None: + hint = self._current_selector(now_ns) + state = self.offer_state + if hint is None or not (self.selector_pending or self.discovery is not None) or state is not None and not state['accepted']: + return None + return hint['group'] + + def _fresh_selected_effect(self, rows: Sequence[TrackingEpisode], name: str, target: dict[str, float]) -> float | None: + effects = [] + for row in rows: + feature = next((item for item in row.sensitivity if item.name == name), None) + if ( + not row.native_gradient_complete + or feature is None + or not feature.native_gradient_complete + or feature.feature_energy <= 1e-10 + or feature.native_endpoints < 4 + or feature.native_separated / feature.native_endpoints < 0.09 + or feature.max_step_physical_rms <= 0 + or _score_scale(row) <= 0 + ): + return None + # No error/error_feature_sum/sign is inspected: the earlier coordinate + # and bounded direction were fixed before these observations existed. + effects.append(feature.max_step_physical_rms * abs(target[name] - self.accepted[name]) / MAX_STEP / _score_scale(row)) + return min(effects) if effects else None + + def _comparison_allocation( + self, episode: TrackingEpisode, expected: float, target: dict[str, float], *, minimum_episodes: int | None = None + ) -> tuple[int, int] | None: + if minimum_episodes is not None and minimum_episodes not in (3, 9): + return None + minimum = max(_required(episode.kind), minimum_episodes or 0) + if type(expected) not in (int, float) or not math.isfinite(expected) or expected <= 0.0001: + return None + # Freeze the NEW baseline size before observing any of its scores. More + # clean baseline observations can reduce time under an experimental target. + # This is an empirical exposure forecast, never an acceptance calculation. + reference = tuple( + row + for row in self.planning_history + if row.binding == self.binding + and row.profile_sha256 == episode.profile_sha256 + and _matches(episode, row) + and 0 <= episode.end_ns - row.end_ns <= 600_000_000_000 + ) + if len(reference) < 2: + return None + if self.rmse is None: + cadence = statistics.median(b.end_ns - a.end_ns for a, b in zip(reference, reference[1:], strict=False)) + slew_ns = math.ceil(max(abs(target[n] - self.accepted[n]) for n in NAMES) / 0.005 * 1e9) + if math.floor((TRIAL_TIMEOUT_NS - slew_ns - self.lag_ns) / cadence) < minimum: + return None + return minimum, minimum + variance, inflation, _ = _noise(reference, self.rmse) + intervals = [b.end_ns - a.end_ns for a, b in zip(reference, reference[1:], strict=False)] + cadence = statistics.median(intervals) + slew_ns = math.ceil(max(abs(target[n] - self.accepted[n]) for n in NAMES) / 0.005 * 1e9) + feasible = min(MAX_EPISODES, math.floor((TRIAL_TIMEOUT_NS - slew_ns - self.lag_ns) / cadence)) + alpha = 0.05 / ((self.sequence + 1) * (self.sequence + 2)) + choices = [] + for before_count, futures in _comparison_thresholds(variance, inflation, feasible, minimum, alpha): + for future_count, threshold in futures: + if expected > threshold: + choices.append((future_count, before_count + future_count, before_count)) + break + if not choices: + # Predicted local native effect is not a reliable forecast of closed-loop + # tracking improvement. A bounded pilot may measure that improvement; + # the unchanged independent final test still decides whether to keep it. + if feasible < minimum: + return None + return minimum, minimum + future_count, _, before_count = min(choices) + return before_count, future_count + + def _revalidate_selector(self, episode: TrackingEpisode, now_ns: int) -> bool: + hint = self._current_selector(now_ns) + if hint is None: + return False + if episode.start_ns <= self.selector_after_ns or not _matches_demand(hint['demand'], episode): + return True + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + try: + validate_response_witness(self.response_witness, self.binding, now_ns) + except (TypeError, ValueError): + self.reason = 'fresh_selector_response_required' + return True + low, high = coordinate_bounds(hint['name']) + target = dict(self.accepted) + target[hint['name']] = min(high, max(low, target[hint['name']] + hint['delta'])) + expected = self._fresh_selected_effect((episode,), hint['name'], target) + if expected is None or expected <= 0.0001: + self.status, self.reason = 'collecting', 'fresh_selected_native_probe_required' + return True + allocation = self._comparison_allocation(episode, expected, target, minimum_episodes=hint.get('minimum_episodes')) + if allocation is None: + self.status, self.reason = 'collecting', 'fresh_recovery_allocation_power_insufficient' + return True + # This fresh source/probe observation is REVALIDATION, never a score row. + # Old selection provenance/expiry is preserved, not retagged as new proof. + # The actual baseline later gets ONE unchanged power check; its honest + # required future quota may exceed this empirical forecast. + self.discovery = { + 'group': hint['group'], + 'name': hint['name'], + 'target': target, + 'expected': expected, + 'anchor': episode, + 'selected_end_ns': episode.end_ns, + 'quota': allocation[0], + 'expires_ns': hint['expires_ns'], + 'recovery': True, + 'minimum_episodes': hint.get('minimum_episodes', _required(episode.kind)), + } + self.selector_pending = False + self.status, self.reason = 'collecting', 'uncensored_independent_comparison_baseline_required' + return True + + def response_current(self, witness, *, now_ns): + """Fresh pose/model residual result; retained model is never permission.""" + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness, prepare_response_witness + + try: + raw = validate_response_witness(witness, self.binding, now_ns) + if raw.certificate != self.certificate: + raise ValueError('response certificate changed') + except (AttributeError, TypeError, ValueError): + self.response_witness = None + self.model_status = 'stale' + self.interrupt('physical_response_context_or_residual_unqualified') + return False + self.response_witness = prepare_response_witness(raw, self.binding, now_ns) + self.model_status = 'valid' + return True + + def observe_episode(self, episode: TrackingEpisode, *, now_ns: int) -> bool: + episode.validate() + if (self.rmse is None) != (episode.normalization_accel is not None): + return False # Never mix legacy force episodes with direct kinematic windows. + if ( + episode.binding != self.binding + or self.model_status != 'valid' + or episode.episode_id in self.seen + or not episode.end_ns <= now_ns + or self.baseline + and episode.start_ns <= self.baseline[-1].end_ns + ): + return False + self.seen.append(episode.episode_id) + self.last_episode_end_ns = max(self.last_episode_end_ns, episode.end_ns) + state = self.offer_state + if state is None or state.get('accepted', False): + if episode.profile_sha256 != self._hash(self.accepted): + return False + # Planning-only evidence retains completed accepted-profile geometry, + # including discovery and zero-error comparison episodes. It never + # supplies the scored baseline or any future trial outcome. + if not self.planning_history or episode.start_ns > self.planning_history[-1].end_ns: + self.planning_history.append(episode) + self._current_selector(now_ns) + if self.selector_pending and self._revalidate_selector(episode, now_ns): + return False + if self.discovery is not None: + discovery = self.discovery + if now_ns > discovery['expires_ns']: + self._discard_selector() + self.status, self.reason = 'collecting', 'fresh_comparison_baseline_timeout' + return False + if ( + not _matches(discovery['anchor'], episode) + or episode.start_ns <= discovery['selected_end_ns'] + or self.comparison + and episode.start_ns <= self.comparison[-1].end_ns + ): + return False + # These observations were requested BEFORE their error was measured. + # Low/zero tracking errors remain in the comparison baseline. + self.comparison.append(episode) + if len(self.comparison) == discovery['quota']: + rows = tuple(self.comparison) + self.discovery = None + self.comparison.clear() + expected = discovery['expected'] + if discovery['recovery']: + fresh_effect = self._fresh_selected_effect(rows, discovery['name'], discovery['target']) + if fresh_effect is None: + self.selector_pending = self.selector_hint is not None + self.selector_after_ns = rows[-1].end_ns + self.status, self.reason = 'collecting', 'fresh_selected_native_probe_required' + return False + expected = min(expected, fresh_effect) + plan = self._trial_plan(rows, expected, discovery['target'], minimum_episodes=discovery.get('minimum_episodes')) + if plan is None: + self._discard_selector() + self.status, self.reason = 'not_identifiable', 'independent_baseline_trial_power_insufficient' + return False + return self._launch(discovery['group'], discovery['name'], discovery['target'], rows, plan, now_ns) + self.status, self.reason = 'collecting', 'uncensored_independent_comparison_baseline_required' + return False + self.baseline.append(episode) + return self._propose(now_ns) + if state['phase'] == 'trial' and now_ns > state['deadline_ns']: + self._rollback('trial_future_support_timeout') + return False + if state['phase'] != 'trial' or self.settled_ns is None: + return False + if ( + episode.start_ns <= self.settled_ns + self.lag_ns + or episode.profile_sha256 != self._hash(state['target']) + or not _matches_demand(state.get('demand', _demand(self.before[0])), episode) + or self.future + and episode.start_ns <= self.future[-1].end_ns + ): + return False + baseline_error = statistics.mean(_score(row) for row in self.before) + baseline_reversals = statistics.mean(row.reversal_hz for row in self.before) + reversal_limit = max(1.25 * baseline_reversals, baseline_reversals + 0.3) + # A single turn is compared with the observed baseline envelope. Comparing + # one noisy crossing count with a mean incorrectly labels baseline variation + # as deterioration. The complete future batch still has to meet the original + # mean-rate limit below, independently of its tracking-error improvement. + if _score(episode) > 1.5 * max(baseline_error, 0.03) or episode.reversal_hz > max(reversal_limit, max(row.reversal_hz for row in self.before)): + self._rollback('trial_tracking_or_oscillation_worse') + return False + self.future.append(episode) + if len(self.future) == state['required_episodes']: + if statistics.mean(row.reversal_hz for row in self.future) > reversal_limit: + self._rollback('trial_oscillation_batch_worse') + return False + # Predeclared looks only; no outcome-selected quota or scale search. + old = [_score(row) for row in self.before] + new = [_score(row) for row in self.future] + if self.rmse is None: + result = _feedback_result(old, new) + if result > 0: + confirmed = state.copy() + confirmed['phase'] = 'confirmed' + self.offer_state = confirmed + self.status, self.reason = 'confirmed', 'consistent_later_tracking_improved' + else: + demand = state.get('demand', _demand(self.before[0])) + for name in state['names']: + direction = 1 if state['target'][name] > state['baseline'][name] else -1 + if result < 0: + self.closed_loop_worse.append((name, direction, self._hash(state['baseline']), demand, episode.end_ns)) + else: + self.inconclusive_trials.append((name, direction, self._hash(state['baseline']), demand, episode.end_ns)) + self._rollback('matched_tracking_not_consistently_improved') + return True + improvement = statistics.mean(old) - statistics.mean(new) + before_var, before_inflation, before_df = _noise(self.before, self.rmse) + future_var, future_inflation, future_df = _noise(self.future, self.rmse) + left_variance = before_var * before_inflation / len(old) + right_variance = future_var * future_inflation / len(new) + noise = math.sqrt(left_variance + right_variance) + looks = state.get('look_quotas', (state['required_episodes'],)) + critical = _critical(_welch_df(left_variance, before_df, right_variance, future_df), state['alpha'] / len(looks)) + if improvement > max(0.0001, critical * noise): + confirmed = state.copy() + confirmed['phase'] = 'confirmed' + self.offer_state = confirmed + self.status, self.reason = 'confirmed', 'independent_future_tracking_improved' + else: + later = next((quota for quota in looks if quota > len(self.future)), None) + if later is not None and improvement >= -max(0.0001, critical * noise): + pending = state.copy() + pending['required_episodes'] = later + self.offer_state = pending + self.status, self.reason = 'trial_active', 'predeclared_independent_future_look_required' + return True + if improvement < -max(0.0001, critical * noise) and len(state['names']) == 1: + name = state['names'][0] + direction = 1 if state['target'][name] > state['baseline'][name] else -1 + self.closed_loop_worse.append((name, direction, self._hash(state['baseline']), state.get('demand', _demand(self.before[0])), episode.end_ns)) + self._rollback('independent_future_tracking_not_improved') + return True + + def _comparison_demand(self, rows): + # Keep the input class fixed from selection through noise estimation, + # baseline and future scoring. Approximate demand matching is symmetric, + # not transitive; choosing the first baseline as a new anchor changes it. + hint = self.selector_hint + if ( + self.rmse is None + and hint is not None + and hint['binding'] == self.binding + and hint['model_revision'] == self.certificate.model_revision + and hint['profile_sha256'] == rows[0].profile_sha256 + and hint['selected_ns'] <= rows[-1].end_ns <= hint['expires_ns'] + ): + return hint['demand'] + return _demand(rows[0]) + + def _trial_plan( + self, rows: tuple[TrackingEpisode, ...], expected: float, target: dict[str, float], *, minimum_episodes: int | None = None + ) -> TrialPlan | None: + if minimum_episodes is not None and minimum_episodes not in (3, 9): + return None + minimum = max(_required(rows[0].kind), minimum_episodes or 0) + if type(expected) not in (int, float) or not math.isfinite(expected) or expected <= 0.0001: + return None + if self.rmse is None: + if len(rows) != minimum: + return None + cadence = statistics.median(b.end_ns - a.end_ns for a, b in zip(rows, rows[1:], strict=False)) + slew_ns = math.ceil(max(abs(target[n] - self.accepted[n]) for n in NAMES) / 0.005 * 1e9) + if math.floor((TRIAL_TIMEOUT_NS - slew_ns - self.lag_ns) / cadence) < minimum: + return None + # Source-only empirical feedback makes NO alpha/confidence claim. + return minimum, 0.0, 0.0 + # Predeclared family-wise budget over arbitrarily many attempts. Selecting + # a coordinate on baseline evidence does not select any future outcome. + alpha = 0.05 / ((self.sequence + 1) * (self.sequence + 2)) + variance, inflation, before_df = _noise(rows, self.rmse) + # A short, unusually quiet comparison batch must not be the only forecast + # of future road noise/correlation. Retain the larger empirical estimate + # from earlier comparable geometry under this EXACT accepted profile. + # Selection can affect that history: it is used only conservatively for + # quota planning; the independent baseline and final test are unchanged. + reference = tuple( + row + for row in self.planning_history + if row.binding == self.binding + and row.profile_sha256 == rows[0].profile_sha256 + and _matches(rows[0], row) + and 0 <= rows[-1].end_ns - row.end_ns <= 600_000_000_000 + ) + future_variance, future_inflation = variance, inflation + if len(reference) >= 2: + reference_variance, reference_inflation, _ = _noise(reference, self.rmse) + future_variance = max(variance, reference_variance) + future_inflation = max(inflation, reference_inflation) + intervals = [b.end_ns - a.end_ns for a, b in zip(rows, rows[1:], strict=False)] + cadence = statistics.median(intervals) + slew_ns = math.ceil(max(abs(target[n] - self.accepted[n]) for n in NAMES) / 0.005 * 1e9) + feasible = min(MAX_EPISODES, math.floor((TRIAL_TIMEOUT_NS - slew_ns - self.lag_ns) / cadence)) + required = None + # Nominal80% planning estimate under the empirical future-noise forecast. + # The physical effect RMS is not guaranteed error reduction; actual + # future tracking decides acceptance. Baseline variance/df remain FIXED. + for quota in range(minimum, feasible + 1): + left_variance, right_variance = variance * inflation / len(rows), future_variance * future_inflation / quota + look_alpha = alpha / len(_look_quotas(quota, feasible)) + critical = _critical(_welch_df(left_variance, before_df, right_variance, max(1, math.floor(quota / future_inflation) - 1)), look_alpha) + # Welch df need not fall when one variance grows: better balance can + # increase df and lower its critical value. Preserve the original + # per-quota critical as a lower bound so a noisier forecast can NEVER + # make the planning threshold easier than the equal-noise estimate. + original_critical = _critical(_welch_df(left_variance, before_df, variance * inflation / quota, max(1, math.floor(quota / inflation) - 1)), look_alpha) + critical = max(critical, original_critical) + planned_se = math.sqrt(left_variance + right_variance) + if expected > (critical + 0.8416212335729143) * planned_se: + required = quota + break + if required is None: + if feasible < minimum: + return None + # Fixed minimum whole-event pilot, chosen BEFORE any pilot outcome. + # No early positive peeking; actual final Welch/HAC/alpha test below is + # unmodified and uses measured future variance, never this forecast. + required = minimum + left_variance = variance * inflation / len(rows) + right_variance = future_variance * future_inflation / required + critical = _critical( + _welch_df(left_variance, before_df, right_variance, max(1, math.floor(required / future_inflation) - 1)), alpha / len(_look_quotas(required, feasible)) + ) + return required, critical, alpha + + def _propose(self, now_ns: int) -> bool: + if self.last_attempt and now_ns - self.last_attempt < RETRY_NS: + return False + unsupported: frozenset[str] = frozenset() + if self.domain == 'torque_pid': + from openpilot.starpilot.flm.pid_stages import UNSUPPORTED + + unsupported = UNSUPPORTED + choices: list[TrialChoice] = [] + # Geometry is input-only and independent of the coordinate group. Compute + # each cohort once per proposal call, not once for every group. This local + # memo never survives a new observation, binding, source or profile. + matched_cohorts = {} + for group, names in GROUPS.items(): + examined = set() + for episode in reversed(self.baseline): + if not _kind_for_group(group, episode.kind) or not _group_direction_matches(group, episode.kind, episode.direction): + continue + matched = matched_cohorts.get(episode.episode_id) + if matched is None: + demand = _demand(episode) + matched = tuple(row for row in self.baseline if _matches_demand(demand, row) and 0 <= now_ns - row.end_ns <= 600_000_000_000) + matched_cohorts[episode.episode_id] = matched + geometry = tuple(row.episode_id for row in matched) + if geometry in examined: + continue + examined.add(geometry) + for name in names: + if name in unsupported: + continue + # Discovery can use only episodes with an ACTUAL probe for this + # coordinate. Missing probes from another scheduled group are not + # zero sensitivities. The later comparison baseline stays uncensored. + rows = tuple( + row + for row in matched + if row.native_gradient_complete + and any( + feature.name == name + and feature.native_gradient_complete + and feature.feature_energy > 1e-10 + and feature.native_endpoints >= 4 + and feature.native_separated / feature.native_endpoints >= 0.09 + for feature in row.sensitivity + ) + ) + rows, minimum = _discovery_support(rows, episode.kind, self.rmse) + if not rows: + continue + estimates: list[float] = [] + for row in rows: + feature = next((item for item in row.sensitivity if item.name == name), None) + if feature is None or feature.feature_energy <= 1e-10 or feature.native_endpoints < 4 or feature.native_separated / feature.native_endpoints < 0.09: + break + estimates.append(feature.error_feature_sum / feature.feature_energy) + if len(estimates) != len(rows): + continue + sign = 1 if statistics.median(estimates) > 0 else -1 + if self.rmse is None: + # Native secants establish that this coordinate can change actual + # encoded commands. They do not identify a closed-loop plant or + # prove a consistent improvement direction. Use them to select a + # bounded experiment only; a separate uncensored baseline/future + # comparison must still prove improvement before any persistence. + confidence = abs(statistics.median(estimates)) + if confidence <= 0: + continue + else: + if sum(value * sign > 0 for value in estimates) < math.ceil(0.8 * len(estimates)): + continue + mean = statistics.mean(estimates) + confidence = abs(mean) - T95[len(estimates) - 2] * statistics.stdev(estimates) / math.sqrt(len(estimates)) + if confidence <= 0: + continue + + # A one-tick native derivative does not include the PID's later + # closed-loop response. A completed independently measured WORSE + # one-coordinate trial can select the opposite bounded experiment. + # Its old outcomes NEVER become the new baseline or acceptance score. + def observed_worse(direction, name=name, rows=rows): + return any( + old_name == name + and old_direction == direction + and old_profile == self._hash(self.accepted) + and 0 <= now_ns - old_end <= 600_000_000_000 + and _matches_demand(demand, rows[-1]) + for old_name, old_direction, old_profile, demand, old_end in self.closed_loop_worse + ) + + if observed_worse(sign): + if observed_worse(-sign): + continue # Both directions failed; do not oscillate guesses. + sign = -sign + if self.rmse is None and self._same_inconclusive_trial(name, sign, rows[-1], now_ns): + continue + delta = sign * min(MAX_STEP, 0.1 * confidence) + predicted = min(next(item.max_step_physical_rms for item in row.sensitivity if item.name == name) for row in rows) + expected = predicted * abs(delta) / MAX_STEP / max(_score_scale(row) for row in rows) + if expected <= 0.0001: + continue # The original native effect must be visible. + # Fresh comparison allocation and _trial_plan independently enforce + # the full noise/power bound before any target can be offered. Do not + # also reject on outcome-selected discovery-score variability. + low, high = coordinate_bounds(name) + target = dict(self.accepted) + target[name] = min(high, max(low, target[name] + delta)) + if target[name] == self.accepted[name]: + continue + expected *= abs(target[name] - self.accepted[name]) / abs(delta) + self.ready_since.setdefault(group, now_ns) + benefit = sum(abs(next(item.error_feature_sum for item in row.sensitivity if item.name == name)) for row in rows) + allocation = self._comparison_allocation(episode, expected, target, minimum_episodes=minimum) + if allocation is not None: + choices.append((confidence, group, name, target, rows, benefit, expected, allocation, episode)) + if not choices: + self.status, self.reason = 'not_identifiable', 'qualified_consistent_hook_sensitivity_required' + return False + + # Try another qualified coordinate before repeating a completed failure + # on the same accepted profile and comparable maneuver. Fault-interrupted + # trials supply no negative outcome and cannot affect this preference. + def rejected_count(choice): + name, target, rows = choice[2], choice[3], choice[4] + direction = 1 if target[name] > self.accepted[name] else -1 + profile = self._hash(self.accepted) + return sum( + old_name == name and old_direction == direction and old_profile == profile and _matches_demand(demand, rows[-1]) + for old_name, old_direction, old_profile, demand in self.rejected_directions + ) + + least_rejected = min(map(rejected_count, choices)) + choices = [choice for choice in choices if rejected_count(choice) == least_rejected] + _, group, name, target, rows, _, expected, allocation, input_anchor = max( + choices, key=lambda value: (now_ns - self.ready_since[value[1]] >= TRIAL_TIMEOUT_NS, value[5], -self.ready_since[value[1]], value[1], value[2]) + ) + # Discovery may select an error/direction, but those outcome-selected + # episodes are NEVER the score baseline for a later keep/revert decision. + self.discovery = { + 'group': group, + 'name': name, + 'target': target, + 'expected': expected, + 'anchor': input_anchor if self.rmse is None else rows[-1], + 'selected_end_ns': max(now_ns, self.last_episode_end_ns), + 'quota': allocation[0], + 'expires_ns': now_ns + 600_000_000_000, + 'recovery': False, + 'minimum_episodes': len(rows), + } + self.selector_hint = { + 'group': group, + 'name': name, + 'delta': target[name] - self.accepted[name], + 'demand': _demand(input_anchor if self.rmse is None else rows[-1]), + 'binding': self.binding, + 'profile_sha256': self._hash(self.accepted), + 'model_revision': self.certificate.model_revision, + 'controller_domain': self.domain, + 'selected_ns': now_ns, + 'expires_ns': self.discovery['expires_ns'], + 'source_episode_ids': tuple(row.episode_id for row in rows), + 'quota': len(rows), + 'minimum_episodes': len(rows), + } + self.selector_pending = False + self.comparison.clear() + self.baseline.clear() + self.status, self.reason = 'collecting', 'uncensored_independent_comparison_baseline_required' + return False + + def _launch(self, group: str, name: str, target: dict[str, float], rows: tuple[TrackingEpisode, ...], plan: TrialPlan, now_ns: int) -> bool: + required, critical, alpha = plan + self.ready_since[group] = now_ns + self.sequence += 1 + trial_id = hashlib.sha256(repr((self.binding, self.sequence, now_ns, self._hash(self.accepted), name)).encode()).hexdigest() + # Fix every look BEFORE any trial outcome exists. Small-error pilots can + # use a second whole-episode look inside the ORIGINAL deadline; each look + # spends an equal share of this attempt's unchanged family-wise budget. + looks = (required,) + if self.rmse is not None and required >= 9 and len(rows) >= 2: + cadence = statistics.median(b.end_ns - a.end_ns for a, b in zip(rows, rows[1:], strict=False)) + slew_ns = math.ceil(max(abs(target[n] - self.accepted[n]) for n in NAMES) / 0.005 * 1e9) + feasible = min(MAX_EPISODES, math.floor((TRIAL_TIMEOUT_NS - slew_ns - self.lag_ns) / cadence)) + looks = _look_quotas(required, feasible) + prior = self.offer_state if self.offer_state is not None and self.offer_state.get('accepted', False) else None + self.offer_state = { + 'trial_id': trial_id, + 'phase': 'trial', + 'group': group, + 'names': (name,), + 'baseline': dict(self.accepted), + 'target': target, + 'started_ns': now_ns, + 'deadline_ns': now_ns + TRIAL_TIMEOUT_NS, + 'required_episodes': required, + 'critical': critical, + 'alpha': alpha, + 'accepted': False, + 'target_sequence': self.sequence, + 'published': False, + 'prior_accepted': prior, + 'look_quotas': looks, + } + if self.rmse is None: + self.offer_state['demand'] = self._comparison_demand(rows) + self.last_attempt, self.before, self.future = now_ns, rows, [] + # Every attempted baseline is spent once, including failed/interrupted + # trials. Subsequent directions need new independent baseline geometry. + self.baseline.clear() + self.settled_ns = None + self.status, self.reason = 'trial_pending', 'actual_target_application_acknowledgement_required' + return True + + def _same_inconclusive_trial(self, name, direction, episode, now_ns): + # A completed noisy trial supplies no sign inversion, score, acceptance or + # current permission. Avoid repeatedly spending the same driving exposure + # when the other supported coordinates may be informative instead. + return any( + old_name == name + and old_direction == direction + and old_profile == self._hash(self.accepted) + and 0 <= now_ns - finished <= 600_000_000_000 + and _matches_demand(demand, episode) + for old_name, old_direction, old_profile, demand, finished in self.inconclusive_trials + ) + + def _rollback(self, reason: str) -> None: + if self.offer_state is None: + raise ValueError('rollback requires a pending live trial') + state = self.offer_state.copy() + if state['phase'] == 'trial' and state.get('published', True) is False: + self.interrupt(reason) + return + if ( + state['phase'] == 'trial' + and self.before + and reason in ('independent_future_tracking_not_improved', 'matched_tracking_not_consistently_improved', 'trial_tracking_or_oscillation_worse') + ): + for name in state['names']: + direction = 1 if state['target'][name] > state['baseline'][name] else -1 + self.rejected_directions.append((name, direction, self._hash(state['baseline']), state.get('demand', _demand(self.before[0])))) + state['phase'], state['target'] = 'rollback', dict(state['baseline']) + self.offer_state = state + self._discard_selector() + self.status, self.reason = 'rollback', reason + + def acknowledge_application( + self, + *, + trial_id: str, + target_sequence: int, + phase: str, + target_sha256: str, + applied_full30: tuple[tuple[str, float], ...], + control_ns: int, + binding: LiveTrialBinding, + now_ns: int, + status: str, + confirmed_target_sha256: str = '', + ) -> bool: + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + try: + validate_response_witness(self.response_witness, self.binding, now_ns) + except (TypeError, ValueError): + return False + state = self.offer_state + if ( + state is None + or binding != self.binding + or trial_id != state['trial_id'] + or target_sequence != state.get('target_sequence', self.sequence) + or phase != state['phase'] + or target_sha256 != self._hash(state['target']) + or type(control_ns) is not int + or type(now_ns) is not int + or not state['started_ns'] < control_ns <= now_ns <= control_ns + 100_000_000 + or control_ns < self.last_ack_ns + or status != 'applied' + or (phase in ('trial', 'confirmed') and not state.get('accepted', False) and now_ns > state['deadline_ns']) + or (phase == 'confirmed' and confirmed_target_sha256 != target_sha256) + or self._hash(_map(applied_full30)) != target_sha256 + or self.model_status != 'valid' + ): + return False + state['published'] = True + self.last_ack_ns = control_ns + self.last_ack = (trial_id, target_sequence, phase, target_sha256, control_ns) + if state.get('accepted', False): + return True + if phase == 'trial': + self.settled_ns = self.settled_ns or control_ns + self.status, self.reason = 'trial_active', 'scoring_later_matched_whole_episodes' + else: + if phase == 'confirmed': + self.accepted = dict(state['target']) + self._discard_selector() + self.last_correction = control_ns + accepted = state.copy() + accepted['accepted'] = True + accepted.pop('prior_accepted', None) + self.offer_state = accepted + else: + # Keep leasing the actually restored accepted baseline. Dropping the + # offer here would withdraw earlier accepted nonzero corrections. + accepted = state.copy() + accepted['accepted'] = True + accepted.pop('prior_accepted', None) + self.offer_state = accepted + if self.selector_pending: + self.selector_after_ns = max(self.selector_after_ns, control_ns) + self.baseline.clear() + self.future.clear() + self.settled_ns = None + self.status, self.reason = 'collecting', 'next_independent_tracking_episodes_required' + return True + + def mark_published(self, offer) -> None: + state = self.offer_state + if state is not None and offer.trial_id == state['trial_id'] and offer.target_sequence == state.get('target_sequence', self.sequence): + state['published'] = True # Conservative before-send marker, not an ACK. + + def offer(self, now_ns: int, *, publication=True) -> LiveTrialOffer | None: + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + try: + validate_response_witness(self.response_witness, self.binding, now_ns) + except (TypeError, ValueError): + if self.offer_state is not None: + self.interrupt('fresh_physical_response_witness_unavailable') + return None + state = self.offer_state + if state is None: + return None + if state['phase'] in ('trial', 'confirmed') and not state.get('accepted', False) and now_ns > state['deadline_ns']: + published = state.get('published', True) + self._rollback('trial_future_support_timeout') + state = self.offer_state + if state is None: + if published: + raise ValueError('rollback lost immutable trial state') + return None + baseline = tuple((name, state['baseline'][name]) for name in NAMES) + target = tuple((name, state['target'][name]) for name in NAMES) + self.publication_sequence += 1 + offer = LiveTrialOffer( + VERSION, + self.binding, + state['trial_id'], + self.publication_sequence, + state['phase'], + state['group'], + state['names'], + baseline, + target, + self._hash(state['baseline']), + self._hash(state['target']), + state['started_ns'], + now_ns, + min(now_ns + MAX_LEASE_NS, validate_response_witness(self.response_witness, self.binding, now_ns).valid_until_ns), + self.response_witness, + target_sequence=state.get('target_sequence', self.sequence), + deadline_ns=state['deadline_ns'], + ).validate() + if publication: + self.mark_published(offer) + return offer + + def report(self, now_ns): + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + try: + validate_response_witness(self.response_witness, self.binding, now_ns) + current = True + except (TypeError, ValueError): + current = False + state = self.offer_state + acknowledged = bool( + current + and state is not None + and self.last_ack is not None + and self.last_ack[:4] == (state['trial_id'], state.get('target_sequence', self.sequence), state['phase'], self._hash(state['target'])) + and 0 <= now_ns - self.last_ack[-1] <= 100_000_000 + ) + status, reason = self.status, self.reason + if state is not None and state.get('accepted', False): + if acknowledged: + status = state['phase'] + reason = 'accepted_target_currently_acknowledged' if status == 'confirmed' else 'prior_accepted_baseline_restored' + else: + state = None + + def mean(rows, field): + return statistics.mean(getattr(row, field) for row in rows) if rows else None + + return { + 'version': 1, + 'status': status, + 'reason': reason, + 'group': None if state is None else state['group'], + 'names': [] if state is None else list(state['names']), + 'trialId': None if state is None else state['trial_id'], + 'phase': None if state is None else state['phase'], + 'requiredEpisodes': None if state is None else state['required_episodes'], + 'beforeEpisodes': len(self.before), + 'futureEpisodes': len(self.future), + 'modelStatus': self.model_status if current else 'stale', + 'lagMs': self.lag_ns / 1e6, + 'baselineErrorRms': mean(self.before, 'error_rms'), + 'futureErrorRms': mean(self.future, 'error_rms'), + 'baselineReversalHz': mean(self.before, 'reversal_hz'), + 'futureReversalHz': mean(self.future, 'reversal_hz'), + 'applicationAcknowledged': acknowledged, + 'applicationSettledMonoTime': self.last_ack[-1] if acknowledged and self.last_ack is not None else self.settled_ns, + 'lastCorrectionMonoTime': self.last_correction or None, + 'bindingCurrent': current, + } + + def tracking_values(self): + state = self.offer_state + pending = None if state is None or state.get('accepted', False) else state['target'] + return { + 'version': 1, + 'accepted': [{'name': name, 'value': self.accepted[name]} for name in NAMES], + 'pending': None if pending is None else [{'name': name, 'value': pending[name]} for name in NAMES], + 'acceptedSha256': self._hash(self.accepted), + 'pendingSha256': None if pending is None else self._hash(pending), + } diff --git a/openpilot/starpilot/flm/live_tracking_actor.py b/openpilot/starpilot/flm/live_tracking_actor.py new file mode 100644 index 0000000000..adf4ed34af --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_actor.py @@ -0,0 +1,931 @@ +"""Response bootstrap and live observation actor; no Controls or native owner. + +No production bootstrap policy is inferred from the diagnostic180-second limit. +The daemon supplies a separately qualified process, otherwise collection holds. +""" + +from dataclasses import replace +from typing import cast + +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, full_profile_sha256 +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding +from openpilot.starpilot.flm.live_tracking_ingress import LiveTrackingIngress +from openpilot.starpilot.flm.live_tracking_response import certificate_from_dict, validate_certificate, validate_response_witness + + +def _observed_probe_candidates( + ready, + prefix, + points, + *, + after_ns, + verify_limited=None, + checkpoint_indices=None, + stratified=False, + same_checkpoint=False, + phase_windows=(), + phase_fallback_after_ns=None, + fresh_limit=4, + discovery_headroom=False, +): + """Only distinct original emissions with actual independent endpoints.""" + if type(discovery_headroom) is not bool: + raise ValueError('explicit numerical discovery sampling policy required') + if type(fresh_limit) is not int or not 1 <= fresh_limit <= 4: + raise ValueError('one to four original numerical endpoints') + by_stamp = {row.control_ns: (index, row) for index, row in enumerate(prefix)} + eligible = [] + starts = (0,) if checkpoint_indices is None else tuple(checkpoint_indices) + for control, output in reversed(ready): + entry = by_stamp.get(control.control_ns) + if ( + entry is None + or control.control_ns <= after_ns + or (control.control_ns, output.apply_ns) not in points + or output.limited + and (verify_limited is None or not verify_limited(control, output)) + or control != entry[1] + or any(control.control_ns == row[0].control_ns for row in eligible) + ): + continue + anchors = tuple(index for index in starts if 0 <= index <= entry[0]) + if not anchors: + continue + original_prefix = prefix[anchors[-1] : entry[0] + 1] + if not 1 <= len(original_prefix) <= 100: + continue + eligible.append((control, output, original_prefix)) + if phase_windows: + for first, last, requested in phase_windows: + if not first < last: + continue + + def phase(stamp, first=first, last=last): + return min(3, (stamp - first) * 4 // (last - first + 1)) + + missing = set(range(4)) - {phase(stamp) for stamp in requested if first <= stamp <= last} + buckets = { + slot: [row for row in eligible if first <= row[0].control_ns <= last and row[0].control_ns not in requested and phase(row[0].control_ns) == slot] + for slot in missing + } + # Do not spend bounded work on a window whose missing phase is no + # longer available under the ORIGINAL source/replay/age requirements. + if not missing or any(not values for values in buckets.values()): + continue + chosen = [ + min(values, key=lambda row: (row[1].limited, abs(8 * (row[0].control_ns - first) - (2 * slot + 1) * (last - first + 1)), row[0].control_ns)) + for slot, values in sorted(buckets.items()) + ] + return tuple(sorted(chosen, key=lambda row: row[0].control_ns)) + if phase_fallback_after_ns is None: + return () + eligible = [row for row in eligible if row[0].control_ns > phase_fallback_after_ns] + if discovery_headroom and fresh_limit == 1 and eligible: + # Bound serial live-stream work to one recent original measurement. + # Closed-window repairs above still request every missing time stratum. + return (min(eligible, key=lambda row: (row[1].limited, -row[0].control_ns)),) + if same_checkpoint and eligible: + # Each batch must retain one genuine original checkpoint. Process the + # earliest available one first, so its watermark cannot skip a newer one. + first_checkpoint = min(row[2][0].control_ns for row in eligible) + eligible = [row for row in eligible if row[2][0].control_ns == first_checkpoint] + # Prefer real original headroom BEFORE computing any perturbation or loss. + # A native-clipped endpoint still contributes when its original CAN proof is + # complete; no missing observation, changed limit, or numerical effect is filled. + if stratified and len(eligible) > 4: + first = min(row[0].control_ns for row in eligible) + last = max(row[0].control_ns for row in eligible) + selected = [] + for slot in range(4): + bucket = [row for row in eligible if min(3, (row[0].control_ns - first) * 4 // (last - first + 1)) == slot] + if bucket: + selected.append(min(bucket, key=lambda row: (row[1].limited if discovery_headroom else False, row[0].control_ns))) + if len(selected) < 4: + remaining = [row for row in eligible if all(row[0].control_ns != used[0].control_ns for used in selected)] + remaining.sort(key=lambda row: row[0].control_ns) + selected.extend(remaining[: 4 - len(selected)]) + else: + eligible.sort(key=lambda row: row[1].limited) + selected = eligible[:4] + selected = tuple(sorted(selected, key=lambda row: row[0].control_ns)) + if len(selected) > fresh_limit: + # Clock-stratified work only. Missing whole-window phases are repaired + # above without this fresh-stream cap; the original four-point proof stays. + if discovery_headroom: + # Discovery needs observable native response. Choose only among original + # clock strata using limiter flags; no error, estimate or trial outcome + # is read. This never filters the whole physical comparison score. + from itertools import combinations + + return min( + combinations(selected, fresh_limit), + key=lambda rows: (sum(row[1].limited for row in rows), -(rows[-1][0].control_ns - rows[0][0].control_ns), tuple(row[0].control_ns for row in rows)), + ) + indices = (len(selected) - 1,) if fresh_limit == 1 else tuple(i * (len(selected) - 1) // (fresh_limit - 1) for i in range(fresh_limit)) + return tuple(selected[i] for i in indices) + return selected + + +def _original_probe_windows(episodes, profile_hash, group, names): + """Closed ORIGINAL geometry may queue numbers before the last lagged pose. + + Every selected endpoint must ALREADY have its own independent physical pair. + Whole-episode publication still waits for the unchanged full pose watermark. + """ + windows = [] + for row in episodes.completed.values(): + if row['episode'].profile_sha256 == profile_hash and row['group'] == group and row['names'] == names: + windows.append((min(p.control_ns for p in row['points']), max(p.control_ns for p in row['points']), frozenset(row['requested']))) + for start, end, kind, _direction, profile in episodes.pending: + if kind not in ('large', 'subtle') or profile != profile_hash: + continue + source_tracking = getattr(episodes, 'source_tracking', False) + points = tuple(point for point in episodes.endpoints if start <= (point.reference_ns if source_tracking else point.control_ns) <= end) + original_controls = {point.control_ns for point in points} + requests = ( + {stamp: value for stamp, value in episodes.probe_requests.items() if stamp in original_controls} + if source_tracking + else {stamp: value for stamp, value in episodes.probe_requests.items() if start <= stamp <= end} + ) + if len(points) < 4 or any(value[:2] != (group, names) for value in requests.values()): + continue + # Source windows use their reference clock for membership, then retain + # each paired ORIGINAL command clock for actual native probe coverage. + # No point or command is moved/restamped to fit the new window boundary. + last = max(point.control_ns for point in points) if source_tracking else end + windows.append((min(p.control_ns for p in points), last, frozenset(requests))) + return tuple(sorted(windows, key=lambda row: row[0])) + + +def _group_direction_matches(group, kind, direction): + # A large maneuver has one steering direction; opposite-side hooks are + # correctly neutral there. A subtle weave contains both directions. + from openpilot.starpilot.flm.live_tracking import _group_direction_matches as matches + + return matches(group, kind, direction) + + +def _probe_group_coverage(rows, names, *, kind, direction): + from openpilot.starpilot.flm.live_tracking import _required + + matched = tuple(row for row in rows if row.kind == kind and row.direction == direction and getattr(row, 'native_gradient_complete', False)) + counts = { + name: sum( + any( + feature.name == name + and feature.feature_energy > 1e-10 + and feature.native_endpoints >= 4 + and feature.native_separated / feature.native_endpoints >= 0.09 + for feature in row.sensitivity + ) + for row in matched + ) + for name in names + } + # One unexcited maneuver never establishes that a hook cannot affect this car. + # Count repeated WHOLE matching maneuvers, retaining the original3/9 protocol. + unexcited = sum( + all(any(feature.name == name and feature.native_endpoints >= 4 and feature.feature_energy <= 1e-10 for feature in row.sensitivity) for name in names) + for row in matched + ) + return max(counts.values(), default=0), not any(counts.values()) and unexcited >= _required(kind) + + +class LiveTrackingActor: + def __init__(self, bootstrap_process=None, *, probe_process=None, source_tracking=False): + if type(source_tracking) is not bool or source_tracking and bootstrap_process is not None: + raise ValueError('source tracking has no plant bootstrap process') + self.source_tracking = source_tracking + self.process = bootstrap_process + from openpilot.starpilot.flm.live_tracking_bootstrap_producer import ResponseSnapshotProcess + + self.snapshot_process = ResponseSnapshotProcess() + self.probe_preparation = ResponseSnapshotProcess() + self.probe_process = probe_process + self.probe_token = None + self.last_probe_control_ns = 0 + self.last_decode_ns = 0 + self.probe_queue = [] + self.probe_queue_key = None + self.probe_group_index = self.probe_group_samples = 0 + self.probe_group_epoch = None + self.probe_group_profile = None + self.ingress = self.certificate = self.application = None + self.seed = self.token = self.attempted_epoch = None + self.pending_bootstrap_result = None + self.pending_probe_results = [] + self.prepared_probe_context = None + self.reason = 'response_model_required' + self.diagnostic_result = None + self.last_publication_pose_ns = 0 + self.initial_basis = None + self.restore_pending = False + self.response_recovery_generation = 0 + self.native_key = self.hardware_inventory = () + + def interrupt(self, reason, *, context_changed=False): + state = None if self.ingress is None else self.ingress.optimizer.offer_state + retain_numeric = bool(not context_changed and self.seed is not None and self.token is not None and (state is None or state.get('accepted') is True)) + # An immutable numerical job keeps its ORIGINAL token and deadline. This + # preserves no current source/native witness, active trial or command lease. + if not retain_numeric: + self.pending_probe_results.clear() + self.pending_bootstrap_result = None + self.snapshot_process.cancel(reason) + self.probe_preparation.cancel(reason) + if self.process is not None: + self.process.cancel(reason) + if self.probe_process is not None: + self.probe_process.cancel(reason) + self.probe_token = None + self.prepared_probe_context = None + self.probe_queue.clear() + self.probe_queue_key = None + if self.ingress is not None: + self.ingress.interrupt(reason, context_changed=context_changed, preserve_completed=retain_numeric) + self.reason = reason + + def close(self, reason='live_tracking_actor_closed'): + self.pending_probe_results.clear() + self.pending_bootstrap_result = None + # Shutdown may wait for actual reaping; tick/interrupt only cancel/poll. + # Always attempt both reapers, including when one fails unexpectedly. + failure = None + closed = True + for process in (self.snapshot_process, self.probe_preparation, self.process, self.probe_process): + if process is not None: + try: + closed = process.close(reason) and closed + except BaseException as error: + if failure is None: + failure = error + closed = False + self.probe_token = None + self.prepared_probe_context = None + self.probe_queue.clear() + self.probe_queue_key = None + if self.ingress is not None: + self.ingress.interrupt(reason, context_changed=True) + self.reason = reason + if failure is not None: + raise failure + return closed + + @property + def pending(self): + return any(process is not None and process.child is not None for process in (self.process, self.probe_process)) + + def binding_for(self, collector, source): + from openpilot.starpilot.flm.runtime_adapter_registry import digest, control_vm_regime + + control = collector.ready[-1][0] + anchor = control.vehicle_model_anchor + if len(anchor) != 3 or control_vm_regime(control) != anchor: + raise ValueError('genuine current owner vehicle-model anchor required') + return LiveTrialBinding( + control.identity, + source.cp_sha256, + control.controller_revision, + control.baseline_revision, + source.source_epoch, + source.reset, + source.calibration_generation, + digest(anchor), + control.native_identity if control.native_required else '', + 'bootstrap-pending-v1', + source.persistent_source_epoch, + control.native_required, + ).validate() + + def adopt_response(self, certificate, collector, accepted_full30=None): + if self.process is not None and self.process.child is not None: + self.process.cancel('independently_qualified_response_available') + certificate = validate_certificate(certificate) + if self.seed is None or replace(certificate.binding, physical_model_revision=self.seed.physical_model_revision) != self.seed: + raise ValueError('bootstrap response belongs to another current source') + accepted = tuple((name, 0.0) for name in NAMES) if accepted_full30 is None else accepted_full30 + self.ingress = LiveTrackingIngress(certificate.binding, certificate, accepted, controller_domain=collector.adapter.command_family) + self.certificate = certificate + collector.adapter.replay.source_controller_revision = collector.adapter.controller_revision + self.ingress.replay = collector.adapter.replay + self.ingress.episodes.replay = collector.adapter.replay + self.probe_token = None + self.last_probe_control_ns = 0 + collector.live_tracking = self.ingress + self.ingress.scoring_enabled = not self.restore_pending + from openpilot.starpilot.flm.live_tracking_requalification import basis_from_pose + + try: + pose = collector.original_source_capture.rows[-1] + self.initial_basis = basis_from_pose( + certificate.binding, pose, now_ns=pose.completed_ns, native_key=self.native_key, hardware_inventory=self.hardware_inventory + ) + except (AttributeError, IndexError, TypeError, ValueError): + # Missing hardware provenance withholds cross-drive persistence, not + # otherwise valid within-drive learning/current physical monitoring. + self.initial_basis = None + from openpilot.starpilot.flm.live_tracking_response import TrackingSourceContext + + self.reason = 'fresh_current_source_window_required' if type(certificate) is TrackingSourceContext else 'fresh_physical_residual_window_required' + + def hold_restore(self, pending): + self.restore_pending = bool(pending) + if self.ingress is not None: + self.ingress.scoring_enabled = not self.restore_pending + if self.restore_pending: + self.ingress.optimizer.interrupt('historical_restore_pending') + + def _submit_probe_queue(self, token, now_ns): + from openpilot.starpilot.flm.live_tracking_probe_process import make_probe_batch + + # The aggregate literal snapshot bound also applies to a batch. Split only + # numerical work; four genuine endpoints per episode remain mandatory. + for count in range(len(self.probe_queue), 0, -1): + try: + batch = make_probe_batch(tuple(item[0] for item in self.probe_queue[:count]), queued_ns=now_ns) + except ValueError: + continue + if self.probe_process.submit_probe(batch, token=token, now_ns=now_ns): + self.probe_token = token + del self.probe_queue[:count] + if not self.probe_queue: + self.probe_queue_key = None + return True + return False + self.probe_queue.clear() + self.probe_queue_key = None + self.reason = 'bounded_numerical_probe_transport_unavailable' + return False + + def _probe_tick(self, collector, source, *, now_ns): + """Queue optional numerical work; never update a controller in this actor.""" + if self.probe_process is None or self.ingress is None: + return + if self.restore_pending: + self.probe_process.poll(token=None, now_ns=now_ns) + self.probe_queue.clear() + self.probe_queue_key = None + return + from openpilot.starpilot.flm.live_tracking_probe_process import make_probe_job, probe_replay_snapshot + + if self.process is not None and self.process.child is not None: + return # At most one optional reconstruction/identification child. + optimizer = self.ingress.optimizer + accepted = tuple((name, optimizer.accepted[name]) for name in NAMES) + profile_hash = full_profile_sha256(self.certificate.binding.baseline_revision, dict(accepted)) + token = (self.token, self.certificate.model_revision, profile_hash) + try: + result = self.probe_process.poll_probe(token=token, binding=self.certificate.binding, now_ns=now_ns) + completed, self.pending_probe_results = self.pending_probe_results, [] + if result is not None: + completed.append(result) + for result in completed: + for row in result if type(result) is tuple else (result,): + self.ingress.observe_probe(row, now_ns=now_ns) + self.probe_group_samples += 1 + from openpilot.starpilot.flm.live_tracking import discovery_needs_probes + + if optimizer.discovery is not None and not discovery_needs_probes(optimizer.discovery): + # Direction/coordinate selection already has genuine native probes. + # Its NEW uncensored baseline needs measured episodes, not another + # finite-difference replay of each endpoint. Reap optional old work + # nonblocking; no original samples or native grants are synthesized. + self.probe_preparation.cancel('independent_measurement_comparison') + self.probe_process.cancel('independent_measurement_comparison') + self.probe_queue.clear() + self.probe_queue_key = self.prepared_probe_context = self.probe_token = None + return + # Do not consume a one-shot prepared result while the numerical child + # cannot accept it. The original enqueue time/deadline remains binding; + # one existing preparation pipe holds it without blocking ingestion. + prepared = self.probe_preparation.poll(token=token, now_ns=now_ns) if self.probe_process.child is None else None + if prepared is not None and prepared.get('numeric') is not None: + expected, self.prepared_probe_context = self.prepared_probe_context, None + from openpilot.starpilot.flm.live_tracking_probe_process import _context_digest + + if ( + expected is None + or set(prepared) != {'numeric', 'job_sha256', 'context_sha256'} + or prepared['context_sha256'] != _context_digest(expected) + or type(prepared['job_sha256']) is not str + or len(prepared['job_sha256']) != 64 + or any(char not in '0123456789abcdef' for char in prepared['job_sha256']) + or any(not 0 < row['queued_ns'] <= now_ns <= row['queued_ns'] + 30_000_000_000 for row in expected['batch']) + ): + raise ValueError('owned inline numerical context or deadline differs') + numeric = self.probe_process.decode_report( + prepared['numeric'], expected=(prepared['job_sha256'], expected), prepared_sha=prepared['context_sha256'], binding=self.certificate.binding + ) + for row in numeric if type(numeric) is tuple else (numeric,): + self.ingress.observe_probe(row, now_ns=now_ns) + self.probe_group_samples += 1 + if prepared is not None and prepared.get('prepared') is not None: + expected, self.prepared_probe_context = self.prepared_probe_context, None + if expected is None: + raise ValueError('original parent numerical context unavailable') + if self.probe_process.submit_prepared_probe(prepared['prepared'], expected=expected, token=token, now_ns=now_ns): + self.probe_token = token + if self.probe_preparation.pid is not None: + return + if self.probe_queue and (cast(tuple[object, ...], self.probe_queue_key)[0] != token or now_ns - self.probe_queue[0][1] > 2_000_000_000): + self.probe_queue.clear() + self.probe_queue_key = None + if self.probe_process.child is not None or optimizer.offer_state is not None and optimizer.offer_state['phase'] == 'trial': + return + pair = collector.live_response_pair + if pair is None: + return + control, output, _pose = pair + from openpilot.starpilot.flm.live_tracking_response import verified_clipped_input + + def known_input(candidate, applied): + return verified_clipped_input(collector.adapter.replay, candidate, applied, self.certificate.binding) + + if output.limited and not known_input(control, output): + return # Missing original CAN/limiter proof is never a numerical input. + if control.control_ns <= self.last_probe_control_ns or self.ingress.profile(control) != profile_hash: + return + witness = self.ingress.monitor.witness(now_ns) + if witness is None: + return + history = tuple(row for row in self.ingress.control_history if row.control_ns <= control.control_ns) + checkpoints = [index for index, row in enumerate(history) if row.replay_snapshot] + if not checkpoints: + return + # Retain every genuine checkpoint in the existing bounded history. Each + # endpoint still selects its OWN latest preceding<=100-frame checkpoint. + # Restricting the parent view to two anchors can hide the middle of a + # maneuver while the serial numerical worker finishes an earlier batch. + first_checkpoint = checkpoints[0] + prefix = history[first_checkpoint:] + checkpoint_indices = tuple(index - first_checkpoint for index in checkpoints if index >= first_checkpoint) + # Use the worker's existing bounded full-checkpoint prefix. Restricting + # admission to its first120ms starved4Hz sampling after170ms physical + # lag. Every original frame is replayed; no missing endpoint is inferred. + if not prefix or prefix[-1] != control or not 1 <= len(prefix) <= 301: + if self.probe_queue: + self._submit_probe_queue(token, now_ns) + return + unsupported = () + if collector.adapter.command_family == 'torque_pid': + from openpilot.starpilot.flm.pid_stages import UNSUPPORTED + + unsupported = UNSUPPORTED + kind = self.ingress.episodes.mode + # Original prefix verification dominates optional numerical CPU. Measure + # every supported hook from that same genuine replay, rather than letting + # an unexcited first group starve all other coordinates. PID's four + # unmeasurable friction hooks remain excluded, never reported as measured. + group, names = 'all30', tuple(name for name in NAMES if name not in unsupported) + if not names: + return + ep = self.ingress.episodes + phase_windows = _original_probe_windows(ep, profile_hash, group, names) + if kind not in ('large', 'subtle') and not phase_windows: + return + self.probe_group_epoch = (kind, self.ingress.episodes.start) + queue_key = (token, prefix[0].control_ns, group, names) + if self.probe_queue and (len(self.probe_queue) >= 2 or self.probe_queue_key != queue_key or now_ns - self.probe_queue[0][1] >= 250_000_000): + self._submit_probe_queue(token, now_ns) + return + # Freeze up to four DISTINCT already-observed physical endpoints from + # the same original checkpoint. A4Hz coordinator must not require four + # separate process roundtrips within each short real maneuver. + points = {(point.control_ns, point.apply_ns) for point in self.ingress.episodes.endpoints} + selected = [] + original_ready = tuple( + (candidate, applied) for candidate, applied in getattr(collector, 'probe_ready', collector.ready) if 0 <= now_ns - applied.apply_ns <= 2_000_000_000 + ) + # A scalar newest-selected watermark cannot describe unsampled holes. + # Repair a known whole window's missing ORIGINAL time phases first, + # including older unrequested endpoints still inside all existing bounds. + # Selection inspects only source clocks/request identities, never error, + # improvement, native effect magnitude or a predicted favorable result. + # A completed/drained prior window cannot accept fresh requests. + # Do not let its unrequested points consume the limited fresh-stream + # batch slots; explicit whole-window repairs above remain independent. + fresh_after = max(self.last_probe_control_ns, ep.start - 1) + if phase_windows: + candidates = _observed_probe_candidates( + original_ready, + prefix, + points, + after_ns=0, + verify_limited=known_input, + checkpoint_indices=checkpoint_indices, + stratified=True, + phase_windows=phase_windows, + phase_fallback_after_ns=fresh_after if kind == 'large' else None, + fresh_limit=4, + discovery_headroom=True, + ) + elif kind == 'large': + candidates = _observed_probe_candidates( + original_ready, + prefix, + points, + after_ns=fresh_after, + verify_limited=known_input, + checkpoint_indices=checkpoint_indices, + stratified=True, + fresh_limit=4, + discovery_headroom=True, + ) + else: + candidates = () # Short subtle windows are sampled as whole original batches. + for candidate, applied, original_prefix in candidates: + if self.ingress.episodes.request_probe(candidate.control_ns, group=group, names=names, queued_ns=now_ns): + selected.append((candidate, applied, original_prefix)) + if not selected: + return + # Fork freezes the genuine source prefix without copying its nested + # checkpoint graph on the ingestion loop. The original worker still + # recomputes the complete controller/native evidence independently. + from openpilot.starpilot.flm.live_tracking_probe_process import original_probe_context, prepare_probe_wire + + expected = original_probe_context(selected, accepted, self.certificate.binding, source, group=group, names=names, now_ns=now_ns) + + def prepare_probe(): + jobs = [] + for endpoint, emission, original_prefix in selected: + snapshot = probe_replay_snapshot(collector.adapter.replay, original_prefix, endpoint_output=emission) + jobs.append( + make_probe_job( + replay_snapshot=snapshot, + control=endpoint, + output=emission, + accepted_full30=accepted, + binding=self.certificate.binding, + response_witness=witness, + source=source, + evidence_ns=now_ns, + queued_ns=now_ns, + current_vehicle_model_anchor=endpoint.vehicle_model_anchor, + group=group, + names=names, + prefix_controls=original_prefix, + ) + ) + # This existing child already owns a frozen original source snapshot. + # Reconstruct/validate the same numerical job HERE instead of paying a + # second Python startup and streaming it through another pipe. + import json + import os + import resource + import time + + os.nice(10) + from openpilot.starpilot.flm.live_tracking_probe_worker import evaluate_job, MAX_OUTPUT + + soft, hard = resource.getrlimit(resource.RLIMIT_CPU) + limit = min([20] + [value for value in (soft, hard) if value >= 0]) + resource.setrlimit(resource.RLIMIT_CPU, (limit, hard)) + memory_soft, memory_hard = resource.getrlimit(resource.RLIMIT_AS) + memory_limit = min([2 * 1024 * 1024 * 1024] + [value for value in (memory_soft, memory_hard) if value >= 0]) + resource.setrlimit(resource.RLIMIT_AS, (memory_limit, memory_hard)) + prepared = prepare_probe_wire(jobs, queued_ns=now_ns, expected=expected) + started_ns = time.monotonic_ns() + if not now_ns <= started_ns <= now_ns + 2_000_000_000: + raise ValueError('original numerical input deadline expired') + numeric = evaluate_job(json.loads(prepared['wire']), now_ns=started_ns) + if len(json.dumps(numeric, separators=(',', ':'), allow_nan=False).encode()) > MAX_OUTPUT: + raise ValueError('original numerical output bound') + return {'numeric': numeric, 'job_sha256': prepared['job_sha256'], 'context_sha256': prepared['context_sha256']} + + if self.probe_preparation.submit_preparation(prepare_probe, token=token, epoch=self.probe_group_epoch, now_ns=now_ns): + self.prepared_probe_context = expected + self.last_probe_control_ns = max(self.last_probe_control_ns, max(item[0].control_ns for item in selected)) + + except (AttributeError, KeyError, TypeError, ValueError, OverflowError): + # Missing/unsupported source-state reconstruction is no sensitivity proof. + self.reason = 'genuine_native_sensitivity_unavailable' + + def _retire_identified_history(self, collector): + """Retain live prefixes after independent identification has completed. + + Pending workers own fork-frozen originals. Live endpoints are at most2s + old with at most1s checkpoint history; retain4s plus its preceding original + checkpoint. No decoder state, unconsumed CAN, certificate or episode score + is replaced. Release at most32 rows per structure per tick. + """ + if self.ingress is None or self.certificate is None: + return + replay = collector.adapter.replay + binding = self.certificate.binding + if ( + replay.raw_mode is not True + or replay.identity != binding.identity + or replay.baseline_revision != binding.baseline_revision + or not replay.records + or getattr(replay, 'trace_archive', None) is not None + ): + return + cutoff = replay.records[-1].control_ns - 4_000_000_000 + anchors = [stamp for stamp, _ in replay.checkpoints if stamp <= cutoff] + if not anchors: + return + anchor = anchors[-1] + for _ in range(32): + if not replay.records or replay.records[0].control_ns >= anchor: + break + removed = replay.records.popleft() + replay.normalized.pop(removed.control_ns, None) + oldest = replay.records[0].control_ns + for _ in range(2): + if len(replay.checkpoints) < 2 or replay.checkpoints[1][0] > oldest: + break + replay.checkpoints.popleft() + emissions = replay.emissions + for _ in range(32): + if not emissions.events: + break + stamp = next(iter(emissions.events)) + event = emissions.events[stamp] + if event.control_ns >= oldest or (emissions.decode_queue and emissions.decode_queue[0] <= stamp): + break # Never skip original input to the stateful CAN decoder. + emissions.events.popitem(last=False) + emissions.decode_errors.pop(stamp, None) + if emissions.decoder is not None: + emissions.decoder.verified.pop(stamp, None) + if emissions.by_control.get(event.control_ns) is event: + emissions.by_control.pop(event.control_ns) + if emissions.events: + first_frame = next(iter(emissions.events.values())).witness.frame + 1 + for _ in range(32): + if not emissions.carriers or next(iter(emissions.carriers)) >= first_frame: + break + emissions.carriers.popitem(last=False) + context = getattr(collector.adapter, 'original_context_receipts', None) + if context is not None: + # An unconsumed native emission can still require an older original + # control. Keep it together with every retained replay/checkpoint row. + keep_from = min(oldest, next(iter(emissions.events.values())).control_ns) if emissions.events else oldest + for _ in range(32): + if not context.controls or next(iter(context.controls)) >= keep_from: + break + context.controls.popitem(last=False) + for _ in range(32): + if not context.outputs or next(iter(context.outputs.values())).control_ns >= keep_from: + break + context.outputs.popitem(last=False) + archive = getattr(collector.adapter, 'original_feedback_archive', None) + if archive is not None: + for records in (archive.publications, archive.consumptions): + for _ in range(32): + if not records or next(iter(records)) >= oldest: + break + records.popitem(last=False) + + def _same_bootstrap_source(self, source, now_ns): + """Only retain historical computation; this is never native admission.""" + from openpilot.starpilot.flm.runtime_adapter_registry import SourceState + + return bool( + self.seed is not None + and self.token is not None + and type(source) is SourceState + and source.enabled is True + and source.manual is False + and source.configuration_known is True + and all(type(stamp) is int for stamp in (source.observed_ns, source.expires_ns, now_ns)) + and 0 < source.observed_ns <= now_ns <= source.expires_ns <= source.observed_ns + 500_000_000 + and (source.cp_sha256, source.source_epoch, source.reset, source.calibration_generation, source.persistent_source_epoch) + == (self.seed.cp_sha256, self.seed.source_epoch, self.seed.reset_generation, self.seed.calibration_generation, self.seed.persistent_source_epoch) + ) + + def tick(self, collector, source, *, now_ns, current_native, native_key=(), hardware_inventory=(), allow_bootstrap=True): + self.native_key, self.hardware_inventory = native_key, hardware_inventory + snapshot_result = self.snapshot_process.poll(token=self.token if self._same_bootstrap_source(source, now_ns) else None, now_ns=now_ns) + if self.probe_process is not None and self.ingress is None: + # Continue actual reaping after a previous source change removed ingress. + self.probe_process.poll(token=None, now_ns=now_ns) + if collector is None or source is None or not collector.ready or current_native is not True: + # A frozen response job describes ORIGINAL source/native observations, + # not permission to apply a correction now. Temporary input/native gaps + # must not cancel that computation after expensive snapshot preparation. + # Keep any completed report inert until a fresh exact binding and current + # native admission below can independently admit the numerical model. + same_source = collector is not None and self._same_bootstrap_source(source, now_ns) + retain = same_source and self.ingress is None + retain_probe = same_source and self.ingress is not None and self.certificate is not None and self.probe_token is not None + if self.process is not None: + result = self.process.poll(token=self.token if retain else None, now_ns=now_ns) + if retain and type(result) is dict: + self.pending_bootstrap_result = result + if not retain: + self.pending_bootstrap_result = None + if self.probe_process is not None: + if retain_probe: + # This is only an immutable historical derivative, never current + # permission. Keep its original 30s deadline and exact job binding. + result = self.probe_process.poll_probe(token=self.probe_token, binding=self.certificate.binding, now_ns=now_ns) + if result is not None and len(self.pending_probe_results) < 4: + self.pending_probe_results.append(result) + else: + self.pending_probe_results.clear() + self.probe_process.poll(token=None, now_ns=now_ns) + self.probe_queue.clear() + self.probe_queue_key = None + if self.ingress is not None: + self.ingress.interrupt('current_source_or_native_permission_unavailable', preserve_completed=retain_probe) + return None + try: + seed = self.binding_for(collector, source) + except (AttributeError, TypeError, ValueError): + if self.process is not None: + self.process.poll(token=None, now_ns=now_ns) + if self.probe_process is not None: + self.probe_process.poll(token=None, now_ns=now_ns) + self.probe_queue.clear() + self.probe_queue_key = None + if self.ingress is not None: + self.ingress.interrupt('actual_response_source_unavailable', context_changed=True) + self.reason = 'actual_response_source_unavailable' + return None + token = tuple(vars(seed).values()) + if self.seed is not None and seed != self.seed: + self.prepared_probe_context = None + self.pending_probe_results.clear() + self.pending_bootstrap_result = None + if self.probe_process is not None: + self.probe_process.poll(token=None, now_ns=now_ns) + self.probe_queue.clear() + self.probe_queue_key = None + if self.ingress is not None: + self.ingress.interrupt('source_configuration_changed', context_changed=True) + self.ingress = self.certificate = None + collector.live_tracking = None + self.attempted_epoch = None + self.seed, self.token = seed, token + # Keep the stateful CAN decoder current independently of which physical + # samples happen to be clipped. At most64 original messages per4Hz tick; + # no skipped counter/checksum input, no endpoint or authority fabrication. + if now_ns - self.last_decode_ns >= 250_000_000: + self.last_decode_ns = now_ns + history = collector.adapter.replay.emissions + if history.events: + try: + history.decode(next(reversed(history.events.values()))) + except (AttributeError, KeyError, TypeError, ValueError, OverflowError): + pass # Pending/invalid bytes still fail their exact verification. + if self.ingress is not None and self.ingress.monitor.persistent_residual_failure(now_ns): + # This is numeric-model withdrawal, never admission of a replacement. + # Preserve the per-epoch attempt latch: no outcome-based cohort search. + self.interrupt('persistent_physical_response_drift', context_changed=True) + self.response_recovery_generation += 1 + self.hold_restore(False) + self.ingress = self.certificate = self.application = self.initial_basis = None + collector.live_tracking = None + capture = collector.original_source_capture + if capture is not None: + self.attempted_epoch = capture.epoch + return None + if self.process is not None: + result, self.pending_bootstrap_result = self.pending_bootstrap_result, None + polled = self.process.poll(token=token, now_ns=now_ns) + if polled is not None: + result = polled + if type(result) is dict: + if result.get('production_admissible') is not True: + self.diagnostic_result = result + self.reason = 'diagnostic_response_not_production_admission' + elif result.get('status') == 'response_identified' and result.get('certificate') is not None: + self.adopt_response(certificate_from_dict(result['certificate']), collector) + else: + self.reason = str(result.get('reason', 'response_identification_unavailable')) + if self.ingress is None and self.source_tracking and allow_bootstrap: + from openpilot.starpilot.flm.live_tracking_response import source_context + + capture = collector.original_source_capture + if capture is not None and capture.rows: + try: + context = source_context(seed, collector.ready[-1][0], capture.rows[-1], now_ns=now_ns) + self.adopt_response(context, collector) + except (AttributeError, KeyError, TypeError, ValueError, OverflowError): + self.reason = 'current_original_tracking_sources_required' + if self.ingress is None and self.source_tracking: + self.reason = 'fresh_current_tracking_sources_required' + return None + if self.ingress is None: + if not allow_bootstrap: + self.reason = 'retained_response_requalification_pending' + return None + if self.process is None: + self.reason = 'production_bootstrap_budget_not_yet_qualified' + return None + capture = collector.original_source_capture + if not self.pending and capture is not None and self.attempted_epoch != capture.epoch: + if type(snapshot_result) is dict and self.snapshot_process.epoch == capture.epoch and self.snapshot_process.token == token: + snapshot = snapshot_result.get('snapshot') + if snapshot is not None and self.process.submit(snapshot, token=token, now_ns=now_ns): + self.attempted_epoch = capture.epoch + self.reason = 'initial_independent_response_identification' + elif snapshot is None: + self.reason = str(snapshot_result.get('error', 'original_response_snapshot_not_ready')) + elif self.snapshot_process.pid is None: + self.snapshot_process.submit(collector, seed, token=token, now_ns=now_ns) + self.reason = self.snapshot_process.reason + return None + self._retire_identified_history(collector) + self._probe_tick(collector, source, now_ns=now_ns) + if self.restore_pending: + return None + offer = self._current_publication_offer(now_ns) + if offer is None: + return None + pose = validate_response_witness(offer.response_witness, self.certificate.binding, now_ns).last_pose_ns + if pose <= self.last_publication_pose_ns: + return None + self.last_publication_pose_ns = pose + return offer + + def _current_publication_offer(self, now_ns): + from openpilot.starpilot.flm.live_tracking_response import response_witness_value, validate_response_witness + + offer = self.ingress.latest_offer + if offer is None: + return None + try: + # Structural validation of the owned old message is not current permission. + offer.validate() + old = response_witness_value(offer.response_witness) + optimizer = self.ingress.optimizer + current = validate_response_witness(optimizer.response_witness, self.certificate.binding, now_ns) + state = optimizer.offer_state + if offer.binding != self.certificate.binding or old.certificate != self.certificate or state is None: + raise ValueError('trial publication identity changed') + if offer.trial_id != state['trial_id']: + prior = state.get('prior_accepted') + if ( + prior is None + or prior.get('accepted') is not True + or (offer.trial_id, offer.phase, offer.target_sequence, offer.target_sha256) + != (prior['trial_id'], prior['phase'], prior.get('target_sequence', optimizer.sequence), optimizer._hash(prior['target'])) + ): + raise ValueError('cached offer is not the owned predecessor') + state_changed = ( + offer.trial_id != state['trial_id'] + or offer.phase != state['phase'] + or offer.target_sequence != state.get('target_sequence', optimizer.sequence) + or offer.target_sha256 != optimizer._hash(state['target']) + ) + if state_changed or current.last_pose_ns > old.last_pose_ns: + # Issue from a genuinely newer qualified pose; never renew the old + # witness, the original trial deadline, or any native command lease. + offer = optimizer.offer(now_ns, publication=False) + if offer is None: + raise ValueError('current trial publication unavailable') + self.ingress.latest_offer = offer + validate_response_witness(offer.response_witness, self.certificate.binding, now_ns) + return offer + except (AttributeError, TypeError, ValueError): + self.ingress.interrupt('fresh_response_witness_unavailable') + return None + + def mark_publication(self, offer): + if self.ingress is not None: + self.ingress.optimizer.mark_published(offer) + + def observe_application(self, application, *, now_ns): + if self.ingress is None or not self.ingress.observe_application(application, now_ns=now_ns): + return False + self.application = application + return True + + def observe_restored(self, application, offer, *, now_ns): + from openpilot.starpilot.flm.live_trial_owner import LiveTrialApplication + + if self.ingress is None or type(application) is not LiveTrialApplication: + return False + if ( + application.status != 'restored' + or application.binding != self.certificate.binding + or application.trial_id != offer.history_trial_id + or application.applied != offer.target + or application.applied_sha256 != offer.target_sha256 + or not 0 < application.control_ns <= now_ns <= application.control_ns + 100_000_000 + ): + return False + validate_response_witness(offer.response_witness, self.certificate.binding, now_ns) + if full_profile_sha256(self.certificate.binding.baseline_revision, dict(application.applied)) != offer.target_sha256: + return False + self.ingress.latest_restore_offer = offer + self.application = application + self.ingress.optimizer.accepted = dict(application.applied) + self.ingress.optimizer.baseline.clear() + self.ingress.optimizer.offer_state = None + self.ingress.optimizer.future.clear() + self.hold_restore(False) + return True + + def persistence_inputs(self, now_ns): + if self.ingress is None or self.application is None or self.ingress.latest_offer is None: + return None + offer = self.ingress.latest_offer + if ( + offer.phase != 'confirmed' + or self.application.confirmed_target_sha256 != offer.target_sha256 + or not self.ingress.optimizer.report(now_ns)['applicationAcknowledged'] + ): + return None + return offer, self.application, self.certificate diff --git a/openpilot/starpilot/flm/live_tracking_bootstrap.py b/openpilot/starpilot/flm/live_tracking_bootstrap.py new file mode 100644 index 0000000000..f424f9a706 --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_bootstrap.py @@ -0,0 +1,103 @@ +"""One initial genuine response experiment; no knob fit or profile admission. + +The live tracking path retains this numerical model across ordinary capture +epochs while independent current residual/source checks remain qualified. +Full candidate replay remains available offline and is not called here. +""" + +import base64 +import pickle + +VERSION = 'live-response-bootstrap-v1' + + +def bootstrap_snapshot(snapshot): + from openpilot.starpilot.flm.controller_replay import ( + unpack_primitive_snapshot, + pack_primitive_snapshot, + factory_from_snapshot, + MAX_FIT_RAW_BYTES, + ) + from openpilot.starpilot.flm.primitive_source_memo import independent_primitive_graph + from openpilot.starpilot.flm.normalized_trace_archive import reserve_for_packet + from openpilot.starpilot.flm.feedback_replay import PROTOCOL as FEEDBACK_PROTOCOL, EVENT_VERSION, verify_archive + from openpilot.starpilot.flm.event_mean_evidence import VERSION as MEAN_VERSION + from openpilot.starpilot.flm.candidate_feedback import CLOSED_POLICY + from openpilot.starpilot.flm.torque_adapter import Objective + from openpilot.starpilot.flm.full_surface_profile import envelope_from_dict + from openpilot.starpilot.flm.source_plant_pipeline import _template, replay_data + from openpilot.starpilot.flm.original_context_receipts import verify_original_replay + from openpilot.starpilot.flm.source_plant_fit import source_binding, prepare, PreparedPlant, SOURCE_COMPLETE_SLOT_POLICY as BOOTSTRAP_SLOT_POLICY + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + from openpilot.starpilot.flm.live_tracking_response import certificate_from_prepared, certificate_to_dict + + if ( + type(snapshot) is not dict + or snapshot.get('live_job_version') != VERSION + or snapshot.get('version') != 1 + or type(snapshot.get('snapshot_b64')) is not str + or len(snapshot['snapshot_b64']) > 6 * 1024 * 1024 + ): + raise ValueError('explicit bounded live response bootstrap wrapper') + payload = unpack_primitive_snapshot(base64.b64decode(snapshot['snapshot_b64'], validate=True)) + if ( + payload.get('live_job_version') != VERSION + or payload.get('version') != 1 + or payload.get('family') not in ('torque', 'torque_pid') + or payload.get('family') != snapshot.get('family') + or payload.get('plant_protocol') != FEEDBACK_PROTOCOL + or payload.get('event_version') != EVENT_VERSION + or payload.get('event_mean_version') != MEAN_VERSION + or payload.get('candidate_feedback_policy') != CLOSED_POLICY + or payload.get('event_operation') != 'trial' + or tuple(snapshot.get('objective_key', ())) != tuple(payload.get('objective_key', ())) + ): + raise ValueError('current original source/native response bootstrap protocol') + payload = independent_primitive_graph(payload, reserved_bytes=reserve_for_packet(payload['source_packet'])) + packet = payload['source_packet'] + envelope = envelope_from_dict(payload['effect_envelope']) + replay = replay_data(packet, objective=Objective(**payload['objective']), identity=payload['identity'], baseline_revision=envelope.selected_baseline_revision) + if len(pickle.dumps(payload, protocol=5)) + len(pickle.dumps(replay, protocol=5)) > MAX_FIT_RAW_BYTES: + raise ValueError('bootstrap complete original raw byte bound') + replay = pack_primitive_snapshot(replay) + factory, _ = factory_from_snapshot(replay) + template, _, _ = _template(payload, factory) + actual = verify_original_replay(packet['context_receipts'], template, replay) + binding = source_binding(packet, actual) + verify_archive(packet, actual) + if payload.get('live_slot_policy') != BOOTSTRAP_SLOT_POLICY: + raise ValueError('explicit live response calibration selection policy') + from openpilot.starpilot.flm.live_tracking_bootstrap_cohorts import VERSION as COHORT_VERSION, validate_selected_cohort + + if payload.get('live_cohort_version') != COHORT_VERSION: + raise ValueError('explicit outcome-blind checkpoint cohort protocol required') + current = LiveTrialBinding(**payload['live_binding']) + selection = validate_selected_cohort(payload.get('live_cohort_selection'), packet, actual, current, payload['now_ns'], source_complete=True) + from openpilot.starpilot.flm.plant_lag import TRAINING_DOMAIN_ONLY + + prepared = prepare(packet, actual, binding, slot_policy=BOOTSTRAP_SLOT_POLICY, support_policy=TRAINING_DOMAIN_ONLY) + if type(prepared) is not PreparedPlant: + return { + 'live_job_version': VERSION, + 'status': 'collecting', + 'reason': prepared.get('reason', 'response_unavailable'), + 'certificate': None, + 'profile': None, + 'current_permission': False, + } + if ( + tuple(tuple(p.mono_ns for p in prepared.source_poses if w.start_ns <= p.mono_ns <= w.end_ns) for w in prepared.original_windows) + != selection['source_pose_ns'] + ): + raise ValueError('independent native-verified source pose eligibility differs') + certificate = certificate_from_prepared(prepared, current, original_packet=packet) + # No finite differences, group rotation, trial scoring, replay validation, + # profile, live lease or current/native permission is manufactured here. + return { + 'live_job_version': VERSION, + 'status': 'response_identified', + 'reason': 'independent_train_heldout_response_passed', + 'certificate': certificate_to_dict(certificate), + 'profile': None, + 'current_permission': False, + } diff --git a/openpilot/starpilot/flm/live_tracking_bootstrap_cohorts.py b/openpilot/starpilot/flm/live_tracking_bootstrap_cohorts.py new file mode 100644 index 0000000000..2ed029cc47 --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_bootstrap_cohorts.py @@ -0,0 +1,581 @@ +"""Outcome-blind scheduling of a FIRST complete original calibration cohort. + +This is source/command availability, never plant identification or permission. +Every candidate is an actual captured checkpoint, ordered by source clock. All +response yaw/roll/error/residual values are deliberately absent from selection. +""" + +from bisect import bisect_left, bisect_right +import math +import pickle +from typing import cast + +VERSION = 'first-complete-physical-feedback-cohort-v7' +MAX_CHECKPOINTS = 128 +CHECK_INTERVAL_NS = 1_000_000_000 + + +def _decline(diagnostics, reason): + if type(diagnostics) is dict: + diagnostics['reason'] = reason + return None + + +def _literal(value): + kind = type(value) + if kind in (type(None), bool, int, str, bytes): + return value + if kind is float and math.isfinite(value): + return value + if kind is tuple: + return tuple(_literal(item) for item in value) + raise ValueError('closed source-only cohort scheduling value required') + + +def source_quality_keys(frames, poses, emissions, checkpoints, controls, *, diagnostics=None): + """Fresh exact values invalidate scheduling memo; never reads a response. + + The memo avoids rescanning every sealed cohort, NOT original source/native + verification. Its keys contain all fields that scheduling reads, with Float64 + bits preserved by pickle5. Unknown leaves are rejected before pickle callbacks. + """ + from openpilot.starpilot.flm.controller_replay import ReplayFrame, Snapshot + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + from openpilot.starpilot.flm.torque_emission_replay import Emission + from opendbc.car.steering_projection import TorqueProjection + + if ( + len(frames) > 12000 + or len(poses) > 5120 + or len(emissions) > 12000 + or any(type(row) is not ReplayFrame for row in frames) + or any(type(row) is not OriginalSourcePose for row in poses) + or any(type(row) is not Emission or type(row.witness) is not TorqueProjection for row in emissions) + or len(controls) > 12000 + or any(type(row) is not ControlEvidence for row in controls) + ): + raise ValueError('exact bounded original cohort source') + if len(checkpoints) > MAX_CHECKPOINTS: + _decline(diagnostics, 'checkpoint_count_exceeds_128_bound') + raise ValueError('genuine bounded checkpoint metadata') + if any(type(row) is not tuple or len(row) != 2 or type(row[1]) is not Snapshot for row in checkpoints): + _decline(diagnostics, 'checkpoint_metadata_type_unavailable') + raise ValueError('genuine bounded checkpoint metadata') + raw = ( + tuple((row.control_ns, row.healthy, row.manual, row.frame_sequence) for row in frames), + tuple( + ( + row.mono_ns, + row.source_lower_ns, + row.source_upper_ns, + row.completed_ns, + row.speed_mps, + row.valid, + row.native_allowed, + row.native_required, + row.identity, + row.source_epoch, + row.baseline_revision, + row.controller_revision, + row.calibration, + row.native_identity, + row.context, + ) + for row in poses + ), + tuple( + ( + row.control_ns, + row.state_ns, + row.apply_ns, + tuple(row.witness), + row.valid, + row.active, + row.override, + row.fault, + row.limited, + row.native_allowed, + row.native_required, + row.identity, + row.source_epoch, + row.baseline_revision, + row.native_identity, + row.native_observed_ns, + row.native_expires_ns, + ) + for row in emissions + ), + tuple((stamp, snapshot.revision, snapshot.values) for stamp, snapshot in checkpoints), + tuple( + ( + row.control_ns, + row.state_ns, + row.enabled, + row.active, + row.valid, + row.manual, + row.override, + row.fault, + row.identity, + row.source_epoch, + row.baseline_revision, + row.controller_revision, + row.native_identity, + row.native_required, + row.native_allowed, + row.native_observed_ns, + row.native_expires_ns, + row.vehicle_model_anchor, + row.saturated, + row.model_wobble, + row.controller_chatter, + row.limited, + row.source_receipt_ns, + row.domain, + row.stage_schema_revision, + row.replay_inputs, + row.requested, + ) + for row in controls + ), + ) + keys = tuple(tuple(pickle.dumps(_literal(row), protocol=5) for row in group) for group in raw) + for stamps in ( + tuple(row.control_ns for row in frames), + tuple(row.mono_ns for row in poses), + tuple(row.apply_ns for row in emissions), + tuple(row.control_ns for row in controls), + ): + if any(type(value) is not int or value <= 0 for value in stamps) or any(right <= left for left, right in zip(stamps, stamps[1:], strict=False)): + raise ValueError('ordered original source cohort clocks') + if any(type(row.control_ns) is not int or row.control_ns <= 0 for row in emissions): + raise ValueError('original emitted command clock type') + if any(right.control_ns < left.control_ns for left, right in zip(emissions, emissions[1:], strict=False)): + raise ValueError('original emitted command clock regression') + return keys + + +def new_schedule_cache(old, token, keys): + """Only fresh append-only exact-value source can retain examined anchors.""" + if ( + type(old) is dict + and old.get('token') == token + and type(old.get('keys')) is tuple + and len(old['keys']) == len(keys) + and type(old.get('examined')) is dict + and 'selected' in old + and all(len(new) >= len(before) and new[: len(before)] == before for new, before in zip(keys, old['keys'], strict=True)) + ): + return dict(old, keys=keys) + return {'token': token, 'keys': keys, 'examined': {}, 'selected': None} + + +def _directions(rows): + for positive in (True, False): + selected = [(slot, value) for slot, value in rows if (value > 0 if positive else value < 0)] + if len(selected) < 80: + return False + if sum(sum(slot == index for slot, _ in selected) >= 20 for index in range(8)) < 2: + return False + return True + + +def _command_quality(groups): + from openpilot.starpilot.flm.plant_lag import MIN_EXCITATION + + train = tuple((slot, value) for slot, values in enumerate(groups[:4]) for value in values) + heldout = tuple((slot + 4, value) for slot, values in enumerate(groups[4:]) for value in values) + values = tuple(value for _, value in train) + mean = sum(values) / len(values) + variance = sum((value - mean) ** 2 for value in values) / len(values) + if variance < MIN_EXCITATION**2 or not _directions(train) or not _directions(heldout): + return None + low, high = min(values), max(values) + # Scheduling is availability only. Match the response worker's supported + # TRAINING domain instead of requiring every future input to lie inside it. + # Outside inputs neither widen that range nor count as supported evidence; + # the child still validates ALL original heldout errors and native/source data. + supported = tuple((slot, value) for slot, value in heldout if low <= value <= high) + if any(sum(slot == index for slot, _ in supported) < 40 for index in range(4, 8)) or not _directions(supported): + return None + return {'training_command_std': math.sqrt(variance), 'command_low': low, 'command_high': high} + + +def source_complete_groups(applies, commands, groups): + """Input-only eligibility shared by parent scheduling and independent child. + + Keep the fixed eight slot boundaries. An unavailable command excludes every + pose whose FULL all-lag lookback includes it. No response value is read; no + command is interpolated, restamped or made known by omitting it from a tape. + """ + from openpilot.starpilot.flm.plant_lag import LAGS_NS, JOIN_NS + + gap_prefix, invalid_prefix = [0], [0] + for index, stamp in enumerate(applies): + gap_prefix.append(gap_prefix[-1] + int(index > 0 and stamp - applies[index - 1] > JOIN_NS)) + invalid_prefix.append(invalid_prefix[-1] + int(commands[index] is None)) + previous = [0] * len(LAGS_NS) + accepted = [] + for group in groups: + rows = [] + for row in group: + first = bisect_right(applies, row.source_lower_ns - LAGS_NS[-1]) - 1 + last = bisect_right(applies, row.source_upper_ns) - 1 + if ( + first < 0 + or last < first + or row.source_lower_ns - LAGS_NS[-1] - applies[first] > JOIN_NS + or gap_prefix[last + 1] - gap_prefix[first + 1] + or invalid_prefix[last + 1] - invalid_prefix[first] + ): + continue + endpoints = [] + for index, lag_ns in enumerate(LAGS_NS): + lower, upper = row.source_lower_ns - lag_ns, row.source_upper_ns - lag_ns + left, right = bisect_right(applies, lower) - 1, bisect_right(applies, upper) - 1 + if left < 0 or left != right or upper - applies[left] > JOIN_NS or applies[left] <= previous[index]: + break + endpoints.append(applies[left]) + if len(endpoints) == len(LAGS_NS): + rows.append(row) + previous = endpoints + accepted.append(tuple(rows)) + return tuple(accepted) + + +def _common_command_source(applies, commands, groups): + filtered = source_complete_groups(applies, commands, groups) + return all(len(before) == len(after) for before, after in zip(groups, filtered, strict=True)) + + +def quality_for_start(start_ns, *, frames, poses, emissions, controls, binding, now_ns, diagnostics=None, source_complete=False): + """Fresh literal quality of eight fixed slots; no response values inspected.""" + from openpilot.starpilot.flm.controller_replay import ReplayFrame + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence + from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current + from openpilot.starpilot.flm.runtime_adapter_registry import digest + from openpilot.starpilot.flm.torque_emission_replay import Emission + from openpilot.starpilot.flm.source_plant_fit import FIRST_SLOT_NS, BLOCK_NS, PURGE_NS + from openpilot.starpilot.flm.plant_lag import LAGS_NS, JOIN_NS + from openpilot.starpilot.flm.live_tracking_response import physical_calibration_generation + from opendbc.car.steering_projection import TorqueProjection, normalized_applied_command + + end_ns = start_ns + FIRST_SLOT_NS + 7 * (BLOCK_NS + PURGE_NS) + BLOCK_NS + if type(start_ns) is not int or start_ns <= 0 or end_ns > now_ns: + return _decline(diagnostics, 'fixed_cohort_clock_incomplete') + if ( + type(frames) is not tuple + or type(poses) is not tuple + or type(emissions) is not tuple + or len(frames) > 12000 + or len(poses) > 5120 + or len(emissions) > 12000 + or any(type(row) is not ReplayFrame for row in frames) + or any(type(row) is not OriginalSourcePose for row in poses) + or any(type(row) is not Emission for row in emissions) + or type(controls) is not tuple + or len(controls) > 12000 + or any(type(row) is not ControlEvidence for row in controls) + ): + raise ValueError('exact bounded original cohort source required') + stamps = tuple(frame.control_ns for frame in frames) + first, crossing = bisect_left(stamps, start_ns), bisect_left(stamps, end_ns) + if first == len(frames) or stamps[first] != start_ns or crossing == len(frames): + return _decline(diagnostics, 'fixed_cohort_frame_coverage_unavailable') + selected_frames = frames[first : crossing + 1] + if any(frame.healthy is not True or frame.manual is not False for frame in selected_frames) or any( + not 0 < right.control_ns - left.control_ns <= 50_000_000 or right.frame_sequence != left.frame_sequence + 1 + for left, right in zip(selected_frames, selected_frames[1:], strict=False) + ): + return _decline(diagnostics, 'fixed_cohort_frame_health_or_contiguity_unavailable') + by_stamp = {row.control_ns: row for row in controls} + if len(by_stamp) != len(controls) or any(frame.control_ns not in by_stamp for frame in selected_frames): + return _decline(diagnostics, 'fixed_cohort_original_control_coverage_unavailable') + selected_controls = tuple(by_stamp[frame.control_ns] for frame in selected_frames) + if any( + not row.enabled + or not row.active + or not row.valid + or row.manual + or row.override + or row.fault + or row.native_required is not binding.native_required + or (row.identity, row.source_epoch, row.baseline_revision, row.controller_revision, row.native_identity) + != (binding.identity, binding.source_epoch, binding.baseline_revision, binding.controller_revision, binding.native_identity) + or row.native_required + and (not row.native_allowed or not 0 < row.native_observed_ns <= row.control_ns <= row.native_expires_ns) + for row in selected_controls + ): + return _decline(diagnostics, 'fixed_cohort_control_source_or_native_unqualified') + if not source_complete and any( + row.saturated + or row.model_wobble + or row.controller_chatter + or row.limited + and not (row.stage_schema_revision == 'torque-selected-stage-snapshot-v1' and dict(row.replay_inputs).get('curvature_limited') == 0.0) + for row in selected_controls + ): + # Preserve the conservative legacy controller-row selector. Only the + # explicit versioned PHYSICAL input policy below may retain a delivered + # tagged input; its independent native/CAN verifier remains mandatory. + return _decline(diagnostics, 'fixed_cohort_controller_row_quality_unqualified') + anchors = tuple(dict.fromkeys(row.vehicle_model_anchor for row in selected_controls)) + if any(type(anchor) is not tuple or len(anchor) != 3 or digest(anchor) != binding.vehicle_model_generation for anchor in anchors): + return _decline(diagnostics, 'fixed_cohort_vehicle_model_anchor_unavailable') + selected = [] + calibration = physical_calibration_generation(binding) + context = None + for index in range(8): + lower = start_ns + FIRST_SLOT_NS + index * (BLOCK_NS + PURGE_NS) + upper = lower + BLOCK_NS + raw = tuple(row for row in poses if lower <= row.mono_ns < upper) + if any( + any(type(value) is not int for value in (row.source_lower_ns, row.source_upper_ns, row.completed_ns)) + or not 0 < row.source_lower_ns <= row.mono_ns <= row.source_upper_ns <= row.completed_ns + for row in raw + ): + return _decline(diagnostics, 'fixed_slot_pose_source_clock_unavailable') + group = tuple(row for row in raw if lower <= row.source_lower_ns <= row.source_upper_ns < upper) + if not 40 <= len(group) <= 240: + return _decline(diagnostics, 'fixed_slot_pose_count_unavailable') + for row in group: + # These are source/context/physical QUALITY conditions, not residuals. + if ( + row.valid is not True + or row.native_allowed is not True + or row.native_required is not binding.native_required + or (row.identity, row.source_epoch, row.baseline_revision, row.controller_revision, row.calibration, row.native_identity) + != (binding.identity, binding.source_epoch, binding.baseline_revision, binding.controller_revision, calibration, binding.native_identity) + or type(row.speed_mps) not in (float, int) + or not math.isfinite(row.speed_mps) + or row.speed_mps <= 0 + or not 0 < row.source_lower_ns <= row.mono_ns <= row.source_upper_ns <= row.completed_ns <= now_ns + ): + return _decline(diagnostics, 'fixed_slot_pose_source_or_native_unqualified') + if context is None: + context = row.context + if row.context != context: + return _decline(diagnostics, 'fixed_slot_pose_context_changed') + selected.append(group) + events = tuple(event for event in emissions if start_ns <= event.control_ns <= selected_frames[-1].control_ns) + applies = tuple(event.apply_ns for event in events) + if not events or any(right <= left for left, right in zip(applies, applies[1:], strict=False)): + return _decline(diagnostics, 'fixed_cohort_original_emission_order_unavailable') + commands = [] + for event in events: + if ( + event.valid is not True + or event.active is not True + or event.override is not False + or event.fault is not False + or event.native_allowed is not True + or event.native_required is not binding.native_required + or (event.identity, event.source_epoch, event.baseline_revision, event.native_identity) + != (binding.identity, binding.source_epoch, binding.baseline_revision, binding.native_identity) + or type(event.witness) is not TorqueProjection + or not 0 < event.state_ns <= event.control_ns <= event.apply_ns <= now_ns + or event.native_required + and not 0 < event.native_observed_ns <= event.control_ns <= event.apply_ns <= event.native_expires_ns + ): + commands.append(None) + continue + control = by_stamp.get(event.control_ns) + # This immutable view copies the ACTUAL emitted source/codec fields only; + # it supplies no native permission. The child still decodes actual CAN. + output = AppliedEvidence( + event.control_ns, + event.state_ns, + event.apply_ns, + 'normalized_torque', + event.witness.reported_torque, + event.valid, + event.active, + event.override, + event.fault, + event.limited, + event.native_allowed, + projection=event.witness, + ) + # This selector schedules PHYSICAL calibration only. Original saturation, + # chatter and limiting remain in the full immutable packet. They do not + # erase a known emitted input. The independent physical-input owner still + # verifies its exact CAN/limiter result, source identity and native leases. + # Current witnesses, trial eligibility and application remain unchanged. + if control is None or not consumed_sources_current(control, output): + commands.append(None) + continue + value = normalized_applied_command(event.witness) + commands.append(-cast(float, value) if type(value) in (float, int) and math.isfinite(cast(float, value)) else None) + source_support = {} + if source_complete: + selected = source_complete_groups(applies, commands, selected) + if any(not 40 <= len(group) <= 240 for group in selected): + return _decline(diagnostics, 'fixed_slots_source_complete_pose_count_unavailable') + source_support = {'source_pose_ns': tuple(tuple(row.mono_ns for row in group) for group in selected)} + if not _common_command_source(applies, commands, selected): + return _decline(diagnostics, 'fixed_cohort_common_command_source_unavailable') + # This is NOT lag identification: only whether ANY declared lag has adequate + # original command support. No response/correlation/error chooses this lag. + for lag_ns in LAGS_NS: + groups = [] + for group in selected: + values = [] + for row in group: + lower, upper = row.source_lower_ns - lag_ns, row.source_upper_ns - lag_ns + left, right = bisect_right(applies, lower) - 1, bisect_right(applies, upper) - 1 + if left < 0 or left != right or upper - applies[left] > JOIN_NS or commands[left] is None: + break + values.append(commands[left]) + if len(values) != len(group): + break + groups.append(tuple(values)) + if len(groups) != 8: + continue + summary = _command_quality(groups) + if summary is not None: + return dict( + summary, **source_support, start_ns=start_ns, end_ns=end_ns, pose_counts=tuple(len(group) for group in selected), command_support_lag_ns=lag_ns + ) + return _decline(diagnostics, 'fixed_slots_command_support_or_excitation_unavailable') + + +def select_first_cohort( + *, + checkpoints, + frames, + poses, + emissions, + controls, + binding, + epoch, + epoch_start_ns, + cutoff_ns, + now_ns, + cache=None, + diagnostics=None, + unavailable_anchors=(), + source_complete=False, +): + """Actual checkpoint anchors are declared independently of response outcomes.""" + from openpilot.starpilot.flm.controller_replay import Snapshot + + if ( + type(checkpoints) is not tuple + or len(checkpoints) > MAX_CHECKPOINTS + or not poses + or type(epoch) is not int + or epoch < 0 + or not 0 < epoch_start_ns < cutoff_ns + or cutoff_ns - epoch_start_ns > 100_000_000_000 + ): + raise ValueError('bounded original source epoch required') + binding.validate() + if any(type(row) is not tuple or len(row) != 2 or type(row[0]) is not int or type(row[1]) is not Snapshot for row in checkpoints): + raise ValueError('genuine captured checkpoint anchors required') + anchors = tuple(stamp for stamp, _ in checkpoints) + if any(right <= left for left, right in zip(anchors, anchors[1:], strict=False)): + raise ValueError('original checkpoint order') + preceding = tuple(stamp for stamp in anchors if stamp <= poses[0].mono_ns) + if not preceding: + return _decline(diagnostics, 'original_checkpoint_before_first_pose_unavailable') + origin = preceding[-1] + if cache is None: + cache = {'examined': {}, 'selected': None} + if cache['selected'] is not None: + if type(diagnostics) is dict: + diagnostics['reason'] = 'cached_source_cohort_selected_not_child_qualified' + return dict(cache['selected']) + from openpilot.starpilot.flm.source_plant_fit import FIRST_SLOT_NS, BLOCK_NS, PURGE_NS + + duration_ns = FIRST_SLOT_NS + 7 * (BLOCK_NS + PURGE_NS) + BLOCK_NS + if type(diagnostics) is dict: + diagnostics.update( + { + 'reason': 'fixed_cohort_still_collecting_or_previously_declined', + 'requiredCohortNs': duration_ns, + 'originCheckpointNs': origin, + 'examinedAnchors': len(cast(dict[int, bool], cache['examined'])), + } + ) + for start_ns in anchors: + if start_ns < origin or start_ns + duration_ns >= cutoff_ns: + continue + if ( + start_ns + duration_ns > now_ns + or not frames + or not poses + or frames[-1].control_ns < start_ns + duration_ns + or poses[-1].source_lower_ns < start_ns + duration_ns + or not emissions + or emissions[-1].control_ns < start_ns + duration_ns + or not controls + or controls[-1].control_ns < start_ns + duration_ns + or start_ns in cast(dict[int, bool], cache['examined']) + ): + continue + if start_ns in unavailable_anchors: + _decline(diagnostics, 'fixed_cohort_feedback_source_incomplete') + continue + if type(diagnostics) is dict: + diagnostics['lastExaminedAnchorNs'] = start_ns + result = quality_for_start( + start_ns, + frames=frames, + poses=poses, + emissions=emissions, + controls=controls, + binding=binding, + now_ns=now_ns, + diagnostics=diagnostics, + source_complete=source_complete, + ) + cast(dict[int, bool], cache['examined'])[start_ns] = result is not None + if type(diagnostics) is dict: + diagnostics['examinedAnchors'] = len(cast(dict[int, bool], cache['examined'])) + if result is not None and result['end_ns'] < cutoff_ns: + selected = dict(result, version=VERSION, epoch=epoch, epoch_start_ns=epoch_start_ns, cutoff_ns=cutoff_ns, origin_checkpoint_ns=origin) + cache['selected'] = selected + if type(diagnostics) is dict: + diagnostics['reason'] = 'source_cohort_selected_not_child_qualified' + return dict(selected) + return None + + +def validate_selected_cohort(selection, packet, engine, binding, now_ns, *, source_complete=False): + """Fresh child recomputation of selected command support; no parent trust.""" + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose + + if ( + type(selection) is not dict + or selection.get('version') != VERSION + or not engine.records + or selection.get('epoch') != packet['capture_epoch'] + or selection.get('epoch_start_ns') != packet['capture_epoch_start_ns'] + or selection.get('cutoff_ns') != packet['capture_epoch_cutoff_ns'] + or selection.get('start_ns') != engine.records[0].control_ns + or type(selection.get('origin_checkpoint_ns')) is not int + or not 0 < selection['origin_checkpoint_ns'] <= selection['start_ns'] < selection['end_ns'] < selection['cutoff_ns'] + or not 0 <= selection['epoch_start_ns'] - selection['origin_checkpoint_ns'] <= 1_100_000_000 + or selection['start_ns'] not in dict(engine.checkpoints) + ): + raise ValueError('explicit original checkpoint cohort selection provenance') + poses = tuple(OriginalSourcePose(**row) for row in packet['source_poses']) + quality = quality_for_start( + selection['start_ns'], + frames=tuple(engine.records), + poses=poses, + emissions=tuple(engine.emissions.events.values()), + controls=tuple(engine.original_context_controls.values()), + binding=binding, + now_ns=now_ns, + source_complete=source_complete, + ) + if ( + quality is None + or set(selection) != set(quality) | {'version', 'epoch', 'epoch_start_ns', 'cutoff_ns', 'origin_checkpoint_ns'} + or any(selection[name] != value for name, value in quality.items()) + ): + raise ValueError('current independently verified cohort command/source quality changed') + return selection diff --git a/openpilot/starpilot/flm/live_tracking_bootstrap_process.py b/openpilot/starpilot/flm/live_tracking_bootstrap_process.py new file mode 100644 index 0000000000..a6c1a226ce --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_bootstrap_process.py @@ -0,0 +1,18 @@ +"""Nonblocking response-only diagnostic actor, separate from legacy fitter.""" +from openpilot.starpilot.flm.fit_process import FitProcess +from openpilot.starpilot.flm.live_tracking_bootstrap_worker import PURPOSE, MAX_WALL_NS + + +class LiveBootstrapProcess(FitProcess): + worker_module = 'openpilot.starpilot.flm.live_tracking_bootstrap_worker' + wall_ns = MAX_WALL_NS + interval_ns = 0 + + def submit(self, snapshot, *, token, now_ns): + # The parent makes purpose explicit in its child environment. This actor + # never changes parent scheduling, legacy fitter limits or native policy. + import os + if os.environ.get('STARPILOT_LIVE_BOOTSTRAP_PURPOSE') != PURPOSE: + self.reason = 'production_bootstrap_budget_not_yet_qualified' + return False + return super().submit(snapshot, token=token, now_ns=now_ns) diff --git a/openpilot/starpilot/flm/live_tracking_bootstrap_producer.py b/openpilot/starpilot/flm/live_tracking_bootstrap_producer.py new file mode 100644 index 0000000000..b4766d60b2 --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_bootstrap_producer.py @@ -0,0 +1,422 @@ +"""Initial response-only export, independent of old group-fit cadence. + +Slot counts are availability, never an identification/admission declaration. +The child still verifies the complete authentic prefix/native/source experiment. +""" + +import base64 +from dataclasses import asdict + +from openpilot.starpilot.flm.live_tracking_bootstrap import VERSION + + +def _epoch_checkpoints(checkpoints, poses, cutoff_ns): + """Retain the original first-pose anchor and every checkpoint in its epoch.""" + from openpilot.starpilot.flm.controller_replay import Snapshot + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose + + if ( + type(checkpoints) is not tuple + or len(checkpoints) > 600 + or type(poses) is not tuple + or not poses + or type(poses[0]) is not OriginalSourcePose + or type(poses[0].mono_ns) is not int + or poses[0].mono_ns <= 0 + or type(cutoff_ns) is not int + or cutoff_ns <= poses[0].mono_ns + or any(type(row) is not tuple or len(row) != 2 or type(row[0]) is not int or row[0] <= 0 or type(row[1]) is not Snapshot for row in checkpoints) + ): + raise ValueError('genuine bounded epoch checkpoint metadata') + stamps = tuple(stamp for stamp, _ in checkpoints) + if any(right <= left for left, right in zip(stamps, stamps[1:], strict=False)): + raise ValueError('original checkpoint order') + preceding = tuple(stamp for stamp in stamps if stamp <= poses[0].mono_ns) + if not preceding: + return () + origin = preceding[-1] + return tuple(row for row in checkpoints if origin <= row[0] < cutoff_ns) + + +def _epoch_rows(rows, kind, stamp_name, start_ns, cutoff_ns, *, order_name=None): + """A fixed epoch view, preserving every original row and validating history.""" + if ( + type(rows) is not tuple + or len(rows) > 60000 + or type(start_ns) is not int + or type(cutoff_ns) is not int + or not 0 < start_ns < cutoff_ns + or any(type(row) is not kind for row in rows) + ): + raise ValueError('bounded original epoch source metadata') + stamps = tuple(getattr(row, stamp_name) for row in rows) + if any(type(stamp) is not int or stamp <= 0 for stamp in stamps): + raise ValueError('ordered original epoch source metadata') + if order_name is None: + ordered = stamps + else: + from openpilot.starpilot.flm.torque_emission_replay import Emission + + if kind is not Emission or stamp_name != 'control_ns' or order_name != 'apply_ns': + raise ValueError('only actual emissions have separate application ordering') + ordered = tuple(row.apply_ns for row in rows) + if any(right < left for left, right in zip(stamps, stamps[1:], strict=False)): + raise ValueError('original emitted command clock regression') + if any(type(stamp) is not int or stamp <= 0 for stamp in (*stamps, *ordered)) or any( + right <= left for left, right in zip(ordered, ordered[1:], strict=False) + ): + raise ValueError('ordered original epoch source metadata') + return tuple(row for row, stamp in zip(rows, stamps, strict=True) if start_ns <= stamp < cutoff_ns) + + +def response_snapshot(collector, binding, now_ns): + from openpilot.starpilot.flm.controller_replay import pack_primitive_snapshot, unpack_primitive_snapshot + from openpilot.starpilot.flm.source_plant_fit import SOURCE_COMPLETE_SLOT_POLICY as BOOTSTRAP_SLOT_POLICY + from openpilot.starpilot.flm.source_plant_runtime import _current_fit_pair_reason, _snapshot_from_packet + + adapter, capture = collector.adapter, collector.original_source_capture + binding.validate() + readiness = { + 'reason': 'raw_prerequisites_unavailable', + 'observedMonoTime': now_ns, + 'rawPoses': 0 if capture is None else len(capture.rows), + 'readyPairs': len(collector.ready), + 'captureEpoch': None if capture is None else capture.epoch, + 'captureStartNs': None if capture is None else capture.epoch_start_ns, + 'captureCutoffNs': None if capture is None else capture.epoch_cutoff_ns, + } + collector._live_bootstrap_readiness = readiness + if capture is None or not capture.rows or not collector.ready or adapter.envelope is None: + return None + current_decline = _current_fit_pair_reason(collector, now_ns, physical_binding=binding) + if current_decline is not None: + readiness['reason'] = current_decline + return None + from openpilot.starpilot.flm.live_tracking_bootstrap_cohorts import ( + VERSION as COHORT_VERSION, + CHECK_INTERVAL_NS, + select_first_cohort, + source_quality_keys, + new_schedule_cache, + _literal, + ) + import pickle + + # Bound OPTIONAL scheduling work to1Hz, never the100Hz control loop. A memo + # retains examined anchors only under fresh exact append-only source values. + if getattr(collector, '_live_bootstrap_selection_epoch', None) == capture.epoch and now_ns < getattr(collector, '_live_bootstrap_selection_next_ns', 0): + previous = getattr(collector, '_live_bootstrap_last_selection', None) + if type(previous) is dict: + readiness['previousSelection'] = previous.copy() + readiness['reason'] = 'selection_throttled' + readiness['throttled'] = True + return None + collector._live_bootstrap_selection_epoch = capture.epoch + collector._live_bootstrap_selection_next_ns = now_ns + CHECK_INTERVAL_NS + retained_checkpoints = tuple(adapter.replay.checkpoints) + checkpoints = _epoch_checkpoints(retained_checkpoints, tuple(capture.rows), capture.epoch_cutoff_ns) + readiness['retainedCheckpoints'] = len(retained_checkpoints) + # Before a fixed cohort can exist, rebuilding/pickling the entire archive + # cannot produce a job. Use only the original checkpoint and pose clocks to + # defer that optional work; admission and every full quality check stay below. + from openpilot.starpilot.flm.controller_replay import Snapshot + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose + from openpilot.starpilot.flm.source_plant_fit import FIRST_SLOT_NS, BLOCK_NS, PURGE_NS + + duration_ns = FIRST_SLOT_NS + 7 * (BLOCK_NS + PURGE_NS) + BLOCK_NS + if ( + 0 < len(checkpoints) <= 128 + and all(type(row) is tuple and len(row) == 2 and type(row[0]) is int and row[0] > 0 and type(row[1]) is Snapshot for row in checkpoints) + and all(right[0] > left[0] for left, right in zip(checkpoints, checkpoints[1:], strict=False)) + and type(capture.rows[0]) is OriginalSourcePose + and type(capture.rows[-1]) is OriginalSourcePose + ): + preceding = tuple(stamp for stamp, _ in checkpoints if stamp <= capture.rows[0].mono_ns) + if preceding: + earliest_end_ns = preceding[-1] + duration_ns + if now_ns < earliest_end_ns or capture.rows[-1].source_lower_ns < earliest_end_ns: + readiness.update( + reason='fixed_cohort_clock_incomplete', + requiredCohortNs=duration_ns, + earliestCompleteNs=earliest_end_ns, + checkpoints=len(checkpoints), + throttled=False, + ) + collector._live_bootstrap_last_selection = readiness.copy() + return None + if not checkpoints: + readiness['reason'] = 'original_epoch_checkpoint_unavailable' + return None + # The recorder serves all objectives for ten minutes. This response job is + # one original100s epoch, including its preceding genuine checkpoint. Old + # history must not consume that epoch's unchanged12000-row source budget. + from openpilot.starpilot.flm.controller_replay import ReplayFrame + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + from openpilot.starpilot.flm.torque_emission_replay import Emission + + origin_ns = checkpoints[0][0] + frames = _epoch_rows(tuple(adapter.replay.records), ReplayFrame, 'control_ns', origin_ns, capture.epoch_cutoff_ns) + poses = tuple(capture.rows) + emissions = _epoch_rows(tuple(adapter.replay.emissions.events.values()), Emission, 'control_ns', origin_ns, capture.epoch_cutoff_ns, order_name='apply_ns') + controls = _epoch_rows(tuple(adapter.original_context_receipts.controls.values()), ControlEvidence, 'control_ns', origin_ns, capture.epoch_cutoff_ns) + readiness.update( + { + 'reason': 'checking_fresh_source_keys', + 'checkpoints': len(checkpoints), + 'frames': len(frames), + 'emissions': len(emissions), + 'controls': len(controls), + 'throttled': False, + } + ) + collector._live_bootstrap_last_selection = readiness.copy() + try: + keys = source_quality_keys(frames, poses, emissions, checkpoints, controls, diagnostics=readiness) + finally: + collector._live_bootstrap_last_selection = readiness.copy() + token = pickle.dumps(_literal((capture.epoch, capture.epoch_start_ns, capture.epoch_cutoff_ns, tuple(vars(binding).items()))), protocol=5) + # Feedback is original source input, not a fit outcome. Select the first + # complete source experiment instead of repeatedly exporting an early prefix + # with missing consumption receipts. The child still verifies every receipt. + feedback = adapter.original_feedback_archive + if feedback is None or feedback.failed: + readiness['reason'] = 'original_feedback_source_unavailable' + return None + frame_stamps = tuple(frame.control_ns for frame in frames) + from bisect import bisect_left + + unavailable = [] + for stamp, _ in checkpoints: + crossing = bisect_left(frame_stamps, stamp + duration_ns) + first = bisect_left(frame_stamps, stamp) + if crossing >= len(frames) or first >= len(frames): + continue + source_ids = frame_stamps[first : crossing + 1] + if ( + feedback.poison_through_ns + and source_ids[0] <= feedback.poison_through_ns + 100_000_000 + or any(source_ids[0] - 100_000_000 <= bad <= source_ids[-1] + 100_000_000 for bad in feedback.failures) + or any(identity not in feedback.consumptions for identity in source_ids) + ): + unavailable.append(stamp) + unavailable = tuple(unavailable) + token = pickle.dumps((token, unavailable), protocol=5) + cache = new_schedule_cache(getattr(collector, '_live_bootstrap_cohort_cache', None), token, keys) + collector._live_bootstrap_cohort_cache = cache + selection = select_first_cohort( + checkpoints=checkpoints, + frames=frames, + poses=poses, + emissions=emissions, + controls=controls, + binding=binding, + epoch=capture.epoch, + epoch_start_ns=capture.epoch_start_ns, + cutoff_ns=capture.epoch_cutoff_ns, + now_ns=now_ns, + cache=cache, + diagnostics=readiness, + unavailable_anchors=unavailable, + source_complete=True, + ) + collector._live_bootstrap_last_selection = readiness.copy() + if selection is None: + return None + start = selection['start_ns'] + # Freeze the already selected experiment. Later frames belong to live + # collection, not this response-only job, and may contain unrelated gaps. + end_frame = next((frame for frame in frames if frame.control_ns >= selection['end_ns']), None) + records = () if end_frame is None else tuple(frame for frame in frames if start <= frame.control_ns <= end_frame.control_ns) + if not records or len(records) > 12000 or records[-1].control_ns < selection['end_ns']: + readiness['reason'] = 'selected_original_frame_coverage_unavailable' + return None + # Keep the actual currently selected factory/objective/FullSurfaceEnvelope. + # A response-only job never rotates the capture selector or invents an + # objective in order to bypass the original factory/native checks. + schedule = {'version': 'live-response-bootstrap-source-v1', 'objective_key': adapter.key} + wire = adapter.original_source_packet(collector, ((start, records[-1].control_ns),), now_ns) + wrapper = _snapshot_from_packet(collector, wire, now_ns, schedule, response_only=True) + if wrapper is None: + readiness['reason'] = 'original_response_packet_wrapper_unavailable' + return None + payload = unpack_primitive_snapshot(base64.b64decode(wrapper['snapshot_b64'], validate=True)) + payload['live_job_version'] = VERSION + payload['live_slot_policy'] = BOOTSTRAP_SLOT_POLICY + payload['live_binding'] = asdict(binding) + payload['live_cohort_version'] = COHORT_VERSION + payload['live_cohort_selection'] = selection + packed = pack_primitive_snapshot(payload) + result = dict(wrapper, live_job_version=VERSION, snapshot_b64=base64.b64encode(packed).decode('ascii')) + if len(result['snapshot_b64']) > 6 * 1024 * 1024: + readiness['reason'] = 'response_transport_bound_exceeded' + raise ValueError('complete response bootstrap transport bound') + readiness['reason'] = 'response_snapshot_ready_not_child_qualified' + return result + + +class ResponseSnapshotProcess: + """Prepare immutable historical input off the live ingestion loop. + + Linux copy-on-write captures a single-threaded estimator's exact objects. + The child never reads IPC or grants authority; its bounded JSON output still + goes through the original independent response worker and current admission. + """ + + def __init__(self): + self.pid = None + self.fd = None + self.token = None + self.epoch = None + self.started = self.last_started = 0 + self.output = bytearray() + self.reason = 'idle' + self.canceled = False + + def submit(self, collector, binding, *, token, now_ns): + if self.pid is not None or now_ns - self.last_started < 5_000_000_000: + return False + capture = collector.original_source_capture + if capture is None or not capture.rows: + return False + from openpilot.starpilot.flm.source_plant_fit import FIRST_SLOT_NS, BLOCK_NS, PURGE_NS + + duration = FIRST_SLOT_NS + 7 * (BLOCK_NS + PURGE_NS) + BLOCK_NS + if capture.rows[-1].source_lower_ns - capture.rows[0].mono_ns < duration: + return False + + def prepare(): + snapshot = response_snapshot(collector, binding, now_ns) + return {'snapshot': snapshot, 'readiness': getattr(collector, '_live_bootstrap_readiness', None)} + + return self.submit_preparation(prepare, token=token, epoch=capture.epoch, now_ns=now_ns) + + def submit_preparation(self, prepare, *, token, epoch, now_ns): + """One frozen optional estimator job; never an IPC or authority reader.""" + import os + + if self.pid is not None or not callable(prepare): + return False + read_fd, write_fd = os.pipe2(os.O_CLOEXEC) + parent = os.getpid() + try: + pid = os.fork() + except OSError: + os.close(read_fd) + os.close(write_fd) + self.reason = 'snapshot_child_unavailable' + self.last_started = now_ns + return False + if pid == 0: + import ctypes + import json + import resource + import signal + + try: + os.close(read_fd) + os.setsid() + signal.signal(signal.SIGTERM, signal.SIG_DFL) + if ctypes.CDLL(None).prctl(1, signal.SIGKILL, 0, 0, 0) != 0 or os.getppid() != parent: + os._exit(1) + os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0)) + soft, hard = resource.getrlimit(resource.RLIMIT_CPU) + limit = min([30] + [x for x in (soft, hard) if x >= 0]) + resource.setrlimit(resource.RLIMIT_CPU, (limit, hard)) + try: + result = prepare() + except Exception as error: + result = {'snapshot': None, 'error': type(error).__name__ + ': ' + str(error)[:512]} + payload = json.dumps(result, separators=(',', ':'), allow_nan=False).encode() + if len(payload) > 8 * 1024 * 1024: + os._exit(1) + position = 0 + while position < len(payload): + position += os.write(write_fd, payload[position : position + 65536]) + os.close(write_fd) + os._exit(0) + except BaseException: + os._exit(1) + os.close(write_fd) + os.set_blocking(read_fd, False) + self.pid, self.fd, self.token, self.epoch = pid, read_fd, token, epoch + self.started = self.last_started = now_ns + self.output = bytearray() + self.canceled = False + self.reason = 'preparing_historical_snapshot' + return True + + def cancel(self, reason): + import os + import signal + + if self.pid is not None: + try: + os.kill(self.pid, signal.SIGKILL) + except ProcessLookupError: + pass + self.reason = reason + self.canceled = True + + def poll(self, *, token, now_ns): + import os + import json + + if self.pid is None: + return None + if token != self.token or now_ns - self.started > 30_000_000_000: + self.cancel('snapshot_context_changed' if token != self.token else 'snapshot_wall_limit') + eof = False + for _ in range(16): + try: + part = os.read(self.fd, 65536) + except BlockingIOError: + break + if not part: + eof = True + break + self.output.extend(part) + if len(self.output) > 8 * 1024 * 1024: + self.cancel('snapshot_output_bound') + break + pid, status = os.waitpid(self.pid, os.WNOHANG) + if not pid: + return None + # Drain the bounded remainder after exit; never accept partial output. + while not eof and len(self.output) <= 8 * 1024 * 1024: + try: + part = os.read(self.fd, 65536) + except BlockingIOError: + break + if not part: + eof = True + break + self.output.extend(part) + accepted = ( + not self.canceled + and token == self.token + and now_ns - self.started <= 30_000_000_000 + and os.waitstatus_to_exitcode(status) == 0 + and eof + and len(self.output) <= 8 * 1024 * 1024 + ) + result = None + if accepted: + try: + result = json.loads(self.output) + except (ValueError, UnicodeError): + pass + os.close(self.fd) + self.pid = self.fd = None + self.output.clear() + return result + + def close(self, reason='snapshot_closed'): + import os + + self.cancel(reason) + if self.pid is not None: + os.waitpid(self.pid, 0) + os.close(self.fd) + self.pid = self.fd = None + self.output.clear() + return True diff --git a/openpilot/starpilot/flm/live_tracking_bootstrap_worker.py b/openpilot/starpilot/flm/live_tracking_bootstrap_worker.py new file mode 100644 index 0000000000..307fa7d9a9 --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_bootstrap_worker.py @@ -0,0 +1,67 @@ +"""Explicit response-only purpose worker; production budget awaits measurement. + +This version is an isolated diagnostic purpose, not the legacy20-second fitter. +Its result is numerical calibration only and cannot authorize actuation/save. +""" +import gc +import json +import os +import resource +import sys + +PURPOSE = 'live-response-bootstrap-diagnostic-v1' +MAX_CPU = 180 +MAX_WALL_NS = 300_000_000_000 +MAX_AS = 2*1024*1024*1024 +MAX_INPUT = 8*1024*1024 +MAX_OUTPUT = 65536 + + +def main(): + try: + if os.environ.get('STARPILOT_LIVE_BOOTSTRAP_PURPOSE') != PURPOSE: + raise ValueError('explicit isolated response measurement purpose required') + if not sys.platform.startswith('linux'): + raise ValueError('qualified Linux response worker required') + import ctypes + import signal + parent = int(os.environ['STARPILOT_FLM_PARENT_PID']) + if ctypes.CDLL(None).prctl(1, signal.SIGKILL, 0, 0, 0) != 0 or os.getppid() != parent: + raise ValueError('bootstrap parent disappeared') + os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0)) + os.nice(10) + for kind, ceiling in ((resource.RLIMIT_AS, MAX_AS), (resource.RLIMIT_CPU, MAX_CPU)): + soft, hard = resource.getrlimit(kind) + limit = min((ceiling, *[value for value in (soft, hard) if value >= 0])) + resource.setrlimit(kind, (limit, hard)) + if any(os.environ.get(name) != '1' for name in ('OPENBLAS_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS', 'NUMEXPR_NUM_THREADS')): + raise ValueError('one numerical thread required') + raw = sys.stdin.buffer.read(MAX_INPUT+1) + if len(raw) > MAX_INPUT: + raise ValueError('response worker input bound') + from openpilot.starpilot.flm.live_tracking_bootstrap import bootstrap_snapshot + was_enabled = gc.isenabled() + try: + gc.disable() + # Native accelerator remains disabled under this separate180-second + # purpose until an explicit new native-worker policy is qualified. Pure + # Python fallback is the original algorithm, not a retagged binary proof. + result = bootstrap_snapshot(json.loads(raw)) + finally: + if was_enabled: + gc.enable() + result = dict(result, diagnostic_purpose=PURPOSE, production_admissible=False) + encoded = json.dumps(result, separators=(',', ':'), allow_nan=False).encode() + if len(encoded) > MAX_OUTPUT: + raise ValueError('response worker output bound') + sys.stdout.buffer.write(encoded) + sys.stdout.buffer.flush() + return 0 + except (Exception, MemoryError) as error: + # Bounded diagnostic metadata only, never snapshot/source content. + sys.stderr.write(type(error).__name__+': '+str(error)[:240]+'\n') + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/openpilot/starpilot/flm/live_tracking_episodes.py b/openpilot/starpilot/flm/live_tracking_episodes.py new file mode 100644 index 0000000000..2de65a771a --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_episodes.py @@ -0,0 +1,879 @@ +"""Whole fullrate geometry joined to independent, unmodified physical endpoints. + +This adapter grants no driving permission and never interpolates a pose. Its +inputs must come from the collector's original source pair, not UI telemetry. +""" + +from collections import OrderedDict, deque +from typing import cast +from dataclasses import dataclass, replace +import hashlib +import math +import struct + +from openpilot.common.constants import ACCELERATION_DUE_TO_GRAVITY +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, PROBE_GROUPS, probe_names, full_profile_sha256 +from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence +from openpilot.starpilot.flm.live_tracking import TrackingEpisode, HookSensitivity +from openpilot.starpilot.flm.live_tracking_response import validate_certificate, validate_response_witness, physical_calibration_generation +from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose, validate_pose +from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current, exact_unclipped_pair, asynchronous_hint + +MAX_CONTROL_GAP_NS = 50_000_000 +MAX_POSE_GAP_NS = 100_000_000 +MAX_EVENT_NS = 10_000_000_000 +HISTORY_NS = 12_000_000_000 +PROBE_DEADLINE_NS = 30_000_000_000 +MAX_COMPLETED = 16 +MAX_PROBE_REQUESTS = MAX_COMPLETED * 240 + + +def _finite(value): + return type(value) in (int, float) and math.isfinite(value) + + +@dataclass(frozen=True) +class _Tick: + control: ControlEvidence + output: AppliedEvidence | None + profile: str + geometry: float + goal: float + + +@dataclass(frozen=True) +class _Endpoint: + control_ns: int + apply_ns: int + pose_ns: int + speed: float + goal: float + error: float + measured: float + reference_ns: int = 0 + + +def tracking_reference_force(control): + """Actual selected controller target, not its incoming future FF request. + + The torque controller's request buffer already aligns its PID target with + actuator delay. PID torque publishes its own actual kinematic target in the + same stage field. Retain the existing original roll/force convention and + independent actual-pose conversion; no reference time is shifted or inferred. + """ + stage = dict(control.stage_values) + inputs = dict(control.replay_inputs) + setpoint, roll = stage['setpoint'], inputs['roll'] + if len(stage) != len(control.stage_values) or not _finite(setpoint) or not _finite(roll): + raise ValueError('actual finite controller tracking target required') + return float(setpoint) - ACCELERATION_DUE_TO_GRAVITY * math.sin(float(roll)) + + +def tracking_reference_acceleration(control): + setpoint = dict(control.stage_values)['setpoint'] + if not _finite(setpoint): + raise ValueError('actual finite controller tracking target required') + return float(setpoint) + + +def tracking_reversal_rate(errors, duration_s, model_rmse): + # Count complete excursions above the SAME minimum error that discovery + # can act on. Milligravity crossings of a sampled/quantized reference are + # not evidence of vehicle weave. Original RMS and drift tests still see + # every unfiltered endpoint; this changes only the oscillation counter. + threshold = max(0.015, 3 * model_rmse) + signs = [1 if error > threshold else -1 if error < -threshold else 0 for error in errors] + nonzero = [sign for sign in signs if sign] + return sum(left != right for left, right in zip(nonzero, nonzero[1:], strict=False)) / duration_s + + +class TrackingEpisodeBuilder: + """Bounded operation-local joins, retaining true100Hz geometry and pose rate. + + observe_source_control is called for EVERY actual owner tick. Only actual + emissions enter observe_control; no transmission is inferred. observe_pose + receives the collector's exact lag-associated control/output pair and fresh + response witness. Completed windows wait for a real physical watermark; + time passing alone can never invent their missing independent endpoints. + """ + + def __init__(self, binding, response_certificate, *, controller_domain='torque'): + binding.validate() + certificate = validate_certificate(response_certificate) + if certificate.binding != binding: + raise ValueError('episode physical model binding') + if type(controller_domain) is not str or controller_domain not in ('torque', 'torque_pid'): + raise ValueError('actual controller domain required') + from openpilot.starpilot.flm.pid_stages import UNSUPPORTED + + self.numeric_probe_names = tuple(name for name in NAMES if controller_domain != 'torque_pid' or name not in UNSUPPORTED) + self.defer_unprobed = False + self.numerical_observed_ns = 0 + from openpilot.starpilot.flm.live_tracking_response import TrackingSourceContext + + self.source_tracking = type(certificate) is TrackingSourceContext + self.binding, self.certificate = binding, certificate + self.replay = None + self.ticks = OrderedDict() + self.emissions = {} + self.emission_count = 0 + self.endpoints = deque(maxlen=320) + self.probes = OrderedDict() + self.probe_received = {} + self.profile_keys = OrderedDict() + self.probe_requests = OrderedDict() + self.completed = OrderedDict() + self.closed_through_ns = 0 + self.pending = deque(maxlen=16) + self.ready = deque(maxlen=16) + self.last_pose_ns = self.watermark = 0 + self.reason = 'whole_tracking_geometry_required' + self.last_input_status = 'ignored' + self.plot = None + self._clear_geometry() + + def _clear_geometry(self): + self.start = self.last = self.neutral = self.prior_sign = self.first_sign = self.reversals = 0 + self.mode = self.profile = self.last_frame = None + if not hasattr(self, 'input_window_origin'): + self.input_window_origin = None + self.input_window_boundary = self.input_window_wait_until = None + self.input_window_min = self.input_window_max = self.input_window_sum = 0.0 + self.input_window_count = 0 + + def interrupt(self, reason, *, preserve_completed=False): + # Completed windows contain ORIGINAL endpoints and original probe deadlines. + # Keeping those numbers supplies no current witness, offer or permission. + completed = self.completed.copy() if preserve_completed else OrderedDict() + retained_keys = set().union(*(row['requested'] for row in completed.values())) if completed else set() + requests = OrderedDict((key, value) for key, value in self.probe_requests.items() if key in retained_keys) + ready = tuple(self.ready) if preserve_completed else () + closed = self.closed_through_ns if preserve_completed else 0 + self.ticks.clear() + self.emissions.clear() + self.emission_count = 0 + self.endpoints.clear() + self.probes.clear() + self.probe_received.clear() + self.probe_requests.clear() + self.completed.clear() + self.closed_through_ns = 0 + self.pending.clear() + self.ready.clear() + self.last_pose_ns = self.watermark = 0 + self.numerical_observed_ns = 0 + self.plot = None + self._clear_geometry() + self.completed.update(completed) + self.probe_requests.update(requests) + self.ready.extend(ready) + self.closed_through_ns = closed + self.reason = reason + self.last_input_status = 'invalid' + + def _source_control(self, control, current_native): + b = self.binding + if ( + type(control) is not ControlEvidence + or current_native is not True + or (control.identity, control.source_epoch, control.baseline_revision, control.controller_revision, control.native_identity, control.native_required) + != (b.identity, b.source_epoch, b.baseline_revision, b.controller_revision, b.native_identity, b.native_required) + or control.domain != 'normalized_torque' + or not control.enabled + or not control.active + or not control.valid + or control.manual + or control.override + or control.fault + or control.saturated + or control.model_wobble + or control.limited + and not asynchronous_hint(control) + or not 0 < control.state_ns <= control.control_ns <= control.state_ns + 30_000_000 + or not 0 < control.source_receipt_ns <= control.control_ns <= control.source_receipt_ns + 500_000_000 + or b.native_required + and ( + not control.native_allowed + or not 0 < control.native_observed_ns <= control.control_ns <= control.native_expires_ns <= control.native_observed_ns + 200_000_000 + ) + ): + raise ValueError('episode original source/driver/controller/native exclusion') + + def _pair(self, control, output, current_native): + self._source_control(control, current_native) + b = self.binding + from openpilot.starpilot.flm.live_tracking_response import verified_clipped_input + + observed_clip = verified_clipped_input(self.replay, control, output, b) if type(output) is AppliedEvidence and output.limited else False + if ( + type(output) is not AppliedEvidence + or control.limited + and not exact_unclipped_pair(control, output) + and not observed_clip + or not output.active + or not output.valid + or output.override + or output.fault + or output.limited + and not observed_clip + or output.domain != control.domain + or output.control_ns != control.control_ns + or not consumed_sources_current(control, output) + or b.native_required + and (not output.native_allowed or output.apply_ns > control.native_expires_ns) + ): + raise ValueError('episode actual native pair/driver/limiter exclusion') + + def _actual_coordinates(self, control): + # Legacy selected coordinates remain neutral TWO-space even while a full30 + # owner is active. Never reinterpret those two numbers as a full profile. + legacy = control.applied_coordinates + if type(legacy) is not tuple or len(legacy) != 2 or any(not _finite(value) or value != 0 for value in legacy): + raise ValueError('old selected residual cannot impersonate full30 state') + live = control.live_applied_coordinates + if type(live) is not tuple: + raise ValueError('exact numerical live consumed map') + if live: + if control.effect_trace is not None or len(live) != len(NAMES) or not all(_finite(value) for value in live): + raise ValueError('one actual full30 owner required') + return live + if control.effect_trace is not None: + from openpilot.starpilot.flm.effect_trace import FullEffectTrace + + trace = control.effect_trace + if type(trace) is not FullEffectTrace: + raise ValueError('actual old full30 trace required') + trace.validate() + envelope = trace.envelope + b = self.binding + if ( + ( + envelope.cp_sha256, + envelope.controller_revision, + envelope.source_epoch, + envelope.reset_generation, + envelope.selected_baseline_revision, + envelope.persistent_source_epoch, + ) + != (b.cp_sha256, b.controller_revision, b.source_epoch, b.reset_generation, b.baseline_revision, b.persistent_source_epoch) + or trace.control_ns != control.effect_prepare_ns + or not control.state_ns <= trace.control_ns <= control.control_ns + or trace.qualified + and ( + control.applied_profile_sequence <= 0 + or not 0 + < control.applied_profile_observed_ns + <= control.control_ns + <= control.applied_profile_expires_ns + <= control.applied_profile_observed_ns + 500_000_000 + ) + ): + raise ValueError('actual old full30 trace source or lease changed') + return tuple(value for _, value in trace.applied) + # No full owner and genuinely neutral legacy inputs mean neutral full30; + # a pending offer/report is NEVER used to fill unseen applied coordinates. + return (0.0,) * len(NAMES) + + def observe_source_control(self, control, *, profile_sha256, now_ns, current_native, preserve_completed=False): + """EVERY genuine controller tick, independently of CAN transmit cadence.""" + self.last_input_status = 'ignored' + try: + self._source_control(control, current_native) + if type(now_ns) is not int or not 0 <= now_ns - control.control_ns <= MAX_POSE_GAP_NS or control.domain != 'normalized_torque': + raise ValueError('episode current actual complete profile') + actual_coordinates = self._actual_coordinates(control) + profile_key = struct.pack('!30d', *actual_coordinates) + if profile_key not in self.profile_keys: + self.profile_keys[profile_key] = full_profile_sha256(self.binding.baseline_revision, dict(zip(NAMES, actual_coordinates, strict=True))) + if len(self.profile_keys) > 4: + self.profile_keys.popitem(last=False) + if self.profile_keys[profile_key] != profile_sha256: + raise ValueError('episode current actual complete profile') + stage = dict(control.stage_values) + inputs = dict(control.replay_inputs) + frame = inputs['control_frame'] + if not _finite(frame) or frame < 0 or int(frame) != frame: + raise ValueError('actual original owner frame sequence') + geometry = stage['setpoint'] + goal = tracking_reference_acceleration(control) if self.source_tracking else tracking_reference_force(control) + if len(stage) != len(control.stage_values) or not _finite(geometry): + raise ValueError('episode original stage geometry') + stamp = control.control_ns + if self.last and stamp <= self.last: + old = self.ticks.get(stamp) + if old is not None and old.control == control and old.profile == profile_sha256 and old.geometry == geometry and old.goal == goal: + return False + raise ValueError('episode control clock regression/duplicate mutation') + if self.last and (stamp - self.last > MAX_CONTROL_GAP_NS or profile_sha256 != self.profile or frame != cast(float, self.last_frame) + 1): + self.interrupt('episode control gap or actual profile changed', preserve_completed=preserve_completed) + self.ticks[stamp] = _Tick(control, None, profile_sha256, geometry, goal) + self.profile = profile_sha256 + self._geometry(stamp, geometry) + self.last = stamp + self.last_frame = frame + if len(self.ticks) > 1400: + raise ValueError('bounded original fullrate history') + while self.ticks and stamp - next(iter(self.ticks)) > HISTORY_NS: + old, _ = self.ticks.popitem(last=False) + self.emission_count -= len(self.emissions.pop(old, {})) + self.probes.pop(old, None) + self.probe_received.pop(old, None) + if not any(old in row['requested'] for row in self.completed.values()): + self.probe_requests.pop(old, None) + self._finish() + self.last_input_status = 'accepted' + return True + except (AttributeError, KeyError, TypeError, ValueError, OverflowError): + self.interrupt('episode original control unqualified', preserve_completed=preserve_completed) + return False + + def observe_control(self, control, output, *, profile_sha256, now_ns, current_native, preserve_completed=False): + """Attach only an actual emitted native/CarOutput pair, never fill a tick.""" + self.last_input_status = 'ignored' + try: + self._pair(control, output, current_native) + if type(now_ns) is not int or not 0 <= now_ns - output.apply_ns <= MAX_POSE_GAP_NS: + raise ValueError('episode actual emission receipt stale') + # Backwards compatibility for genuine100Hz emitting platforms. Slower + # platforms MUST feed observe_source_control separately for EVERY tick. + if control.control_ns > self.last: + if not self.observe_source_control( + control, profile_sha256=profile_sha256, now_ns=now_ns, current_native=current_native, preserve_completed=preserve_completed + ): + return False + tick = self.ticks.get(control.control_ns) + if tick is None: + return False # An old emission cannot recreate discarded source time. + if tick.control != control or tick.profile != profile_sha256: + raise ValueError('actual emission differs from original source tick') + # One consumed CC can be emitted on multiple genuine Card ticks. Bind + # each original application time rather than replacing or inventing one. + prior = self.emissions.get(control.control_ns, {}).get(output.apply_ns) + if prior is not None: + if prior == output: + return False + raise ValueError('changed duplicate actual emission') + if self.emission_count >= 2800: + raise ValueError('bounded original emission history') + self.emissions.setdefault(control.control_ns, {})[output.apply_ns] = output + self.emission_count += 1 + self.ticks[control.control_ns] = replace(tick, output=output) + self.last_input_status = 'accepted' + return True + except (AttributeError, KeyError, TypeError, ValueError, OverflowError): + self.interrupt('episode actual native join unqualified', preserve_completed=preserve_completed) + return False + + def _complete(self, end): + if self.start and 200_000_000 <= end - self.start <= MAX_EVENT_NS: + self.pending.append((self.start, end, self.mode, self.first_sign, self.profile)) + + def _input_window(self, stamp, goal): + if self.input_window_origin is None: + self.input_window_origin = stamp + self.input_window_boundary = stamp + 2_000_000_000 + self.start = stamp + elif not self.start: + if self.input_window_wait_until is None: + elapsed = stamp - self.input_window_origin + self.input_window_wait_until = self.input_window_origin + ((elapsed + 1_999_999_999) // 2_000_000_000) * 2_000_000_000 + if stamp < self.input_window_wait_until: + return # Skip the interrupted partial window; never re-anchor on its outcome. + self.start = stamp + self.input_window_boundary = self.input_window_origin + ((stamp - self.input_window_origin) // 2_000_000_000 + 1) * 2_000_000_000 + self.input_window_wait_until = None + if stamp >= self.input_window_boundary: + if self.input_window_count: + signed = self.input_window_sum / self.input_window_count + self.first_sign = 1 if signed > 0.03 else -1 if signed < -0.03 else 0 + same_direction = self.input_window_min >= 0 or self.input_window_max <= 0 + self.mode = 'large' if self.first_sign and same_direction and max(abs(self.input_window_min), abs(self.input_window_max)) >= 0.25 else 'subtle' + self._complete(self.last) # Last ORIGINAL observed control, not a synthetic boundary tick. + self.start = stamp + self.input_window_boundary = self.input_window_origin + ((stamp - self.input_window_origin) // 2_000_000_000 + 1) * 2_000_000_000 + self.input_window_min = self.input_window_max = self.input_window_sum = 0.0 + self.input_window_count = 0 + self.input_window_min = min(self.input_window_min, goal) + self.input_window_max = max(self.input_window_max, goal) + self.input_window_sum += goal + self.input_window_count += 1 + self.mode = 'subtle' + + def _geometry(self, stamp, goal): + if self.source_tracking: + self._input_window(stamp, goal) + return + sign = 1 if goal > 0.03 else -1 if goal < -0.03 else 0 + if abs(goal) >= 0.25: + self.neutral = 0 + if self.mode != 'large': + self.start, self.mode, self.first_sign, self.reversals = stamp, 'large', sign, 0 + elif sign != self.first_sign: + self._complete(stamp - 1) + self.start, self.first_sign = stamp, sign + self.prior_sign = sign + return + if self.mode == 'large': + if not sign: + self.neutral = self.neutral or stamp + if stamp - self.neutral >= 500_000_000: + self._complete(stamp) + self.start = self.neutral = self.prior_sign = self.first_sign = self.reversals = 0 + self.mode = None + elif sign != self.first_sign: + self._complete(stamp - 1) + self.start = self.neutral = self.prior_sign = self.first_sign = self.reversals = 0 + self.mode = None + else: + self.neutral = 0 + return + if sign and self.prior_sign and sign != self.prior_sign: + if self.mode is None: + self.start, self.mode, self.first_sign, self.reversals = stamp, 'subtle', sign, 0 + else: + self.reversals += 1 + if self.reversals == 2 and sign == self.first_sign: + self._complete(stamp - 1) + self.start, self.first_sign, self.reversals = stamp, sign, 0 + if self.mode == 'subtle' and stamp - self.start > MAX_EVENT_NS: + self.start = self.first_sign = self.reversals = 0 + self.mode = None + if sign: + self.prior_sign = sign + + def request_probe(self, control_ns, *, group, names, queued_ns): + """Register ONLY after owned immutable prepared-job enqueue; no permission. + + The actor queue is bounded to four genuine prepared endpoint jobs. This + reserves numerical work only, never child execution or native permission. + queued_ns is the real parent clock, independent of archived source stamps. + First request fixes a 30s numerical deadline; subsequent requests cannot + extend it. Register during the window, before it has been drained. + """ + if ( + type(control_ns) is not int + or type(queued_ns) is not int + or queued_ns <= 0 + or type(group) is not str + or group not in PROBE_GROUPS + or type(names) is not tuple + or not 1 <= len(names) <= len(PROBE_GROUPS[group]) + or any(type(name) is not str for name in names) + or probe_names(names) != names + or any(name not in PROBE_GROUPS[group] for name in names) + ): + return False + tick = self.ticks.get(control_ns) + if tick is None or tick.output is None: + return False + old = self.probe_requests.get(control_ns) + if old is not None: + return old[:2] == (group, names) # Never refresh the original deadline. + if len(self.probe_requests) >= MAX_PROBE_REQUESTS: + return False + for row in self.completed.values(): + if row['episode'].start_ns <= control_ns <= row['episode'].end_ns: + if row['group'] != group or row['names'] != names: + return False + row['requested'].add(control_ns) + self.probe_requests[control_ns] = (group, names, queued_ns) + return True # Existing first-request deadline remains immutable. + # A ready/drained episode cannot be amended or counted a second time. + if control_ns <= self.closed_through_ns or any(e.start_ns <= control_ns <= e.end_ns for e in self.ready): + return False + self.probe_requests[control_ns] = (group, names, queued_ns) + return True + + def observe_probe(self, result, *, now_ns=None): + from openpilot.starpilot.flm.live_tracking_probe import validate_probe_result + + try: + validate_probe_result(result, self.binding) + request = self.probe_requests.get(result.control_ns) + if request is not None and ( + type(now_ns) is not int + or not request[2] <= now_ns <= request[2] + PROBE_DEADLINE_NS + or result.group != request[0] + or tuple(row.name for row in result.rows) != request[1] + ): + return False + for row in self.completed.values(): + point = next((point for point in row['points'] if point.control_ns == result.control_ns), None) + if point is None: + continue + if ( + request is None + or result.control_ns not in row['requested'] + or now_ns > row['deadline'] + or result.apply_ns != point.apply_ns + or result.profile_sha256 != row['episode'].profile_sha256 + or result.model_revision != self.certificate.model_revision + ): + return False + row['probes'][result.control_ns] = result + row.pop('_wait_signature', None) + self._release_completed(now_ns) + return True + tick = self.ticks.get(result.control_ns) + if ( + tick is None + or result.apply_ns not in self.emissions.get(result.control_ns, {}) + or result.profile_sha256 != tick.profile + or result.model_revision != self.certificate.model_revision + ): + return False + self.probes[result.control_ns] = result + if request is not None: + self.probe_received[result.control_ns] = now_ns + return True + except (AttributeError, TypeError, ValueError): + return False + + @staticmethod + def _gradient_rows(points, probes): + rows = {name: [] for name in NAMES} + seen = set() + for point in points: + key = point.control_ns, point.apply_ns + probe = probes.get(point.control_ns) + if key in seen or probe is None or getattr(probe, 'apply_ns', None) != point.apply_ns: + continue + seen.add(key) + for row in probe.rows: + if row.name in rows and getattr(row, 'status', None) in ('separated', 'quantized'): + rows[row.name].append((point, row)) + return rows + + @staticmethod + def _gradient_coverage(points, probes, requested, rows): + if not points or set(probes) != set(requested): + return {} + first, last = min(p.control_ns for p in points), max(p.control_ns for p in points) + slots = {p.control_ns: min(3, (p.control_ns - first) * 4 // (last - first + 1)) for p in points} + required = set(slots.values()) + return {name: len(values) >= 4 and {slots[point.control_ns] for point, _ in values} == required for name, values in rows.items()} + + @staticmethod + def _native_gradient_complete(points, probes, requested, names): + rows = TrackingEpisodeBuilder._gradient_rows(points, probes) + coverage = TrackingEpisodeBuilder._gradient_coverage(points, probes, requested, rows) + return bool(names) and all(coverage.get(name, False) for name in names) + + def _sensitivities(self, points, probes, *, requested=None): + rows = self._gradient_rows(points, probes) + coverage = self._gradient_coverage(points, probes, set(probes) if requested is None else requested, rows) + return tuple( + HookSensitivity( + name, + sum(point.error * row.physical_derivative for point, row in values), + sum(row.physical_derivative**2 for _, row in values), + sum(row.native_separated for _, row in values), + len(values), + math.sqrt(sum(row.max_step_physical_delta**2 for _, row in values) / len(values)), + coverage.get(name, False), + ) + for name, values in rows.items() + if values + ) + + def _release_completed(self, now_ns): + if type(now_ns) is not int or now_ns <= 0: + return # Missing real clock holds; never fabricate a numerical timeout. + from openpilot.starpilot.flm.live_tracking_probe import validate_probe_result + + for key, row in tuple(self.completed.items()): + wait_signature = len(row['probes']), len(row['requested']) + if now_ns < row['deadline'] and (set(row['probes']) != row['requested'] or row.get('_wait_signature') == wait_signature): + continue # Only cache a decision to WAIT, never a proof to publish. + valid = {} + for control_ns, probe in row['probes'].items(): + try: + validate_probe_result(probe, self.binding) + point = next(point for point in row['points'] if point.control_ns == control_ns) + if ( + control_ns in row['requested'] + and probe.apply_ns == point.apply_ns + and probe.profile_sha256 == row['episode'].profile_sha256 + and probe.model_revision == self.certificate.model_revision + and probe.group == row['group'] + and tuple(item.name for item in probe.rows) == row['names'] + ): + valid[control_ns] = probe + except (AttributeError, TypeError, ValueError, StopIteration): + pass + supported = all(sum(any(item.name == name for item in probe.rows) for probe in valid.values()) >= 4 for name in row['names']) + sensitivities = self._sensitivities(row['points'], valid, requested=row['requested']) + # One unavailable coordinate cannot invalidate another independently + # complete native gradient. Every proposed coordinate checks its own flag. + gradient_complete = any(feature.native_gradient_complete for feature in sensitivities) + if not gradient_complete and now_ns < row['deadline']: + row['_wait_signature'] = wait_signature + continue + if (supported and gradient_complete) or now_ns >= row['deadline']: + episode = replace(row['episode'], sensitivity=sensitivities, native_gradient_complete=gradient_complete).validate() + self.ready.append(episode) + self.closed_through_ns = max(self.closed_through_ns, episode.end_ns) + self.plot = (episode, row['points']) + del self.completed[key] + for point in row['points']: + self.probe_requests.pop(point.control_ns, None) + self.reason = 'qualified_whole_tracking_episode' if gradient_complete else 'whole_episode_probe_deadline_partial_support' + + def observe_pose(self, pose, control, output, *, witness, now_ns, current_native, preserve_completed=False): + self.last_input_status = 'ignored' + try: + self._pair(control, output, current_native) + raw_witness = validate_response_witness(witness, self.binding, now_ns) + if raw_witness.certificate != self.certificate or (raw_witness.last_control_ns, raw_witness.last_apply_ns, raw_witness.last_pose_ns) != ( + control.control_ns, + output.apply_ns, + pose.mono_ns, + ): + raise ValueError('episode current physical model or exact residual pair changed') + if type(pose) is not OriginalSourcePose: + raise ValueError('exact original pose') + validate_pose(pose, now_ns) + b, lag = self.binding, self.certificate.lag_ns + if ( + (pose.identity, pose.source_epoch, pose.baseline_revision, pose.controller_revision, pose.calibration, pose.native_identity, pose.native_required) + != (b.identity, b.source_epoch, b.baseline_revision, b.controller_revision, physical_calibration_generation(b), b.native_identity, b.native_required) + or not 0 <= pose.source_upper_ns - lag - output.apply_ns <= 30_000_000 + or output.apply_ns > pose.source_lower_ns - lag + or not 0 <= now_ns - pose.mono_ns <= MAX_POSE_GAP_NS + ): + raise ValueError('episode lag/source interval changed') + # A genuine lagged pair from BEFORE a source/profile boundary cannot + # count toward the new lineage. It also must not repeatedly erase its + # new 100Hz history while the physical lag catches up. + if self.ticks and control.control_ns < next(iter(self.ticks)): + return False + tick = self.ticks.get(control.control_ns) + if ( + tick is None + or tick.control != control + or self.emissions.get(control.control_ns, {}).get(output.apply_ns) != output + or dict(control.stage_values).get('setpoint') != tick.geometry + or (tracking_reference_acceleration(control) if self.source_tracking else tracking_reference_force(control)) != tick.goal + ): + raise ValueError('episode pose not in exact observed owner history') + if pose.mono_ns <= self.last_pose_ns: + return False + if self.endpoints and control.control_ns <= self.endpoints[-1].control_ns: + return False + if self.last_pose_ns and pose.mono_ns - self.last_pose_ns > MAX_POSE_GAP_NS: + raise ValueError('episode independent pose gap') + # Plant identification associates a command with its later response. + # Tracking accuracy instead compares the desired and actual state at + # the SAME observed time, as the existing controller reports do. + # Keep the causal command/native probe below; use a distinct original + # reference control, never an interpolated or future desired sample. + reference = next((self.ticks[stamp] for stamp in reversed(self.ticks) if stamp <= pose.source_lower_ns), None) + if reference is None or pose.source_upper_ns - reference.control.control_ns > 30_000_000 or reference.profile != tick.profile: + self.last_input_status = 'pending' + return False + self._source_control(reference.control, current_native) + goal = tracking_reference_acceleration(reference.control) if self.source_tracking else tracking_reference_force(reference.control) + if goal != reference.goal: + raise ValueError('original tracking reference changed') + measured = pose.speed_mps * pose.yaw_rate + if not self.source_tracking: + measured -= ACCELERATION_DUE_TO_GRAVITY * math.sin(pose.roll_rad) + if not _finite(measured) or pose.speed_mps <= 0: + raise ValueError('episode finite independent response') + self.endpoints.append( + _Endpoint(control.control_ns, output.apply_ns, pose.mono_ns, pose.speed_mps, goal, goal - measured, measured, reference.control.control_ns) + ) + self.last_pose_ns = pose.mono_ns + self.watermark = pose.source_lower_ns - lag + self.numerical_observed_ns = now_ns # Already validated original pose/witness receipt clock. + self._finish() + self.last_input_status = 'accepted' + return True + except (AttributeError, KeyError, TypeError, ValueError, OverflowError): + self.interrupt('episode original physical join unqualified', preserve_completed=preserve_completed) + return False + + def _finish(self): + while self.pending and self.watermark > self.pending[0][1] + 30_000_000: + start, end, kind, direction, profile = self.pending.popleft() + ticks = [tick for stamp, tick in self.ticks.items() if start <= stamp <= end] + points = [point for point in self.endpoints if start <= (point.reference_ns if self.source_tracking else point.control_ns) <= end] + if not ticks or not 4 <= len(points) <= 240: + self.reason = 'whole_episode_independent_endpoints_required' + continue + stamps = [tick.control.control_ns for tick in ticks] + gaps = [right - left for left, right in zip(stamps, stamps[1:], strict=False)] + max_gap = max(gaps, default=0) + if ( + stamps[0] != start + or not 0 < max_gap <= MAX_CONTROL_GAP_NS + or any(tick.profile != profile for tick in ticks) + or max(point.speed for point in points) > 1.15 * min(point.speed for point in points) + ): + continue + from openpilot.starpilot.flm.live_tracking_probe import validate_probe_result + + qualified_probes = {} + for point in points: + probe = self.probes.get(point.control_ns) + if probe is not None: + try: + validate_probe_result(probe, self.binding) + if ( + probe.control_ns == point.control_ns + and probe.apply_ns == point.apply_ns + and probe.profile_sha256 == profile + and probe.model_revision == self.certificate.model_revision + ): + qualified_probes[point.control_ns] = probe + except (AttributeError, TypeError, ValueError): + pass # Numerical support disappears; never invent a derivative. + goal_rms = math.sqrt(sum(point.goal**2 for point in points) / len(points)) + bins = [[] for _ in range(8)] + for tick in ticks: + index = min(7, (tick.control.control_ns - start) * 8 // (end - start + 1)) + bins[index].append(tick.goal) + scale = max(0.1, goal_rms) if self.source_tracking else goal_rms + if scale <= 0 or any(not values for values in bins): + self.reason = 'whole_episode_actual_goal_shape_required' + continue + goal_shape = tuple(sum(values) / len(values) / scale for values in bins) + sensitivities = self._sensitivities(points, qualified_probes) + errors = [point.error for point in points] + from openpilot.starpilot.flm.live_tracking_response import TrackingSourceContext + + # Prediction residuals do not define the size of observed tracking weave. + floor = 0.0 if type(self.certificate) is TrackingSourceContext else self.certificate.heldout_rmse + reversal_hz = tracking_reversal_rate(errors, (end - start) / 1e9, floor) + digest = hashlib.sha256( + repr( + ( + ('fixed-input-kinematic-windows-v1' if self.source_tracking else 'live-whole-tracking-v4-actionable-error-excursions'), + self.binding, + profile, + start, + end, + kind, + tuple((point.control_ns, point.apply_ns, point.pose_ns, point.reference_ns) for point in points), + ) + ).encode() + ).hexdigest() + episode = TrackingEpisode( + self.binding, + profile, + digest, + start, + end, + kind, + direction, + min(point.speed for point in points), + max(point.speed for point in points), + goal_rms, + math.sqrt(sum(error**2 for error in errors) / len(errors)), + reversal_hz, + len(points), + max_gap, + sensitivities, + goal_shape, + normalization_accel=scale if self.source_tracking else None, + ) + episode.validate() + requests = {point.control_ns: self.probe_requests[point.control_ns] for point in points if point.control_ns in self.probe_requests} + defer = ( + kind == 'subtle' + and getattr(self, 'defer_unprobed', False) is True + and type(getattr(self, 'numerical_observed_ns', 0)) is int + and self.numerical_observed_ns > 0 + ) + if requests or defer: + if requests: + group, names, _ = next(iter(requests.values())) + deadline = min(value[2] for value in requests.values()) + PROBE_DEADLINE_NS + else: + # A short whole maneuver fits inside the EXISTING historical input + # age. Keep its ORIGINAL endpoints for one bounded numerical batch, + # rather than closing it before the4Hz coordinator can request them. + group, names = 'all30', self.numeric_probe_names + deadline = self.numerical_observed_ns + PROBE_DEADLINE_NS + if any(value[:2] != (group, names) for value in requests.values()) or len(self.completed) >= MAX_COMPLETED: + self.reason = 'whole_episode_probe_group_or_backlog_unavailable' + continue + self.completed[episode.episode_id] = { + 'episode': episode, + 'points': tuple(points), + 'group': group, + 'names': names, + 'requested': set(requests), + 'deadline': deadline, + 'probes': { + key: value for key, value in qualified_probes.items() if key in requests and requests[key][2] <= self.probe_received.get(key, 0) <= deadline + }, + } + else: + self.ready.append(episode) + self.closed_through_ns = max(self.closed_through_ns, episode.end_ns) + self.plot = (episode, tuple(points)) + self.reason = 'qualified_whole_tracking_episode' + + def release_measured_only(self): + """Release already validated measurements without inventing derivatives. + + Only the coordinator's post-selection comparison path calls this. Every + episode already passed its original control/native/CAN/pose/clock checks. + Optional counterfactual probes are not actual actuation permission and + cannot delay an uncensored baseline or a later measured keep/revert score. + """ + if len(self.ready) + len(self.completed) > self.ready.maxlen: + self.interrupt('measured_episode_backlog') + return False + for key, row in tuple(self.completed.items()): + episode = replace(row['episode'], sensitivity=(), native_gradient_complete=False).validate() + self.ready.append(episode) + self.closed_through_ns = max(self.closed_through_ns, episode.end_ns) + self.plot = (episode, row['points']) + for point in row['points']: + self.probe_requests.pop(point.control_ns, None) + del self.completed[key] + self.probe_requests.clear() + return True + + def drain(self, *, now_ns=None): + self._release_completed(now_ns) + # Optional native probes may finish after a later unprobed maneuver. + # Preserve source order before the optimizer's strict chronological check; + # never replace source time with numerical completion time. The original + # probe deadline and bounded ready deque still bound this wait. + barrier = min((row['episode'].start_ns for row in self.completed.values()), default=None) + ordered = sorted(self.ready, key=lambda episode: episode.start_ns) + result = tuple(episode for episode in ordered if barrier is None or episode.end_ns < barrier) + self.ready.clear() + self.ready.extend(episode for episode in ordered if barrier is not None and episode.end_ns >= barrier) + return result + + def tracking_plot(self): + """Last completed episode's actual paired samples; never an interpolated line.""" + if self.plot is None: + return None + episode, points = self.plot + indices = tuple(range(len(points))) if len(points) <= 64 else tuple(index * (len(points) - 1) // 63 for index in range(64)) + selected = tuple(points[index] for index in indices) + return { + 'version': 'live-tracking-paired-plot-v1', + 'episodeId': episode.episode_id, + 'profileSha256': episode.profile_sha256, + 'lagNs': self.certificate.lag_ns, + 'units': 'm/s²', + 'sampled': len(selected) != len(points), + 'totalEndpoints': len(points), + 'points': [ + { + 'controlMonoTime': point.control_ns, + 'applyMonoTime': point.apply_ns, + 'poseMonoTime': point.pose_ns, + 'desiredForceAccel': point.goal, + 'measuredForceAccel': point.measured, + 'errorForceAccel': point.error, + } + for point in selected + ], + } diff --git a/openpilot/starpilot/flm/live_tracking_ingress.py b/openpilot/starpilot/flm/live_tracking_ingress.py new file mode 100644 index 0000000000..48f9d3869c --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_ingress.py @@ -0,0 +1,164 @@ +"""Optional learner adapter for genuine fullrate controls and physical poses. + +No controller update, native encoding, file polling or process wait occurs here. +Snapshot probe work is submitted separately; absent probes hold identification. +""" +from collections import deque +from openpilot.starpilot.flm.live_tracking import LiveTrackingOptimizer +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, full_profile_sha256 +from openpilot.starpilot.flm.live_tracking_episodes import TrackingEpisodeBuilder +from openpilot.starpilot.flm.live_tracking_response import ResponseMonitor + + +class LiveTrackingIngress: + def __init__(self, binding, certificate, accepted_full30, *, controller_domain): + self.binding = binding + self.replay = None + self.optimizer = LiveTrackingOptimizer(binding, accepted_full30, controller_domain=controller_domain, + response_certificate=certificate) + self.monitor = ResponseMonitor(certificate) + self.episodes = TrackingEpisodeBuilder(binding, certificate,controller_domain=controller_domain) + self.last_pose_ns = 0 + self.latest_offer = None + self.scoring_enabled = True + # Original2s-old numeric endpoint plus its actual<=1s prestate prefix. + # This ring grants no current permission; per-job prefix remains<=100ticks. + self.control_history = deque(maxlen=301) + self.last_coordinates = self.last_profile = None + + def profile(self, control): + values = control.live_applied_coordinates + if not values: + if control.effect_trace is not None: + control.effect_trace.validate() + values = tuple(dict(control.effect_trace.applied)[name] for name in NAMES) + elif control.applied_coordinates == (0., 0.): + values = (0.,)*30 + else: + raise ValueError('actual complete consumed tracking coordinates unavailable') + if type(values) is not tuple or len(values) != 30: + raise ValueError('actual thirty hook coordinates') + if values != self.last_coordinates: + self.last_profile = full_profile_sha256(self.binding.baseline_revision, dict(zip(NAMES, values, strict=True))) + self.last_coordinates = values + return self.last_profile + + def interrupt(self, reason, *, context_changed=False, preserve_completed=True): + self.monitor.reset(reason) + state = self.optimizer.offer_state + preserve = bool(preserve_completed and not context_changed and (state is None or state.get('accepted') is True)) + self.episodes.interrupt(reason, preserve_completed=preserve) + self.optimizer.interrupt(reason, context_changed=context_changed) + self.control_history.clear() + self.latest_offer = None + + def _retain_completed(self): + state = self.optimizer.offer_state + return state is None or state.get('accepted') is True + + def observe_source_control(self, control, *, now_ns, current_native): + try: + profile = self.profile(control) + except (AttributeError, TypeError, ValueError): + self.interrupt('actual_tracking_profile_unavailable') + return False + accepted = self.episodes.observe_source_control(control, profile_sha256=profile, + now_ns=now_ns, current_native=current_native, preserve_completed=self._retain_completed()) + if not accepted and self.episodes.last_input_status == 'invalid': + self.interrupt(self.episodes.reason) + if accepted: + state = self.optimizer.offer_state + settled = self.optimizer.settled_ns + if (state is not None and state['phase'] == 'trial' and state.get('accepted') is not True and + settled is not None and control.control_ns > settled and + profile != self.optimizer._hash(state['target'])): + # An actual newer consumer has left an acknowledged target. Older + # queued controls and the initial slew are not withdrawal evidence. + # Discard the experiment; neither its lease nor its scores can resume. + self.interrupt('acknowledged_trial_profile_withdrawn') + return False + self.control_history.append(control) + return accepted + + def observe_control(self, control, output, *, profile_sha256, now_ns, current_native): + # EVERY joined owner tick enters here, including the ticks between poses. + accepted = self.episodes.observe_control(control, output, profile_sha256=profile_sha256, + now_ns=now_ns, current_native=current_native, preserve_completed=self._retain_completed()) + if not accepted and getattr(self.episodes, 'last_input_status', 'invalid') == 'invalid': + self.interrupt(self.episodes.reason) + return accepted + + def observe_probe(self, result, *, now_ns): + accepted = self.episodes.observe_probe(result, now_ns=now_ns) + witness = self.monitor.witness(now_ns) + if witness is not None and self.optimizer.response_current(witness, now_ns=now_ns): + for episode in self.episodes.drain(now_ns=now_ns): + if self.scoring_enabled: + self.optimizer.observe_episode(episode, now_ns=now_ns) + return accepted + + def observe_original_pose(self, pose, collector, *, now_ns, current_native): + lag = self.optimizer.lag_ns + lower = [(c, o) for c, o in collector.ready if o.apply_ns <= pose.source_lower_ns-lag] + upper = [(c, o) for c, o in collector.ready if o.apply_ns <= pose.source_upper_ns-lag] + if not lower or not upper or lower[-1][1].apply_ns != upper[-1][1].apply_ns: + return None + return self.observe_pose(pose, *lower[-1], now_ns=now_ns, current_native=current_native) + + def observe_pose(self, pose, control, output, *, now_ns, current_native): + # The caller passes the collector's original lag-associated pair. Neither + # this adapter nor a 4Hz report chooses/interpolates a replacement pose. + if pose.mono_ns <= self.last_pose_ns: + return None + self.last_pose_ns = pose.mono_ns + if not self.monitor.observe(control, output, pose, now_ns=now_ns, + binding=self.binding, current_native=current_native, replay=self.replay): + state = self.optimizer.offer_state + preserve = (self.monitor.reason == 'response_command_outside_identified_range' and + (state is None or state.get('accepted') is True)) + self.optimizer.interrupt(self.monitor.reason) + self.episodes.interrupt(self.monitor.reason, preserve_completed=preserve) + return None + witness = self.monitor.witness(now_ns) + if witness is None: + self.optimizer.interrupt('fresh_physical_residual_window_required') + return None + if not self.optimizer.response_current(witness, now_ns=now_ns): + return None + state=self.optimizer.offer_state + from openpilot.starpilot.flm.live_tracking import discovery_needs_probes + measurement_only = bool(self.optimizer.discovery is not None and not discovery_needs_probes(self.optimizer.discovery) or + state is not None and state['phase']=='trial') + self.episodes.defer_unprobed=bool(self.scoring_enabled and not measurement_only) + if measurement_only and not self.episodes.release_measured_only(): + self.interrupt('measured_episode_backlog') + return None + self.episodes.observe_pose(pose, control, output, witness=witness, now_ns=now_ns, + current_native=current_native, preserve_completed=self._retain_completed()) + for episode in self.episodes.drain(now_ns=now_ns): + if self.scoring_enabled: + self.optimizer.observe_episode(episode, now_ns=now_ns) + # Renew only on a genuine new physical pose; no restamping stale evidence. + self.latest_offer = self.optimizer.offer(now_ns, publication=False) + return self.latest_offer + + def observe_application(self, application, *, now_ns): + """Only the actual typed wire reader's current-owner acknowledgement.""" + from openpilot.starpilot.flm.live_trial_owner import LiveTrialApplication + if (type(application) is not LiveTrialApplication or self.optimizer.offer_state is None or + self.latest_offer is None or application.version != 1 or not 0 < application.sequence <= self.latest_offer.sequence): + return False + state = self.optimizer.offer_state + return self.optimizer.acknowledge_application(trial_id=application.trial_id, + target_sequence=application.target_sequence, phase=state['phase'], + target_sha256=application.applied_sha256, applied_full30=application.applied, + control_ns=application.control_ns, binding=application.binding, now_ns=now_ns, + status=application.status, confirmed_target_sha256=application.confirmed_target_sha256) + + def report(self, now_ns): + tracking = self.optimizer.report(now_ns) + plot = self.episodes.tracking_plot() + if plot is not None: + plot = dict(plot, bindingCurrent=tracking['bindingCurrent'], + ageSeconds=max(0., (now_ns-plot['points'][-1]['poseMonoTime'])/1e9)) + return {'tracking': tracking, 'trackingValues': self.optimizer.tracking_values(), 'trackingPlot': plot} diff --git a/openpilot/starpilot/flm/live_tracking_probe.py b/openpilot/starpilot/flm/live_tracking_probe.py new file mode 100644 index 0000000000..ac89a5956d --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_probe.py @@ -0,0 +1,968 @@ +"""Bounded original-prefix controller/native secants; numerical evidence only. + +No live controller is changed and no driving permission or causal plant model +is produced. Legacy model mode uses measured physical units. Source-context mode uses +normalized native-command secants solely for proposal direction/visibility; +it does NOT claim a plant prediction or physical effect magnitude. +""" + +from dataclasses import dataclass, fields +import math +import struct +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + +from openpilot.starpilot.flm.canonical_surface_adapter import GROUPS, PROBE_GROUPS, probe_names, coordinate_bounds +from openpilot.starpilot.flm.live_trial_contract import MAX_STEP + +_OWNER = object() +MAX_FRESH_NS = 100_000_000 + + +def _finite(value): + return (type(value) is int or type(value) is float) and math.isfinite(value) + + +@dataclass(frozen=True) +class ProbeCoordinate: + name: str + baseline_coordinate: float + minus_coordinate: float + plus_coordinate: float + baseline_delivered: float + minus_delivered: float + plus_delivered: float + physical_derivative: float # (m/s^2)/canonical coordinate, with measured sign. + native_separated: bool + max_step_physical_delta: float # m/s^2 at MAX_STEP, NOT a derivative. + status: str + + +@dataclass(frozen=True) +class ProbeResult: + binding: object + control_ns: int + apply_ns: int + profile_sha256: str + model_revision: str + rows: tuple[ProbeCoordinate, ...] + controller_domain: str + group: str + _seal: object + + +def validate_probe_result(value, binding=None): + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + from openpilot.starpilot.flm.pid_stages import SUPPORTED + + if ( + type(value) is not ProbeResult + or type(value._seal) is not tuple + or len(value._seal) != 2 + or value._seal[0] is not _OWNER + or type(value.binding) is not LiveTrialBinding + ): + raise ValueError('owned actual-controller probe result required') + value.binding.validate() + if ( + binding is not None + and value.binding != binding + or type(value.controller_domain) is not str + or value.controller_domain not in ('torque', 'torque_pid') + or type(value.rows) is not tuple + or any(type(row) is not ProbeCoordinate or type(row.name) is not str for row in value.rows) + or type(value.group) is not str + or value.group not in PROBE_GROUPS + or not 1 <= len(value.rows) <= len(PROBE_GROUPS[value.group]) + or tuple(row.name for row in value.rows) != probe_names(tuple(row.name for row in value.rows)) + or any(row.name not in PROBE_GROUPS[value.group] or value.controller_domain == 'torque_pid' and row.name not in SUPPORTED for row in value.rows) + or any(type(stamp) is not int for stamp in (value.control_ns, value.apply_ns)) + or not 0 < value.control_ns <= value.apply_ns <= value.control_ns + 30_000_000 + or any(type(text) is not str or len(text) != 64 or any(c not in '0123456789abcdef' for c in text) for text in (value.profile_sha256, value.model_revision)) + or value.model_revision != value.binding.physical_model_revision + ): + raise ValueError('owned actual-controller probe result required') + for row in value.rows: + if ( + type(row) is not ProbeCoordinate + or type(row.status) is not str + or row.status not in ('separated', 'quantized', 'native_unavailable') + or type(row.native_separated) is not bool + or any( + not _finite(number) + for number in ( + row.baseline_coordinate, + row.minus_coordinate, + row.plus_coordinate, + row.baseline_delivered, + row.minus_delivered, + row.plus_delivered, + row.physical_derivative, + row.max_step_physical_delta, + ) + ) + ): + raise ValueError('finite exact native coordinate probe required') + low, high = coordinate_bounds(row.name) + if ( + not low <= row.minus_coordinate <= row.baseline_coordinate <= row.plus_coordinate <= high + or not row.minus_coordinate < row.plus_coordinate + or max(row.baseline_coordinate - row.minus_coordinate, row.plus_coordinate - row.baseline_coordinate) > MAX_STEP + 1e-15 + or row.plus_coordinate - row.minus_coordinate > 2 * MAX_STEP + 1e-15 + or row.max_step_physical_delta != abs(row.physical_derivative) * MAX_STEP + or row.native_separated is not (row.status == 'separated') + or row.native_separated is not (row.minus_delivered != row.plus_delivered) + or not row.native_separated + and (row.physical_derivative != 0 or row.max_step_physical_delta != 0) + ): + raise ValueError('bounded separated native physical secant required') + if value._seal[1] != _probe_key(value): + raise ValueError('owned probe result contents changed') + return value + + +def _probe_key(value): + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + + return ( + _record(value.binding, LiveTrialBinding), + value.control_ns, + value.apply_ns, + value.profile_sha256, + value.model_revision, + value.controller_domain, + value.group, + tuple(_record(row, ProbeCoordinate) for row in value.rows), + ) + + +def _primitive(value, depth=0): + """Fresh exact bytes for source mutation detection; never invokes reducers.""" + if depth > 40: + raise ValueError('bounded probe source graph') + kind = type(value) + if value is None: + return ('none',) + if kind is bool or kind is int or kind is str or kind is bytes: + if kind is int and value.bit_length() > 512: + raise ValueError('bounded probe source integer') + if (kind is str or kind is bytes) and len(value) > 131072: + raise ValueError('bounded probe source scalar') + return kind.__name__, value + if kind is float and math.isfinite(value): + return 'float64', struct.pack('!d', value) + if kind is tuple or kind is list: + if len(value) > 8192: + raise ValueError('bounded probe source sequence') + return kind.__name__, tuple(_primitive(child, depth + 1) for child in value) + if kind is dict and len(value) <= 8192 and all(type(key) is str for key in value): + return 'dict', tuple((key, _primitive(value[key], depth + 1)) for key in sorted(value)) + raise ValueError('exact primitive probe source required') + + +def _record(value, cls): + if type(value) is not cls: + raise ValueError('exact source record required') + # No asdict/deepcopy/custom reducer. Fields are individually type-checked by + # the enclosing domain validation before this complete fresh key is formed. + return tuple((field.name, _primitive(getattr(value, field.name))) for field in fields(cls)) + + +def _physical_row(name, current, minus, plus, base, low, high, slope, status): + derivative = -slope * (high - low) / (plus - minus) if status == 'separated' else 0.0 + return ProbeCoordinate(name, current, minus, plus, base, low, high, derivative, status == 'separated', abs(derivative) * MAX_STEP, status) + + +def _projection_literal(witness): + from opendbc.car.steering_projection import TorqueProjection, projection_to_dict + + if type(witness) is not TorqueProjection: + raise ValueError('exact actual native projection required') + # TorqueProjection is an exact NamedTuple, not a dataclass. Read builtin + # tuple slots before the stock converter can examine unknown field values. + _primitive(tuple(tuple.__iter__(witness))) + value = projection_to_dict(witness) + if type(value) is not dict: + raise ValueError('bounded actual native projection required') + _primitive(value) + return value + + +def _factory_configuration(factory): + """Close mutable constructor metadata BEFORE coercions or native CP access.""" + from capnp.lib.capnp import _DynamicStructReader + from opendbc.car import structs + from openpilot.starpilot.flm.controller_replay import SelectedControllerFactory + from openpilot.starpilot.flm.pid_controller_replay import PidControllerFactory + from openpilot.starpilot.lateral.controller_selection import ControllerMode + + if type(factory) is SelectedControllerFactory: + mode, policy = factory.mode, factory.policy + if type(mode) is not ControllerMode or type(mode.value) is not str or type(policy) not in (str, type(None)): + raise ValueError('exact selected constructor mode/policy required') + mode, policy = mode.value, str(policy) + elif type(factory) is PidControllerFactory: + mode, policy = factory.mode, None + if type(mode) is not str or mode != 'native': + raise ValueError('exact stock PID constructor mode required') + else: + raise ValueError('exact registered constructor required') + if ( + type(factory.cp) is not _DynamicStructReader + or factory.cp.schema != structs.CarParams.schema + or type(factory.turn_assist) is not bool + or type(factory.manual_baseline) is not tuple + or type(factory.manual_source_epoch) is not str + ): + raise ValueError('closed native constructor inputs required') + _primitive(factory.manual_baseline) + return mode, policy + + +# Optional worker only: exact complete bytes are reconstructed on EVERY lookup. +# Values are immutable snapshot (revision, numeric tuple) only, never permission. +_PREFIX_CACHE = {} +_PREFIX_CACHE_MAX_KEY = 2 * 1024 * 1024 + + +def _trace_key(trace): + from openpilot.starpilot.flm.effect_trace import FullEffectTrace + from openpilot.starpilot.flm.full_surface_profile import FullSurfaceEnvelope + + if trace is None: + return None + if type(trace) is not FullEffectTrace or type(trace.envelope) is not FullSurfaceEnvelope: + raise ValueError('exact actual full30 trace required') + value = tuple( + (field.name, _record(trace.envelope, FullSurfaceEnvelope) if field.name == 'envelope' else _primitive(getattr(trace, field.name))) + for field in fields(FullEffectTrace) + ) + trace.validate() + return value + + +def _frame_key(frame, binding): + from openpilot.starpilot.flm.controller_replay import ReplayFrame + + if type(frame) is not ReplayFrame: + raise ValueError('exact original prefix frame required') + if frame.effect_trace is not None: + _trace_key(frame.effect_trace) + envelope = frame.effect_trace.envelope + if ( + envelope.cp_sha256, + envelope.controller_revision, + envelope.source_epoch, + envelope.reset_generation, + envelope.selected_baseline_revision, + envelope.persistent_source_epoch, + ) != ( + binding.cp_sha256, + binding.controller_revision, + binding.source_epoch, + binding.reset_generation, + binding.baseline_revision, + binding.persistent_source_epoch, + ): + raise ValueError('original full30 prefix source binding differs') + return tuple( + (field.name, _trace_key(frame.effect_trace) if field.name == 'effect_trace' else _primitive(getattr(frame, field.name))) for field in fields(ReplayFrame) + ) + + +def _consumed_coordinates(control, binding): + """Actual same-frame owner map; never substitute a target or stored prior.""" + from openpilot.starpilot.flm.live_trial_contract import full_coordinates + from openpilot.starpilot.flm.canonical_surface_adapter import NAMES + + live = getattr(control, 'live_applied_coordinates', ()) + if type(live) is not tuple: + raise ValueError('exact actual consumed live full30 required') + trace = control.effect_trace + if trace is not None: + _trace_key(trace) + envelope = trace.envelope + if ( + envelope.cp_sha256, + envelope.controller_revision, + envelope.source_epoch, + envelope.reset_generation, + envelope.selected_baseline_revision, + envelope.persistent_source_epoch, + ) != ( + binding.cp_sha256, + binding.controller_revision, + binding.source_epoch, + binding.reset_generation, + binding.baseline_revision, + binding.persistent_source_epoch, + ): + raise ValueError('actual consumed full30 source differs') + if live: + if len(live) != len(NAMES) or any(not _finite(value) for value in live): + raise ValueError("exact finite actual consumed live vector30 required") + rows = tuple(zip(NAMES, live, strict=True)) + full_coordinates(rows, binding.baseline_revision) + if trace is not None and _primitive(rows) != _primitive(trace.applied): + raise ValueError('two actual full30 owner receipts disagree') + return rows, True + if trace is not None: + return trace.applied, True + return tuple((name, 0.0) for name in NAMES), False + + +def _consumed_source(rows, binding): + from openpilot.starpilot.flm.selected_effects import EffectReplaySource + + values = dict(rows) + name = GROUPS['ff.left'][0] + return EffectReplaySource.from_coordinates({name: values[name]}, binding.baseline_revision, accumulated=values) + + +def _control_key(control, binding): + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + from openpilot.starpilot.flm.runtime_adapter_registry import vm_regime, digest + + if type(control) is not ControlEvidence: + raise ValueError('exact original100Hz control receipt required') + key = tuple( + (field.name, _trace_key(control.effect_trace) if field.name == 'effect_trace' else _primitive(getattr(control, field.name))) + for field in fields(ControlEvidence) + ) + if ( + any( + type(stamp) is not int + for stamp in ( + control.state_ns, + control.control_ns, + control.source_receipt_ns, + control.native_observed_ns, + control.native_expires_ns, + control.snapshot_sequence, + ) + ) + or any(type(flag) is not bool for flag in (control.native_required, control.native_allowed)) + or not 0 < control.state_ns <= control.control_ns <= control.state_ns + 30_000_000 + or not 0 < control.source_receipt_ns <= control.control_ns <= control.source_receipt_ns + 500_000_000 + or (control.identity, control.source_epoch, control.baseline_revision, control.controller_revision, control.native_identity, control.native_required) + != (binding.identity, binding.source_epoch, binding.baseline_revision, binding.controller_revision, binding.native_identity, binding.native_required) + ): + raise ValueError('original100Hz source/controller/native receipt differs') + anchor = getattr(control, 'vehicle_model_anchor', None) + if ( + type(anchor) is not tuple + or len(anchor) != 3 + or any(not _finite(value) or value == 0 for value in anchor) + or digest(vm_regime(dict(control.replay_inputs), anchor)) != binding.vehicle_model_generation + ): + raise ValueError('actual original owner vehicle-model anchor differs') + if control.native_allowed and ( + not binding.native_required + or not 0 < control.native_observed_ns <= control.control_ns <= control.native_expires_ns <= control.native_observed_ns + 200_000_000 + ): + raise ValueError('actual original native receipt clock differs') + if ( + binding.native_required + and control.enabled + and control.active + and control.valid + and not control.manual + and not control.override + and not control.fault + and not control.native_allowed + ): + raise ValueError('actual healthy original native permission unavailable') + _consumed_coordinates(control, binding) + return key + + +def _prefix_before(replay, control, output, binding, factory, prefix_controls: tuple["ControlEvidence", ...] = ()): + """Actual latest checkpoint through EVERY original tick, at most100 updates. + + The donor is the genuine optional learner recorder. A separate bounded owner + reconstructs it; the donor actual/probe controllers are never advanced here. + Native permission/packets are freshly decoded even on a snapshot cache hit. + """ + from itertools import islice + import pickle + from openpilot.starpilot.flm.controller_replay import ( + ControllerReplay, + Snapshot, + MAX_FRAME_GAP_NS, + restore_snapshot, + capture_snapshot, + _run, + same_final_command, + ) + from openpilot.starpilot.flm.torque_emission_replay import Emission, EmissionHistory + from openpilot.starpilot.flm.learner_factory import _frame + from openpilot.starpilot.flm.original_context_receipts import _same_original_frame + from openpilot.starpilot.flm.torque_adapter import Objective + from opendbc.car.steering_projection import projection_to_dict + + if ( + type(replay) is not ControllerReplay + or type(replay.emissions) is not EmissionHistory + or replay.factory is not factory + or replay.failed + or not replay.raw_mode + or replay.identity != binding.identity + or replay.baseline_revision != binding.baseline_revision + or type(replay.objective) is not Objective + ): + raise ValueError('actual original recorder/selected constructor required') + + def source(): + checkpoints = [(stamp, snapshot) for stamp, snapshot in replay.checkpoints if stamp <= control.control_ns] + if not checkpoints: + raise ValueError('genuine original prefix checkpoint missing') + start, checkpoint = checkpoints[-1] + if type(checkpoint) is not Snapshot or not 0 <= control.control_ns - start <= 1_000_000_000: + raise ValueError('bounded actual prefix checkpoint required') + frames = tuple(reversed(tuple(row for row in islice(reversed(replay.records), 201) if start <= row.control_ns <= control.control_ns))) + if ( + not 1 <= len(frames) <= 100 + or frames[0].control_ns != start + or frames[-1].control_ns != control.control_ns + or any( + right.frame_sequence != left.frame_sequence + 1 or not 0 < right.control_ns - left.control_ns <= MAX_FRAME_GAP_NS + for left, right in zip(frames, frames[1:], strict=False) + ) + ): + raise ValueError('complete bounded original100Hz prefix required') + from opendbc.car.car_helpers import interfaces + + mode, policy = _factory_configuration(factory) + interface = interfaces.get(str(factory.cp.carFingerprint)) + if factory.interface_class is not interface or interface is None: + raise ValueError("exact current registered constructor interface required") + constructor = (interface.__module__, interface.__qualname__, policy) + domain = 'torque_pid' if factory.cp.lateralTuning.which() == 'pid' else 'torque' + if any(row.controller_domain != domain for row in frames): + raise ValueError('original controller prefix domain differs') + frame_keys = tuple(_frame_key(row, binding) for row in frames) + if type(prefix_controls) is not tuple or len(prefix_controls) != len(frames) or any(type(row) is not type(control) for row in prefix_controls): + raise ValueError('every actual100Hz source control receipt required') + control_keys = tuple(_control_key(row, binding) for row in prefix_controls) + if any( + type(row.turn_assist) is not bool + or row.turn_assist is not factory.turn_assist + or (row.controller_mode, row.policy) != (control.controller_mode, control.policy) + for row in prefix_controls + ): + raise ValueError("actual prefix constructor/policy/assist configuration changed") + if ( + _primitive(prefix_controls[0].replay_snapshot) != _primitive(checkpoint.values) + or prefix_controls[0].snapshot_sequence <= 0 + or any(row.snapshot_sequence != prefix_controls[0].snapshot_sequence or row.generation != control.generation for row in prefix_controls) + or any(row.replay_snapshot for row in prefix_controls[1:]) + or any(not _same_original_frame(frame, _frame(row)) for frame, row in zip(frames, prefix_controls, strict=True)) + or _control_key(prefix_controls[-1], binding) != _control_key(control, binding) + ): + raise ValueError('complete genuine checkpoint/source-control prefix differs') + if not _same_original_frame(frames[-1], _frame(control)): + raise ValueError('current endpoint differs from original recorded tick') + events = tuple(reversed(tuple(row for row in islice(reversed(replay.emissions.events.values()), 201) if start <= row.control_ns <= control.control_ns))) + if not events or events[-1].apply_ns != output.apply_ns or events[-1].control_ns != control.control_ns: + raise ValueError('actual endpoint native chronology incomplete') + native = [] + for event in events: + if type(event) is not Emission: + raise ValueError('exact actual native prefix emission required') + if (event.identity, event.source_epoch, event.baseline_revision, event.native_identity, event.native_required) != ( + binding.identity, + binding.source_epoch, + binding.baseline_revision, + binding.native_identity, + binding.native_required, + ): + raise ValueError('original native prefix source differs') + projection = projection_to_dict(event.witness) + if projection is None: + raise ValueError('original native prefix projection missing') + body = {field.name: projection if field.name == 'witness' else getattr(event, field.name) for field in fields(Emission)} + _primitive(body) + native.append(body) + first, last = events[0].witness.frame + 1, events[-1].witness.frame + 1 + if not 0 <= last - first <= 200: + raise ValueError('bounded actual native carrier prefix required') + carriers = tuple(reversed(tuple((stamp, *row) for stamp, row in islice(reversed(replay.emissions.carriers.items()), 301) if first <= stamp <= last))) + native_data = {'events': tuple(native), 'carriers': carriers} + if _primitive(projection_to_dict(events[-1].witness)) != _primitive(projection_to_dict(output.projection)): + raise ValueError('endpoint emitted packet differs from recorded prefix') + key = pickle.dumps( + ( + _record(checkpoint, Snapshot), + frame_keys, + control_keys, + _primitive(native_data), + _record(binding, type(binding)), + factory.cp.as_builder().to_bytes(), + mode, + constructor, + tuple(factory.manual_baseline), + factory.manual_source_epoch, + factory.turn_assist, + _record(replay.objective, Objective), + ), + protocol=5, + ) + return key, start, checkpoint, frames, native_data + + key, start, checkpoint, frames, native_data = source() + # The authoritative loader rechecks every actual CAN event, all Card carrier + # ticks, source/native leases, hidden filter state and actual encoder chronology. + native = EmissionHistory(factory.cp, identity=binding.identity, baseline_revision=binding.baseline_revision) + native.load(native_data, ((start, control.control_ns),)) + requests = {row.control_ns: row.selected_command for row in frames} + from openpilot.starpilot.flm.torque_emission_replay import EncoderBranch + + branch = EncoderBranch() + for event in native.events.values(): + if event.control_ns not in requests: + raise ValueError('native prefix has no actual controller tick') + branch.step(event, requests[event.control_ns], actual=True) + saved = _PREFIX_CACHE.get(key) if len(key) <= _PREFIX_CACHE_MAX_KEY else None + if saved is None: + owner = ControllerReplay( + factory, replay.objective, identity=binding.identity, baseline_revision=binding.baseline_revision, model_revision='live-prefix-numerical-v1' + ) + from openpilot.starpilot.flm.runtime_adapter_registry import baseline_values + from openpilot.starpilot.flm.native_baseline import checkpoint_native_valid, static_values + + original = baseline_values(owner.probe) + static = () if factory.cp.lateralTuning.which() == 'pid' else static_values(owner.probe) + native_baseline = () if factory.cp.lateralTuning.which() == 'pid' else tuple(owner.probe.native_learning_baseline) + restore_snapshot(owner.probe, owner.probe_vm, checkpoint) + if factory.cp.lateralTuning.which() == 'pid': + if baseline_values(owner.probe) != original: + raise ValueError('PID prefix changed immutable constructor gains') + elif ( + static_values(owner.probe) != static + or tuple(owner.probe.native_learning_baseline) != native_baseline + or ( + bool(baseline_values(owner.probe) != original or tuple(owner.probe.native_learning_provenance)) + if factory.manual_baseline + else not checkpoint_native_valid(owner.probe, factory.cp, now_ns=start) + ) + ): + raise ValueError('actual native prefix checkpoint provenance failed') + # Never forge an old FullEffectTrace for a new live-owner map. Replay + # the actual transported consumed map on EVERY original100Hz controller tick. + from openpilot.starpilot.flm.selected_effects import EffectReplayRecordedSource + + snapshot = None + for frame, recorded in zip(frames, prefix_controls, strict=True): + if any(frame.applied_coordinates) or any(frame.centre_values) or any(frame.actual_stage_residual or ()): + raise ValueError('unsupported legacy residual in actual live prefix') + if frame.control_ns == control.control_ns: + snapshot = capture_snapshot(owner.probe, owner.probe_vm) + rows, has_source = _consumed_coordinates(recorded, binding) + effect = ( + ( + EffectReplayRecordedSource(recorded.effect_trace) + if recorded.effect_trace is not None and not getattr(recorded, 'live_applied_coordinates', ()) + else _consumed_source(rows, binding) + ) + if has_source + else None + ) + request, _ = _run( + owner.probe, owner.probe_vm, frame.inputs, replay.objective, (0.0, 0.0), frame.torque_values, actual_stage_residual=(0.0, 0.0), effect_trial=effect + ) + if not same_final_command(request, frame.selected_command): + raise ValueError('actual original100Hz controller request differs') + if type(snapshot) is not Snapshot: + raise ValueError('actual original endpoint before-state missing') + _record(snapshot, Snapshot) + saved = snapshot.revision, snapshot.values + if len(key) <= _PREFIX_CACHE_MAX_KEY: + if len(_PREFIX_CACHE) >= 2: + _PREFIX_CACHE.pop(next(iter(_PREFIX_CACHE))) + _PREFIX_CACHE[key] = saved + if source()[0] != key: + raise ValueError('actual original prefix source changed during reconstruction') + + # Repeated enclosing before/after checks need fresh COMPLETE source keys, + # not another controller reconstruction and native-prefix decoder traversal. + # This closure rereads the original graphs on EVERY invocation; it caches + # neither their content nor a permission/validation verdict. + def fresh_source_key(): + return source()[0] + + return Snapshot(*saved), key, fresh_source_key + + +def _native_lease(control, output, binding, *, now_ns, historical): + if binding.native_required: + if ( + not control.native_allowed + or not output.native_allowed + or not 0 < control.native_observed_ns <= control.control_ns <= output.apply_ns <= control.native_expires_ns + or not historical + and now_ns > control.native_expires_ns + or control.native_expires_ns > control.native_observed_ns + 200_000_000 + ): + raise ValueError('fresh actual native lease required') + elif control.native_allowed or control.native_identity or control.native_observed_ns or control.native_expires_ns: + raise ValueError('ordinary permission cannot impersonate required-native lease') + + +def _evidence_clock(*, evidence_ns, queued_ns, now_ns, current_native, archived_clock=False): + if type(archived_clock) is not bool: + raise ValueError("explicit evidence clock domain required") + historical = evidence_ns is not None + if historical: + if ( + type(evidence_ns) is not int + or evidence_ns <= 0 + or type(queued_ns) is not int + or not 0 < queued_ns <= now_ns <= queued_ns + 30_000_000_000 + or not archived_clock + and evidence_ns > queued_ns + or current_native + ): + raise ValueError("historical numerical job clock/authority invalid") + validation_ns = evidence_ns + else: + if queued_ns is not None or archived_clock or not current_native: + raise ValueError("current native owner permission required") + validation_ns = now_ns + return historical, validation_ns + + +def probe_endpoint( + *, + factory, + control, + output, + accepted_full30, + binding, + response_witness, + source, + now_ns, + current_native, + current_vehicle_model_anchor, + replay, + group, + names, + prefix_controls, + evidence_ns=None, + queued_ns=None, + archived_clock=False, +): + """Probe one real endpoint after bounded actual100Hz prefix reconstruction. + + Caller obtains current_native from the actual native/ordinary Panda owner, + independently of the response model. It is a gate, never a returned grant. + A genuine <=1s checkpoint and every original tick are mandatory; a missing + endpoint checkpoint is reconstructed, never filled with a default/model guess. + """ + from openpilot.starpilot.flm.controller_replay import ( + SelectedControllerFactory, + capture_snapshot, + restore_snapshot, + _run, + same_final_command, + ) + from openpilot.starpilot.flm.pid_controller_replay import PidControllerFactory + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence, replay_values + from openpilot.starpilot.flm.runtime_adapter_registry import ( + SourceState, + car_params_digest, + selected_descriptor, + selected_identity, + baseline_revision, + baseline_values, + vm_regime, + digest, + ) + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding, full_coordinates + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness, _certificate_key, _witness_key + from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current, exact_unclipped_pair + from openpilot.starpilot.flm.learner_factory import _frame + from openpilot.starpilot.flm.torque_adapter import Objective + from openpilot.starpilot.flm.torque_emission_replay import Emission, EmissionDecoder, EncoderBranch, delivered + from opendbc.car.steering_projection import TorqueProjection, projection_to_dict + from openpilot.starpilot.flm.pid_stages import SUPPORTED, coordinates_supported + from openpilot.starpilot.flm.selected_effects import EffectReplaySource + + if ( + (type(factory) is not SelectedControllerFactory and type(factory) is not PidControllerFactory) + or type(control) is not ControlEvidence + or type(output) is not AppliedEvidence + or type(binding) is not LiveTrialBinding + or type(source) is not SourceState + or type(current_native) is not bool + or type(now_ns) is not int + ): + raise ValueError('actual selected checkpoint/source/native owners required') + _factory_configuration(factory) + historical, validation_ns = _evidence_clock( + evidence_ns=evidence_ns, queued_ns=queued_ns, now_ns=now_ns, current_native=current_native, archived_clock=archived_clock + ) + binding.validate() + if any( + type(flag) is not bool + for flag in ( + control.enabled, + control.active, + control.valid, + control.manual, + control.override, + control.fault, + control.saturated, + control.limited, + control.model_wobble, + control.native_required, + control.native_allowed, + control.turn_assist, + output.valid, + output.active, + output.override, + output.fault, + output.limited, + output.native_allowed, + source.enabled, + source.manual, + source.configuration_known, + ) + ) or any( + type(stamp) is not int + for stamp in ( + control.control_ns, + control.state_ns, + control.source_receipt_ns, + control.native_observed_ns, + control.native_expires_ns, + output.control_ns, + output.state_ns, + output.apply_ns, + source.observed_ns, + source.expires_ns, + ) + ): + raise ValueError('exact actual source clocks and flags required') + # Validate complete primitive input records before mapping, restoring or any + # controller/native call. Effect trace has its own exact typed validator. + for field in fields(ControlEvidence): + if field.name != 'effect_trace': + _primitive(getattr(control, field.name)) + for field in fields(AppliedEvidence): + if field.name != 'projection': + _primitive(getattr(output, field.name)) + _record(source, SourceState) + witness = validate_response_witness(response_witness, binding, validation_ns) + certificate = witness.certificate + profile = full_coordinates(accepted_full30, binding.baseline_revision) + prior = dict(accepted_full30) + domain = 'torque_pid' if type(factory) is PidControllerFactory else 'torque' + if ( + type(group) is not str + or group not in PROBE_GROUPS + or type(names) is not tuple + or not 1 <= len(names) <= len(PROBE_GROUPS[group]) + or any(type(name) is not str for name in names) + or probe_names(names) != names + or any(name not in PROBE_GROUPS[group] for name in names) + or domain == 'torque_pid' + and any(name not in SUPPORTED for name in names) + ): + raise ValueError('canonical bounded supported numerical probe coordinates required') + if domain == 'torque_pid' and not coordinates_supported(prior): + raise ValueError('PID cannot probe unsupported friction thresholds') + from opendbc.car.car_helpers import interfaces + + if factory.interface_class is not interfaces.get(str(factory.cp.carFingerprint)): + raise ValueError("exact current registered constructor interface required") + from openpilot.starpilot.flm.controller_replay import ControllerReplay + + if type(replay) is not ControllerReplay: + raise ValueError('exact original numerical replay required') + controller, vm = factory.create() + descriptor = selected_descriptor(factory.cp, factory.interface_class(factory.cp), controller) + if ( + descriptor is None + or descriptor.domain != domain + or descriptor.controller_mode != control.controller_mode + or descriptor.policy != control.policy + or descriptor.turn_assist is not control.turn_assist + or descriptor.controller_revision != binding.controller_revision + or selected_identity(descriptor, binding.source_epoch, binding.baseline_revision) != binding.identity + or baseline_revision(descriptor, baseline_values(controller)) != binding.baseline_revision + ): + raise ValueError('actual constructor/source baseline differs') + # This exact constructor revision is independently checked above. Original + # CAN verification admits a known limited INPUT only to numeric replay; + # EncoderBranch still applies the unchanged recorded limiter to every probe. + replay.source_controller_revision = descriptor.controller_revision + from openpilot.starpilot.flm.live_tracking_response import verified_clipped_input + + known_limited = bool(output.limited and verified_clipped_input(replay, control, output, binding)) + if ( + not source.enabled + or source.manual + or not source.configuration_known + or not 0 < source.observed_ns <= validation_ns <= source.expires_ns <= source.observed_ns + 500_000_000 + or (source.source_epoch, source.reset, source.cp_sha256, source.calibration_generation, source.persistent_source_epoch) + != (binding.source_epoch, binding.reset_generation, binding.cp_sha256, binding.calibration_generation, binding.persistent_source_epoch) + or type(current_vehicle_model_anchor) is not tuple + or len(current_vehicle_model_anchor) != 3 + or any(not _finite(number) or number == 0 for number in current_vehicle_model_anchor) + or current_vehicle_model_anchor != getattr(control, 'vehicle_model_anchor', None) + or digest(vm_regime(dict(control.replay_inputs), current_vehicle_model_anchor)) != binding.vehicle_model_generation + or tuple(factory.manual_baseline) != tuple(source.manual_baseline) + or car_params_digest(factory.cp) != binding.cp_sha256 + or (control.identity, control.source_epoch, control.baseline_revision, control.controller_revision, control.native_identity) + != (binding.identity, binding.source_epoch, binding.baseline_revision, binding.controller_revision, binding.native_identity) + or control.native_required is not binding.native_required + or not control.enabled + or not control.active + or not control.valid + or control.manual + or control.override + or control.fault + or control.saturated + or control.limited + and not exact_unclipped_pair(control, output) + and not known_limited + or control.model_wobble + or not output.valid + or not output.active + or output.override + or output.fault + or output.limited + and not known_limited + or control.domain != 'normalized_torque' + or output.domain != control.domain + or output.control_ns != control.control_ns + or not consumed_sources_current(control, output) + or not 0 < control.source_receipt_ns <= control.control_ns <= control.source_receipt_ns + 500_000_000 + or not control.control_ns <= output.apply_ns <= validation_ns <= output.apply_ns + (2_000_000_000 if historical else MAX_FRESH_NS) + or not replay_values(control.replay_inputs, 96) + or not replay_values(control.stage_values, 96) + or type(control.snapshot_sequence) is not int + or control.snapshot_sequence <= 0 + ): + raise ValueError('current genuine checkpoint/control/source pair unavailable') + _native_lease(control, output, binding, now_ns=now_ns, historical=historical) + projection = output.projection + if type(projection) is not TorqueProjection or projection_to_dict(projection) is None or not projection.steer_requested: + raise ValueError('exact native prestate required') + frame = _frame(control) + if frame.controller_domain != domain or any(frame.applied_coordinates) or any(frame.centre_values) or any(frame.actual_stage_residual or ()): + raise ValueError('legacy residuals cannot be erased by a full30 probe') + actual_full30, _ = _consumed_coordinates(control, binding) + if _primitive(actual_full30) != _primitive(accepted_full30): + raise ValueError('probe baseline differs from actual same-frame consumed full30') + snapshot, prefix_key, fresh_prefix_key = _prefix_before(replay, control, output, binding, factory, prefix_controls) + event = Emission( + output.control_ns, + output.state_ns, + output.apply_ns, + projection, + control.identity, + control.source_epoch, + control.baseline_revision, + output.valid, + output.active, + output.override, + output.fault, + output.limited, + output.native_allowed, + control.native_required, + control.native_observed_ns, + control.native_expires_ns, + control.native_identity, + ) + EmissionDecoder(factory.cp).check(event) + from openpilot.starpilot.flm.clipping_quality import QUALIFIED_EVENT_CLIPPING_QUALITY + + actual_delivered = EncoderBranch().step(event, control.requested, actual=True) + if actual_delivered != delivered(output): + raise ValueError('actual decoded CAN delivery differs') + # The frame retains exact original tuning and source inputs; the cloned + # controller is restored to this immutable before-state for EVERY run. + + def key(): + # effect_trace already validated above; convert its complete field graph + # explicitly because _record intentionally rejects arbitrary objects. + trace_key = _trace_key(control.effect_trace) + mode, policy = _factory_configuration(factory) + controls = tuple((field.name, _primitive(getattr(control, field.name))) for field in fields(ControlEvidence) if field.name != 'effect_trace') + return ( + controls, + trace_key, + _primitive(projection_to_dict(output.projection)), + tuple((field.name, _primitive(getattr(output, field.name))) for field in fields(AppliedEvidence) if field.name != 'projection'), + _record(source, SourceState), + factory.cp.as_builder().to_bytes(), + mode, + tuple(factory.manual_baseline), + factory.manual_source_epoch, + factory.interface_class, + factory.turn_assist, + policy, + _primitive(accepted_full30), + _primitive(current_vehicle_model_anchor), + _record(binding, LiveTrialBinding), + _certificate_key(certificate), + _witness_key(witness), + fresh_prefix_key(), + ) + + before = key() + if before[-1] != prefix_key: + raise ValueError('actual prefix changed before coordinate probing') + + def run(name, coordinate): + restore_snapshot(controller, vm, snapshot) + if capture_snapshot(controller, vm) != snapshot: + raise ValueError('probe checkpoint restore differs') + request, log = _run( + controller, + vm, + frame.inputs, + Objective('feedforward', 'low', 'steady'), + (0.0, 0.0), + frame.torque_values, + actual_stage_residual=(0.0, 0.0), + effect_trial=EffectReplaySource.from_coordinates({name: coordinate}, binding.baseline_revision, accumulated=prior), + ) + if not _finite(request) or bool(log.saturated): + raise ValueError('actual controller probe clipped or nonfinite') + return request + + baseline = run(names[0], prior[names[0]]) + if not same_final_command(baseline, control.requested): + raise ValueError('immutable actual checkpoint failed baseline replay') + base = EncoderBranch().step(event, baseline) + from openpilot.starpilot.flm.live_tracking_response import TrackingSourceContext + + source_only = type(certificate) is TrackingSourceContext + if base != actual_delivered or not source_only and not certificate.command_low <= -base <= certificate.command_high: + raise ValueError('actual baseline outside measured command support') + rows = [] + for name in names: + lower, upper = coordinate_bounds(name) + minus, plus = max(lower, prior[name] - MAX_STEP), min(upper, prior[name] + MAX_STEP) + if not minus < plus: + raise ValueError('nonempty original canonical probe interval required') + try: + low = EncoderBranch(quality_contract=QUALIFIED_EVENT_CLIPPING_QUALITY).step(event, run(name, minus)) + high = EncoderBranch(quality_contract=QUALIFIED_EVENT_CLIPPING_QUALITY).step(event, run(name, plus)) + if not source_only and not all(certificate.command_low <= -number <= certificate.command_high for number in (low, high)): + raise ValueError('native secant outside measured physical model support') + status = 'separated' if low != high else 'quantized' + except (ValueError, OverflowError): + low = high = base + status = 'native_unavailable' + rows.append(_physical_row(name, prior[name], minus, plus, base, low, high, 1.0 if source_only else certificate.slope, status)) + validate_response_witness(response_witness, binding, validation_ns) + if key() != before: + raise ValueError('actual probe source/controller/native inputs changed') + result = ProbeResult(binding, control.control_ns, output.apply_ns, profile, certificate.model_revision, tuple(rows), domain, group, None) + object.__setattr__(result, '_seal', (_OWNER, _probe_key(result))) + return validate_probe_result(result, binding) diff --git a/openpilot/starpilot/flm/live_tracking_probe_process.py b/openpilot/starpilot/flm/live_tracking_probe_process.py new file mode 100644 index 0000000000..0a012b7ff0 --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_probe_process.py @@ -0,0 +1,586 @@ +"""One bounded numerical probe child; results never carry current permission.""" + +from dataclasses import fields +import hashlib + +from openpilot.starpilot.flm.fit_process import FitProcess + +VERSION = 'live-probe-numerical-job-v1' +MAX_PACKED = 3 * 1024 * 1024 + + +def _digest(payload): + return hashlib.sha256(payload).hexdigest() + + +def _plain(value, cls): + from openpilot.starpilot.flm.live_tracking_probe import _record + + _record(value, cls) + return {field.name: getattr(value, field.name) for field in fields(cls)} + + +def _literal_control(row): + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + from openpilot.starpilot.flm.live_tracking_probe import _primitive, _trace_key + + if type(row) is not ControlEvidence: + raise ValueError('exact original control receipt required') + _trace_key(row.effect_trace) + result = {} + for field in fields(ControlEvidence): + value = getattr(row, field.name) + if field.name == 'effect_trace': + result[field.name] = _literal_trace(value) + else: + _primitive(value) + result[field.name] = value + return result + + +def _literal_trace(value): + from openpilot.starpilot.flm.live_tracking_probe import _primitive, _trace_key + + _trace_key(value) + if value is None: + return None + trace = {item.name: getattr(value, item.name) for item in fields(type(value))} + trace['envelope'] = {item.name: getattr(value.envelope, item.name) for item in fields(type(value.envelope))} + _primitive(trace) + return trace + + +def probe_replay_snapshot(replay, prefix_controls, *, endpoint_output=None): + """Copy one genuine bounded raw prefix, without blocks or recorder mutation. + + This is a restricted numerical transport, never an original/native acceptance. + The child recomputes every actual controller and native chronology check. + """ + from itertools import islice + from collections import OrderedDict, deque + from openpilot.starpilot.flm.controller_replay import ( + ControllerReplay, + SelectedControllerFactory, + Snapshot, + ReplayFrame, + pack_primitive_snapshot, + ) + from openpilot.starpilot.flm.pid_controller_replay import PidControllerFactory + from openpilot.starpilot.flm.torque_emission_replay import EmissionHistory, Emission + from openpilot.starpilot.flm.torque_adapter import Objective + from openpilot.starpilot.flm.live_tracking_probe import _primitive, _factory_configuration, _projection_literal + from openpilot.starpilot.flm.learner_factory import _frame + from openpilot.starpilot.flm.original_context_receipts import _same_original_frame + from opendbc.car.steering_projection import TorqueProjection + + if ( + type(replay) is not ControllerReplay + or type(replay.emissions) is not EmissionHistory + or type(replay.factory) not in (SelectedControllerFactory, PidControllerFactory) + or type(prefix_controls) is not tuple + or not 1 <= len(prefix_controls) <= 100 + ): + raise ValueError('genuine bounded raw prefix owner required') + + def source(): + if ( + type(replay.records) is not deque + or type(replay.checkpoints) is not deque + or type(replay.emissions.events) is not OrderedDict + or type(replay.emissions.carriers) is not OrderedDict + or type(replay.failed) is not bool + or type(replay.raw_mode) is not bool + ): + raise ValueError('exact bounded recorder containers/flags required') + controls = tuple(_literal_control(row) for row in prefix_controls) + if any(type(row['control_ns']) is not int for row in controls): + raise ValueError('exact actual prefix clocks required') + first, last = controls[0]['control_ns'], controls[-1]['control_ns'] + endpoint_apply_ns = None if endpoint_output is None else endpoint_output.apply_ns + if endpoint_output is not None and endpoint_output.control_ns != last: + raise ValueError('exact original endpoint application required') + if replay.failed or replay.raw_mode is not True or not 0 < first <= last <= first + 1_000_000_000: + raise ValueError('genuine bounded raw prefix unavailable') + candidates = [] + for row in islice(reversed(replay.checkpoints), 600): + if type(row) is not tuple or len(row) != 2 or type(row[0]) is not int or row[0] <= 0 or type(row[1]) is not Snapshot: + raise ValueError('exact genuine checkpoint row required') + if row[0] <= last: + candidates.append(row) + latest = candidates[0] if candidates else None + if latest is None or latest[0] != first: + raise ValueError("latest genuine actual prefix checkpoint required") + checkpoint = latest[1] + if type(checkpoint) is not Snapshot or _primitive(checkpoint.values) != _primitive(prefix_controls[0].replay_snapshot): + raise ValueError('genuine actual prefix checkpoint required') + selected = [] + # The endpoint can be up to2s old and its ORIGINAL checkpoint up to1s + # before it. Scan that already-retained context; the emitted job STILL + # contains at most100 original ticks and all old freshness guards remain. + for row in islice(reversed(replay.records), 301): + if type(row) is not ReplayFrame or type(row.control_ns) is not int: + raise ValueError('exact original prefix frame clock required') + if first <= row.control_ns <= last: + selected.append(row) + frames = tuple(reversed(selected)) + stored = [] + for frame in frames: + row = {field.name: _literal_trace(frame.effect_trace) if field.name == 'effect_trace' else getattr(frame, field.name) for field in fields(ReplayFrame)} + _primitive(row) + stored.append(row) + # The original comparator's unknown fallback may deepcopy. Only a fully + # closed literal frame graph may reach it, including all tuning fields. + if ( + len(frames) != len(controls) + or not frames + or frames[0].control_ns != first + or frames[-1].control_ns != last + or any(not _same_original_frame(frame, _frame(control)) for frame, control in zip(frames, prefix_controls, strict=True)) + ): + raise ValueError('genuine complete original prefix differs') + selected_events = [] + for event in islice(reversed(replay.emissions.events.values()), 301): + if type(event) is not Emission or type(event.control_ns) is not int: + raise ValueError('exact original native emission clock required') + if first <= event.control_ns <= last and (endpoint_apply_ns is None or event.apply_ns <= endpoint_apply_ns): + selected_events.append(event) + events = tuple(reversed(selected_events)) + native = [] + for event in events: + if type(event) is not Emission or type(event.witness) is not TorqueProjection: + raise ValueError('exact actual native prefix event required') + row = {field.name: _projection_literal(event.witness) if field.name == 'witness' else getattr(event, field.name) for field in fields(Emission)} + _primitive(row) + native.append(row) + if not events or events[-1].control_ns != last: + raise ValueError('actual endpoint native chronology missing') + first_frame, last_frame = events[0].witness.frame + 1, events[-1].witness.frame + 1 + selected_carriers = [] + for frame, row in islice(reversed(replay.emissions.carriers.items()), 301): + if type(frame) is not int or type(row) is not tuple or len(row) != 2 or any(type(value) is not int for value in row): + raise ValueError('exact actual native carrier row required') + if first_frame <= frame <= last_frame: + selected_carriers.append((frame, *row)) + carriers = tuple(reversed(selected_carriers)) + factory = replay.factory + mode, policy = _factory_configuration(factory) + from opendbc.car.car_helpers import interfaces + + interface = interfaces.get(str(factory.cp.carFingerprint)) + if type(factory) not in (SelectedControllerFactory, PidControllerFactory) or interface is None or factory.interface_class is not interface: + raise ValueError("exact registered prefix factory required") + constructor = (interface.__module__, interface.__qualname__, policy) + data = { + 'version': 1, + 'raw_mode': True, + 'cp_bytes': factory.cp.as_builder().to_bytes(), + 'mode': mode, + 'turn_assist': factory.turn_assist, + 'manual_baseline': factory.manual_baseline, + 'manual_source_epoch': factory.manual_source_epoch, + 'objective': _plain(replay.objective, Objective), + 'identity': replay.identity, + 'baseline_revision': replay.baseline_revision, + 'model_revision': replay.model_revision, + 'windows': ((first, last),), + 'checkpoints': ((first, checkpoint.revision, checkpoint.values),), + 'frames': tuple(stored), + 'emissions': {'events': tuple(native), 'carriers': carriers}, + } + # Includes genuine complete source receipts as well as transported raw data. + return data, _primitive((data, controls, constructor)), factory + + data, key, factory = source() + packed = pack_primitive_snapshot(data) + if len(packed) > MAX_PACKED: + raise ValueError('bounded raw probe snapshot byte limit') + _, after, current_factory = source() + if after != key or current_factory is not factory: + raise ValueError('actual raw prefix source changed while copying') + return packed + + +def make_probe_job( + *, + replay_snapshot, + control, + output, + accepted_full30, + binding, + response_witness, + source, + evidence_ns, + queued_ns, + current_vehicle_model_anchor, + group, + names, + prefix_controls, + archived_clock=False, +): + """Freeze genuine owner evidence, independently of later application authority. + + evidence_ns is the ACTUAL parent observation time, never a fabricated emission + clock. Child wall time remains real; historical leases are checked at emission. + """ + from openpilot.starpilot.flm.controller_replay import pack_primitive_snapshot + from openpilot.starpilot.flm.fullrate_collector import AppliedEvidence + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding, full_coordinates + from openpilot.starpilot.flm.live_tracking_response import witness_to_dict + from openpilot.starpilot.flm.live_tracking_probe import _primitive, _projection_literal + from openpilot.starpilot.flm.runtime_adapter_registry import SourceState + from opendbc.car.steering_projection import TorqueProjection + + if ( + type(archived_clock) is not bool + or type(evidence_ns) is not int + or evidence_ns <= 0 + or type(queued_ns) is not int + or queued_ns <= 0 + or not archived_clock + and evidence_ns > queued_ns + or type(replay_snapshot) is not bytes + or not 0 < len(replay_snapshot) <= MAX_PACKED + or type(prefix_controls) is not tuple + or not 1 <= len(prefix_controls) <= 100 + or type(output) is not AppliedEvidence + or type(output.projection) is not TorqueProjection + ): + raise ValueError('bounded genuine numerical probe evidence required') + binding.validate() + full_coordinates(accepted_full30, binding.baseline_revision) + + applied = { + field.name: _projection_literal(output.projection) if field.name == 'projection' else getattr(output, field.name) for field in fields(AppliedEvidence) + } + _primitive(applied) + payload = { + 'version': VERSION, + 'replay': replay_snapshot, + 'control': _literal_control(control), + 'output': applied, + 'accepted_full30': accepted_full30, + 'binding': _plain(binding, LiveTrialBinding), + 'witness': witness_to_dict(response_witness, now_ns=evidence_ns), + 'source': _plain(source, SourceState), + 'evidence_ns': evidence_ns, + 'queued_ns': queued_ns, + 'archived_clock': archived_clock, + 'anchor': current_vehicle_model_anchor, + 'group': group, + 'names': names, + 'prefix_controls': tuple(_literal_control(row) for row in prefix_controls), + } + _primitive(dict(payload, replay=None)) + packed = pack_primitive_snapshot(payload) + if len(packed) > MAX_PACKED: + raise ValueError('numerical probe job byte bound') + return {'version': VERSION, 'payload': packed.hex(), 'sha256': _digest(packed)} + + +def decode_probe_job(job): + from openpilot.starpilot.flm.controller_replay import unpack_primitive_snapshot + from openpilot.starpilot.flm.live_tracking_probe import _primitive + + if ( + type(job) is not dict + or set(job) != {'version', 'payload', 'sha256'} + or job['version'] != VERSION + or type(job['payload']) is not str + or not 0 < len(job['payload']) <= 2 * MAX_PACKED + or type(job['sha256']) is not str + ): + raise ValueError('restricted numerical probe envelope required') + packed = bytes.fromhex(job['payload']) + if _digest(packed) != job['sha256']: + raise ValueError('numerical job content changed') + payload = unpack_primitive_snapshot(packed) + if type(payload) is not dict or payload.get('version') != VERSION: + raise ValueError('numerical job schema differs') + batch = payload.get('batch') + if batch is not None: + if type(batch) is not tuple or not 1 <= len(batch) <= 4: + raise ValueError('bounded genuine numerical endpoint batch required') + _primitive(dict(payload, batch=None, replay=None)) + rows = batch + else: + rows = (payload,) + for row in rows: + if ( + type(row) is not dict + or row.get('version') != VERSION + or 'batch' in row + or type(row.get('replay')) is not bytes + or not 0 < len(row['replay']) <= MAX_PACKED + ): + raise ValueError('bounded literal replay bytes required') + _primitive(dict(row, replay=None)) + return payload + + +def make_probe_batch(jobs, *, queued_ns): + """One child, up to four REAL independently anchored original endpoints. + + Each original evidence clock and literal snapshot is retained. queued_ns is + only the actual batch lifecycle clock; no stale evidence becomes a grant. + """ + from openpilot.starpilot.flm.controller_replay import pack_primitive_snapshot + from openpilot.starpilot.flm.live_tracking_probe import _primitive + + if type(jobs) is not tuple or not 1 <= len(jobs) <= 4 or type(queued_ns) is not int or queued_ns <= 0: + raise ValueError('bounded actual probe batch required') + rows = tuple(decode_probe_job(job) for job in jobs) + if any('batch' in row or type(row.get('prefix_controls')) is not tuple or not row['prefix_controls'] for row in rows): + raise ValueError("single genuine endpoint inputs required") + common = ('binding', 'accepted_full30', 'anchor', 'group', 'names', 'archived_clock') + first = rows[0] + previous = previous_apply = 0 + from openpilot.starpilot.flm.controller_replay import MAX_FRAME_GAP_NS + + for row in rows: + if ( + 'batch' in row + or any(_primitive(row[name]) != _primitive(first[name]) for name in common) + or type(row['control']['control_ns']) is not int + or not previous < row['control']['control_ns'] + or not 0 < row['queued_ns'] <= queued_ns <= row['queued_ns'] + 2_000_000_000 + ): + raise ValueError('genuine same-checkpoint/profile/group probe batch required') + prefix = row['prefix_controls'] + if any(type(item) is not dict for item in prefix) or type(row.get('control')) is not dict or type(row.get('output')) is not dict: + raise ValueError('exact original endpoint dictionaries required') + stamps = tuple(item.get('control_ns') for item in prefix) + apply_ns = row.get('output', {}).get('apply_ns') + if ( + not 1 <= len(prefix) <= 100 + or any(type(stamp) is not int for stamp in stamps) + or not 0 < stamps[0] <= stamps[-1] <= stamps[0] + 1_000_000_000 + or any(not 0 < right - left <= MAX_FRAME_GAP_NS for left, right in zip(stamps, stamps[1:], strict=False)) + or _primitive(prefix[-1]) != _primitive(row['control']) + or not prefix[0].get('replay_snapshot') + or any(item.get('replay_snapshot') for item in prefix[1:]) + or type(apply_ns) is not int + or not previous_apply < apply_ns + ): + raise ValueError('independent complete original prefix or application order differs') + previous = row['control']['control_ns'] + previous_apply = apply_ns + payload = {'version': VERSION, 'queued_ns': queued_ns, 'batch': rows} + packed = pack_primitive_snapshot(payload) + if len(packed) > MAX_PACKED: + raise ValueError('numerical batch byte bound') + return {'version': VERSION, 'payload': packed.hex(), 'sha256': _digest(packed)} + + +def _prepared_context(data): + """Small original request identity, independently recomputed by the worker.""" + rows = data.get('batch', (data,)) + return { + 'batch': tuple( + { + 'binding': row['binding'], + 'control': {key: row['control'][key] for key in ('control_ns', 'stage_schema_revision')}, + 'output': {'apply_ns': row['output']['apply_ns']}, + 'accepted_full30': row['accepted_full30'], + 'group': row['group'], + 'names': row['names'], + 'source': row['source'], + 'anchor': row['anchor'], + 'evidence_ns': row['evidence_ns'], + 'queued_ns': row['queued_ns'], + 'prefix_first_ns': row['prefix_controls'][0]['control_ns'], + 'prefix_last_ns': row['prefix_controls'][-1]['control_ns'], + } + for row in rows + ) + } + + +def _context_digest(context): + import json + + raw = json.dumps(context, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + if not 0 < len(raw) <= 65536: + raise ValueError('bounded original numerical request context required') + return _digest(raw) + + +def original_probe_context(selected, accepted, binding, source, *, group, names, now_ns): + from dataclasses import asdict + + rows = tuple( + { + 'binding': asdict(binding), + 'control': {'control_ns': control.control_ns, 'stage_schema_revision': control.stage_schema_revision}, + 'output': {'apply_ns': output.apply_ns}, + 'accepted_full30': accepted, + 'group': group, + 'names': names, + 'source': asdict(source), + 'anchor': control.vehicle_model_anchor, + 'evidence_ns': now_ns, + 'queued_ns': now_ns, + 'prefix_controls': ({'control_ns': prefix[0].control_ns}, {'control_ns': prefix[-1].control_ns}), + } + for control, output, prefix in selected + ) + context = _prepared_context({'batch': rows}) + _context_digest(context) + return context + + +def prepare_probe_wire(jobs, *, queued_ns, expected): + """Called in the preparation CHILD; never walk large replay graphs in parent.""" + import json + + job = make_probe_batch(tuple(jobs), queued_ns=queued_ns) + context = _prepared_context(decode_probe_job(job)) + digest = _context_digest(context) + if digest != _context_digest(expected): + raise ValueError('prepared numerical input differs from original request') + wire = json.dumps(dict(job, preparedContextSha256=digest), separators=(',', ':'), allow_nan=False).encode() + from openpilot.starpilot.flm.fit_process import MAX_INPUT + + if not 0 < len(wire) <= MAX_INPUT: + raise ValueError('prepared numerical IPC byte bound') + return {'wire': wire.decode('ascii'), 'wire_sha256': _digest(wire), 'job_sha256': job['sha256'], 'context_sha256': digest} + + +class ProbeProcess(FitProcess): + worker_module = 'openpilot.starpilot.flm.live_tracking_probe_worker' + wall_ns = 30_000_000_000 + interval_ns = 0 + + def __init__(self, *, archived_diagnostic=False): + if type(archived_diagnostic) is not bool: + raise ValueError("explicit diagnostic clock mode required") + super().__init__() + self.expected = None + self.prepared_context_sha = None + self.archived_diagnostic = archived_diagnostic + + def submit_probe(self, job, *, token, now_ns): + if self.child is not None: + return False + payload = decode_probe_job(job) + if not 0 < payload['queued_ns'] <= now_ns <= payload['queued_ns'] + 100_000_000: + return False + from openpilot.starpilot.flm.live_tracking_probe import _primitive, _evidence_clock + + for row in payload.get('batch', (payload,)): + archived = row.get('archived_clock', False) + if archived and not self.archived_diagnostic: + return False + _evidence_clock(evidence_ns=row['evidence_ns'], queued_ns=row['queued_ns'], now_ns=now_ns, current_native=False, archived_clock=archived) + if not super().submit(job, token=(_primitive(token), self.archived_diagnostic, job['sha256']), now_ns=now_ns): + return False + # Independent immutable exact content; callers may mutate their original job. + self.expected = (job['sha256'], payload) + self.prepared_context_sha = None + return True + + def submit_prepared_probe(self, prepared, *, expected, token, now_ns): + # Only immutable bytes and a small parent-owned original identity cross + # here. The worker recomputes this identity from its fully decoded job. + if self.child is not None: + return False + from openpilot.starpilot.flm.live_tracking_probe import _primitive + + if type(prepared) is not dict or set(prepared) != {'wire', 'wire_sha256', 'job_sha256', 'context_sha256'}: + raise ValueError('prepared numerical envelope differs') + from openpilot.starpilot.flm.fit_process import MAX_INPUT + + wire = prepared['wire'] + if type(wire) is str: + if not 0 < len(wire) <= MAX_INPUT: + raise ValueError('prepared numerical wire bound') + try: + wire = wire.encode('ascii') + except UnicodeEncodeError as error: + raise ValueError('prepared numerical wire is not ASCII') from error + if ( + type(wire) is not bytes + or not 0 < len(wire) <= MAX_INPUT + or _digest(wire) != prepared['wire_sha256'] + or prepared['context_sha256'] != _context_digest(expected) + or type(prepared['job_sha256']) is not str + or len(prepared['job_sha256']) != 64 + or any(char not in '0123456789abcdef' for char in prepared['job_sha256']) + ): + raise ValueError('prepared numerical wire or original context differs') + rows = expected['batch'] + if not 1 <= len(rows) <= 4 or any(not 0 < row['queued_ns'] <= now_ns <= row['queued_ns'] + 2_000_000_000 for row in rows): + return False + from copy import deepcopy + + owned_context = deepcopy(expected) + digest = prepared['job_sha256'] + if not super().submit_bytes(wire, token=(_primitive(token), self.archived_diagnostic, digest), now_ns=now_ns): + return False + self.expected = digest, owned_context + self.prepared_context_sha = prepared['context_sha256'] + return True + + def poll_probe(self, *, token, binding, now_ns): + expected = self.expected + prepared_sha = self.prepared_context_sha + from openpilot.starpilot.flm.live_tracking_probe import _primitive + + digest = expected[0] if expected is not None else None + report = super().poll(token=(_primitive(token), self.archived_diagnostic, digest), now_ns=now_ns) + if self.child is None: + self.expected = None + self.prepared_context_sha = None + if report is None or expected is None: + return None + return self.decode_report(report, expected=expected, prepared_sha=prepared_sha, binding=binding) + + @staticmethod + def decode_report(report, *, expected, prepared_sha, binding): + # Shared exact decoder for the existing numerical subprocess and the + # already-forked snapshot child. Neither result carries live permission. + from openpilot.starpilot.flm.live_tracking_probe import ProbeResult, ProbeCoordinate, _OWNER, _probe_key, validate_probe_result + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding, full_coordinates + + digest, payload = expected + batch = payload.get('batch') + key = 'results' if batch is not None else 'rows' + if ( + type(report) is not dict + or set(report) != ({'version', 'sha256', key, 'contextSha256'} if prepared_sha else {'version', 'sha256', key}) + or report['version'] != VERSION + or report['sha256'] != digest + or prepared_sha is not None + and report.get('contextSha256') != prepared_sha + ): + raise ValueError('owned child numerical result/context differs') + + def seal(data, output): + if ( + LiveTrialBinding(**data['binding']) != binding + or type(output) is not list + or not 1 <= len(output) <= 30 + or any(type(row) is not dict or set(row) != set(ProbeCoordinate.__dataclass_fields__) for row in output) + ): + raise ValueError('owned child numerical result/context differs') + rows = tuple(ProbeCoordinate(**row) for row in output) + if tuple(row.name for row in rows) != data['names']: + raise ValueError('owned child coordinate group differs') + result = ProbeResult( + binding, + data['control']['control_ns'], + data['output']['apply_ns'], + full_coordinates(data['accepted_full30'], binding.baseline_revision), + binding.physical_model_revision, + rows, + 'torque_pid' if data['control']['stage_schema_revision'] == 'pid-selected-stage-snapshot-v2' else 'torque', + data['group'], + None, + ) + object.__setattr__(result, '_seal', (_OWNER, _probe_key(result))) + return validate_probe_result(result, binding) + + if batch is not None: + if type(report['results']) is not list or len(report['results']) != len(batch): + raise ValueError('actual numerical endpoint batch shape differs') + return tuple(seal(data, output) for data, output in zip(batch, report['results'], strict=True)) + return seal(payload, report['rows']) diff --git a/openpilot/starpilot/flm/live_tracking_probe_worker.py b/openpilot/starpilot/flm/live_tracking_probe_worker.py new file mode 100644 index 0000000000..d4dc14e63b --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_probe_worker.py @@ -0,0 +1,129 @@ +"""Optional bounded controller/native numerical work, never command authority.""" +import json +import os +import resource +import sys +import time + +MAX_INPUT = 8*1024*1024 +MAX_OUTPUT = 65536 + + +def _evaluate_data(data, *, now_ns): + from openpilot.starpilot.flm.controller_replay import factory_from_snapshot, ControllerReplay, unpack_primitive_snapshot, Snapshot, ReplayFrame + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence + from openpilot.starpilot.flm.runtime_adapter_registry import SourceState + from openpilot.starpilot.flm.live_tracking_response import witness_from_dict + from openpilot.starpilot.flm.live_tracking_probe import probe_endpoint + from openpilot.starpilot.flm.effect_trace import trace_from_dict + from openpilot.starpilot.flm.torque_emission_replay import projection + + binding = LiveTrialBinding(**data['binding']).validate() + factory, objective = factory_from_snapshot(data['replay']) + # Restore exact raw records, not the old normalized verifier which correctly + # refuses new nonzero live history. probe_endpoint performs the authoritative + # actual per-tick controller/native replay on this genuine bounded prefix. + raw = unpack_primitive_snapshot(data['replay']) + if raw.get('version') != 1 or raw.get('raw_mode') is not True or not 1 <= len(raw['frames']) <= 256: + raise ValueError('bounded literal original recorder required') + end = data['control']['control_ns'] + checkpoints = [row for row in raw['checkpoints'] if row[0] <= end] + if not checkpoints: + raise ValueError('genuine original checkpoint missing') + start, revision, values = checkpoints[-1] + replay = ControllerReplay(factory, objective, identity=raw['identity'], + baseline_revision=raw['baseline_revision'], model_revision=raw['model_revision']) + replay.checkpoint(start, Snapshot(revision, values)) + for row in raw['frames']: + if not start <= row['control_ns'] <= end: + continue + copied = dict(row) + if copied.get('effect_trace') is not None: + copied['effect_trace'] = trace_from_dict(copied['effect_trace']) + if not replay.record_actual_frame(ReplayFrame(**copied)): + raise ValueError('incomplete actual raw prefix') + events = tuple(row for row in raw['emissions']['events'] if start <= row['control_ns'] <= end) + if not events: + raise ValueError('actual native prefix missing') + first, last = events[0]['witness']['frame']+1, events[-1]['witness']['frame']+1 + carriers = tuple(row for row in raw['emissions']['carriers'] if first <= row[0] <= last) + replay.emissions.load({'events': events, 'carriers': carriers}, ((start, end),)) + + def control(row): + copied = dict(row) + if copied['effect_trace'] is not None: + copied['effect_trace'] = trace_from_dict(copied['effect_trace']) + return ControlEvidence(**copied) + + copied = dict(data['output']) + copied['projection'] = projection(copied['projection']) + witness = witness_from_dict(data['witness'], binding=binding, now_ns=data['evidence_ns']) + result = probe_endpoint(factory=factory, control=control(data['control']), output=AppliedEvidence(**copied), + accepted_full30=data['accepted_full30'], binding=binding, response_witness=witness, + source=SourceState(**data['source']), now_ns=now_ns, evidence_ns=data['evidence_ns'], queued_ns=data['queued_ns'], + current_native=False, archived_clock=data.get('archived_clock', False), + current_vehicle_model_anchor=data['anchor'], replay=replay, group=data['group'], names=data['names'], + prefix_controls=tuple(control(row) for row in data['prefix_controls'])) + return result + + +def evaluate_job(job, *, now_ns): + from dataclasses import asdict + from openpilot.starpilot.flm.live_tracking_probe_process import decode_probe_job, VERSION + from openpilot.starpilot.flm.live_tracking_probe_process import _prepared_context, _context_digest + if type(job) is not dict: + raise ValueError('exact numerical envelope required') + context_sha = job.get('preparedContextSha256') + if context_sha is not None: + if set(job) != {'version', 'payload', 'sha256', 'preparedContextSha256'}: + raise ValueError('prepared numerical envelope shape differs') + job = {key: value for key, value in job.items() if key != 'preparedContextSha256'} + data = decode_probe_job(job) + if context_sha is not None and _context_digest(_prepared_context(data)) != context_sha: + raise ValueError('actual decoded numerical job differs from parent request') + batch = data.get('batch') + if batch is not None: + rows = [[asdict(row) for row in _evaluate_data(endpoint, now_ns=now_ns).rows] for endpoint in batch] + report = {'version': VERSION, 'sha256': job['sha256'], 'results': rows} + if context_sha is not None: + report['contextSha256'] = context_sha + return report + result = _evaluate_data(data, now_ns=now_ns) + report = {'version': VERSION, 'sha256': job['sha256'], 'rows': [asdict(row) for row in result.rows]} + if context_sha is not None: + report['contextSha256'] = context_sha + return report + + +def main(): + try: + if sys.platform.startswith('linux'): + import ctypes + import signal + parent = int(os.environ['STARPILOT_FLM_PARENT_PID']) + if ctypes.CDLL(None).prctl(1, signal.SIGKILL, 0, 0, 0) != 0 or os.getppid() != parent: + raise ValueError('probe parent disappeared') + if hasattr(os, 'sched_setscheduler'): + os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0)) + os.nice(10) + for kind, ceiling in ((resource.RLIMIT_AS, 2*1024*1024*1024), (resource.RLIMIT_CPU, 20)): + soft, hard = resource.getrlimit(kind) + limit = min((ceiling, *[value for value in (soft, hard) if value >= 0])) + resource.setrlimit(kind, (limit, hard)) + raw = sys.stdin.buffer.read(MAX_INPUT+1) + if len(raw) > MAX_INPUT: + raise ValueError('probe input bound') + result = evaluate_job(json.loads(raw), now_ns=time.monotonic_ns()) + encoded = json.dumps(result, separators=(',', ':'), allow_nan=False).encode() + if len(encoded) > MAX_OUTPUT: + raise ValueError('probe output bound') + sys.stdout.buffer.write(encoded) + sys.stdout.buffer.flush() + return 0 + except (Exception, MemoryError): + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/openpilot/starpilot/flm/live_tracking_report.py b/openpilot/starpilot/flm/live_tracking_report.py new file mode 100644 index 0000000000..636ad0f0b0 --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_report.py @@ -0,0 +1,98 @@ +"""Bounded telemetry validation; no application or model admission authority.""" +import math +from typing import TypedDict, cast + + +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, coordinate_bounds + + +class CoordinateRow(TypedDict): + name: str + value: int | float + + +def _hex(value): + return type(value) is str and len(value) == 64 and all(char in '0123456789abcdef' for char in value) + + +def _number(value, maximum): + return type(value) in (int, float) and math.isfinite(value) and 0 <= value <= maximum + + +def _coordinates(rows): + if type(rows) is not list or len(rows) != 30: + raise ValueError('complete tracking coordinates') + for name, raw_row in zip(NAMES, rows, strict=True): + row = cast(CoordinateRow, raw_row) + low, high = coordinate_bounds(name) + if (type(row) is not dict or set(row) != {'name', 'value'} or row['name'] != name or + type(row['value']) not in (int, float) or not math.isfinite(row['value']) or not low <= row['value'] <= high): + raise ValueError('actual canonical tracking coordinate') + return rows + + +def tracking_objects(current, *, fresh): + result = {} + tracking = current.get('tracking') + if tracking is not None: + keys = {'version', 'status', 'reason', 'group', 'names', 'trialId', 'phase', 'requiredEpisodes', + 'beforeEpisodes', 'futureEpisodes', 'modelStatus', 'lagMs', 'baselineErrorRms', 'futureErrorRms', + 'baselineReversalHz', 'futureReversalHz', 'applicationAcknowledged', 'applicationSettledMonoTime', + 'lastCorrectionMonoTime', 'bindingCurrent'} + if (type(tracking) is not dict or set(tracking) != keys or tracking['version'] != 1 or + any(type(tracking[name]) is not str or len(tracking[name]) > 128 for name in ('status', 'reason', 'modelStatus')) or + any(tracking[name] is not None and (type(tracking[name]) is not str or len(tracking[name]) > 96) + for name in ('group', 'phase')) or + tracking['trialId'] is not None and not _hex(tracking['trialId']) or + type(tracking['names']) is not list or len(tracking['names']) > 4 or + any(type(name) is not str or name not in NAMES for name in tracking['names']) or + len(set(tracking['names'])) != len(tracking['names']) or + type(tracking['applicationAcknowledged']) is not bool or type(tracking['bindingCurrent']) is not bool or + any(type(tracking[key]) is not int or not 0 <= tracking[key] <= 24 for key in ('beforeEpisodes', 'futureEpisodes')) or + tracking['requiredEpisodes'] is not None and + (type(tracking['requiredEpisodes']) is not int or not 3 <= tracking['requiredEpisodes'] <= 24) or + any(tracking[key] is not None and (type(tracking[key]) is not int or not 0 < tracking[key] < 2**63) + for key in ('applicationSettledMonoTime', 'lastCorrectionMonoTime')) or + any(tracking[key] is not None and not _number(tracking[key], maximum) for key, maximum in + (('lagMs', 500), ('baselineErrorRms', 100), ('futureErrorRms', 100), ('baselineReversalHz', 100), ('futureReversalHz', 100)))): + raise ValueError('bounded tracking state telemetry') + result['tracking'] = dict(tracking, + bindingCurrent=tracking['bindingCurrent'] and fresh, + applicationAcknowledged=tracking['applicationAcknowledged'] and fresh) + values = current.get('trackingValues') + if values is not None: + if (type(values) is not dict or set(values) != {'version', 'accepted', 'pending', 'acceptedSha256', 'pendingSha256'} or + values['version'] != 1 or not _hex(values['acceptedSha256']) or + (values['pending'] is None) != (values['pendingSha256'] is None) or + values['pendingSha256'] is not None and not _hex(values['pendingSha256'])): + raise ValueError('bounded learned tracking values') + _coordinates(values['accepted']) + if values['pending'] is not None: + _coordinates(values['pending']) + result['trackingValues'] = values + plot = current.get('trackingPlot') + if plot is not None: + if (type(plot) is not dict or set(plot) != {'version', 'episodeId', 'profileSha256', 'lagNs', 'units', + 'sampled', 'totalEndpoints', 'points', 'bindingCurrent', 'ageSeconds'} or + plot['version'] != 'live-tracking-paired-plot-v1' or not _hex(plot['episodeId']) or not _hex(plot['profileSha256']) or + type(plot['lagNs']) is not int or not 0 <= plot['lagNs'] <= 500_000_000 or plot['units'] != 'm/s²' or + type(plot['sampled']) is not bool or type(plot['bindingCurrent']) is not bool or + type(plot['totalEndpoints']) is not int or not 4 <= plot['totalEndpoints'] <= 240 or + not _number(plot['ageSeconds'], 1e9) or type(plot['points']) is not list or not 4 <= len(plot['points']) <= 64 or + len(plot['points']) > plot['totalEndpoints']): + raise ValueError('bounded genuine tracking plot') + previous = 0 + for point in plot['points']: + if (type(point) is not dict or set(point) != {'controlMonoTime', 'applyMonoTime', 'poseMonoTime', + 'desiredForceAccel', 'measuredForceAccel', 'errorForceAccel'} or + any(type(point[key]) is not int for key in ('controlMonoTime', 'applyMonoTime', 'poseMonoTime')) or + not 0 < point['controlMonoTime'] <= point['applyMonoTime'] <= point['poseMonoTime'] < 2**63 or + not point['applyMonoTime']+plot['lagNs'] <= point['poseMonoTime'] <= point['applyMonoTime']+plot['lagNs']+30_000_000 or + point['poseMonoTime'] <= previous or + any(type(point[key]) not in (int, float) or not math.isfinite(point[key]) or abs(point[key]) > 100 + for key in ('desiredForceAccel', 'measuredForceAccel', 'errorForceAccel')) or + point['errorForceAccel'] != point['desiredForceAccel']-point['measuredForceAccel']): + raise ValueError('original tracking endpoint timestamps/force values') + previous = point['poseMonoTime'] + result['trackingPlot'] = dict(plot, bindingCurrent=plot['bindingCurrent'] and fresh) + return result diff --git a/openpilot/starpilot/flm/live_tracking_requalification.py b/openpilot/starpilot/flm/live_tracking_requalification.py new file mode 100644 index 0000000000..66d26b44e3 --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_requalification.py @@ -0,0 +1,357 @@ +"""Historical numerical response reuse; never current command permission. + +The original eight identification blocks remain provenance. A new source epoch +requires new independent physical residuals under its own actual native guards, +not eight invented/relabelled identification blocks. Failed checks restart fresh +observation; an unqualified historical model is never returned to a consumer. +""" + +from collections import deque +from dataclasses import asdict, dataclass, replace +import hashlib +import math +import struct + +VERSION = 'retained-current-residual-v1' + + +def _primitive_key(value, depth=0): + if depth > 8: + raise ValueError('bounded stable physical context') + kind = type(value) + if kind is tuple: + if len(value) > 32: + raise ValueError('bounded stable context tuple') + return ('tuple', tuple(_primitive_key(item, depth + 1) for item in value)) + if kind is str and value.isascii() and 0 < len(value) <= 192: + return ('str', value) + if kind is bool: + return ('bool', value) + if kind is int and value.bit_length() <= 64: + return ('int', value) + if kind is float and math.isfinite(value): + return ('float64', struct.pack('!d', value)) + raise ValueError('exact finite stable context primitives') + + +@dataclass(frozen=True) +class StableResponseBasis: + identity: str + cp_sha256: str + controller_revision: str + baseline_revision: str + calibration_rpy: tuple + vehicle_model: tuple + hardware: tuple + native_required: bool + + def key(self): + if ( + type(self) is not StableResponseBasis + or type(self.native_required) is not bool + or type(self.calibration_rpy) is not tuple + or len(self.calibration_rpy) != 3 + or any(type(v) is not float or not math.isfinite(v) for v in self.calibration_rpy) + or type(self.vehicle_model) is not tuple + or len(self.vehicle_model) != 7 + or any(type(v) is not float or not math.isfinite(v) or v == 0 for v in self.vehicle_model) + or type(self.hardware) is not tuple + or not self.hardware + ): + raise ValueError('complete measured stable response basis') + return _primitive_key( + ( + self.identity, + self.cp_sha256, + self.controller_revision, + self.baseline_revision, + self.calibration_rpy, + self.vehicle_model, + self.hardware, + self.native_required, + ) + ) + + +def basis_from_pose(binding, pose, *, now_ns, native_key=(), hardware_inventory=()): + """Use measured pose/calibration/VM and the actual current owner inventory. + + Required-axis identity is checked in full BEFORE omitting its ephemeral axis + session from the retained numerical basis. Ordinary hardware inventory must be + supplied by the real current observer; no serial or receiver contract is made + up here. Missing inventory prevents reuse and leaves bootstrap available. + """ + from openpilot.starpilot.flm.original_source_capture import validate_pose + from openpilot.starpilot.flm.live_tracking_response import physical_calibration_generation + from openpilot.starpilot.flm.runtime_adapter_registry import native_identity_digest + + binding.validate() + validate_pose(pose, now_ns) + if ( + len(pose.context) != 15 + or (pose.identity, pose.source_epoch, pose.baseline_revision, pose.controller_revision, pose.calibration, pose.native_identity) + != ( + binding.identity, + binding.source_epoch, + binding.baseline_revision, + binding.controller_revision, + physical_calibration_generation(binding), + binding.native_identity, + ) + or pose.native_required is not binding.native_required + or pose.context[5] != binding.controller_revision + or type(pose.context[10]) is not tuple + or len(pose.context[10]) != 3 + ): + raise ValueError('current original physical context differs') + if binding.native_required: + if type(native_key) is not tuple or len(native_key) != 5 or native_identity_digest(native_key) != binding.native_identity: + raise ValueError('actual current complete native inventory required') + hardware = ('axis-hardware-v1', *native_key[1:]) + else: + if native_key != () or type(hardware_inventory) is not tuple or not hardware_inventory: + raise ValueError('actual ordinary hardware inventory required') + config = pose.ordinary_safety[4] + if type(config) is not tuple or len(config) != 3: + raise ValueError('ordinary current configuration inventory') + # Preserve config and inventory. controlsAllowed/RX/heartbeat/fault values + # are CURRENT guards in validate_pose/current observer, not historical keys. + hardware = ('ordinary-hardware-v1', hardware_inventory, config[0], config[1], tuple(tuple(row[:4]) for row in config[2])) + value = StableResponseBasis( + binding.identity, + binding.cp_sha256, + binding.controller_revision, + binding.baseline_revision, + pose.calibration_rpy, + (*pose.context[10], *pose.context[11:]), + hardware, + binding.native_required, + ) + value.key() + return value + + +def compatible_basis(historical, current): + from openpilot.starpilot.flm.runtime_adapter_registry import vm_regime + + historical.key() + current.key() + normalized = vm_regime(dict(zip(("vm.sR", "vm.cF", "vm.cR"), current.vehicle_model[:3], strict=True)), historical.vehicle_model[:3]) + if normalized is None: + return False + # selected_identity includes source_epoch. Compare stable physical context, + # while each pose still must match its OWN current canonical binding above. + return historical.key() == replace(current, identity=historical.identity, vehicle_model=(*normalized, *current.vehicle_model[3:])).key() + + +def _rebound(certificate, binding, evidence_digest, *, command_range=None): + from openpilot.starpilot.flm.live_tracking_response import _revision, validate_certificate, TrackingSourceContext + + value = replace( + certificate, + binding=replace(binding, physical_model_revision='0' * 64), + model_revision='0' * 64, + provenance=('retained-current-source-v3' if type(certificate) is TrackingSourceContext else VERSION), + parent_model_revision=certificate.model_revision, + requalification_sha256=evidence_digest, + ) + if command_range is not None: + low, high = command_range + if not certificate.command_low <= low < high <= certificate.command_high: + raise ValueError('local response range cannot extrapolate original measurements') + value = replace(value, command_low=low, command_high=high) + revision = _revision(value) + return validate_certificate(replace(value, model_revision=revision, binding=replace(value.binding, physical_model_revision=revision))) + + +class RetainedResponseRequalifier: + """Fresh independent residual/range witness before exposing a rebound model. + + This object contains no Profile, native lease or actuation admission. It may + run across a new UUID/epoch only when measured stable basis remains identical. + It never reuses the previous monitor's rows, permission or pending trial. + """ + + def __init__(self, certificate, historical_basis, current_binding): + from openpilot.starpilot.flm.live_tracking_response import validate_certificate, ResponseMonitor + + self.original = validate_certificate(certificate) + if type(historical_basis) is not StableResponseBasis: + raise ValueError('exact retained measured basis') + self.basis_key = historical_basis.key() + self._historical_basis = historical_basis + if ( + certificate.binding.identity, + certificate.binding.cp_sha256, + certificate.binding.controller_revision, + certificate.binding.baseline_revision, + certificate.binding.native_required, + ) != ( + historical_basis.identity, + historical_basis.cp_sha256, + historical_basis.controller_revision, + historical_basis.baseline_revision, + historical_basis.native_required, + ): + raise ValueError('retained certificate/basis mismatch') + current_binding.validate() + if (current_binding.cp_sha256, current_binding.controller_revision, current_binding.baseline_revision, current_binding.native_required) != ( + historical_basis.cp_sha256, + historical_basis.controller_revision, + historical_basis.baseline_revision, + historical_basis.native_required, + ): + raise ValueError('car/controller/tune/native domain changed') + # Private provisional certificate is NEVER a returned/reusable result. + self._candidate = _rebound(self.original, current_binding, '0' * 64) + self._monitor = ResponseMonitor(self._candidate) + self._binding = self._candidate.binding + self._rows = deque(maxlen=80) + self._frozen_certificate = None + self.reason = 'fresh_independent_requalification_required' + + def observe(self, control, output, pose, *, now_ns, current_native, native_key=(), hardware_inventory=()): + from openpilot.starpilot.flm.torque_emission_replay import delivered + + try: + if current_native is not True: + raise ValueError('current native permission unavailable') + if self._historical_basis.key() != self.basis_key: + raise ValueError("retained stable basis mutated") + current = basis_from_pose(self._binding, pose, now_ns=now_ns, native_key=native_key, hardware_inventory=hardware_inventory) + if not compatible_basis(self._historical_basis, current): + raise ValueError('measured_calibration_vm_or_hardware_changed') + if control.controller_revision != self._binding.controller_revision or control.generation != pose.context[2]: + raise ValueError("current controller/generation differs from measured pose") + accepted = self._monitor.observe(control, output, pose, now_ns=now_ns, binding=self._binding, current_native=current_native) + if not accepted: + if self._monitor.reason != 'qualified': + self._rows.clear() + self.reason = self._monitor.reason + return False + self._rows.append((pose.mono_ns, control.control_ns, output.apply_ns, float(-delivered(output)), float(pose.speed_mps))) + valid_stamps = {row[0] for row in self._monitor.rows} + self._rows = deque((row for row in self._rows if row[0] in valid_stamps), maxlen=80) + self.reason = 'fresh_independent_requalification_required' + return True + except (AttributeError, KeyError, TypeError, ValueError, OverflowError) as error: + self._monitor.reset(str(error)) + self._rows.clear() + self.reason = str(error) + return False + + def interrupt(self, reason): + """Withdraw fresh samples only; preserve the historical numerical record.""" + self._monitor.reset(reason) + self._rows.clear() + self.reason = reason + + def persistent_residual_failure(self, now_ns): + """Fresh qualified numerical failure, never a current permission verdict.""" + from openpilot.starpilot.flm.live_tracking_response import validate_certificate + + validate_certificate(self.original) + if self._historical_basis.key() != self.basis_key: + raise ValueError("retained stable basis mutated") + return self._monitor.persistent_residual_failure(now_ns) + + def qualified(self, now_ns): + """Return numerical certificate + freshly validated physical witness only.""" + from openpilot.starpilot.flm.live_tracking_response import ResponseWitness, validate_response_witness, validate_certificate, TrackingSourceContext + + validate_certificate(self.original) + if self._historical_basis.key() != self.basis_key: + raise ValueError("retained stable basis mutated") + witness = self._monitor.witness(now_ns) + if witness is None or len(self._rows) < 8: + return None + speeds = [row[4] for row in self._rows] + if min(speeds) <= 0 or max(speeds) > 1.15 * min(speeds): + self.reason = 'fresh_comparable_speed_range_required' + return None + if self._frozen_certificate is not None: + # Ordinary poses cannot silently extend a published numerical scope. + return self._frozen_certificate, witness + commands = [row[3] for row in self._rows] + # Retention is not slope identification. Restrict support to the exact + # distinct delivered codec coordinates seen in this fresh residual window. + # The independent original identification and its thresholds are unchanged. + source_only = type(self.original) is TrackingSourceContext + if not source_only and min(commands) == max(commands): + self.reason = 'fresh_local_range_required' + return None + # Do not freeze support on the leading edge of a startup transient. A + # monotone run's next genuine sample can already lie outside its min/max, + # starving the separate fresh ingress before it can publish any restore. + # Require the ORIGINAL delivered input to revisit its measured range from + # both directions through a complete excursion (two reversals). A single + # startup peak is not a revisited local operating range. This is input-only readiness, not a residual-selected + # window, and neither expands the original support nor renews permission. + changes = [right - left for left, right in zip(commands, commands[1:], strict=False) if right != left] + signs = [value > 0 for value in changes] + reversals = sum(left != right for left, right in zip(signs, signs[1:], strict=False)) + if not source_only and reversals < 2: + self.reason = 'fresh_local_range_revisit_required' + return None + digest = hashlib.sha256(repr((VERSION, self.original.model_revision, self.basis_key, tuple(self._rows), witness.residual_rmse)).encode('ascii')).hexdigest() + cert = _rebound(self.original, self._binding, digest, command_range=None if source_only else (min(commands), max(commands))) + fresh = ResponseWitness( + cert, + cert.binding, + witness.last_control_ns, + witness.last_apply_ns, + witness.last_pose_ns, + witness.window_start_ns, + witness.independent_poses, + witness.residual_rmse, + witness.valid_until_ns, + ) + validate_response_witness(fresh, cert.binding, now_ns) + self._frozen_certificate = cert + self._binding = cert.binding + # Existing rows are real observations inside this exact min/max; model + # coefficients and lag did not change. No new pose or permission is made. + self._monitor.certificate = cert + self.reason = 'retained_response_freshly_requalified' + return cert, fresh + + +def retained_to_dict(certificate, basis): + from openpilot.starpilot.flm.live_tracking_response import certificate_to_dict + + # Constructor validates matching historical context, without claiming freshness. + RetainedResponseRequalifier(certificate, basis, certificate.binding) + return {"version": VERSION, "certificate": certificate_to_dict(certificate), "basis": asdict(basis), "current_permission": False, "pending_trial": None} + + +def retained_from_dict(value): + from openpilot.starpilot.flm.live_tracking_response import certificate_from_dict + + if ( + type(value) is not dict + or set(value) != {"version", "certificate", "basis", "current_permission", "pending_trial"} + or value["version"] != VERSION + or value["current_permission"] is not False + or value["pending_trial"] is not None + or type(value["basis"]) is not dict + or set(value["basis"]) != set(StableResponseBasis.__dataclass_fields__) + ): + raise ValueError("historical numerical model only; no permission or pending trial") + + def freeze(item, depth=0): + if depth > 8: + raise ValueError("bounded historical numerical basis") + if type(item) in (list, tuple): + if len(item) > 32: + raise ValueError("bounded historical numerical inventory") + return tuple(freeze(row, depth + 1) for row in item) + _primitive_key(item) + return item + + raw = dict(value["basis"]) + for key in ("calibration_rpy", "vehicle_model", "hardware"): + raw[key] = freeze(raw[key]) + basis = StableResponseBasis(**raw) + certificate = certificate_from_dict(value["certificate"]) + RetainedResponseRequalifier(certificate, basis, certificate.binding) + return certificate, basis diff --git a/openpilot/starpilot/flm/live_tracking_response.py b/openpilot/starpilot/flm/live_tracking_response.py new file mode 100644 index 0000000000..482dc3e6af --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_response.py @@ -0,0 +1,779 @@ +"""Retained numerical response identification plus fresh real-pose residuals. + +A certificate is never a current/native permission. No desired-model trajectory +is used as measured vehicle response. This code belongs to the optional learner. +""" + +from collections import deque +from dataclasses import asdict, dataclass, replace +import hashlib +import struct +from types import GetSetDescriptorType +import math +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + +VERSION = 'live-tracking-response-v1' +SOURCE_VERSION = 'live-tracking-source-context-v3' +MAX_FRESH_NS = 100_000_000 +WINDOW_NS = 2_000_000_000 + + +def _hex(value): + return type(value) is str and len(value) == 64 and all(c in '0123456789abcdef' for c in value) + + +@dataclass(frozen=True) +class ResponseCertificate: + version: str + binding: LiveTrialBinding + model_revision: str + lag_ns: int + slope: float + intercept: float + command_low: float + command_high: float + heldout_rmse: float + train_block_ids: tuple[str, ...] + heldout_block_ids: tuple[str, ...] + source_inventory_sha256: str + identification_sha256: str + identified_ns: int + provenance: str = "source-calibration-eight-v1" + parent_model_revision: str = "" + requalification_sha256: str = "" + + +@dataclass(frozen=True) +class TrackingSourceContext: + """Current controller/source identity, NOT a vehicle response model. + + lag_ns is the delay consumed by the selected controller, not an identified + plant delay. No slope, residual accuracy or training/heldout blocks exist. + This context never grants native permission or validates a knob correction. + """ + + version: str + binding: LiveTrialBinding + model_revision: str # Legacy binding slot; v3 contains a SOURCE context hash. + lag_ns: int + source_inventory_sha256: str + observed_ns: int + provenance: str = 'current-tracking-source-v3' + parent_model_revision: str = '' + requalification_sha256: str = '' + + +_SOURCE_FIELDS = ( + 'version', + 'binding', + 'model_revision', + 'lag_ns', + 'source_inventory_sha256', + 'observed_ns', + 'provenance', + 'parent_model_revision', + 'requalification_sha256', +) + + +def source_context(binding, control, pose, *, now_ns): + """Called with the actual collector's current pair and original raw pose. + + Fresh independent pose/native/source checks remain in ResponseMonitor; the + optimizer receives no offer until the full current observation window passes. + """ + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose, validate_pose + from openpilot.starpilot.flm.runtime_adapter_registry import digest + + if type(control) is not ControlEvidence or type(pose) is not OriginalSourcePose: + raise ValueError('actual current control and original pose required') + binding.validate() + validate_pose(pose, now_ns) + if ( + not 0 < control.state_ns <= control.control_ns <= now_ns + or control.native_required is not binding.native_required + or (control.identity, control.source_epoch, control.baseline_revision, control.controller_revision, control.native_identity) + != (binding.identity, binding.source_epoch, binding.baseline_revision, binding.controller_revision, binding.native_identity) + or (pose.identity, pose.source_epoch, pose.baseline_revision, pose.controller_revision, pose.native_identity) + != (binding.identity, binding.source_epoch, binding.baseline_revision, binding.controller_revision, binding.native_identity) + ): + raise ValueError('current source context identity differs') + values = dict(control.replay_inputs) + delay = values['lat_delay'] + if type(delay) not in (int, float) or not math.isfinite(delay) or not 0 <= delay <= 0.5: + raise ValueError('actual consumed controller delay required') + inventory = digest( + ('current-tracking-source-v3', tuple(vars(binding).items()), control.control_ns, control.state_ns, control.source_receipt_ns, pose.mono_ns, pose.context) + ) + value = TrackingSourceContext(SOURCE_VERSION, binding, '0' * 64, round(delay * 1e9), inventory, now_ns) + revision = _revision(value) + return validate_certificate(replace(value, model_revision=revision, binding=replace(binding, physical_model_revision=revision))) + + +def _source_key(context): + owned = _owned(context, TrackingSourceContext, _SOURCE_FIELDS) + if ( + any( + type(owned[name]) is not str + for name in ('version', 'model_revision', 'source_inventory_sha256', 'provenance', 'parent_model_revision', 'requalification_sha256') + ) + or type(owned['lag_ns']) is not int + or type(owned['observed_ns']) is not int + ): + raise ValueError('exact source context scalar types') + return ( + owned['version'], + _binding_values(owned['binding']), + owned['model_revision'], + owned['lag_ns'], + owned['source_inventory_sha256'], + owned['observed_ns'], + owned['provenance'], + owned['parent_model_revision'], + owned['requalification_sha256'], + ) + + +def validate_source_context(context): + _source_key(context) + context.binding.validate() + if ( + context.version != SOURCE_VERSION + or not 0 <= context.lag_ns <= 500_000_000 + or context.observed_ns <= 0 + or not all(_hex(value) for value in (context.model_revision, context.source_inventory_sha256)) + or context.binding.physical_model_revision != context.model_revision + or _revision(context) != context.model_revision + ): + raise ValueError('current source context identity required') + if context.provenance == 'current-tracking-source-v3': + if context.parent_model_revision or context.requalification_sha256: + raise ValueError('initial source context cannot claim restoration') + elif context.provenance == 'retained-current-source-v3': + if not _hex(context.parent_model_revision) or not _hex(context.requalification_sha256): + raise ValueError('retained source context needs fresh observations') + else: + raise ValueError('unsupported source context provenance') + return context + + +_CERT_FIELDS = ( + 'version', + 'binding', + 'model_revision', + 'lag_ns', + 'slope', + 'intercept', + 'command_low', + 'command_high', + 'heldout_rmse', + 'train_block_ids', + 'heldout_block_ids', + 'source_inventory_sha256', + 'identification_sha256', + 'identified_ns', + 'provenance', + 'parent_model_revision', + 'requalification_sha256', +) +_BIND_FIELDS = ( + 'identity', + 'cp_sha256', + 'controller_revision', + 'baseline_revision', + 'source_epoch', + 'reset_generation', + 'calibration_generation', + 'vehicle_model_generation', + 'native_identity', + 'physical_model_revision', + 'persistent_source_epoch', + 'native_required', +) +_WITNESS_FIELDS = ( + 'certificate', + 'binding', + 'last_control_ns', + 'last_apply_ns', + 'last_pose_ns', + 'window_start_ns', + 'independent_poses', + 'residual_rmse', + 'valid_until_ns', +) +_VALIDATED = object() + + +def physical_calibration_generation(binding): + from openpilot.starpilot.flm.runtime_adapter_registry import digest + + token = binding.calibration_generation + if type(token) is not str or len(token) != 32 or any(char not in '0123456789abcdef' for char in token): + raise ValueError('actual raw calibrated vehicle-frame lifecycle token required') + return digest(('live_vehicle_frame_v2', token)) + + +def _owned(value, cls, names): + if type(value) is not cls or type(cls) is not type or cls.__bases__ != (object,) or cls.__getattribute__ is not object.__getattribute__: + raise ValueError('exact plain physical witness class') + namespace = vars(cls) + if ( + '__getattr__' in namespace + or type(namespace.get('__dict__')) is not GetSetDescriptorType + or any(name in namespace and type(namespace[name]) not in (type(None), bool, int, float, str, tuple) for name in names) + ): + raise ValueError('changed physical witness field accessor') + owned = object.__getattribute__(value, '__dict__') + if type(owned) is not dict or any(type(key) is not str for key in owned) or any(name not in owned for name in names): + raise ValueError('exact physical witness field dictionary') + return owned + + +def _binding_values(binding): + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + + owned = _owned(binding, LiveTrialBinding, _BIND_FIELDS) + values = tuple(owned[name] for name in _BIND_FIELDS) + if any(type(value) is not str or not value.isascii() or len(value) > 192 for value in values[:-1]) or type(values[-1]) is not bool: + raise ValueError('immutable primitive physical binding') + return values + + +def _certificate_key(certificate): + if type(certificate) is TrackingSourceContext: + return _source_key(certificate) + owned = _owned(certificate, ResponseCertificate, _CERT_FIELDS) + binding = _binding_values(owned['binding']) + if any( + type(owned[name]) is not str + for name in ( + 'version', + 'model_revision', + 'source_inventory_sha256', + 'identification_sha256', + 'provenance', + 'parent_model_revision', + 'requalification_sha256', + ) + ) or any(type(owned[name]) is not int for name in ('lag_ns', 'identified_ns')): + raise ValueError('physical certificate scalar types') + floats = tuple(owned[name] for name in ('slope', 'intercept', 'command_low', 'command_high', 'heldout_rmse')) + if any(type(value) not in (int, float) or not math.isfinite(value) for value in floats): + raise ValueError('finite physical certificate values') + ids = tuple(owned[name] for name in ('train_block_ids', 'heldout_block_ids')) + if any(type(rows) is not tuple or len(rows) != 4 or any(type(v) is not str or len(v) > 128 for v in rows) for rows in ids): + raise ValueError('bounded independent identification blocks') + return ( + owned['version'], + binding, + owned['model_revision'], + owned['lag_ns'], + struct.pack('!5d', *floats), + *ids, + owned['source_inventory_sha256'], + owned['identification_sha256'], + owned['identified_ns'], + owned['provenance'], + owned['parent_model_revision'], + owned['requalification_sha256'], + ) + + +def _revision(certificate): + key = _certificate_key(certificate) + binding = key[1][:9] + key[1][10:] # Own revision cannot recursively bind itself. + return hashlib.sha256(repr((key[0], binding, *key[3:])).encode('utf-8', 'surrogatepass')).hexdigest() + + +def validate_certificate(certificate): + if type(certificate) is TrackingSourceContext: + return validate_source_context(certificate) + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + + if ( + type(certificate) is not ResponseCertificate + or type(certificate.binding) is not LiveTrialBinding + or certificate.version != VERSION + or type(certificate.lag_ns) is not int + or not 0 <= certificate.lag_ns <= 500_000_000 + or any( + type(v) not in (int, float) or not math.isfinite(v) + for v in (certificate.slope, certificate.intercept, certificate.command_low, certificate.command_high, certificate.heldout_rmse) + ) + or certificate.slope <= 0 + or certificate.command_low >= certificate.command_high + or not 0 <= certificate.heldout_rmse <= 0.05 + or type(certificate.identified_ns) is not int + or certificate.identified_ns <= 0 + or type(certificate.train_block_ids) is not tuple + or len(certificate.train_block_ids) != 4 + or type(certificate.heldout_block_ids) is not tuple + or len(certificate.heldout_block_ids) != 4 + or any(type(v) is not str or not 1 <= len(v) <= 128 for v in certificate.train_block_ids + certificate.heldout_block_ids) + or len(set(certificate.train_block_ids + certificate.heldout_block_ids)) != 8 + or not all(_hex(v) for v in (certificate.model_revision, certificate.source_inventory_sha256, certificate.identification_sha256)) + or certificate.binding.physical_model_revision != certificate.model_revision + or _revision(certificate) != certificate.model_revision + ): + raise ValueError('source-bound independently identified response certificate required') + certificate.binding.validate() + if certificate.provenance == 'source-calibration-eight-v1': + if certificate.parent_model_revision or certificate.requalification_sha256: + raise ValueError('initial model cannot claim retained requalification') + elif certificate.provenance == 'retained-current-residual-v1': + if not _hex(certificate.parent_model_revision) or not _hex(certificate.requalification_sha256): + raise ValueError('retained model needs original parent and fresh requalification') + else: + raise ValueError('unsupported response provenance') + return certificate + + +def certificate_from_prepared(prepared, binding, *, original_packet): + """Called only in the actual initial source/native/calibration worker. + + The PreparedPlant comes from existing prepare(), after original controller, + native CAN and independent train/heldout physical identification. This is a + numerical result, not a profile acceptance or transferable native verdict. + """ + from openpilot.starpilot.flm.source_plant_fit import PreparedPlant + from openpilot.starpilot.flm import plant_lag + from openpilot.starpilot.flm.physical_response import PhysicalModel + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + + if type(prepared) is not PreparedPlant or type(binding) is not LiveTrialBinding or type(prepared.model) is not PhysicalModel: + raise ValueError('actual prepared physical experiment required') + source, result, model = prepared.binding, prepared.identification, prepared.model + from openpilot.starpilot.flm.runtime_adapter_registry import digest, canonical_physical_context + from openpilot.starpilot.flm.source_plant_fit import experiment_context + + original_context = original_packet['source_poses'][0]['context'] + if ( + prepared.context != experiment_context(original_packet, source) + or len(original_context) != 15 + or original_context[5] != binding.controller_revision + or source.controller_revision != digest(binding.controller_revision) + or source.vehicle_model_generation != digest((original_context[2], original_context[10], list(original_context[11:]))) + or binding.vehicle_model_generation != digest(tuple(original_context[10])) + or source.native_required is not binding.native_required + ): + raise ValueError('actual raw controller/vehicle/source binding differs') + if ( + result.status != 'physical_affine_test_passed' + or result.decision is None + or not plant_lag.heldout_supported(result) + or (source.identity, source.cp_sha256, source.baseline_revision, source.source_epoch, source.calibration_generation, source.native_identity) + != (binding.identity, binding.cp_sha256, binding.baseline_revision, binding.source_epoch, physical_calibration_generation(binding), binding.native_identity) + or result.binding_digest != plant_lag.binding_digest(source) + or result.decision.protocol != plant_lag.PHYSICAL_PROTOCOL + or model.identity != binding.identity + or model.context != canonical_physical_context(prepared.context) + or model.command_low != result.decision.command_low + or model.command_high != result.decision.command_high + or model.heldout_rmse != max(error for _, error, _ in result.heldout_blocks) + or model.lag_ns != result.decision.lag_ns + or model.slope != result.decision.slope + or model.intercept != result.decision.intercept + or model.train_blocks != result.decision.train_block_ids + or model.heldout_blocks != tuple(block for block, _, _ in result.heldout_blocks) + ): + raise ValueError('original independent physical identification differs') + value = ResponseCertificate( + VERSION, + binding, + '0' * 64, + model.lag_ns, + model.slope, + model.intercept, + model.command_low, + model.command_high, + model.heldout_rmse, + model.train_blocks, + model.heldout_blocks, + prepared.context[2], + plant_lag.digest(asdict(result)), + max(row.mono_ns for row in prepared.source_poses), + ) + revision = _revision(value) + return validate_certificate(replace(value, model_revision=revision, binding=replace(binding, physical_model_revision=revision))) + + +@dataclass(frozen=True) +class ResponseWitness: + certificate: ResponseCertificate | TrackingSourceContext + binding: LiveTrialBinding + last_control_ns: int + last_apply_ns: int + last_pose_ns: int + window_start_ns: int + independent_poses: int + residual_rmse: float | None + valid_until_ns: int + + +def _validate_raw_witness(witness, binding, now_ns, *, content=True): + if type(witness) is not ResponseWitness: + raise ValueError('fresh source physical residual witness required') + cert = validate_certificate(witness.certificate) if content else witness.certificate + source_only = type(cert) is TrackingSourceContext + if ( + _binding_values(witness.binding) != _binding_values(binding) + or _binding_values(cert.binding) != _binding_values(binding) + or any( + type(v) is not int + for v in ( + witness.last_control_ns, + witness.last_apply_ns, + witness.last_pose_ns, + witness.window_start_ns, + witness.independent_poses, + witness.valid_until_ns, + now_ns, + ) + ) + or not 0 < witness.last_control_ns <= witness.last_apply_ns <= witness.last_pose_ns <= now_ns <= witness.valid_until_ns + or not witness.valid_until_ns <= witness.last_pose_ns + MAX_FRESH_NS + or not 0 <= witness.last_pose_ns - cert.lag_ns - witness.last_apply_ns <= 30_000_000 + or source_only + and cert.provenance == 'current-tracking-source-v3' + and cert.observed_ns > witness.last_pose_ns + or not 0 < witness.window_start_ns <= witness.last_pose_ns + or witness.last_pose_ns - witness.window_start_ns > WINDOW_NS + or not 8 <= witness.independent_poses <= 80 + or ( + witness.residual_rmse is not None + if source_only + else type(witness.residual_rmse) not in (int, float) + or not math.isfinite(witness.residual_rmse) + or not 0 <= witness.residual_rmse <= max(0.01, 3 * cert.heldout_rmse) + ) + ): + raise ValueError('physical response witness stale, unqualified or mismatched') + return witness + + +@dataclass(frozen=True) +class ValidatedResponseWitness: + value: ResponseWitness + certificate: ResponseCertificate | TrackingSourceContext + binding: LiveTrialBinding + witness_dictionary: object + certificate_dictionary: object + binding_dictionary: object + certificate_binding: object + certificate_binding_dictionary: object + witness_key: tuple + certificate_key: tuple + _seal: object + + +def _witness_key(witness): + owned = _owned(witness, ResponseWitness, _WITNESS_FIELDS) + source_only = type(owned['certificate']) is TrackingSourceContext + if any(type(owned[name]) is not int for name in _WITNESS_FIELDS[2:7] + ('valid_until_ns',)) or ( + owned['residual_rmse'] is not None if source_only else type(owned['residual_rmse']) not in (int, float) or not math.isfinite(owned['residual_rmse']) + ): + raise ValueError('exact immutable source witness scalars') + return ( + *_binding_values(owned['binding']), + *(owned[name] for name in _WITNESS_FIELDS[2:7]), + None if source_only else struct.pack('!d', owned['residual_rmse']), + owned['valid_until_ns'], + ) + + +def prepare_response_witness(witness, binding, now_ns): + """Content validation once on a new owned delivery, never permission cache.""" + value = _validate_raw_witness(witness, binding, now_ns) + if type(value.certificate) is TrackingSourceContext: + return value # Small source context is revalidated on EVERY consumer tick. + return ValidatedResponseWitness( + value, + value.certificate, + value.binding, + _owned(value, ResponseWitness, _WITNESS_FIELDS), + _owned(value.certificate, ResponseCertificate, _CERT_FIELDS), + _owned(value.binding, type(value.binding), _BIND_FIELDS), + value.certificate.binding, + _owned(value.certificate.binding, type(value.certificate.binding), _BIND_FIELDS), + _witness_key(value), + _certificate_key(value.certificate), + _VALIDATED, + ) + + +def response_witness_value(value): + if type(value) is ResponseWitness: + return value + if type(value) is not ValidatedResponseWitness: + raise ValueError('exact owned physical response wrapper') + _owned( + value, + ValidatedResponseWitness, + ( + 'value', + 'certificate', + 'binding', + 'witness_dictionary', + 'certificate_dictionary', + 'binding_dictionary', + 'certificate_binding', + 'certificate_binding_dictionary', + 'witness_key', + 'certificate_key', + '_seal', + ), + ) + if ( + value._seal is not _VALIDATED + or type(value.value) is not ResponseWitness + or type(value.certificate) is not ResponseCertificate + or value.value.certificate is not value.certificate + or value.value.binding is not value.binding + or value.certificate.binding is not value.certificate_binding + or _owned(value.certificate_binding, type(value.certificate_binding), _BIND_FIELDS) is not value.certificate_binding_dictionary + or _owned(value.value, ResponseWitness, _WITNESS_FIELDS) is not value.witness_dictionary + or _owned(value.certificate, ResponseCertificate, _CERT_FIELDS) is not value.certificate_dictionary + or _owned(value.binding, type(value.binding), _BIND_FIELDS) is not value.binding_dictionary + or _witness_key(value.value) != value.witness_key + or _certificate_key(value.certificate) != value.certificate_key + ): + raise ValueError('owned physical response delivery changed') + return value.value + + +def validate_response_witness(witness, binding, now_ns): + value = response_witness_value(witness) + return _validate_raw_witness(value, binding, now_ns, content=type(witness) is not ValidatedResponseWitness) + + +PERSISTENT_RESIDUAL_NS = 3_000_000_000 +PERSISTENT_RESIDUAL_POSES = 60 + + +def verified_clipped_input(replay, control, output, binding): + """Actual delivered physical input, never an unclipped command or grant. + + A measured/driver native limiter can change a known actuator input without + invalidating the physical observation. Verify its original CAN and preserve + every source flag. Numerical probing and current application remain strict. + """ + from openpilot.starpilot.flm.controller_replay import ControllerReplay + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence + from openpilot.starpilot.flm.runtime_adapter_registry import car_params_digest + from openpilot.starpilot.flm.torque_eligibility import bounded_context_output, asynchronous_hint, consumed_sources_current + + try: + if ( + type(replay) is not ControllerReplay + or type(control) is not ControlEvidence + or type(output) is not AppliedEvidence + or replay.identity != binding.identity + or replay.baseline_revision != binding.baseline_revision + or car_params_digest(replay.factory.cp) != binding.cp_sha256 + or getattr(replay, 'source_controller_revision', None) != binding.controller_revision + or control.controller_revision != binding.controller_revision + or control.identity != binding.identity + or control.source_epoch != binding.source_epoch + or control.baseline_revision != binding.baseline_revision + or control.native_identity != binding.native_identity + or control.native_required is not binding.native_required + or not control.enabled + or not control.active + or not control.valid + or control.manual + or control.override + or control.fault + or control.saturated + or control.model_wobble + or control.limited + and not asynchronous_hint(control) + or not bounded_context_output(output) + or not consumed_sources_current(control, output) + or control.native_required + and ( + not control.native_allowed + or not 0 < control.native_observed_ns <= control.control_ns <= output.apply_ns <= control.native_expires_ns <= control.native_observed_ns + 200_000_000 + ) + ): + return False + return replay.applied_matches(control, output) is True + except (AttributeError, KeyError, TypeError, ValueError, OverflowError): + return False + + +class ResponseMonitor: + def __init__(self, certificate): + self.certificate = validate_certificate(certificate) + self.rows = deque(maxlen=80) + self.reason = 'fresh_residual_samples_required' + self.residual_failure_start_ns = self.residual_failure_count = 0 + + def reset(self, reason): + self.rows.clear() + self.reason = reason + self.residual_failure_start_ns = self.residual_failure_count = 0 + + def observe(self, control, output, pose, *, now_ns, binding, current_native, replay=None): + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose, validate_pose + from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current + from openpilot.starpilot.flm.torque_eligibility import exact_unclipped_pair + from openpilot.starpilot.flm.torque_emission_replay import delivered + from openpilot.starpilot.flm.runtime_adapter_registry import digest, control_vm_regime + + c = self.certificate + try: + if ( + binding != c.binding + or current_native is not True + or type(control) is not ControlEvidence + or type(output) is not AppliedEvidence + or type(pose) is not OriginalSourcePose + ): + raise ValueError('response_current_source_or_native_unavailable') + validate_pose(pose, now_ns) + observed_clip = verified_clipped_input(replay, control, output, binding) if output.limited else False + if ( + type(control.vehicle_model_anchor) is not tuple + or len(control.vehicle_model_anchor) != 3 + or control_vm_regime(control) != control.vehicle_model_anchor + or digest(control.vehicle_model_anchor) != binding.vehicle_model_generation + or len(pose.context) != 15 + or tuple(pose.context[10]) != control.vehicle_model_anchor + or control.generation != pose.context[2] + or control.controller_revision != binding.controller_revision + or pose.controller_revision != binding.controller_revision + or control.native_required is not binding.native_required + or pose.native_required is not binding.native_required + or (control.identity, control.source_epoch, control.baseline_revision, control.native_identity) + != (binding.identity, binding.source_epoch, binding.baseline_revision, binding.native_identity) + or (pose.identity, pose.source_epoch, pose.baseline_revision, pose.calibration, pose.native_identity) + != (binding.identity, binding.source_epoch, binding.baseline_revision, physical_calibration_generation(binding), binding.native_identity) + or not control.enabled + or not control.active + or not control.valid + or control.manual + or control.override + or control.fault + or control.saturated + or (control.limited and not exact_unclipped_pair(control, output) and not observed_clip) + or control.model_wobble + or not output.active + or not output.valid + or output.override + or output.fault + or output.limited + and not observed_clip + or control.native_required + and (not control.native_allowed or not output.native_allowed) + or output.control_ns != control.control_ns + or not consumed_sources_current(control, output) + or not 0 < control.control_ns <= output.apply_ns <= pose.source_lower_ns - c.lag_ns + or not 0 <= pose.source_upper_ns - c.lag_ns - output.apply_ns <= 30_000_000 + or not 0 <= now_ns - pose.mono_ns <= MAX_FRESH_NS + ): + raise ValueError('response_original_source_pair_unqualified') + command = -delivered(output) + measured = pose.speed_mps * pose.yaw_rate - 9.81 * math.sin(pose.roll_rad) + if not math.isfinite(measured): + raise ValueError('finite measured source response required') + source_only = type(c) is TrackingSourceContext + error = None if source_only else measured - (c.slope * command + c.intercept) + if not source_only and not c.command_low <= command <= c.command_high: + raise ValueError('response_command_outside_identified_range') + # Drift is assessed on a complete rolling window, not an isolated 3-sigma + # Gaussian sample. Source/native/driver failures above still reset now. + if self.rows and pose.mono_ns <= self.rows[-1][0]: + return False # Duplicate data neither renews freshness nor erases it. + if self.rows and pose.mono_ns - self.rows[-1][0] > MAX_FRESH_NS: + self.reset('response_source_gap') + self.rows.append((pose.mono_ns, control.control_ns, output.apply_ns, error)) + while self.rows and pose.mono_ns - self.rows[0][0] > WINDOW_NS: + self.rows.popleft() + self.reason = 'qualified' if len(self.rows) >= 8 else 'fresh_residual_samples_required' + # Count ONLY new, source/native/driver/range-qualified physical poses. + # Missing evidence and interrupted data never count as model drift. + residual_bad = not source_only and len(self.rows) >= 8 and sum(row[3] ** 2 for row in self.rows) / len(self.rows) > max(0.01, 3 * c.heldout_rmse) ** 2 + if residual_bad: + if self.residual_failure_start_ns == 0: + self.residual_failure_start_ns = pose.mono_ns + self.residual_failure_count += 1 + else: + self.residual_failure_start_ns = self.residual_failure_count = 0 + return True + except (AttributeError, KeyError, TypeError, ValueError, OverflowError) as error: + self.reset(str(error)) + return False + + def persistent_residual_failure(self, now_ns): + return bool( + type(self.certificate) is not TrackingSourceContext + and self.rows + and type(now_ns) is int + and 0 <= now_ns - self.rows[-1][0] <= MAX_FRESH_NS + and self.residual_failure_start_ns > 0 + and self.rows[-1][0] - self.residual_failure_start_ns >= PERSISTENT_RESIDUAL_NS + and self.residual_failure_count >= PERSISTENT_RESIDUAL_POSES + ) + + def witness(self, now_ns): + if len(self.rows) < 8: + return None + pose, control, apply, _ = self.rows[-1] + value = ResponseWitness( + self.certificate, + self.certificate.binding, + control, + apply, + pose, + self.rows[0][0], + len(self.rows), + None if type(self.certificate) is TrackingSourceContext else math.sqrt(sum(row[3] ** 2 for row in self.rows) / len(self.rows)), + pose + MAX_FRESH_NS, + ) + try: + return validate_response_witness(value, self.certificate.binding, now_ns) + except ValueError: + return None + + +def certificate_to_dict(value): + return asdict(validate_certificate(value)) + + +def certificate_from_dict(value): + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + + if type(value) is dict and value.get('version') == SOURCE_VERSION: + if set(value) != set(TrackingSourceContext.__dataclass_fields__): + raise ValueError('exact source context history shape') + data = dict(value) + if type(data['binding']) is not dict or set(data['binding']) != set(LiveTrialBinding.__dataclass_fields__): + raise ValueError('exact source context binding history shape') + data['binding'] = LiveTrialBinding(**data['binding']) + return validate_source_context(TrackingSourceContext(**data)) + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + + if type(value) is not dict or set(value) != set(ResponseCertificate.__dataclass_fields__): + raise ValueError('response certificate fields') + copied = dict(value) + copied['binding'] = LiveTrialBinding(**copied['binding']) + copied['train_block_ids'], copied['heldout_block_ids'] = tuple(copied['train_block_ids']), tuple(copied['heldout_block_ids']) + return validate_certificate(ResponseCertificate(**copied)) + + +def witness_to_dict(value, *, now_ns): + raw = response_witness_value(value) + return asdict(validate_response_witness(value, raw.binding, now_ns)) + + +def witness_from_dict(value, *, binding, now_ns): + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + + if type(value) is not dict or set(value) != set(ResponseWitness.__dataclass_fields__): + raise ValueError('response witness fields') + copied = dict(value) + copied['certificate'] = certificate_from_dict(copied['certificate']) + copied['binding'] = LiveTrialBinding(**copied['binding']) + return validate_response_witness(ResponseWitness(**copied), binding, now_ns) diff --git a/openpilot/starpilot/flm/live_tracking_response_policy.py b/openpilot/starpilot/flm/live_tracking_response_policy.py new file mode 100644 index 0000000000..32f67d712a --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_response_policy.py @@ -0,0 +1,36 @@ +"""Measured, bounded numerical response job; never a current-control permission.""" + +PURPOSE = 'live-response-bootstrap-v1' +WORKER_MODULE = 'openpilot.starpilot.flm.live_tracking_response_worker' +MAX_CPU = 180 +MAX_WALL_NS = 240_000_000_000 +# The passing device run measured RSS, not VmPeak. Retain its proven AS bound. +MAX_AS = 2*1024*1024*1024 +MAX_INPUT = 8*1024*1024 +MAX_OUTPUT = 65536 + + +def production_result(result): + from openpilot.starpilot.flm.live_tracking_response import certificate_from_dict + if type(result) is not dict or result.get('live_job_version') != PURPOSE: + raise ValueError('exact response bootstrap protocol required') + if result.get('profile') is not None or result.get('current_permission') is not False: + raise ValueError('response calibration cannot grant profile or current permission') + if 'diagnostic_purpose' in result or 'worker_purpose' in result or 'production_admissible' in result: + raise ValueError('bootstrap math cannot supply worker admission metadata') + identified = result.get('status') == 'response_identified' + if identified: + certificate_from_dict(result['certificate']) + elif result.get('certificate') is not None: + raise ValueError('unidentified response cannot carry a certificate') + return dict(result, worker_purpose=PURPOSE, production_admissible=identified) + + +def validate_production_result(result): + if (type(result) is not dict or result.get('worker_purpose') != PURPOSE or + result.get('production_admissible') is not True or 'diagnostic_purpose' in result): + raise ValueError('exact production response worker report required') + body = {key: value for key, value in result.items() if key not in ('worker_purpose', 'production_admissible')} + if production_result(body) != result: + raise ValueError('qualified response report required') + return result diff --git a/openpilot/starpilot/flm/live_tracking_response_process.py b/openpilot/starpilot/flm/live_tracking_response_process.py new file mode 100644 index 0000000000..c8459d8a73 --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_response_process.py @@ -0,0 +1,19 @@ +"""One explicitly selected production response child with inherited real reaping.""" +from openpilot.starpilot.flm.fit_process import FitProcess +from openpilot.starpilot.flm.live_tracking_response_policy import MAX_WALL_NS, WORKER_MODULE, validate_production_result + + +class ProductionBootstrapProcess(FitProcess): + worker_module = WORKER_MODULE + wall_ns = MAX_WALL_NS + interval_ns = 0 + + def poll(self, *, token, now_ns): + result = super().poll(token=token, now_ns=now_ns) + if result is None: + return None + try: + return validate_production_result(result) + except (KeyError, TypeError, ValueError): + self.reason = 'production_response_report_unqualified' + return None diff --git a/openpilot/starpilot/flm/live_tracking_response_worker.py b/openpilot/starpilot/flm/live_tracking_response_worker.py new file mode 100644 index 0000000000..7bd07c3858 --- /dev/null +++ b/openpilot/starpilot/flm/live_tracking_response_worker.py @@ -0,0 +1,59 @@ +"""Fixed production response-calibration purpose, separate from legacy fitting.""" +import gc +import json +import os +import resource +import sys + +from openpilot.starpilot.flm.live_tracking_response_policy import MAX_AS, MAX_CPU, MAX_INPUT, MAX_OUTPUT, PURPOSE, production_result + +from openpilot.starpilot.flm.live_tracking_response_policy import MAX_WALL_NS as MAX_WALL_NS + + +def main(): + try: + if not sys.platform.startswith('linux'): + raise ValueError('qualified Linux response worker required') + import ctypes + import signal + parent = int(os.environ['STARPILOT_FLM_PARENT_PID']) + if ctypes.CDLL(None).prctl(1, signal.SIGKILL, 0, 0, 0) != 0 or os.getppid() != parent: + raise ValueError('bootstrap parent disappeared') + os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0)) + os.nice(10) + for kind, ceiling in ((resource.RLIMIT_AS, MAX_AS), (resource.RLIMIT_CPU, MAX_CPU)): + soft, hard = resource.getrlimit(kind) + limit = min((ceiling, *[value for value in (soft, hard) if value >= 0])) + resource.setrlimit(kind, (limit, hard)) + if any(os.environ.get(name) != '1' for name in ('OPENBLAS_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS', 'NUMEXPR_NUM_THREADS')): + raise ValueError('one numerical thread required') + # This exact module/purpose and actual resource guards are independently + # checked by the native helper; no inherited environment opt-in selects it. + from openpilot.starpilot.flm.native_primitives import enable_response_worker + if not enable_response_worker(PURPOSE): + raise ValueError('qualified response native runtime unavailable') + raw = sys.stdin.buffer.read(MAX_INPUT+1) + if len(raw) > MAX_INPUT: + raise ValueError('response worker input bound') + from openpilot.starpilot.flm.live_tracking_bootstrap import bootstrap_snapshot + was_enabled = gc.isenabled() + try: + gc.disable() + result = bootstrap_snapshot(json.loads(raw)) + finally: + if was_enabled: + gc.enable() + result = production_result(result) + encoded = json.dumps(result, separators=(',', ':'), allow_nan=False).encode() + if len(encoded) > MAX_OUTPUT: + raise ValueError('response worker output bound') + sys.stdout.buffer.write(encoded) + sys.stdout.buffer.flush() + return 0 + except (Exception, MemoryError) as error: + sys.stderr.write(type(error).__name__+': '+str(error)[:240]+'\n') + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/openpilot/starpilot/flm/live_trial_contract.py b/openpilot/starpilot/flm/live_trial_contract.py new file mode 100644 index 0000000000..0ed6b25965 --- /dev/null +++ b/openpilot/starpilot/flm/live_trial_contract.py @@ -0,0 +1,259 @@ +"""Explicit live tracking trial protocol, distinct from replay fit acceptance.""" + +from dataclasses import dataclass +from collections import OrderedDict +import math +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, canonical_names, coordinate_bounds, full_profile_sha256 + +VERSION = 'live-tracking-trial-v1' +MAX_STEP = 0.005 +MAX_LEASE_NS = 500_000_000 +SLEW_PER_SECOND = 0.005 +MAX_TRIAL_NS = 120_000_000_000 + + +def _text(value, maximum=96): + return type(value) is str and 0 < len(value) <= maximum and value.isascii() and all(32 <= ord(c) < 127 for c in value) + + +def _digest(value): + return type(value) is str and len(value) == 64 and all(c in '0123456789abcdef' for c in value) + + +@dataclass(frozen=True) +class LiveTrialBinding: + identity: str + cp_sha256: str + controller_revision: str + baseline_revision: str + source_epoch: str + reset_generation: str + calibration_generation: str + vehicle_model_generation: str + native_identity: str + physical_model_revision: str + persistent_source_epoch: str + native_required: bool = False + + def validate(self): + if type(self) is not LiveTrialBinding or not all(_digest(value) for value in (self.identity, self.cp_sha256, self.baseline_revision, self.source_epoch)): + raise ValueError('live tracking identity binding') + if self.persistent_source_epoch != '' and not _digest(self.persistent_source_epoch): + raise ValueError('optional persistent source identity') + if type(self.native_required) is not bool or (not _digest(self.native_identity) if self.native_required else self.native_identity != ''): + raise ValueError('live ordinary/required native domain') + if not all( + _text(value, 192) + for value in (self.controller_revision, self.reset_generation, self.calibration_generation, self.vehicle_model_generation, self.physical_model_revision) + ): + raise ValueError('live tracking source/model generation') + return self + + +@dataclass(frozen=True) +class LiveTrialOffer: + version: str + binding: LiveTrialBinding + trial_id: str + sequence: int + phase: str + group: str + names: tuple[str, ...] + baseline: tuple[tuple[str, float], ...] + target: tuple[tuple[str, float], ...] + baseline_sha256: str + target_sha256: str + started_ns: int + observed_ns: int + expires_ns: int + response_witness: object = None + target_sequence: int = 1 + deadline_ns: int = 0 + + def validate(self): + return validate_offer(self) + + +def full_coordinates(rows, baseline_revision): + if ( + type(rows) is not tuple + or len(rows) != len(NAMES) + or any(type(row) is not tuple or len(row) != 2 or type(row[0]) is not str or type(row[1]) not in (int, float) or not math.isfinite(row[1]) for row in rows) + or tuple(row[0] for row in rows) != NAMES + ): + raise ValueError('exact immutable live full30 coordinates') + for name, value in rows: + low, high = coordinate_bounds(name) + if not low <= value <= high: + raise ValueError('original baseline25percent canonical bound') + return full_profile_sha256(baseline_revision, rows) + + +_OWNED_OFFER_CONTENT = OrderedDict() + + +def _content_refs(offer): + # Deep values were validated as exact immutable primitives before retention. + # No receiver permission, freshness or time verdict is retained here. + cls = type(offer) + if cls not in (LiveTrialOffer, LiveRestoreOffer) or cls.__mro__ != (cls, object) or LiveTrialBinding.__mro__ != (LiveTrialBinding, object): + return None + if '__getattr__' in vars(cls) or '__getattribute__' in vars(cls): + return None + own = object.__getattribute__(offer, '__dict__') + if type(own) is not dict or type(own.get('binding')) is not LiveTrialBinding: + return None + binding = object.__getattribute__(own['binding'], '__dict__') + if ( + type(own) is not dict + or type(binding) is not dict + or set(own) != set(cls.__dataclass_fields__) + or set(binding) != set(LiveTrialBinding.__dataclass_fields__) + ): + return None + if ( + '__getattr__' in vars(LiveTrialOffer) + or '__getattribute__' in vars(LiveTrialOffer) + or '__getattr__' in vars(LiveTrialBinding) + or '__getattribute__' in vars(LiveTrialBinding) + ): + return None + for checked_class in (cls, LiveTrialBinding): + for name in checked_class.__dataclass_fields__: + value = vars(checked_class).get(name) + if value is not None and hasattr(type(value), '__get__'): + return None + return ( + own, + binding, + tuple(own[name] for name in cls.__dataclass_fields__), + tuple(binding[name] for name in LiveTrialBinding.__dataclass_fields__), + ) + + +def _content_current(offer): + saved = _OWNED_OFFER_CONTENT.get(id(offer)) + current = _content_refs(offer) + return bool( + saved is not None + and saved[0] is offer + and current is not None + and current[0] is saved[1][0] + and current[1] is saved[1][1] + and all(a is b for a, b in zip(current[2], saved[1][2], strict=True)) + and all(a is b for a, b in zip(current[3], saved[1][3], strict=True)) + ) + + +def validate_offer(offer): + if _content_refs(offer) is None: + raise ValueError('owned plain live offer shape') + if _content_current(offer): + return offer + if type(offer) is not LiveTrialOffer or type(offer.version) is not str or offer.version != VERSION or type(offer.binding) is not LiveTrialBinding: + raise ValueError('explicit live tracking protocol') + offer.binding.validate() + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + witness = validate_response_witness(offer.response_witness, offer.binding, offer.observed_ns) + if ( + not _digest(offer.trial_id) + or type(offer.sequence) is not int + or not 0 < offer.sequence < 2**64 + or type(offer.phase) is not str + or offer.phase not in ('trial', 'confirmed', 'rollback') + or not _text(offer.group, 64) + ): + raise ValueError('live trial identity/sequence/phase') + if type(offer.names) is not tuple or any(type(name) is not str for name in offer.names) or canonical_names(offer.names) != offer.names: + raise ValueError('one to four canonical live trial coordinates') + from openpilot.starpilot.flm.canonical_surface_adapter import GROUPS + + if offer.group not in GROUPS or not set(offer.names).issubset(GROUPS[offer.group]): + raise ValueError('live canonical group') + if ( + not _digest(offer.baseline_sha256) + or not _digest(offer.target_sha256) + or full_coordinates(offer.baseline, offer.binding.baseline_revision) != offer.baseline_sha256 + or full_coordinates(offer.target, offer.binding.baseline_revision) != offer.target_sha256 + ): + raise ValueError('live full30 baseline/target hash') + if any( + type(v) is not int for v in (offer.started_ns, offer.observed_ns, offer.expires_ns) + ) or not 0 < offer.started_ns <= offer.observed_ns < offer.expires_ns <= min(offer.observed_ns + MAX_LEASE_NS, witness.valid_until_ns): + raise ValueError('live trial immutable start/current lease') + before = dict(offer.baseline) + for name, value in offer.target: + if name not in offer.names and value != before[name] or abs(value - before[name]) > MAX_STEP + 1e-12: + raise ValueError('small selected live trial step') + if ( + type(offer.target_sequence) is not int + or not 0 < offer.target_sequence < 2**64 + or type(offer.deadline_ns) is not int + or not offer.started_ns < offer.deadline_ns <= offer.started_ns + MAX_TRIAL_NS + ): + raise ValueError('immutable live target sequence/deadline') + if offer.phase == 'trial' and offer.observed_ns > offer.deadline_ns: + raise ValueError('live pending deadline exhausted') + if offer.phase == 'rollback' and offer.target != offer.baseline: + raise ValueError('rollback exact immutable baseline only') + refs = _content_refs(offer) + if refs is not None: + _OWNED_OFFER_CONTENT[id(offer)] = (offer, refs) + _OWNED_OFFER_CONTENT.move_to_end(id(offer)) + while len(_OWNED_OFFER_CONTENT) > 8: + _OWNED_OFFER_CONTENT.popitem(last=False) + return offer + + +RESTORE_VERSION = 'live-tracking-restore-v1' + + +@dataclass(frozen=True) +class LiveRestoreOffer: + version: str + binding: LiveTrialBinding + history_scope: str + history_trial_id: str + target: tuple[tuple[str, float], ...] + target_sha256: str + sequence: int + observed_ns: int + expires_ns: int + response_witness: object + + def validate(self): + if _content_refs(self) is None: + raise ValueError('owned plain restore offer shape') + if _content_current(self): + return self + if type(self) is not LiveRestoreOffer or type(self.version) is not str or self.version != RESTORE_VERSION or type(self.binding) is not LiveTrialBinding: + raise ValueError('separate confirmed-history restore protocol') + self.binding.validate() + if ( + not _digest(self.history_scope) + or not _digest(self.target_sha256) + or not _digest(self.history_trial_id) + or full_coordinates(self.target, self.binding.baseline_revision) != self.target_sha256 + ): + raise ValueError('restore full30 historical target') + if ( + type(self.sequence) is not int + or not 0 < self.sequence < 2**64 + or type(self.observed_ns) is not int + or type(self.expires_ns) is not int + or not 0 < self.observed_ns < self.expires_ns <= self.observed_ns + MAX_LEASE_NS + ): + raise ValueError('restore sequence/lease') + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + witness = validate_response_witness(self.response_witness, self.binding, self.observed_ns) + if self.expires_ns > witness.valid_until_ns: + raise ValueError('restore cannot extend current response witness') + refs = _content_refs(self) + if refs is not None: + _OWNED_OFFER_CONTENT[id(self)] = (self, refs) + _OWNED_OFFER_CONTENT.move_to_end(id(self)) + while len(_OWNED_OFFER_CONTENT) > 8: + _OWNED_OFFER_CONTENT.popitem(last=False) + return self diff --git a/openpilot/starpilot/flm/live_trial_owner.py b/openpilot/starpilot/flm/live_trial_owner.py new file mode 100644 index 0000000000..ff5ba04e58 --- /dev/null +++ b/openpilot/starpilot/flm/live_trial_owner.py @@ -0,0 +1,259 @@ +"""Controls-owned bounded live trial, with no replay validation or persistence IO.""" + +from dataclasses import dataclass, replace +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, CanonicalSurfaceEffects, full_profile_sha256, original_anchor +from openpilot.starpilot.flm.effect_trace import step_full +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding, validate_offer + + +@dataclass(frozen=True) +class LiveTrialApplication: + version: int + binding: LiveTrialBinding + trial_id: str + sequence: int + control_ns: int + status: str + applied: tuple[tuple[str, float], ...] + applied_sha256: str + confirmed_target_sha256: str + reason: str + target_sequence: int = 1 + + +class TrackingTrialOwner: + def __init__(self, binding, *, domain="torque"): + if type(binding) is not LiveTrialBinding: + raise ValueError('typed live binding') + self.binding = binding.validate() + if domain not in ('torque', 'torque_pid'): + raise ValueError('live selected torque domain') + self.domain = domain + self.confirmed_full = tuple((n, 0.0) for n in NAMES) + self.applied_full = dict(self.confirmed_full) + self.last_ns = 0 + self.offer = None + self.source = None + self.trace = None + self.used = self.changed = False + self.original_effect_anchor = '' + self.application = None + self.restore_offer = None + self.interrupted = False + + def neutral(self, reason): + self.source = None + self.trace = None + self.applied_full = dict.fromkeys(NAMES, 0.0) + self.used = self.changed = False + self.application = None + self.interrupted = True + self.last_ns = 0 + + def prepare(self, offer, *, now_ns, permission, binding): + self.source = None + self.application = None + self.used = self.changed = False + if not permission or type(now_ns) is not int or now_ns <= 0 or binding != self.binding: + self.neutral('current_live_permission_lost') + return False + validate_offer(offer) + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + validate_response_witness(offer.response_witness, binding, now_ns) + if offer.binding != self.binding or not offer.observed_ns <= now_ns <= offer.expires_ns: + raise ValueError('live binding/lease changed') + old = self.offer + if self.restore_offer is not None and offer.sequence <= self.restore_offer.sequence: + raise ValueError('trial sequence must follow restored acknowledgement') + if old is not None: + if offer.sequence < old.sequence or offer.sequence == old.sequence and offer != old: + raise ValueError('reordered/changed live offer') + if offer.trial_id == old.trial_id: + if self.interrupted and old.phase == 'trial' and offer.phase != 'rollback': + raise ValueError('interrupted pending trial cannot resume') + if (offer.baseline, offer.baseline_sha256, offer.started_ns, offer.group, offer.names, offer.target_sequence, offer.deadline_ns) != ( + old.baseline, + old.baseline_sha256, + old.started_ns, + old.group, + old.names, + old.target_sequence, + old.deadline_ns, + ): + raise ValueError('live trial immutable baseline changed') + if offer.phase != 'rollback' and (offer.target != old.target or offer.target_sha256 != old.target_sha256): + raise ValueError('live pending target changed') + if old.phase == 'rollback' and offer.phase != 'rollback' or old.phase == 'confirmed' and offer.phase != 'confirmed': + raise ValueError('terminal trial state changed') + elif (old.phase == 'trial' and not self.interrupted) or offer.phase != 'trial' or offer.target_sequence <= old.target_sequence: + raise ValueError('one live trial at a time') + elif offer.phase != 'trial': + raise ValueError('live phase without pending trial') + if old is None or offer.trial_id != old.trial_id: + if offer.baseline != self.confirmed_full or offer.baseline_sha256 != full_profile_sha256(binding.baseline_revision, self.confirmed_full): + raise ValueError('live full30 confirmed baseline CAS') + if old is not None and offer.sequence > old.sequence and offer.observed_ns <= old.observed_ns: + raise ValueError('live publication clock reversal') + if offer.phase == 'trial' and now_ns > offer.deadline_ns: + raise ValueError('live trial original deadline expired') + if old is not None and old.phase == 'trial' and offer.phase == 'confirmed' and offer.observed_ns > offer.deadline_ns: + raise ValueError('confirmation after pending deadline') + if self.last_ns and not 0 < now_ns - self.last_ns <= 50_000_000: + self.neutral('live_control_clock') + self.last_ns = now_ns + return False + if self.binding != binding: + raise ValueError('live current binding changed') + self.applied_full = step_full(self.applied_full, dict(offer.target), now_ns=now_ns, previous_ns=self.last_ns, qualified=True) + self.last_ns = now_ns + self.offer = offer + self.restore_offer = None + self.interrupted = False + self.original_effect_anchor = original_anchor(binding.baseline_revision, offer.names) + self.source = CanonicalSurfaceEffects( + {n: self.applied_full[n] for n in offer.names}, binding.baseline_revision, accumulated=self.applied_full, allow_limited_profile=True + ) + self.source.begin_frame(now_ns) + actual = tuple((n, self.applied_full[n]) for n in NAMES) + actual_hash = offer.target_sha256 if actual == offer.target else full_profile_sha256(binding.baseline_revision, actual) + confirmed = offer.phase == 'confirmed' and actual == offer.target + self.application = LiveTrialApplication( + 1, + binding, + offer.trial_id, + offer.sequence, + now_ns, + 'applied', + actual, + actual_hash, + offer.target_sha256 if confirmed else '', + 'live_current_permission', + offer.target_sequence, + ) + return True + + def prepare_restore(self, offer, prior, *, now_ns, permission, binding): + from openpilot.starpilot.flm.live_trial_contract import LiveRestoreOffer + from openpilot.starpilot.flm.live_trial_store import LiveConfirmedPrior, scope, stable_binding + + self.source = None + self.application = None + self.used = self.changed = False + if not permission or binding != self.binding: + self.neutral('restore_current_permission_lost') + return False + if type(offer) is not LiveRestoreOffer or type(prior) is not LiveConfirmedPrior or prior.historical_only is not True: + raise ValueError('known confirmed startup history required') + offer.validate() + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + validate_response_witness(offer.response_witness, binding, now_ns) + if ( + self.offer is not None + or offer.binding != binding + or not offer.observed_ns <= now_ns <= offer.expires_ns + or offer.history_scope != scope(prior.binding) + or stable_binding(prior.binding) != stable_binding(binding) + or offer.history_trial_id != prior.trial_id + or offer.target != prior.target + or offer.target_sha256 != prior.target_sha256 + ): + raise ValueError('restore differs from startup confirmed history') + old = self.restore_offer + if old is not None and ( + offer.target != old.target or offer.history_scope != old.history_scope or offer.sequence < old.sequence or offer.sequence == old.sequence and offer != old + ): + raise ValueError('restore target/sequence changed') + if self.last_ns and not 0 < now_ns - self.last_ns <= 50_000_000: + self.neutral('restore_control_clock') + self.last_ns = now_ns + return False + self.applied_full = step_full(self.applied_full, dict(offer.target), now_ns=now_ns, previous_ns=self.last_ns, qualified=True) + self.last_ns = now_ns + self.restore_offer = offer + names = (NAMES[0],) + self.original_effect_anchor = original_anchor(binding.baseline_revision, names) + self.source = CanonicalSurfaceEffects( + {n: self.applied_full[n] for n in names}, binding.baseline_revision, accumulated=self.applied_full, allow_limited_profile=True + ) + self.source.begin_frame(now_ns) + actual = tuple((n, self.applied_full[n]) for n in NAMES) + self.application = LiveTrialApplication( + 1, + binding, + offer.history_trial_id, + offer.sequence, + now_ns, + 'applied', + actual, + offer.target_sha256 if actual == offer.target else full_profile_sha256(binding.baseline_revision, actual), + '', + 'current_restore_permission', + ) + return True + + def frame_complete(self, domain): + if self.source is None or self.source.stage_calls != 1: + return False + return self.source.pid_error_calls == 1 if domain == 'torque_pid' else self.source.friction_calls == 1 + + def acknowledge_frame(self, actual): + actual = bool(actual and self.frame_complete(self.domain)) + application = self.application + if application is None: + return None + if ( + actual + and self.restore_offer is not None + and application.applied == self.restore_offer.target + and application.applied_sha256 == self.restore_offer.target_sha256 + ): + self.confirmed_full = self.restore_offer.target + return replace(application, status='restored', confirmed_target_sha256=self.restore_offer.target_sha256, reason='actual_confirmed_history_restored') + if ( + actual + and self.offer is not None + and self.offer.phase == 'confirmed' + and application.applied == self.offer.target + and application.applied_sha256 == self.offer.target_sha256 + ): + self.confirmed_full = self.offer.target + return replace( + application, + status='applied' if actual else 'neutral', + confirmed_target_sha256=application.confirmed_target_sha256 if actual else '', + reason='actual_live_stage_used' if actual else 'live_stage_not_used', + ) + + @property + def applied_coordinates(self): + return () if self.offer is None else tuple(self.applied_full[n] for n in self.offer.names) + + def _record(self, before, after): + self.used = True + self.changed = self.changed or before != after + return after + + def selected_stage(self, frame, *, original_anchor): + before = frame.baseline_ff, frame.baseline_threshold + return before if self.source is None else self._record(before, self.source.selected_stage(frame, original_anchor=original_anchor)) + + @property + def pid_error_calls(self): + return 0 if self.source is None else self.source.pid_error_calls + + def selected_pid_error(self, frame, *, original_anchor): + return frame.error_deg if self.source is None else self._record(frame.error_deg, self.source.selected_pid_error(frame, original_anchor=original_anchor)) + + def selected_pid_stage(self, frame, *, original_anchor): + return frame.baseline_ff if self.source is None else self._record(frame.baseline_ff, self.source.selected_pid_stage(frame, original_anchor=original_anchor)) + + def selected_friction(self, baseline, *, original_anchor): + return baseline if self.source is None else self._record(baseline, self.source.selected_friction(baseline, original_anchor=original_anchor)) + + def selected_deadzone(self, *args, **kwargs): + return args[1] if self.source is None else self._record(args[1], self.source.selected_deadzone(*args, **kwargs)) + + def selected_output(self, *args, **kwargs): + return args[4] if self.source is None else self._record(args[4], self.source.selected_output(*args, **kwargs)) diff --git a/openpilot/starpilot/flm/live_trial_store.py b/openpilot/starpilot/flm/live_trial_store.py new file mode 100644 index 0000000000..3eabf6843f --- /dev/null +++ b/openpilot/starpilot/flm/live_trial_store.py @@ -0,0 +1,244 @@ +"""Learner-only confirmed live targets. Pending offers never enter this document.""" + +from dataclasses import asdict, dataclass +import hashlib +import json +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding, LiveTrialOffer, full_coordinates, validate_offer +from openpilot.starpilot.flm.live_trial_owner import LiveTrialApplication + +KEY = 'AutomaticSteeringLiveTrials' +MAX_BYTES = 32768 +MAX_ROWS = 4 +VERSION = 'live-confirmed-history-v1' + + +def _pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('duplicate live history field') + result[key] = value + return result + + +def stable_binding(binding): + binding.validate() + if not binding.persistent_source_epoch: + raise ValueError('unknown vehicle has no crossdrive history authority') + return (binding.cp_sha256, binding.controller_revision, binding.baseline_revision, binding.reset_generation, binding.persistent_source_epoch) + + +def scope(binding): + return hashlib.sha256(repr(stable_binding(binding)).encode()).hexdigest() + + +def decode(raw): + if raw is None: + return {'version': VERSION, 'confirmed': {}} + if type(raw) is not bytes or not 0 < len(raw) <= MAX_BYTES: + raise ValueError('live history bytes bound') + try: + doc = json.loads(raw, object_pairs_hook=_pairs) + except (UnicodeError, RecursionError) as error: + raise ValueError('live history parse') from error + if ( + type(doc) is not dict + or set(doc) != {'version', 'confirmed'} + or doc['version'] != VERSION + or type(doc['confirmed']) is not dict + or len(doc['confirmed']) > MAX_ROWS + ): + raise ValueError('live confirmed history document') + for key, row in doc['confirmed'].items(): + expected = { + 'binding', + 'target', + 'target_sha256', + 'trial_id', + 'sequence', + 'application_ns', + 'previous_target_sha256', + 'certificate', + } + if type(row) is not dict or set(row) not in (expected, expected | {'retained_response'}): + raise ValueError('confirmed row only; no pending trials') + binding = LiveTrialBinding(**row['binding']).validate() + from openpilot.starpilot.flm.live_tracking_response import certificate_from_dict + + cert = certificate_from_dict(row['certificate']) + if cert.binding != binding or cert.model_revision != binding.physical_model_revision: + raise ValueError('confirmed history model provenance') + if row.get('retained_response') is not None: + from openpilot.starpilot.flm.live_tracking_requalification import retained_from_dict + + retained, _ = retained_from_dict(row['retained_response']) + if retained != cert: + raise ValueError('retained basis belongs to another confirmed certificate') + values = tuple((n, v) for n, v in row['target']) + if ( + key != scope(binding) + or full_coordinates(values, binding.baseline_revision) != row['target_sha256'] + or type(row['trial_id']) is not str + or len(row['trial_id']) != 64 + or type(row['sequence']) is not int + or row['sequence'] <= 0 + or type(row['application_ns']) is not int + or row['application_ns'] <= 0 + ): + raise ValueError('confirmed history identity') + return doc + + +@dataclass(frozen=True) +class LiveConfirmedPrior: + binding: LiveTrialBinding + target: tuple[tuple[str, float], ...] + target_sha256: str + trial_id: str + sequence: int + certificate: object + historical_only: bool = True + response_basis: object | None = None + + +class LiveTrialPersistence: + def __init__(self, params): + self.params = params + + def load_prior(self, binding): + from openpilot.starpilot.saved_source import read_saved + + raw, readable = read_saved(self.params, KEY, MAX_BYTES) + if not readable: + return None + try: + row = decode(raw)['confirmed'].get(scope(binding)) + if row is None: + return None + old = LiveTrialBinding(**row['binding']).validate() + if stable_binding(old) != stable_binding(binding): + return None + return _prior_from_row(row) + except (ValueError, TypeError, KeyError, RecursionError): + return None + + def save_confirmed(self, offer, application, certificate, *, now_ns, current_binding, authorized, response_basis=None): + from openpilot.starpilot.saved_source import read_saved + from openpilot.starpilot.saved_document import commit_exact + + if type(offer) is not LiveTrialOffer or type(application) is not LiveTrialApplication: + return False + try: + validate_offer(offer) + if ( + offer.phase != 'confirmed' + or current_binding() != offer.binding + or not authorized() + or application.binding != offer.binding + or application.trial_id != offer.trial_id + or not 0 < application.sequence <= offer.sequence + or application.target_sequence != offer.target_sequence + or application.status != 'applied' + or application.applied != offer.target + or application.applied_sha256 != offer.target_sha256 + or application.confirmed_target_sha256 != offer.target_sha256 + or not 0 < application.control_ns <= now_ns <= application.control_ns + 100_000_000 + or not offer.observed_ns <= now_ns <= offer.expires_ns + ): + return False + raw, readable = read_saved(self.params, KEY, MAX_BYTES) + if not readable: + return False + doc = decode(raw) + key = scope(offer.binding) + previous = doc['confirmed'].get(key) + retained_response = None + if response_basis is not None: + from openpilot.starpilot.flm.live_tracking_requalification import retained_to_dict + + retained_response = retained_to_dict(certificate, response_basis) + if ( + previous is not None + and previous['target_sha256'] == offer.target_sha256 + and previous['trial_id'] == offer.trial_id + and (retained_response is None or previous.get('retained_response') == retained_response) + ): + return True # Already committed and re-read; publication renewals do not rewrite Params. + if previous is not None and previous['target_sha256'] != offer.baseline_sha256 and previous['target_sha256'] != offer.target_sha256: + return False + if previous is None: + from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, full_profile_sha256 + + if offer.baseline_sha256 != full_profile_sha256(offer.binding.baseline_revision, tuple((n, 0.0) for n in NAMES)): + return False + if len(doc['confirmed']) >= MAX_ROWS: + return False + from openpilot.starpilot.flm.live_tracking_response import certificate_to_dict + + if certificate.binding != offer.binding or certificate.model_revision != offer.binding.physical_model_revision: + return False + certificate = certificate_to_dict(certificate) + doc['confirmed'][key] = { + 'binding': asdict(offer.binding), + 'target': offer.target, + 'target_sha256': offer.target_sha256, + 'trial_id': offer.trial_id, + 'sequence': offer.sequence, + 'application_ns': application.control_ns, + 'previous_target_sha256': offer.baseline_sha256, + 'certificate': certificate, + 'retained_response': retained_response, + } + desired = json.dumps(doc, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + decode(desired) + if len(desired) > MAX_BYTES or current_binding() != offer.binding or not authorized(): + return False + result = commit_exact( + self.params, + key=KEY, + max_bytes=MAX_BYTES, + raw=desired, + expected=raw, + authorized=lambda: authorized() and current_binding() == offer.binding, + temp_prefix='.live-steering-', + ) + return result.committed and result.verified + except (ValueError, TypeError, KeyError, OverflowError, RecursionError): + return False + + +def load_confirmed_history(params): + """Controls startup ONLY: immutable history, no current application permission.""" + from openpilot.starpilot.saved_source import read_saved + + raw, readable = read_saved(params, KEY, MAX_BYTES) + if not readable: + return () + try: + doc = decode(raw) + return tuple(_prior_from_row(row) for row in doc['confirmed'].values()) + except (ValueError, TypeError, KeyError, RecursionError): + return () + + +def _prior_from_row(row): + from openpilot.starpilot.flm.live_tracking_response import certificate_from_dict + + certificate = certificate_from_dict(row['certificate']) + basis = None + if row.get('retained_response') is not None: + from openpilot.starpilot.flm.live_tracking_requalification import retained_from_dict + + retained, basis = retained_from_dict(row['retained_response']) + if retained != certificate: + raise ValueError('retained historical certificate mismatch') + return LiveConfirmedPrior( + LiveTrialBinding(**row['binding']).validate(), + tuple((n, v) for n, v in row['target']), + row['target_sha256'], + row['trial_id'], + row['sequence'], + certificate, + True, + basis, + ) diff --git a/openpilot/starpilot/flm/live_trial_wire.py b/openpilot/starpilot/flm/live_trial_wire.py new file mode 100644 index 0000000000..81c2acd84f --- /dev/null +++ b/openpilot/starpilot/flm/live_trial_wire.py @@ -0,0 +1,343 @@ +"""Additive custom live-trial messages; never parse them as replay profiles.""" + +from openpilot.starpilot.flm.live_trial_contract import VERSION, LiveTrialBinding, LiveTrialOffer, validate_offer +from openpilot.starpilot.flm.live_trial_owner import LiveTrialApplication + +_BINDING = ( + 'identity', + 'cp_sha256', + 'controller_revision', + 'baseline_revision', + 'source_epoch', + 'reset_generation', + 'calibration_generation', + 'vehicle_model_generation', + 'native_identity', + 'physical_model_revision', + 'persistent_source_epoch', +) +_WIRE = ( + 'identity', + 'cpSha256', + 'controllerRevision', + 'baselineRevision', + 'sourceEpoch', + 'resetGeneration', + 'calibrationGeneration', + 'vehicleModelGeneration', + 'nativeIdentity', + 'physicalModelRevision', + 'persistentSourceEpoch', +) + + +def _binding(value): + return {**{wire: getattr(value, name) for name, wire in zip(_BINDING, _WIRE, strict=True)}, 'nativeRequired': value.native_required} + + +def _read_binding(value): + return LiveTrialBinding(*(str(getattr(value, name)) for name in _WIRE), native_required=bool(value.nativeRequired)).validate() + + +def write_trial(event, offer): + validate_offer(offer) + event.starpilotSteeringTracking.version = 1 + event.starpilotSteeringTracking.trial = { + 'version': 1, + 'protocol': VERSION, + 'binding': _binding(offer.binding), + 'trialId': offer.trial_id, + 'sequence': offer.sequence, + 'phase': offer.phase, + 'group': offer.group, + 'names': offer.names, + 'baseline': [{'name': n, 'value': v} for n, v in offer.baseline], + 'target': [{'name': n, 'value': v} for n, v in offer.target], + 'baselineSha256': offer.baseline_sha256, + 'targetSha256': offer.target_sha256, + 'startedMonoTime': offer.started_ns, + 'observedMonoTime': offer.observed_ns, + 'validUntilMonoTime': offer.expires_ns, + 'responseWitness': _write_witness(offer.response_witness, offer.binding, offer.observed_ns), + 'targetSequence': offer.target_sequence, + 'deadlineMonoTime': offer.deadline_ns, + } + + +def read_trial(envelope): + row = envelope.trial + if int(row.version) == 0: + return None + if int(row.version) != 1 or str(row.protocol) != VERSION or len(row.names) > 4 or len(row.baseline) != 30 or len(row.target) != 30: + raise ValueError('live trial wire version/bounds') + return validate_offer( + LiveTrialOffer( + VERSION, + _read_binding(row.binding), + str(row.trialId), + int(row.sequence), + str(row.phase), + str(row.group), + tuple(str(v) for v in row.names), + tuple((str(v.name), float(v.value)) for v in row.baseline), + tuple((str(v.name), float(v.value)) for v in row.target), + str(row.baselineSha256), + str(row.targetSha256), + int(row.startedMonoTime), + int(row.observedMonoTime), + int(row.validUntilMonoTime), + _read_witness(row.responseWitness, _read_binding(row.binding), int(row.observedMonoTime)), + int(row.targetSequence), + int(row.deadlineMonoTime), + ) + ) + + +def write_application(parent, application): + if type(application) is not LiveTrialApplication: + raise ValueError('typed live application') + parent.trackingApplication = { + 'version': 1, + 'binding': _binding(application.binding), + 'trialId': application.trial_id, + 'sequence': application.sequence, + 'controlMonoTime': application.control_ns, + 'status': application.status, + 'applied': [{'name': n, 'value': v} for n, v in application.applied], + 'appliedSha256': application.applied_sha256, + 'confirmedTargetSha256': application.confirmed_target_sha256, + 'reason': application.reason, + 'targetSequence': application.target_sequence, + } + + +def read_application(parent, *, now_ns, offer): + from openpilot.starpilot.flm.live_trial_contract import full_coordinates + + row = parent.trackingApplication + if ( + int(row.version) != 1 + or len(row.applied) != 30 + or _read_binding(row.binding) != offer.binding + or str(row.trialId) != offer.trial_id + or not 0 < int(row.sequence) <= offer.sequence + or int(row.targetSequence) != offer.target_sequence + or not 0 < int(row.controlMonoTime) <= now_ns <= int(row.controlMonoTime) + 100_000_000 + or str(row.status) not in ('applied', 'neutral', 'rejected') + or len(str(row.reason)) > 96 + ): + return None + values = tuple((str(v.name), float(v.value)) for v in row.applied) + if full_coordinates(values, offer.binding.baseline_revision) != str(row.appliedSha256): + return None + confirmed = str(row.confirmedTargetSha256) + if confirmed and (offer.phase != 'confirmed' or confirmed != offer.target_sha256 or values != offer.target or str(row.appliedSha256) != offer.target_sha256): + return None + return LiveTrialApplication( + 1, + offer.binding, + offer.trial_id, + int(row.sequence), + int(row.controlMonoTime), + str(row.status), + values, + str(row.appliedSha256), + confirmed, + str(row.reason), + int(row.targetSequence), + ) + + +def write_restore(event, offer): + offer.validate() + event.starpilotSteeringTracking.version = 1 + event.starpilotSteeringTracking.restore = { + 'version': 1, + 'protocol': offer.version, + 'binding': _binding(offer.binding), + 'historyScope': offer.history_scope, + 'historyTrialId': offer.history_trial_id, + 'target': [{'name': n, 'value': v} for n, v in offer.target], + 'targetSha256': offer.target_sha256, + 'sequence': offer.sequence, + 'observedMonoTime': offer.observed_ns, + 'validUntilMonoTime': offer.expires_ns, + 'responseWitness': _write_witness(offer.response_witness, offer.binding, offer.observed_ns), + } + + +def read_restore(envelope): + from openpilot.starpilot.flm.live_trial_contract import RESTORE_VERSION, LiveRestoreOffer + + row = envelope.restore + if int(row.version) == 0: + return None + if int(row.version) != 1 or str(row.protocol) != RESTORE_VERSION or len(row.target) != 30: + raise ValueError('confirmed restore wire version/bounds') + return LiveRestoreOffer( + RESTORE_VERSION, + _read_binding(row.binding), + str(row.historyScope), + str(row.historyTrialId), + tuple((str(v.name), float(v.value)) for v in row.target), + str(row.targetSha256), + int(row.sequence), + int(row.observedMonoTime), + int(row.validUntilMonoTime), + _read_witness(row.responseWitness, _read_binding(row.binding), int(row.observedMonoTime)), + ).validate() + + +def read_restore_application(parent, *, now_ns, offer): + from openpilot.starpilot.flm.live_trial_contract import full_coordinates + + row = parent.trackingApplication + if ( + int(row.version) != 1 + or len(row.applied) != 30 + or _read_binding(row.binding) != offer.binding + or str(row.trialId) != offer.history_trial_id + or not 0 < int(row.sequence) <= offer.sequence + or not 0 < int(row.controlMonoTime) <= now_ns <= int(row.controlMonoTime) + 100_000_000 + or str(row.status) not in ('applied', 'restored', 'neutral', 'rejected') + or len(str(row.reason)) > 96 + ): + return None + values = tuple((str(v.name), float(v.value)) for v in row.applied) + if full_coordinates(values, offer.binding.baseline_revision) != str(row.appliedSha256): + return None + confirmed = str(row.confirmedTargetSha256) + if str(row.status) == 'restored' and (values != offer.target or confirmed != offer.target_sha256 or str(row.appliedSha256) != offer.target_sha256): + return None + if str(row.status) != 'restored' and confirmed: + return None + return LiveTrialApplication( + 1, + offer.binding, + offer.history_trial_id, + int(row.sequence), + int(row.controlMonoTime), + str(row.status), + values, + str(row.appliedSha256), + confirmed, + str(row.reason), + ) + + +def _write_witness(witness, binding, now_ns): + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + witness = validate_response_witness(witness, binding, now_ns) + c = witness.certificate + from openpilot.starpilot.flm.live_tracking_response import TrackingSourceContext + if type(c) is TrackingSourceContext: + return {'certificate':{'version':c.version,'modelRevision':c.model_revision,'lagNs':c.lag_ns, + 'sourceInventorySha256':c.source_inventory_sha256,'identifiedMonoTime':c.observed_ns, + 'provenance':c.provenance,'parentModelRevision':c.parent_model_revision, + 'requalificationSha256':c.requalification_sha256}, + 'lastControlMonoTime':witness.last_control_ns,'lastApplyMonoTime':witness.last_apply_ns, + 'lastPoseMonoTime':witness.last_pose_ns,'windowStartMonoTime':witness.window_start_ns, + 'independentPoses':witness.independent_poses,'validUntilMonoTime':witness.valid_until_ns} + return { + 'certificate': { + 'version': c.version, + 'modelRevision': c.model_revision, + 'lagNs': c.lag_ns, + 'slope': c.slope, + 'intercept': c.intercept, + 'commandLow': c.command_low, + 'commandHigh': c.command_high, + 'heldoutRmse': c.heldout_rmse, + 'trainBlockIds': c.train_block_ids, + 'heldoutBlockIds': c.heldout_block_ids, + 'sourceInventorySha256': c.source_inventory_sha256, + 'identificationSha256': c.identification_sha256, + 'identifiedMonoTime': c.identified_ns, + 'provenance': c.provenance, + 'parentModelRevision': c.parent_model_revision, + 'requalificationSha256': c.requalification_sha256, + }, + 'lastControlMonoTime': witness.last_control_ns, + 'lastApplyMonoTime': witness.last_apply_ns, + 'lastPoseMonoTime': witness.last_pose_ns, + 'windowStartMonoTime': witness.window_start_ns, + 'independentPoses': witness.independent_poses, + 'residualRmse': witness.residual_rmse, + 'validUntilMonoTime': witness.valid_until_ns, + } + + +def _read_witness(row, binding, now_ns): + from openpilot.starpilot.flm.live_tracking_response import ResponseCertificate, ResponseWitness, prepare_response_witness + + c = row.certificate + from openpilot.starpilot.flm.live_tracking_response import SOURCE_VERSION,TrackingSourceContext + if str(c.version)==SOURCE_VERSION: + if (len(c.trainBlockIds) or len(c.heldoutBlockIds) or str(c.identificationSha256) or + any(float(value)!=0. for value in(c.slope,c.intercept,c.commandLow,c.commandHigh,c.heldoutRmse,row.residualRmse))): + raise ValueError('source context cannot claim fitted model fields') + context=TrackingSourceContext(str(c.version),binding,str(c.modelRevision),int(c.lagNs), + str(c.sourceInventorySha256),int(c.identifiedMonoTime),str(c.provenance), + str(c.parentModelRevision),str(c.requalificationSha256)) + return prepare_response_witness(ResponseWitness(context,binding,int(row.lastControlMonoTime), + int(row.lastApplyMonoTime),int(row.lastPoseMonoTime),int(row.windowStartMonoTime), + int(row.independentPoses),None,int(row.validUntilMonoTime)),binding,now_ns) + if len(c.trainBlockIds) != 4 or len(c.heldoutBlockIds) != 4: + raise ValueError('bounded response certificate blocks') + if any(len(str(v)) > 128 for v in (*c.trainBlockIds, *c.heldoutBlockIds)): + raise ValueError('bounded response certificate block ids') + certificate = ResponseCertificate( + str(c.version), + binding, + str(c.modelRevision), + int(c.lagNs), + float(c.slope), + float(c.intercept), + float(c.commandLow), + float(c.commandHigh), + float(c.heldoutRmse), + tuple(str(v) for v in c.trainBlockIds), + tuple(str(v) for v in c.heldoutBlockIds), + str(c.sourceInventorySha256), + str(c.identificationSha256), + int(c.identifiedMonoTime), + str(c.provenance), + str(c.parentModelRevision), + str(c.requalificationSha256), + ) + return prepare_response_witness( + ResponseWitness( + certificate, + binding, + int(row.lastControlMonoTime), + int(row.lastApplyMonoTime), + int(row.lastPoseMonoTime), + int(row.windowStartMonoTime), + int(row.independentPoses), + float(row.residualRmse), + int(row.validUntilMonoTime), + ), + binding, + now_ns, + ) + + +def write_tracking_source(event, source): + # Optional producer only: reuse exact existing source validation, not a fake publisher. + from openpilot.cereal import messaging + from openpilot.starpilot.flm.learning_wire import write_source_receipt + + temporary = messaging.new_message('autoSteeringLearning') + write_source_receipt(temporary, source) + event.starpilotSteeringTracking.version = 1 + event.starpilotSteeringTracking.sourceReceipt = temporary.autoSteeringLearning.learning.sourceReceipt + + +def read_tracking_source(envelope, *, now_ns, cp_sha256): + from openpilot.starpilot.flm.learning_wire import _read_native_source_receipt + from openpilot.starpilot.flm.runtime_adapter_registry import SourceState + + if int(envelope.version) != 1: + return None + return _read_native_source_receipt(envelope.sourceReceipt, now_ns, cp_sha256, SourceState) diff --git a/openpilot/starpilot/flm/native_primitives.py b/openpilot/starpilot/flm/native_primitives.py new file mode 100644 index 0000000000..19f7bb554e --- /dev/null +++ b/openpilot/starpilot/flm/native_primitives.py @@ -0,0 +1,169 @@ +"""Source-bound optional primitive acceleration in the bounded fitter only.""" +from functools import cache +import hashlib +import importlib.util +from importlib.machinery import ExtensionFileLoader +import json +from pathlib import Path +import platform +import struct +import sys +import sysconfig +from typing import cast + +HERE = Path(__file__).resolve().parent +REVISION = 'flm-exact-primitives-v1' +FALLBACK = object() +_enabled = False + + +def source_paths(): + return {'_primitive_walk.pyx': HERE / '_primitive_walk.pyx', 'native_primitives.py': HERE / 'native_primitives.py', + 'SConscript': HERE / 'SConscript', 'SConstruct': HERE.parents[2] / 'SConstruct'} + + +def runtime_abi(): + return {'implementation': sys.implementation.name, 'cache_tag': sys.implementation.cache_tag, + 'python': list(sys.version_info[:2]), 'SOABI': sysconfig.get_config_var('SOABI'), + 'machine': platform.machine(), 'pointer_bits': struct.calcsize('P') * 8, 'platform': sys.platform} + + +def _digest(path, maximum): + try: + if path.is_symlink() or not path.is_file() or path.stat().st_size > maximum: + return None + return hashlib.sha256(path.read_bytes()).hexdigest() + except OSError: + return None + + +@cache +def native_identity(): + sources = {name: _digest(path, 2 * 1024 * 1024) for name, path in source_paths().items()} + artifact, receipt = HERE / '_primitive_walk.so', HERE / '_primitive_walk.binding.json' + binary_hash = _digest(artifact, 32 * 1024 * 1024) + receipt_hash = _digest(receipt, 65536) + result = {'revision': REVISION, 'sources': sources, 'artifact_sha256': binary_hash, + 'receipt_sha256': receipt_hash, 'abi': runtime_abi(), 'status': 'python-missing'} + if artifact.exists() or receipt.exists(): + result['status'] = 'python-invalid-binding' + if binary_hash is not None and receipt_hash is not None and all(sources.values()): + try: + binding = json.loads(receipt.read_text()) + if (type(binding) is dict and binding.get('schema') == 1 and binding.get('revision') == REVISION and + binding.get('sources') == sources and binding.get('artifact_sha256') == binary_hash): + result['status'] = 'python-unsupported-abi' + if binding.get('abi') == result['abi'] and sys.platform.startswith('linux'): + result['status'] = 'native-bound' + except (OSError, ValueError, TypeError): + pass + return json.dumps(result, sort_keys=True, separators=(',', ':'), allow_nan=False) + + +def enable_fit_worker(): + import os + import resource + if not sys.platform.startswith('linux'): + return + cpu = resource.getrlimit(resource.RLIMIT_CPU)[0] + memory = resource.getrlimit(resource.RLIMIT_AS)[0] + if (not 0 < cpu <= 20 or not 0 < memory <= 2 * 1024 * 1024 * 1024 or + os.getpriority(os.PRIO_PROCESS, 0) < 10 or os.sched_getscheduler(0) != os.SCHED_OTHER): + raise ValueError('FLM native primitive worker context') + global _enabled + _enabled = True + + +@cache +def _extension(): + identity = json.loads(native_identity()) + if identity['status'] != 'native-bound': + return None + path = HERE / '_primitive_walk.so' + expected = identity['artifact_sha256'] + if _digest(path, 32 * 1024 * 1024) != expected: + raise ValueError('FLM native primitive artifact changed') + name = cast(str, __package__) + '._primitive_walk' + if name in sys.modules: + raise ValueError('FLM native primitive unowned preloaded module') + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ValueError('FLM native primitive module missing') + try: + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + except Exception as error: + sys.modules.pop(name, None) + raise ValueError('FLM native primitive import failed') from error + spec = getattr(module, '__spec__', None) + if (spec is None or not isinstance(spec.loader, ExtensionFileLoader) or + Path(spec.origin).resolve() != path.resolve() or Path(cast(str, module.__file__)).resolve() != path.resolve() or path.is_symlink() or + _digest(path, 32 * 1024 * 1024) != expected or + module.compiled_abi() != (sys.version_info.major, sys.version_info.minor, struct.calcsize('P') * 8, REVISION)): + raise ValueError('FLM native primitive executable binding') + return module + + +def try_native(operation, value): + if not _enabled: + return FALLBACK + module = _extension() + if module is None: + return FALLBACK + result = getattr(module, operation)(value) + return FALLBACK if result is module.FALLBACK else result + + +def enable_response_worker(purpose: str) -> bool: + """Enable exact primitives only in the explicitly bounded production child. + + This verifies execution context, not native availability or model authority. + The unchanged extension loader still requires the exact fresh source/ABI/SO + receipt; a qualified context with no valid receipt keeps Python fallback. + """ + import ctypes + from importlib.machinery import ModuleSpec, SourceFileLoader + import os + import resource + import signal + from types import ModuleType + + module_name = 'openpilot.starpilot.flm.live_tracking_response_worker' + worker_path = HERE / 'live_tracking_response_worker.py' + if (type(purpose) is not str or purpose != 'live-response-bootstrap-v1' or + not sys.platform.startswith('linux')): + raise ValueError('FLM native response purpose/platform') + main = sys.modules.get('__main__') + if type(main) is not ModuleType: + raise ValueError('FLM native response main module') + namespace = vars(main) + spec, main_file = namespace.get('__spec__'), namespace.get('__file__') + original = getattr(sys, 'orig_argv', None) + if (type(original) is not list or any(type(value) is not str for value in original) or + tuple(original[1:]) != ('-B', '-m', module_name) or + type(namespace.get('__package__')) is not str or namespace['__package__'] != 'openpilot.starpilot.flm' or + type(spec) is not ModuleSpec or type(spec.name) is not str or spec.name != module_name or + type(spec.loader) is not SourceFileLoader or + type(spec.origin) is not str or type(main_file) is not str or + worker_path.is_symlink() or not worker_path.is_file() or + Path(spec.origin).resolve() != worker_path.resolve() or Path(main_file).resolve() != worker_path.resolve()): + raise ValueError('FLM native response executable context') + expected = {'PURPOSE': purpose, 'MAX_CPU': 180, 'MAX_AS': 2*1024*1024*1024, 'MAX_WALL_NS': 240_000_000_000} + if any(type(namespace.get(name)) is not type(value) or namespace[name] != value for name, value in expected.items()): + raise ValueError('FLM native response fixed policy') + cpu = resource.getrlimit(resource.RLIMIT_CPU)[0] + memory = resource.getrlimit(resource.RLIMIT_AS)[0] + parent = os.environ.get('STARPILOT_FLM_PARENT_PID', '') + if (not 0 < cpu <= 180 or not 0 < memory <= 2*1024*1024*1024 or + os.getpriority(os.PRIO_PROCESS, 0) < 10 or os.sched_getscheduler(0) != os.SCHED_OTHER or + any(os.environ.get(name) != '1' for name in + ('OPENBLAS_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS', 'NUMEXPR_NUM_THREADS')) or + not parent.isascii() or not parent.isdecimal() or not 1 < int(parent) < 2**31 or os.getppid() != int(parent)): + raise ValueError('FLM native response bounded worker context') + death_signal = ctypes.c_int() + if ctypes.CDLL(None).prctl(2, ctypes.byref(death_signal), 0, 0, 0) != 0 or death_signal.value != signal.SIGKILL: + raise ValueError('FLM native response parent death signal') + global _enabled + _enabled = True + return True diff --git a/openpilot/starpilot/flm/native_refinement_wire.py b/openpilot/starpilot/flm/native_refinement_wire.py new file mode 100644 index 0000000000..2caa012381 --- /dev/null +++ b/openpilot/starpilot/flm/native_refinement_wire.py @@ -0,0 +1,81 @@ +"""Bounded original native receipt metadata; never tuning or authority.""" +import base64 +import binascii +from dataclasses import dataclass +import math +import struct + +FIELD = 'native_refinement_v1' +PREFIX = 'nr1:' +FORMAT = '!BQQBd9f' +ORDINARY_INPUTS = 32 +EXTENDED_INPUTS = 33 + + +@dataclass(frozen=True) +class NativeProof: + source_ns: int + applied_ns: int + host: bool + dt: float + target: tuple[float, ...] + before: tuple[float, ...] + after: tuple[float, ...] + + +def encode(values): + """Copy existing original provenance; no source lookup/fit/controller step.""" + if not values: + return None + if (type(values) is not tuple or len(values) < 14 or values[0] != 1. or + any(type(value) not in (int,float) or not math.isfinite(value) for value in values)): + raise ValueError('native compact original provenance') + count = int(values[7]) + if (values[7] != count or not 5 <= count <= 32 or len(values) != 11+2*count or + values[5] not in (0.,1.) or not values[6] > 0 or + any(value < 0 or value > 0xffffffff or int(value) != value for value in values[1:5])): + raise ValueError('native compact original shape') + source = (int(values[1]) << 32) | int(values[2]) + applied = (int(values[3]) << 32) | int(values[4]) + native = (*values[8:11],*values[12:15],*values[12+count:15+count]) + if any(struct.unpack('!f',struct.pack('!f',value))[0] != value for value in native): + raise ValueError('native compact exact Float32 values') + raw = struct.pack(FORMAT,1,source,applied,int(values[5]),values[6],*native) + return PREFIX+base64.b64encode(raw).decode('ascii') + + +def decode(value): + if type(value) is not str or len(value) != 88 or not value.startswith(PREFIX): + raise ValueError('native compact version/size') + try: + raw = base64.b64decode(value[len(PREFIX):],validate=True) + version,source,applied,host,dt,*native = struct.unpack(FORMAT,raw) + except (ValueError,struct.error,binascii.Error) as error: + raise ValueError('native compact original bytes') from error + if PREFIX+base64.b64encode(raw).decode('ascii') != value: + raise ValueError('native compact canonical original bytes') + if (version != 1 or host not in (0,1) or not 0 < source <= applied <= source+500_000_000 or + not math.isfinite(dt) or dt <= 0 or host and dt > .05 or + any(not math.isfinite(item) for item in native)): + raise ValueError('native compact original fields') + return NativeProof(source,applied,bool(host),dt,tuple(native[:3]),tuple(native[3:6]),tuple(native[6:9])) + + +def valid_inputs(values): + """Legacy32 ordinary inputs, plus ONLY one recognized bounded extension.""" + try: + if (type(values) is not tuple or len(values) > EXTENDED_INPUTS or + any(type(row) is not tuple or len(row) != 2 for row in values) or + len({name for name,_ in values}) != len(values)): + return False + ordinary = tuple((name,value) for name,value in values if name != FIELD) + if len(ordinary) > ORDINARY_INPUTS or any(type(name) is not str or not 0 < len(name) <= 96 or + not (type(value) is bool or type(value) in (int,float) and math.isfinite(value) or + type(value) is str and len(value) <= 96) for name,value in ordinary): + return False + proof = tuple(value for name,value in values if name == FIELD) + if proof: + decode(proof[0]) + return True + except (TypeError,ValueError,OverflowError): + return False diff --git a/openpilot/starpilot/flm/normalized_trace_archive.py b/openpilot/starpilot/flm/normalized_trace_archive.py new file mode 100644 index 0000000000..439d598309 --- /dev/null +++ b/openpilot/starpilot/flm/normalized_trace_archive.py @@ -0,0 +1,345 @@ +"""Literal normalized learner snapshot semantics; references confer no authority.""" +from dataclasses import asdict, dataclass, fields, replace +import hashlib +import math +import pickle + +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, MANIFEST_SHA256 + +VERSION = 'full30-normalized-literal-archive-v1' +SOURCE_VERSION = 'unpaired-original-source-normalized-v9' +REF_VERSION = 'full30-literal-row-reference-v1' +MAX_ROWS = 24001 +RESERVED_BYTES = 512 * 1024 +_ENVELOPE_ARRAYS = ('prior_coordinates', 'coordinates') + + +@dataclass(frozen=True) +class TraceRef: + index: int + prepare_ns: int + + +def _numbers(rows): + if (type(rows) is not tuple or len(rows) != 30 or + any(type(row) is not tuple or len(row) != 2 or type(row[0]) is not str for row in rows) or + tuple(row[0] for row in rows) != NAMES): + raise ValueError('normalized trace exact canonical names') + values = tuple(row[1] for row in rows) + _values(values) + return values + + +def _values(values): + if type(values) is not tuple or len(values) != 30 or any( + type(value) not in (int, float) or + type(value) is int and abs(value).bit_length() > 64 or not math.isfinite(value) for value in values): + raise ValueError('normalized trace exact bounded numeric slots') + + +def ref_wire(ref): + if type(ref) is not TraceRef or type(ref.index) is not int or type(ref.prepare_ns) is not int: + raise ValueError('normalized exact row reference') + return {'version': REF_VERSION, 'index': ref.index, 'prepare_ns': ref.prepare_ns} + + +def read_ref(value, archive): + if (type(value) is not dict or any(type(key) is not str for key in value) or set(value) != {'version', 'index', 'prepare_ns'} or + type(value['version']) is not str or value['version'] != REF_VERSION or type(archive) is not TraceArchive): + raise ValueError('explicit normalized row reference and archive') + ref = TraceRef(value['index'], value['prepare_ns']) + archive.row(ref) + return ref + + +class ArchiveBuilder: + def __init__(self): + self.envelopes, self.rows = [], [] + self._envelope_keys, self._row_keys = {}, {} + + def encode(self, trace): + from openpilot.starpilot.flm.effect_trace import FullEffectTrace + if type(trace) is not FullEffectTrace: + raise ValueError('normalized original full trace required') + trace.validate() + key = pickle.dumps(tuple((field.name,getattr(trace.envelope,field.name)) + for field in fields(trace.envelope)),protocol=5) + index = self._envelope_keys.get(key) + if index is None: + envelope = asdict(trace.envelope) + for name in _ENVELOPE_ARRAYS: + envelope[name] = _numbers(getattr(trace.envelope, name)) + if len(self.envelopes) >= MAX_ROWS: + raise ValueError('normalized declaration bound') + index = len(self.envelopes) + self.envelopes.append(envelope) + self._envelope_keys[key] = index + row = (index, trace.version, trace.control_ns, trace.previous_ns, trace.qualified, + _numbers(trace.before), _numbers(trace.target), _numbers(trace.applied), trace.applied_full_profile_sha256) + key = pickle.dumps(row, protocol=5) + row_index = self._row_keys.get(key) + if row_index is None: + if len(self.rows) >= MAX_ROWS: + raise ValueError('normalized trace row bound') + row_index = len(self.rows) + self.rows.append(row) + self._row_keys[key] = row_index + return ref_wire(TraceRef(row_index, trace.control_ns)) + + def record(self, value): + trace = getattr(value, 'effect_trace', None) + if trace is None: + return asdict(value) + body = asdict(replace(value,effect_trace=None)) + body['effect_trace'] = self.encode(trace) + return body + + def literal(self): + return {'version': VERSION, 'manifest': MANIFEST_SHA256, + 'envelopes': tuple(self.envelopes), 'rows': tuple(self.rows)} + + +class TraceArchive: + def __init__(self, literal): + if reservation_upper_bound() > RESERVED_BYTES: + raise ValueError('normalized transient reservation insufficient') + self.literal = literal + self.validate_shape() + + def _header(self): + literal = self.literal + if (type(literal) is not dict or any(type(key) is not str for key in literal) or + set(literal) != {'version','manifest','envelopes','rows'} or + type(literal['version']) is not str or type(literal['manifest']) is not str or + literal['version'] != VERSION or literal['manifest'] != MANIFEST_SHA256 or + type(literal['envelopes']) is not tuple or type(literal['rows']) is not tuple or + not 1 <= len(literal['envelopes']) <= len(literal['rows']) <= MAX_ROWS): + raise ValueError('normalized literal archive source changed') + + @staticmethod + def _declaration(declaration): + from openpilot.starpilot.flm.full_surface_profile import FullSurfaceEnvelope + keys = tuple(field.name for field in fields(FullSurfaceEnvelope)) + if (type(declaration) is not dict or any(type(key) is not str for key in declaration) or + set(declaration) != set(keys)): + raise ValueError('normalized exact literal declaration fields') + for name, value in declaration.items(): + if name in _ENVELOPE_ARRAYS: + _values(value) + elif name == 'trial_names': + if type(value) is not tuple or not 1 <= len(value) <= 30 or any(type(item) is not str or item not in NAMES for item in value): + raise ValueError('normalized declaration trial names') + elif name == 'version': + if type(value) is not int or value not in (1,2): + raise ValueError('normalized declaration version') + elif type(value) is not str or len(value.encode('utf8', 'surrogatepass')) > 256: + raise ValueError('normalized declaration bounded metadata') + + def _row(self, ref): + if (type(ref) is not TraceRef or type(ref.index) is not int or type(ref.prepare_ns) is not int or + not 0 <= ref.index < len(self.literal['rows'])): + raise ValueError('normalized trace row membership') + row = self.literal['rows'][ref.index] + if (type(row) is not tuple or len(row) != 9 or type(row[0]) is not int or + not 0 <= row[0] < len(self.literal['envelopes']) or type(row[1]) is not int or row[1] != 1 or + type(row[2]) is not int or type(row[3]) is not int or row[2] <= 0 or row[3] < 0 or + row[2].bit_length() > 64 or row[3].bit_length() > 64 or + row[2] != ref.prepare_ns or type(row[4]) is not bool or type(row[8]) is not str or + len(row[8]) != 64 or any(char not in '0123456789abcdef' for char in row[8])): + raise ValueError('normalized literal trace clocks/body') + for values in row[5:8]: + _values(values) + return row + + def validate_shape(self): + self._header() + for declaration in self.literal['envelopes']: + self._declaration(declaration) + used = set() + for index, row in enumerate(self.literal['rows']): + if type(row) is not tuple or len(row) != 9: + raise ValueError('normalized literal trace row') + checked = self._row(TraceRef(index,row[2])) + used.add(checked[0]) + if used != set(range(len(self.literal['envelopes']))): + raise ValueError('normalized unused declaration') + + def row(self, ref): + self._header() + row = self._row(ref) + declaration = self.literal['envelopes'][row[0]] + self._declaration(declaration) + return row, declaration + + def resolve(self, ref): + """One unverified typed body; original consumer checks still run fresh.""" + from openpilot.starpilot.flm.effect_trace import FullEffectTrace + from openpilot.starpilot.flm.full_surface_profile import FullSurfaceEnvelope + row, declaration = self.row(ref) + envelope = dict(declaration) + for name in _ENVELOPE_ARRAYS: + envelope[name] = tuple(zip(NAMES, envelope[name], strict=True)) + trace = FullEffectTrace(row[1], FullSurfaceEnvelope(**envelope), row[2], row[3], row[4], + tuple(zip(NAMES, row[5], strict=True)), tuple(zip(NAMES, row[6], strict=True)), + tuple(zip(NAMES, row[7], strict=True)), row[8]) + return trace + + +def archive_of(owner): + archive = getattr(owner, 'trace_archive', None) + if archive is not None and type(archive) is not TraceArchive: + raise ValueError('exact normalized source archive owner') + return archive + + +def archive_key(owner): + archive = archive_of(owner) + if archive is None: + return None + archive.validate_shape() + return VERSION, archive.literal + + +def trace_for(record, owner): + trace = getattr(record, 'effect_trace', None) + if type(trace) is TraceRef: + archive = archive_of(owner) + if archive is None: + raise ValueError('normalized trace missing literal source archive') + if trace.prepare_ns != record.effect_prepare_ns: + raise ValueError('normalized trace actual prepare membership') + return archive.resolve(trace) + return trace + + +def record_body(record, owner): + ref = getattr(record, 'effect_trace', None) + if type(ref) is not TraceRef: + return asdict(record) + archive = archive_of(owner) + if archive is None: + raise ValueError('normalized record missing source archive') + row, declaration = archive.row(ref) + body = asdict(replace(record, effect_trace=None)) + body['effect_trace'] = {'version': VERSION, 'declaration': declaration, 'row': row} + return body + + +def stored_record(record): + ref = getattr(record, 'effect_trace', None) + if type(ref) is not TraceRef: + return asdict(record) + body = asdict(replace(record, effect_trace=None)) + body['effect_trace'] = ref_wire(ref) + return body + + +def current_record(value, literal): + builder = ArchiveBuilder() + body = builder.record(value) + if not builder.rows: + return body + needle_row = builder.rows[0] + needle_declaration = builder.envelopes[needle_row[0]] + needle = pickle.dumps((needle_declaration, needle_row[1:]), protocol=5) + archive = TraceArchive(literal) + for index, row in enumerate(literal['rows']): + if row[2] == needle_row[2] and pickle.dumps((literal['envelopes'][row[0]], row[1:]), protocol=5) == needle: + body['effect_trace'] = ref_wire(TraceRef(index, row[2])) + archive.row(TraceRef(index, row[2])) + return body + raise ValueError('current full trace missing from literal source archive') + + +def normalized_packet(packet): + return type(packet) is dict and type(packet.get('version')) is str and packet['version'] == SOURCE_VERSION + + +def reserve_for_packet(packet): + if not normalized_packet(packet): + if type(packet) is dict and 'trace_archive' in packet: + raise ValueError('legacy source cannot implicitly adopt normalized traces') + return 0 + validate_packet(packet) + return RESERVED_BYTES + + +def guarded_archive(*, position=0, keyword='engine'): + """Rebind complete literal values across mutable work; never cache a verdict.""" + from functools import wraps + def decorate(function): + @wraps(function) + def call(*args, **kwargs): + owner = args[position] if len(args) > position else kwargs[keyword] + before = archive_key(owner) + if before is None: + return function(*args, **kwargs) + digest = hashlib.sha256(pickle.dumps(before, protocol=5)).digest() + result = function(*args, **kwargs) + if hashlib.sha256(pickle.dumps(archive_key(owner), protocol=5)).digest() != digest: + raise ValueError('normalized literal source changed during mutable work') + return result + return call + return decorate + + +def validate_packet(packet): + """Complete table/reference membership; shape only, never source admission.""" + if not normalized_packet(packet): + if type(packet) is dict and 'trace_archive' in packet: + raise ValueError('legacy source cannot implicitly adopt normalized traces') + return None + archive = TraceArchive(packet['trace_archive']) + controls = packet['context_receipts']['controls'] + frames = packet['actual_frames'] + if (type(controls) is not tuple or type(frames) is not tuple or + not 1 <= len(controls) <= 12000 or len(frames) != len(controls)): + raise ValueError('normalized complete original reference inventory') + used = set() + for bodies in (controls,frames): + for body in bodies: + if type(body) is not dict: + raise ValueError('normalized exact original record body') + value = body.get('effect_trace') + if value is None: + continue + if (type(value) is not dict or any(type(key) is not str for key in value) or + set(value) != {'version','index','prepare_ns'} or type(value['version']) is not str or + value['version'] != REF_VERSION or type(value['index']) is not int or + type(value['prepare_ns']) is not int or not 0 <= value['index'] < len(archive.literal['rows']) or + type(body.get('effect_prepare_ns')) is not int or type(body.get('control_ns')) is not int or + value['prepare_ns'] != body['effect_prepare_ns'] or + not 0 < value['prepare_ns'] <= body['control_ns'] or + archive.literal['rows'][value['index']][2] != value['prepare_ns']): + raise ValueError('normalized actual row/prepare reference membership') + used.add(value['index']) + if used != set(range(len(archive.literal['rows']))): + raise ValueError('normalized unused or omitted literal trace row') + return archive + + +def reservation_upper_bound(): + """Worst logical contents of working trace and existing bounded content LRUs.""" + from openpilot.starpilot.flm.full_surface_profile import FullSurfaceEnvelope + from openpilot.starpilot.flm.effect_trace import FullEffectTrace + from openpilot.starpilot.flm import effect_admission_cache as admission + from openpilot.starpilot.flm import canonical_surface_adapter as canonical + if (admission._verified.cache_parameters()['maxsize'] != 16 or + canonical._full_profile_sha256_cached.cache_parameters()['maxsize'] != 128 or + canonical._full_original_anchor_cached.cache_parameters()['maxsize'] != 32 or + canonical._original_anchor_cached.cache_parameters()['maxsize'] != 32): + raise ValueError('normalized retained-content reservation source changed') + envelope_fields = tuple(field.name for field in fields(FullSurfaceEnvelope)) + trace_fields = tuple(field.name for field in fields(FullEffectTrace)) + name_cost = sum(9 + len(name.encode('utf8')) for name in NAMES) + coordinates = 9 + 30 * (9 + 17) + name_cost + names = 9 + name_cost + # Exact integer<=64bits costs at most17; every metadata string<=256UTF8 bytes. + envelope_values = 17 + 2 * coordinates + names + (len(envelope_fields) - 4) * (9 + 256) + envelope = 9 + sum(9 + len(name.encode()) for name in envelope_fields) + envelope_values + envelope_key = 9 + envelope_values + trace = 9 + sum(9 + len(name.encode()) for name in trace_fields) + 17 + envelope + 2 * 17 + 2 + 3 * coordinates + 73 + # The16 envelope proofs retain both key and value. Include construction dicts, + # 128 packed profile keys/results and both existing32-entry source-anchor LRUs. + return (16 * (envelope + envelope_key) + 2 * envelope + trace + + 128 * (9 + 73 + 249 + 73) + 32 * (9 + 73 + 73 + 73) + 32 * (9 + 73 + names + 73)) diff --git a/openpilot/starpilot/flm/original_context_receipts.py b/openpilot/starpilot/flm/original_context_receipts.py new file mode 100644 index 0000000000..925dbb2da8 --- /dev/null +++ b/openpilot/starpilot/flm/original_context_receipts.py @@ -0,0 +1,655 @@ +"""Bounded ORIGINAL owner receipts for context replay only. + +This archive deliberately precedes healthy collection gates. Nothing here is an +EmissionHistory entry, a plant observation, a profile or a native permission. +Child verification consumes full original bodies, never a declared ready/hash. +""" +from collections import OrderedDict +from dataclasses import asdict, dataclass, fields +import math +import hashlib +import json +import struct + +MAX_CONTEXT_RECORDS = 12000 +LEGACY_REVISION = 'original-context-owner-receipts-v1' +REVISION = 'original-context-owner-receipts-v1-emission-v2' + + +def _content_bytes(value): + def typed(item): + kind = type(item) + if item is None or kind in (bool,int,str): + return [kind.__name__,item] + if kind is float and math.isfinite(item): + return ['float64',struct.pack('!d',item).hex()] + if kind is bytes: + return ['bytes',item.hex()] + if kind in (tuple,list): + return [kind.__name__,[typed(child) for child in item]] + if kind is dict and all(type(key) is str for key in item): + return ['dict',[[key,typed(item[key])] for key in sorted(item)]] + raise ValueError('original context exact primitive body') + return json.dumps(typed(value),separators=(',',':'),allow_nan=False).encode() + + +# Equality shortcut only; the original JSON digest/codec remains unchanged. +# Every call rereads all fields into independent type-tagged immutable snapshots. +_FRAME_NAMES = ('control_ns','inputs','selected_command','applied_coordinates','block_id','cohort', + 'desired_physical_accel','healthy','manual','limited','actual_lac_command','frame_sequence', + 'torque_values','centre_values','actual_stage_residual','applied_parameter_names','effect_trace', + 'effect_prepare_ns','controller_domain','desired_kinematic_accel','vehicle_model_anchor','live_applied_coordinates') + + +class _OriginalFrameFallback(Exception): + pass + + +def _original_frame_snapshot(frame, frame_type): + if (type(frame) is not frame_type or frame_type.__getattribute__ is not object.__getattribute__ or + tuple(field.name for field in fields(frame_type)) != _FRAME_NAMES): + raise _OriginalFrameFallback + # A replaced class descriptor could run callbacks unlike the closed graph. + namespace = vars(frame_type) + if '__getattr__' in namespace or any(type(namespace[name]) not in (type(None),bool,int,float,str,bytes,tuple,list,dict) + for name in _FRAME_NAMES if name in namespace): + raise _OriginalFrameFallback + remaining = 4096 + def snapshot(item, depth): + nonlocal remaining + remaining -= 1 + if remaining < 0 or depth > 32: + raise _OriginalFrameFallback + kind = type(item) + if item is None: + return (0,) + if kind is bool: + return (1,item) + if kind is int and item.bit_length() <= 512: + return (2,item) + if kind is float and math.isfinite(item): + return (3,struct.pack('!d',item)) + # ensure_ascii JSON identifies some non-BMP and literal-surrogate strings. + # Restrict this shortcut; whole-original fallback preserves those aliases. + if kind is str and len(item) <= 65536 and item.isascii(): + return (4,item) + if kind is bytes and len(item) <= 65536: + return (5,item) + if kind in (tuple,list): + return (6 if kind is tuple else 7,tuple(snapshot(child,depth+1) for child in item)) + if kind is dict and all(type(key) is str and len(key) <= 65536 and key.isascii() for key in item): + return (8,tuple((key,snapshot(item[key],depth+1)) for key in sorted(item))) + raise _OriginalFrameFallback + return tuple((name,snapshot(getattr(frame,name),0)) for name in _FRAME_NAMES) + + +def _same_original_frame(left, right): + from openpilot.starpilot.flm.controller_replay import ReplayFrame + try: + left_body = _original_frame_snapshot(left,ReplayFrame) + right_body = _original_frame_snapshot(right,ReplayFrame) + except _OriginalFrameFallback: + # The entire original expression, including left deepcopy/serialization + # before right deepcopy, is retained for every unsupported graph. + return _content_bytes(asdict(left)) == _content_bytes(asdict(right)) + return left_body == right_body + + +class OriginalContextReceipts: + def __init__(self): + self.controls = OrderedDict() + self.outputs = OrderedDict() + self.failed = False + + def record_control(self, value): + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + if not isinstance(value, ControlEvidence): + self.failed = True + return + prior = self.controls.get(value.control_ns) + if prior is not None: + if prior != value: + self.failed = True + return + if self.controls and value.control_ns <= next(reversed(self.controls)): + self.failed = True + return + self.controls[value.control_ns] = value + while len(self.controls) > MAX_CONTEXT_RECORDS: + stamp, _ = self.controls.popitem(last=False) + while self.outputs and next(iter(self.outputs.values())).control_ns <= stamp: + self.outputs.popitem(last=False) + + def record_output(self, value): + from openpilot.starpilot.flm.fullrate_collector import AppliedEvidence + if not isinstance(value, AppliedEvidence): + self.failed = True + return + # Repeated carrier observations may refer to one consumed command. Preserve + # the original emission, not an invented new command/application instant. + prior = self.outputs.get(value.apply_ns) + if prior is not None: + from dataclasses import replace + if replace(value, carrier_frame=prior.carrier_frame, + carrier_observed_ns=prior.carrier_observed_ns) != prior: + self.failed = True + return + if self.outputs and value.apply_ns <= next(reversed(self.outputs)): + self.failed = True + return + self.outputs[value.apply_ns] = value + while len(self.outputs) > MAX_CONTEXT_RECORDS: + self.outputs.popitem(last=False) + + def export(self, frames, *, trace_builder=None): + if self.failed or not frames: + raise ValueError('original context archive unavailable') + stamps = tuple(frame.control_ns for frame in frames) + if any(stamp not in self.controls for stamp in stamps): + raise ValueError('original context receipt omitted') + selected_stamps=set(stamps) + return {'revision': REVISION, 'qualified': False, + 'controls': tuple(asdict(self.controls[stamp]) if trace_builder is None else trace_builder.record(self.controls[stamp]) for stamp in stamps), + 'outputs': tuple(asdict(value) for value in self.outputs.values() if value.control_ns in selected_stamps)} + + +def _same_context_frame(left, right, engine): + from openpilot.starpilot.flm.normalized_trace_archive import TraceRef,record_body + if type(left.effect_trace) is TraceRef or type(right.effect_trace) is TraceRef: + return _content_bytes(record_body(left,engine)) == _content_bytes(record_body(right,engine)) + return _same_original_frame(left,right) + + +def verify(packet, engine, frames): + """Reconstruct original control frames and decode optional actual CAN bytes. + + Native disallowed/expired and unhealthy states are exclusions retained only as + boundary context. They never satisfy native evidence or plant admission. + """ + from openpilot.starpilot.flm.learner_factory import _frame + from openpilot.starpilot.flm.torque_raw_fit import control_from_dict, output_from_dict + from openpilot.starpilot.flm.torque_emission_replay import Emission, EmissionDecoder + from opendbc.car.steering_projection import float32 + if (type(packet) is not dict or set(packet) != {'revision','qualified','controls','outputs'} or + packet['revision'] not in (LEGACY_REVISION,REVISION) or packet['qualified'] is not False or + type(packet['controls']) not in (tuple,list) or type(packet['outputs']) not in (tuple,list) or + len(packet['controls']) != len(frames) or len(frames) > MAX_CONTEXT_RECORDS or + len(packet['outputs']) > (len(frames) if packet['revision']==LEGACY_REVISION else MAX_CONTEXT_RECORDS)): + raise ValueError('original context receipt packet') + from openpilot.starpilot.flm.normalized_trace_archive import archive_of + controls = tuple(control_from_dict(row, trace_archive=archive_of(engine)) for row in packet['controls']) + if tuple(row.control_ns for row in controls) != tuple(row.control_ns for row in frames): + raise ValueError('original context receipt original order/omission') + native_identity = None + original_configuration = None + for control, frame in zip(controls, frames, strict=True): + baseline = control.baseline_values + if frame.controller_domain == 'torque' and not engine.factory.manual_baseline: + if len(baseline) < 7: + raise ValueError('original native baseline shape') + # Exact existing torque baseline schema: steer_max, factor/offset/friction, + # deadzone, derived neg/pos limits, immutable PID curves. Only the actual + # native-owned tuple may vary; child native receipt checks follow below. + baseline = (baseline[0],baseline[4],*baseline[7:]) + configuration = (control.generation,control.domain,control.stage_revision,control.response_revision, + control.stage_schema_revision,control.policy,control.controller_mode,control.turn_assist, + baseline,control.native_required) + if original_configuration is None: + original_configuration = configuration + flags = (control.enabled, control.manual, control.active, control.valid, control.override, + control.fault, control.saturated, control.limited, control.model_wobble, + control.controller_chatter, control.turn_assist, control.native_required, control.native_allowed) + if (any(type(flag) is not bool for flag in flags) or + any(type(stamp) is not int for stamp in (control.control_ns,control.state_ns,control.source_receipt_ns, + control.native_observed_ns,control.native_expires_ns)) or + not 0 < control.state_ns <= control.control_ns <= control.state_ns+30_000_000 or + not 0 < control.source_receipt_ns <= control.control_ns <= control.source_receipt_ns+500_000_000 or + control.identity != engine.identity or control.baseline_revision != engine.baseline_revision or + control.source_epoch != engine.objective.context_key or + type(control.generation) is not int or control.generation < 0 or configuration != original_configuration or + control.controller_revision != engine.original_anchor['controller_revision'] or + control.controller_mode != str(engine.factory.mode) or control.turn_assist != engine.factory.turn_assist or + type(control.native_identity) is not str or len(control.native_identity) not in (0,64) or + any(char not in '0123456789abcdef' for char in control.native_identity) or + not 0 <= control.native_observed_ns <= control.control_ns or + not control.native_observed_ns <= control.native_expires_ns <= control.native_observed_ns+200_000_000 or + not _same_context_frame(_frame(control),frame,engine)): + raise ValueError('original context selected owner/body/source') + if not control.native_required and control.native_allowed: + raise ValueError('original optional native receipt cannot grant permission') + if control.native_allowed and (len(control.native_identity) != 64 or + not 0 < control.native_observed_ns <= control.control_ns <= control.native_expires_ns): + raise ValueError('original context native status/clock inconsistent') + if native_identity is None and control.native_identity: + native_identity = control.native_identity + if control.native_identity and control.native_identity != native_identity: + raise ValueError('original context native identity changed') + if control.native_required and frame.healthy and not frame.manual and (not control.native_allowed or len(control.native_identity) != 64 or + not 0 < control.native_observed_ns <= control.control_ns <= control.native_expires_ns): + raise ValueError('original healthy context native unavailable') + by_control = {row.control_ns: row for row in controls} + outputs = tuple(output_from_dict(row) for row in packet['outputs']) + if ((packet['revision']==LEGACY_REVISION and len({row.control_ns for row in outputs}) != len(outputs)) or + len({(row.control_ns,row.apply_ns) for row in outputs}) != len(outputs) or + len({row.apply_ns for row in outputs}) != len(outputs) or + any(right.control_ns < left.control_ns or right.apply_ns <= left.apply_ns or + packet['revision']==LEGACY_REVISION and right.control_ns==left.control_ns + for left,right in zip(outputs,outputs[1:], strict=False))): + raise ValueError('original context duplicate actual application') + decoder = EmissionDecoder(engine.factory.cp) + for output in outputs: + control = by_control.get(output.control_ns) + if (control is None or any(type(flag) is not bool for flag in + (output.valid,output.active,output.override,output.fault,output.limited,output.native_allowed)) or + any(type(stamp) is not int for stamp in (output.state_ns,output.control_ns,output.apply_ns)) or + not control.state_ns <= output.state_ns <= output.apply_ns or + not output.control_ns <= output.apply_ns <= output.control_ns+100_000_000 or + not math.isfinite(output.applied) or output.domain != control.domain): + raise ValueError('original context actual applied owner/clock') + witness = output.projection + if witness is None: + continue # Original absence is retained, never usable for a plant pair. + if (witness.family != str(engine.factory.cp.brand) or witness.fingerprint != str(engine.factory.cp.carFingerprint) or + witness.flags != int(engine.factory.cp.flags) or witness.requested_torque != float32(control.requested) or + witness.reported_torque != output.applied): + raise ValueError('original context actual CAN/request source') + # Decoder verifies actual packet bytes and steer-request state, including + # inactive packets. No healthy record()/load() or trial EncoderBranch call. + decoder.check(Emission(output.control_ns,output.state_ns,output.apply_ns,witness, + control.identity,control.source_epoch,control.baseline_revision,output.valid,output.active, + output.override,output.fault,output.limited,output.native_allowed,control.native_required, + control.native_observed_ns,control.native_expires_ns,control.native_identity)) + return {row.control_ns:row for row in controls} + + +@dataclass(frozen=True) +class OriginalSourceWindow: + window_id: str + start_ns: int + end_ns: int + before_ns: int + after_ns: int + control_ns: tuple[int, ...] + + +def verified_closed_windows(packet, engine, frames): + """Original command/source boundaries ONLY; no poses, seed lag or y access. + + Returns (tuple[OriginalSourceWindow,...], dict[int,ControlEvidence]). The + independent plant caller owns pose membership/common-lag support and its fixed + first-eight design. This helper confers no model or physical qualification. + """ + from types import SimpleNamespace + from openpilot.starpilot.flm.controller_replay import MAX_FRAME_GAP_NS + from openpilot.starpilot.flm.event_storage import source_windows + controls = verify(packet,engine,frames) + if any(right.frame_sequence != left.frame_sequence+1 or + not 0 < right.control_ns-left.control_ns <= MAX_FRAME_GAP_NS + for left,right in zip(frames,frames[1:], strict=False)): + raise ValueError('original context incomplete chronological prefix') + view = SimpleNamespace(records=frames,replay_windows=(),_frame_context=engine._frame_context) + indices = {frame.control_ns:index for index,frame in enumerate(frames)} + if len(indices) != len(frames): + raise ValueError('original context duplicate frame') + result = [] + for start,end in source_windows(view): + first,last = indices[start],indices[end] + if first == 0 or last+1 == len(frames): + continue # Censored prefix/tail are never relabeled complete boundaries. + stamps = tuple(frame.control_ns for frame in frames[first:last+1]) + before,after = frames[first-1].control_ns,frames[last+1].control_ns + from openpilot.starpilot.flm.normalized_trace_archive import record_body + digest = hashlib.sha256(_content_bytes((REVISION,engine.identity,engine.baseline_revision, + engine.original_anchor,tuple(record_body(controls[stamp],engine) for stamp in (before,*stamps,after))))).hexdigest() + result.append(OriginalSourceWindow('original-source-window:'+digest,start,end,before,after,stamps)) + return tuple(result),controls + + +def verify_original_replay(packet, template_engine, replay_bytes): + """Rebuild actual original context, including nonadmissible boundary ticks. + + template_engine supplies factory/objective/full original anchor and the fresh + underlying ControllerReplay. It need not falsely mark a fresh owner valid or + construct an EventControllerReplay before actual checkpoints. Caller counts the union with any + later stage replay once; this function enforces the original prefix bounds. + No candidate coordinates, physical target or fitted model is consumed here. + """ + from openpilot.starpilot.flm.controller_replay import ( + ControllerReplay,ReplayFrame,Snapshot,MAX_CHECKPOINT_NS,MAX_FRAME_GAP_NS, + _VM_FIELDS,_run,capture_snapshot,snapshot_from_values,restore_snapshot, + same_float,same_final_command,unpack_primitive_snapshot) + from openpilot.starpilot.flm.effect_trace import trace_from_dict + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.selected_effects import EffectReplayRecordedSource + from openpilot.starpilot.flm.clipping_quality import ZERO_REFERENCE_QUALITY + from openpilot.starpilot.flm.event_storage import source_windows + from types import SimpleNamespace + engine = template_engine + data = unpack_primitive_snapshot(replay_bytes) + from openpilot.starpilot.flm.normalized_trace_archive import TraceArchive,read_ref,trace_for,archive_key + normalized = data.get('version') == 2 + if normalized: + from openpilot.starpilot.flm.primitive_source_memo import independent_primitive_graph + from openpilot.starpilot.flm.normalized_trace_archive import RESERVED_BYTES + data = independent_primitive_graph(data,reserved_bytes=RESERVED_BYTES) + engine.engine.trace_archive = TraceArchive(data['trace_archive']) + elif 'trace_archive' in data: + raise ValueError('legacy replay cannot implicitly adopt normalized traces') + if (data.get('version') != (2 if normalized else 1) or data.get('raw_mode') is not True or + data['cp_bytes'] != engine.factory.cp.as_builder().to_bytes() or + data['identity'] != engine.identity or data['baseline_revision'] != engine.baseline_revision or + data['mode'] != str(engine.factory.mode) or data['turn_assist'] != engine.factory.turn_assist or + tuple(data.get('manual_baseline',())) != engine.factory.manual_baseline or + data.get('manual_source_epoch','') != engine.factory.manual_source_epoch or + data['objective'] != asdict(engine.engine.objective)): + raise ValueError('original context actual selected replay factory') + frames = [] + for body in data['frames']: + body = dict(body) + for name in ('inputs','torque_values'): + body[name] = tuple(tuple(row) for row in body.get(name,())) + for name in ('applied_coordinates','centre_values','applied_parameter_names','vehicle_model_anchor','live_applied_coordinates'): + if name in body: + body[name] = tuple(body[name]) + if body.get('actual_stage_residual') is not None: + body['actual_stage_residual'] = tuple(body['actual_stage_residual']) + if body.get('effect_trace') is not None: + body['effect_trace'] = read_ref(body['effect_trace'],engine.trace_archive) if normalized else trace_from_dict(body['effect_trace']) + if any(body.get('live_applied_coordinates', ())): + raise ValueError('live nonzero history requires explicit tracking prefix verifier') + frames.append(ReplayFrame(**body)) + frames = tuple(frames) + windows = tuple(data['windows']) + if (not 1 <= len(frames) <= MAX_CONTEXT_RECORDS or not 1 <= len(windows) <= 24 or + len(data['checkpoints']) > 600 or + any(right.frame_sequence != left.frame_sequence+1 or + not 0 < right.control_ns-left.control_ns <= MAX_FRAME_GAP_NS for left,right in zip(frames,frames[1:], strict=False)) or + windows != ((frames[0].control_ns,frames[-1].control_ns),)): + raise ValueError('original context complete prefix replay bound') + normalized_key = hashlib.sha256(__import__('pickle').dumps(archive_key(engine),protocol=5)).digest() if normalized else None + controls = verify(packet,engine,frames) + owner = ControllerReplay(engine.factory,engine.engine.objective,identity=engine.identity, + baseline_revision=engine.baseline_revision,model_revision=data['model_revision']) + if normalized: + owner.trace_archive = engine.trace_archive + owner.raw_mode = True + owner.emissions.load(data['emissions'],windows) # Existing healthy gates unchanged. + checkpoints = {stamp:Snapshot(revision,tuple(values)) for stamp,revision,values in data['checkpoints']} + if (len(checkpoints) != len(data['checkpoints']) or frames[0].control_ns not in checkpoints or + any(type(stamp) is not int or stamp not in controls for stamp in checkpoints)): + raise ValueError('original context original checkpoint omitted/duplicate') + previous_sequence = None + for frame in frames: + control = controls[frame.control_ns] + if type(control.snapshot_sequence) is not int or control.snapshot_sequence <= 0: + raise ValueError('original context owner checkpoint sequence') + if frame.control_ns in checkpoints: + if not control.replay_snapshot or previous_sequence is not None and control.snapshot_sequence != previous_sequence+1: + raise ValueError('original context owner checkpoint receipt omitted/sequence') + if frame.controller_domain == 'torque_pid': + from openpilot.starpilot.flm.pid_controller_replay import pid_snapshot_from_values + expected = pid_snapshot_from_values(control.replay_snapshot) + else: + expected = snapshot_from_values(control.replay_snapshot) + if _content_bytes(asdict(expected)) != _content_bytes(asdict(checkpoints[frame.control_ns])): + raise ValueError('original context owner checkpoint body/revision') + previous_sequence = control.snapshot_sequence + elif control.replay_snapshot or control.snapshot_sequence != previous_sequence: + raise ValueError('original context undeclared owner checkpoint/sequence') + last_checkpoint,requests = None,{} + is_native_torque = engine.factory.cp.lateralTuning.which() == 'torque' and not engine.factory.manual_baseline + from openpilot.starpilot.flm.runtime_adapter_registry import baseline_values + if is_native_torque: + from openpilot.starpilot.flm.native_baseline import checkpoint_native_valid,effective_native_valid,static_values + native_probe,native_vm = engine.factory.create() + native_anchor = tuple(native_probe.native_learning_baseline) + native_static = static_values(native_probe) + + from openpilot.starpilot.flm.native_refinement_wire import FIELD,encode,decode,valid_inputs + last_compact = None + + def advance_original_native(frame): + nonlocal last_compact + compact = dict(frame.inputs).get(FIELD) + if not is_native_torque: + if compact is not None: + raise ValueError('original native receipt on immutable/manual configuration') + return + before,after = baseline_values(owner.actual),controls[frame.control_ns].baseline_values + if before == after: + # Old initial checkpoints may already contain a full original receipt. + # Mirroring that SAME receipt adds no state transition or new authority. + if compact != last_compact and compact != encode(tuple(owner.actual.native_learning_provenance)): + raise ValueError('original native receipt without actual transition') + last_compact = compact + return + if compact is None or compact == last_compact: + raise ValueError('original native changed row missing new receipt') + proof = decode(compact) + if (_content_bytes(proof.before) != _content_bytes(tuple(before[1:4])) or + _content_bytes(proof.after) != _content_bytes(tuple(after[1:4])) or + not owner.last_ns < proof.applied_ns <= frame.control_ns): + raise ValueError('original context native per-row transition continuity') + for name,value in zip(('latAccelFactor','latAccelOffset','friction'),proof.after,strict=True): + setattr(owner.actual.torque_params,name,value) + owner.actual.pid.set_limits(after[6],after[5]) + def words(ns): + return float(ns >> 32),float(ns & 0xffffffff) + owner.actual.native_learning_provenance = (1.,*words(proof.source_ns),*words(proof.applied_ns), + float(proof.host),proof.dt,float(len(before)),*proof.target,*before,*after) + if (tuple(owner.actual.native_learning_baseline) != native_anchor or + static_values(owner.actual) != native_static or + not checkpoint_native_valid(owner.actual,engine.factory.cp,now_ns=frame.control_ns)): + raise ValueError('original context native per-row receipt rejected') + last_compact = compact + def original_source_matches(frame): + trace = trace_for(frame,engine) + if trace is None: + # The real owner clears automatic effects when unhealthy/manual. The + # original body and exact baseline controller rerun establish that context; + # it is never a normalized physical/trial or current-authority row. + prior = dict(engine.original_anchor['accumulated']) + return (not frame.healthy or frame.manual) or (not any(prior.values()) and not any(frame.applied_coordinates)) + try: + from openpilot.starpilot.flm.effect_trace import FullEffectTrace + if type(trace) is not FullEffectTrace: + return False + trace.validate() + fields = ('cp_sha256','controller_revision','source_epoch','reset_generation', + 'selected_baseline_revision','full_original_anchor','context_key','persistent_source_epoch') + envelope = trace.envelope + return (envelope.source_binding() == tuple(engine.original_anchor[name] for name in fields) and + envelope.manifest_sha256 == engine.original_anchor['manifest_sha256'] and + envelope.selected_baseline_revision == engine.baseline_revision and + envelope.source_epoch == engine.objective.context_key and + type(frame.effect_prepare_ns) is int and 0 < frame.effect_prepare_ns == trace.control_ns <= frame.control_ns) + except (AttributeError,KeyError,TypeError,ValueError,OverflowError): + return False + for frame in frames: + if not valid_inputs(frame.inputs): + raise ValueError('original context versioned input budget') + if not is_native_torque and FIELD in dict(frame.inputs): + raise ValueError('original native receipt on immutable/manual configuration') + if last_checkpoint is not None: + advance_original_native(frame) + if frame.control_ns in checkpoints: + if is_native_torque: + restore_snapshot(native_probe,native_vm,checkpoints[frame.control_ns]) + if (tuple(native_probe.native_learning_baseline) != native_anchor or static_values(native_probe) != native_static or + not checkpoint_native_valid(native_probe,engine.factory.cp,now_ns=frame.control_ns) or + baseline_values(native_probe) != controls[frame.control_ns].baseline_values): + raise ValueError('original context native checkpoint provenance') + compact = dict(frame.inputs).get(FIELD) + if compact is not None and compact != encode(tuple(native_probe.native_learning_provenance)): + raise ValueError('original native compact/checkpoint receipt mismatch') + if last_checkpoint is not None: + # VM state is an original current-frame input set before producer + # capture. No PID/gain/filter/jerk state may silently reset at a saved + # checkpoint: require the ongoing original rerun to reach it exactly. + values = dict(frame.inputs) + for name in _VM_FIELDS: + setattr(owner.vm,name,values['vm.'+name]) + ongoing = capture_snapshot(owner.actual,owner.vm) + if _content_bytes(asdict(ongoing)) != _content_bytes(asdict(checkpoints[frame.control_ns])): + raise ValueError('original context hidden intermediate state reset') + owner.checkpoint(frame.control_ns,checkpoints[frame.control_ns]) + last_checkpoint = frame.control_ns + if owner.last_sequence == 0: + last_compact = dict(frame.inputs).get(FIELD) + if (last_checkpoint is None or frame.control_ns-last_checkpoint > MAX_CHECKPOINT_NS or + type(frame.frame_sequence) is not int or frame.frame_sequence <= 0 or + not math.isfinite(frame.selected_command) or not valid_inputs(frame.inputs) or + not original_source_matches(frame) or any(frame.applied_coordinates) or any(frame.centre_values) or + frame.actual_stage_residual is not None and any(frame.actual_stage_residual)): + raise ValueError('original context original source/checkpoint') + trace = trace_for(frame,engine) + source = None if trace is None else EffectReplayRecordedSource(trace) + trace = None + output,_ = _run(owner.actual,owner.vm,frame.inputs,owner.objective,frame.applied_coordinates, + frame.torque_values,actual_stage_residual=frame.actual_stage_residual,effect_trial=source) + if (not same_final_command(output,frame.selected_command) or + frame.actual_lac_command is not None and not same_float(output,frame.actual_lac_command)): + raise ValueError('original context actual Float64/controller output') + if baseline_values(owner.actual) != controls[frame.control_ns].baseline_values: + raise ValueError('original context exact current baseline body') + if is_native_torque and not effective_native_valid(owner.actual,engine.factory.cp): + raise ValueError('original context actual native parameter envelope') + if not owner.record_actual_frame(frame): + raise ValueError('original context actual record lifecycle rejected') + requests[frame.control_ns] = frame.selected_command + healthy = source_windows(SimpleNamespace(records=owner.records,replay_windows=(),_frame_context=owner._frame_context)) + for start,end in healthy: + owner.emissions.rollout(requests,start,end,((start,end),),quality_contract=ZERO_REFERENCE_QUALITY) + if normalized and hashlib.sha256(__import__('pickle').dumps(archive_key(owner),protocol=5)).digest() != normalized_key: + raise ValueError('normalized original source changed during actual replay') + result = EventControllerReplay(owner,engine.objective,engine.original_anchor) + result.original_context_controls = controls + result._original_verified_context = _VerifiedOriginalContext( + _REUSE_OWNER,_verified_context_key(packet,result),tuple(controls.items())) + return result + + +# Created only after verify_original_replay completes the ORIGINAL actual +# controller/checkpoint/native lifecycle. This object never crosses classless +# worker IPC and is not a certificate accepted from a caller or saved report. +_REUSE_OWNER = object() + + +@dataclass(frozen=True) +class _VerifiedOriginalContext: + owner: object + key: bytes + controls: tuple + + +def _verified_context_key(packet, result): + """Exact type/bit owned bytes; no identity-only or payload-qualified cache.""" + import hashlib + import pickle + from openpilot.starpilot.flm.normalized_trace_archive import archive_key + from openpilot.starpilot.flm.controller_replay import ControllerReplay, SelectedControllerFactory, capture_snapshot + from openpilot.starpilot.flm.pid_controller_replay import PidControllerFactory + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.torque_emission_replay import EmissionHistory + if (type(result) is not EventControllerReplay or type(result.engine) is not ControllerReplay or + type(result.factory) not in (SelectedControllerFactory,PidControllerFactory) or + type(result.emissions) is not EmissionHistory or result.failed or not result.raw_mode or + type(result.original_context_controls) is not dict): + raise ValueError('original replay reuse requires actual verified owners') + owner,factory,emissions = result.engine,result.factory,result.emissions + # Same-byte mutation remains equivalent; any changed frame, receipt, CP, + # source, checkpoint, window or actual native packet invalidates the reuse. + selected = (type(factory).__module__,type(factory).__qualname__, + factory.cp.as_builder().to_bytes(),str(factory.mode),str(getattr(factory,'policy','stock-pid')),factory.turn_assist, + factory.manual_baseline,factory.manual_source_epoch, + factory.interface_class.__module__,factory.interface_class.__qualname__) + native = (emissions.cp.as_builder().to_bytes(),emissions.identity,emissions.baseline_revision, + tuple(emissions.events.items()),tuple(emissions.carriers.items())) + value = (packet,owner.failed,owner.raw_mode,owner.identity,owner.baseline_revision,owner.model_revision, + owner.objective,result.objective,result.original_anchor,selected, + tuple(owner.records),tuple(owner.checkpoints),tuple(owner.replay_windows), + owner.last_ns,owner.last_sequence,owner.checkpoint_ns,capture_snapshot(owner.actual,owner.vm), + tuple(result.original_context_controls.items()),native) + if archive_key(result) is not None: + value = (*value,archive_key(result)) + return hashlib.sha256(pickle.dumps(value,protocol=5)).digest() + + +def reuse_verified_context(packet, result, frames): + """Reuse only a completed child-local proof after full owned-byte rebinding. + + A missing proof takes the existing verifier path. A changed owned input is + rejected rather than silently recertified by a cached/native boolean. + """ + proof = result.__dict__.get('_original_verified_context') + if proof is None: + return None + if (type(proof) is not _VerifiedOriginalContext or proof.owner is not _REUSE_OWNER or + tuple(frames) != tuple(result.records) or _verified_context_key(packet,result) != proof.key): + raise ValueError('original verified replay owned bytes changed') + return dict(proof.controls) + + +def verified_replay_subset(packet, result, windows: tuple[tuple[int, int], ...]): + """Child-only exact stage view of already replayed original source/state. + + This avoids serializing and rerunning the SAME original neutral controller a + second time. It transfers no profile, feature, candidate/native verdict or + admission. EventControllerReplay still executes every strict trial and CAN + rollout from the original pre-frame checkpoints. + """ + from collections import deque, OrderedDict + from copy import copy + from openpilot.starpilot.flm.controller_replay import capture_snapshot,restore_snapshot + from openpilot.starpilot.flm.event_storage import source_windows + from types import SimpleNamespace + if reuse_verified_context(packet,result,result.records) is None: + return None + owner = result.engine + frames,checkpoints = tuple(owner.records),dict(owner.checkpoints) + healthy = source_windows(SimpleNamespace(records=frames,replay_windows=(),_frame_context=owner._frame_context)) + if (type(windows) is not tuple or not 1 <= len(windows) <= 24 or + any(type(row) is not tuple or len(row) != 2 or row[0] not in checkpoints or row[0] > row[1] or + i and row[0] <= windows[i-1][1] or + not any(a <= row[0] <= row[1] <= b for a,b in healthy) for i,row in enumerate(windows))): + raise ValueError('original verified replay subset windows') + records = tuple(frame for frame in frames if any(a <= frame.control_ns <= b for a,b in windows)) + if (not records or len(records) > 12000 or records[-1].control_ns != owner.last_ns or + tuple((next(frame.control_ns for frame in records if frame.control_ns >= a), + max(frame.control_ns for frame in records if frame.control_ns <= b)) for a,b in windows) != windows): + # A nonterminal view needs its own actual ending state; retain the existing + # full reconstruction for that unsupported case, without inventing state. + return None + view = copy(owner) # Preserve the completed real checkpoint/record lifecycle. + view.records = deque(records,maxlen=owner.records.maxlen) + view.checkpoints = deque(((stamp,state) for stamp,state in owner.checkpoints + if any(a <= stamp <= b for a,b in windows)),maxlen=owner.checkpoints.maxlen) + view.replay_windows = windows + view.emissions = copy(owner.emissions) + view.emissions.events = OrderedDict((stamp,event) for stamp,event in owner.emissions.events.items() + if any(a <= event.control_ns <= b for a,b in windows)) + view.emissions.by_control = {event.control_ns:event for event in view.emissions.events.values()} + carrier_ranges = [] + for a,b in windows: + emitted = tuple(event for event in view.emissions.events.values() if a <= event.control_ns <= b) + if emitted: + carrier_ranges.append((emitted[0].witness.frame+1,emitted[-1].witness.frame+1)) + view.emissions.carriers = OrderedDict((frame,value) for frame,value in owner.emissions.carriers.items() + if any(a <= frame <= b for a,b in carrier_ranges)) + # The existing exact-event decoder cache proves ORIGINAL bytes only. Every + # counterfactual still creates/advances its own EncoderBranch in rollout. + # Derived current/feature caches are never moved across this stage view. + view.normalized,view.projected = {},{} + view.recent_states,view.actual_centre_friction_values = OrderedDict(),OrderedDict() + view.cached_validation = view.cached_key = None + # Separate mutable controller objects. The terminal state is the exact state + # already produced by the original full replay, never a reset/False flag. + state = capture_snapshot(owner.actual,owner.vm) + view.actual,view.vm = owner.factory.create() + # PID.reset uses integer0, while update stores the continuous Float64 output. + # Existing restore_snapshot preserves an old integer leaf's type. Seed this + # exact continuously valued leaf before restoring ALL original state fields. + view.actual.pid.control = float(owner.actual.pid.control) + restore_snapshot(view.actual,view.vm,state) + view.probe,view.probe_vm = owner.factory.create() + return view diff --git a/openpilot/starpilot/flm/original_source_capture.py b/openpilot/starpilot/flm/original_source_capture.py new file mode 100644 index 0000000000..84470bd17b --- /dev/null +++ b/openpilot/starpilot/flm/original_source_capture.py @@ -0,0 +1,500 @@ +"""Private opt-in unpaired ORIGINAL observations; no plant join or admission. + +Only pose rows are new retained state. Commands/encoders remain in the existing +actual replay engine and emission recorder. Export does bounded work explicitly +in the optional learner, never in Controls/Card. This is not fit qualification. +""" +from dataclasses import asdict, dataclass +import math + +MAX_SOURCE_POSES = 5120 # Shared raw retention, including legacy rows + pending. +# Existing raw collector can retain 24*80 sealed +80 partial rows; reserve its +# full allowance and four pending original poses even while legacy keeps growing. +MAX_RETAINED_SOURCE_POSES = MAX_SOURCE_POSES - (24 + 1) * 80 - 4 +MAX_PACKET_FRAMES = 12000 +MAX_PACKET_WINDOWS = 24 +# Reuse existing classless transport limits; no silent private cap relaxation. +from openpilot.starpilot.flm.controller_replay import ( + MAX_FIT_SNAPSHOT_BYTES, MAX_FIT_RAW_BYTES, pack_primitive_snapshot, unpack_primitive_snapshot, +) +SOURCE_PACKET_VERSION = 'unpaired-original-source-v8' +SOURCE_PACKET_ENCODING = 'primitive-snapshot-v1' +MAX_PACKET_NS = 600_000_000_000 +MAX_CAPTURE_SPAN_NS = 100_000_000_000 # <=10000 fullrate frames, leaves prefix budget. +MAX_GAP_NS = 100_000_000 + + +@dataclass(frozen=True) +class OriginalSourceClockContract: + """External producer provenance, never inferred from timestamp proximity. + + Construction is not verification. Production caller must independently pin + the producer revision and proof; fixture contracts qualify only source tests. + """ + source_domain: str + producer_revision: str + proof_sha256: str + scope: str + + def validate(self): + if (self.source_domain not in ('monotonic', 'boottime', 'unknown-equivalent-at-receipt', 'mono_boot_equivalent_interval_v1') or + not _hex(self.producer_revision) or not _hex(self.proof_sha256) or + self.scope not in ('independently-pinned-producer', 'synthetic-fixture', 'pinned-source-abi')): + raise ValueError('explicit original producer-clock proof required') + if self.scope == 'pinned-source-abi': + from openpilot.starpilot.flm.producer_clock_equivalence import CLOSURE_SHA256, CONVENTION + if (self.source_domain, self.producer_revision, self.proof_sha256) != (CONVENTION, CLOSURE_SHA256, CLOSURE_SHA256): + raise ValueError('unsupported pinned mixed producer closure') + + +@dataclass(frozen=True) +class OriginalSourcePose: + mono_ns: int + received_ns: int + source_boot_ns: int + clock_offset_ns: int + clock_uncertainty_ns: int + calibration: str + calibration_estimate: str + context: tuple + yaw_rate: float + roll_rad: float + native_identity: str + native_allowed: bool + valid: bool + source_epoch: str + identity: str + baseline_revision: str + controller_revision: str + source_state_ns: int + native_apply_ns: int + observed_ns: int = 0 + completed_ns: int = 0 + native_required: bool = True + native_control_ns: int = 0 + native_observed_ns: int = 0 + native_expires_ns: int = 0 + ordinary_safety: tuple = () + clock_equivalence: tuple = () + clock_generation: int = 0 + source_lower_ns: int = 0 + source_upper_ns: int = 0 + source_timestamp_ns: int = 0 + source_clock_domain: str = '' + source_clock_contract: tuple = () + boot_mono_offset_ns: int = 0 + angular_velocity_device: tuple = () + orientation_ned: tuple = () + calibration_rpy: tuple = () + speed_mps: float = 0.0 + steering_angle_deg: float = 0.0 + left_state_ns: int = 0 + right_state_ns: int = 0 + left_state: tuple = () + right_state: tuple = () + + +def _positive_int(value): + return type(value) is int and value > 0 + + +def _hex(value): + return type(value) is str and len(value) == 64 and all(c in '0123456789abcdef' for c in value) + + +def validate_pose(row, now_ns): + if not isinstance(row, OriginalSourcePose): + raise ValueError('typed original pose required') + if (not all(_positive_int(x) for x in (row.mono_ns, row.received_ns, row.source_timestamp_ns, + row.source_state_ns, row.native_apply_ns, row.left_state_ns, row.right_state_ns, row.observed_ns, row.completed_ns, now_ns)) or + not 0 < row.mono_ns <= row.received_ns <= row.observed_ns <= row.completed_ns <= now_ns or now_ns - row.mono_ns > MAX_GAP_NS or + type(row.clock_offset_ns) is not int or type(row.clock_uncertainty_ns) is not int or + not 0 <= row.clock_uncertainty_ns <= 1_000_000 or + row.source_timestamp_ns - row.clock_offset_ns != row.mono_ns or + type(row.source_boot_ns) is not int or type(row.boot_mono_offset_ns) is not int or + row.source_clock_domain not in ('monotonic', 'boottime', 'unknown-equivalent-at-receipt', 'mono_boot_equivalent_interval_v1') or + row.source_boot_ns != (row.source_timestamp_ns if row.source_clock_domain == 'boottime' else 0) or + type(row.source_clock_contract) is not tuple or len(row.source_clock_contract) != 4 or + not row.left_state_ns <= row.mono_ns <= row.right_state_ns <= row.completed_ns or + max(row.mono_ns - row.left_state_ns, row.right_state_ns - row.mono_ns) > MAX_GAP_NS or + not 0 <= row.observed_ns - row.native_apply_ns <= MAX_GAP_NS or + not 0 <= now_ns - row.native_apply_ns <= MAX_GAP_NS or + not 0 <= row.observed_ns - row.source_state_ns <= MAX_GAP_NS or + not 0 <= now_ns - row.source_state_ns <= MAX_GAP_NS or + not all(_hex(x) for x in (row.source_epoch, row.identity, row.calibration, row.calibration_estimate)) or + not row.baseline_revision or not row.controller_revision or type(row.context) is not tuple or not row.context or + row.valid is not True or row.native_allowed is not True or + any(type(x) not in (int, float) or not math.isfinite(x) for x in + (row.speed_mps, row.steering_angle_deg, row.yaw_rate, row.roll_rad, *row.left_state, *row.right_state)) or + type(row.left_state) is not tuple or type(row.right_state) is not tuple or + len(row.left_state) != 2 or len(row.right_state) != 2): + raise ValueError('original source clock/context/brackets/validity') + if type(row.native_required) is not bool: + raise ValueError('original required/ordinary authority branch missing') + if row.native_required: + if (not _hex(row.native_identity) or + not all(_positive_int(value) for value in (row.native_control_ns,row.native_observed_ns,row.native_expires_ns)) or + not row.native_observed_ns <= row.native_control_ns <= row.native_apply_ns <= row.observed_ns <= row.native_expires_ns or + row.native_expires_ns-row.native_observed_ns > 200_000_000): + raise ValueError('required original native identity/lease unavailable') + else: + if (row.native_identity != '' or type(row.native_observed_ns) is not int or + type(row.native_expires_ns) is not int or row.native_observed_ns != 0 or row.native_expires_ns != 0): + raise ValueError('ordinary source must preserve empty native identity and zero axis clocks') + witness = row.ordinary_safety + if (type(witness) is not tuple or len(witness) != 5 or witness[0] != 'ordinary-panda-current-v1' or + not _hex(witness[1]) or not _positive_int(witness[2]) or not 0 <= row.observed_ns-witness[2] <= 1_000_000_000 or + type(witness[3]) is not int or witness[3] < 0 or type(witness[4]) is not tuple or not witness[4]): + raise ValueError('ordinary original CP/Panda current witness unavailable') + contract = OriginalSourceClockContract(*row.source_clock_contract) + contract.validate() + if contract.source_domain != row.source_clock_domain: + raise ValueError('original source clock contract mismatch') + if row.source_clock_domain == 'monotonic' and row.clock_offset_ns != 0: + raise ValueError('MONO source cannot be BOOT adjusted') + if row.source_clock_domain == 'boottime' and row.clock_offset_ns != row.boot_mono_offset_ns: + raise ValueError('BOOT source bridge mismatch') + if row.source_clock_domain == 'unknown-equivalent-at-receipt' and ( + row.clock_offset_ns != 0 or abs(row.boot_mono_offset_ns) > row.clock_uncertainty_ns): + raise ValueError('unknown source requires receipt-specific clock equivalence') + if row.source_clock_domain == 'mono_boot_equivalent_interval_v1': + from openpilot.starpilot.flm.producer_clock_equivalence import ClockEquivalence, source_interval + if type(row.clock_equivalence) is not tuple or len(row.clock_equivalence) != 6: + raise ValueError('original three-sample clock witness missing') + witness = ClockEquivalence(*row.clock_equivalence) + lower, upper = source_interval(row.source_timestamp_ns, row.received_ns, row.observed_ns, witness) + source_interval(row.source_timestamp_ns, row.received_ns, row.completed_ns, witness) + if (row.clock_offset_ns != 0 or row.mono_ns != row.source_timestamp_ns or row.source_boot_ns != 0 or + row.clock_uncertainty_ns != witness.uncertainty_ns or + (row.source_lower_ns, row.source_upper_ns) != (lower, upper) or + not row.left_state_ns <= lower <= upper <= row.right_state_ns or + type(row.clock_generation) is not int or row.clock_generation < 0): + raise ValueError('original clock interval/source bracket mismatch') + for vector in (row.angular_velocity_device, row.orientation_ned, row.calibration_rpy): + if type(vector) is not tuple or len(vector) != 3 or any(type(x) not in (int, float) or not math.isfinite(x) for x in vector): + raise ValueError('original pose/vehicle-frame primitives') + weight = 0.0 if row.left_state_ns == row.right_state_ns else (row.mono_ns - row.left_state_ns) / (row.right_state_ns - row.left_state_ns) + expected = tuple(a + (b - a) * weight for a, b in zip(row.left_state, row.right_state, strict=True)) + if expected != (row.speed_mps, row.steering_angle_deg): + raise ValueError('original state interpolation mismatch') + + +class OriginalSourceCapture: + def __init__(self): + self.rows = [] + self.context = None + self.reason = 'empty' + self.rejected = 0 + self.sealed = False + self.epoch = 0 + self.segments = [] + self.epoch_start_ns = self.epoch_cutoff_ns = 0 + self.parent_epoch_receipt = None + + def reset(self, reason, *, hard=False): + self.reason = reason + if hard: + self.rows.clear() + self.context = None + self.sealed = False + self.segments.clear() + self.epoch += 1 + + def begin_epoch(self, receipt): + """Consume the shared parent's fixed experiment boundary, never fit result. + + The parent must authenticate this boundary from original source records; + this bookkeeping receipt itself grants no native/plant/current authority. + """ + required = {'epoch_id', 'start_ns', 'cutoff_ns', 'identity', 'source_epoch', 'baseline_revision'} + if type(receipt) is not dict or set(receipt) != required or not all(_positive_int(receipt[x]) for x in ('start_ns', 'cutoff_ns')): + raise ValueError('original shared epoch receipt shape') + if (type(receipt['epoch_id']) is not str or not 1 <= len(receipt['epoch_id']) <= 96 or + not 0 < receipt['cutoff_ns'] - receipt['start_ns'] <= MAX_CAPTURE_SPAN_NS or + not _hex(receipt['identity']) or not _hex(receipt['source_epoch']) or not receipt['baseline_revision']): + raise ValueError('original shared epoch receipt bounds') + if self.parent_epoch_receipt == receipt: + return + if self.epoch_cutoff_ns and receipt['start_ns'] < self.epoch_cutoff_ns: + raise ValueError('cannot reselect an existing original experiment') + self.reset('previous original experiment closed unqualified at fixed cutoff', hard=True) + self.epoch_start_ns, self.epoch_cutoff_ns = receipt['start_ns'], receipt['cutoff_ns'] + self.parent_epoch_receipt = dict(receipt) + + def add(self, row, now_ns, *, retained_raw_rows=0): + try: + validate_pose(row, now_ns) + if self.context is not None and row.context != self.context: + self.reset('original source binding changed', hard=True) + if self.rows and row.mono_ns <= self.rows[-1].mono_ns: + raise ValueError('original source duplicate or reorder') + if not self.epoch_start_ns: + self.epoch_start_ns = row.mono_ns + self.epoch_cutoff_ns = row.mono_ns + MAX_CAPTURE_SPAN_NS + if row.mono_ns >= self.epoch_cutoff_ns: + # Outcome-independent source clock cutoff. Do not preserve an expired + # prefix forever after tape eviction or a row-budget freeze. A shared + # parent receipt requires its NEXT authenticated future boundary. + cutoff = self.epoch_cutoff_ns + if self.parent_epoch_receipt is not None: + self.sealed = True + self.reason = 'awaiting next original shared parent epoch receipt' + return False + start = cutoff + ((row.mono_ns - cutoff) // MAX_CAPTURE_SPAN_NS) * MAX_CAPTURE_SPAN_NS + self.reset('expired original experiment discarded unqualified at fixed cutoff', hard=True) + self.epoch_start_ns, self.epoch_cutoff_ns = start, start + MAX_CAPTURE_SPAN_NS + if row.mono_ns < self.epoch_start_ns: + raise ValueError('pose predates original experiment boundary') + receipt = self.parent_epoch_receipt + if receipt is not None and (row.identity, row.source_epoch, row.baseline_revision) != ( + receipt['identity'], receipt['source_epoch'], receipt['baseline_revision']): + raise ValueError('original shared epoch source mismatch') + if self.sealed: + return False + if type(retained_raw_rows) is not int or not 0 <= retained_raw_rows <= MAX_SOURCE_POSES: + raise ValueError('shared raw-row budget count') + if len(self.rows) >= MAX_RETAINED_SOURCE_POSES or len(self.rows) + retained_raw_rows >= MAX_SOURCE_POSES: + # Freeze the original prefix. Never silently rotate/drop rows. + self.sealed = True + self.reason = 'original source row budget frozen' + return False + if not self.rows or row.mono_ns - self.rows[-1].mono_ns > MAX_GAP_NS: + if len(self.segments) >= MAX_PACKET_WINDOWS: + self.sealed = True + self.reason = 'original source segment budget frozen' + return False + self.segments.append(row.mono_ns) + self.context = row.context + self.rows.append(row) + self.reason = 'unqualified original source captured' + return True + except (AttributeError, TypeError, ValueError, OverflowError) as error: + self.rejected += 1 + self.reset(str(error)) + return False + + +def export_packet(adapter, collector, windows, now_ns): + """Whole selected chronology, explicitly bounded and never thinned. + + Native decoding/replay and train/heldout lag qualification remain required in + a separate child. The command tape carries the original native leases and CAN + witnesses; merely exporting those witnesses does not verify the plant. + """ + capture = getattr(collector, 'original_source_capture', None) + if capture is None or not capture.rows: + raise ValueError('original source capture unavailable') + retained_legacy = len(getattr(collector, 'raw_points', ())) + sum(len(block.samples) for block in getattr(collector, 'blocks', ())) + retained_pending = len(getattr(collector, '_original_source_pose_queue', ())) + if len(capture.rows) + retained_legacy + retained_pending > MAX_SOURCE_POSES: + raise ValueError('shared 5120 raw-row retention bound') + if type(windows) is not tuple or len(windows) != 1 or not _positive_int(now_ns): + raise ValueError('original source window bound') + previous = 0 + for window in windows: + if type(window) is not tuple or len(window) != 2: + raise ValueError('original source window shape') + a, b = window + if not all(_positive_int(x) for x in window) or not previous < a < b <= now_ns or now_ns - a > MAX_PACKET_NS: + raise ValueError('original source window clock/order/duration') + previous = b + if windows[-1][1] - windows[0][0] > MAX_PACKET_NS: + raise ValueError('original source span bound') + engine = adapter.replay + if engine.raw_mode is not True: + raise ValueError('original independent capture requires actual raw replay mode') + if engine.failed: + raise ValueError('actual replay recorder failed') + # Single bounded pass over existing records; no new command history buffer. + if len(engine.records) > 60000 or len(engine.emissions.events) > 60000: + raise ValueError('actual recorder retention bound') + frames = tuple(frame for frame in engine.records if any(a <= frame.control_ns <= b for a, b in windows)) + if not 1 <= len(frames) <= MAX_PACKET_FRAMES: + raise ValueError('original source frame bound') + frame_stamps = {frame.control_ns for frame in frames} + if len(frame_stamps) != len(frames): + raise ValueError('actual duplicate frame clock') + for a, b in windows: + selected = tuple(frame for frame in frames if a <= frame.control_ns <= b) + if not selected or selected[0].control_ns != a or selected[-1].control_ns != b: + raise ValueError('original whole source window missing boundary') + for left, right in zip(selected, selected[1:], strict=False): + if not 0 < right.control_ns - left.control_ns <= MAX_GAP_NS or right.frame_sequence != left.frame_sequence + 1: + raise ValueError('actual source frame gap') + checkpoints = tuple((stamp, checkpoint.revision, checkpoint.values) for stamp, checkpoint in engine.checkpoints + if any(a <= stamp <= b for a, b in windows)) + if any(a not in {stamp for stamp, _, _ in checkpoints} for a, _ in windows): + raise ValueError('original source checkpoint prefix required') + cp_bytes = engine.factory.cp.as_builder().to_bytes() + if not 0 < len(cp_bytes) <= 131072: + raise ValueError('original source CarParams bound') + tape = engine.emissions.snapshot(windows) + if not tape['events'] or len(tape['events']) > MAX_PACKET_FRAMES or len(tape['carriers']) > MAX_PACKET_FRAMES: + raise ValueError('actual command tape bound') + last_apply = 0 + for event in tape['events']: + if (event['control_ns'] not in frame_stamps or event['identity'] != adapter.identity or + event['source_epoch'] != adapter.source_epoch or event['baseline_revision'] != adapter.baseline_revision or + event['native_allowed'] is not True or type(event['native_required']) is not bool or + event['native_required'] and not _hex(event['native_identity']) or + not event['native_required'] and (event['native_identity'] != '' or event['native_observed_ns'] != 0 or event['native_expires_ns'] != 0) or + event['valid'] is not True or event['active'] is not True or event['override'] or event['fault'] or + not event['control_ns'] <= event['apply_ns'] <= now_ns or + event['native_required'] and not 0 < event['native_observed_ns'] <= event['control_ns'] <= event['apply_ns'] <= event['native_expires_ns']): + raise ValueError('actual command tape source/native invalid') + if last_apply and event['apply_ns'] <= last_apply: + raise ValueError('actual command tape clock reorder') + last_apply = event['apply_ns'] + rows = tuple(row for row in capture.rows if any(a <= row.mono_ns <= b for a, b in windows)) + if not rows or len(rows) > MAX_SOURCE_POSES: + raise ValueError('original pose row bound') + for row in rows: + # Historical receipt must remain original; validate at bracket completion, + # then enforce separate job age rather than falsely renewing that receipt. + validate_pose(row, row.completed_ns) + if row.completed_ns > now_ns: + raise ValueError('original completion stamp is future to packet export') + if (row.identity != adapter.identity or row.source_epoch != adapter.source_epoch or + row.baseline_revision != adapter.baseline_revision or now_ns - row.mono_ns > MAX_PACKET_NS or + (not row.native_required and row.ordinary_safety[1] != adapter.cp_sha256)): + raise ValueError('original pose source binding') + archive = getattr(adapter, 'original_context_receipts', None) + if archive is None: + raise ValueError('original context-only receipt archive required') + from openpilot.starpilot.flm.normalized_trace_archive import ArchiveBuilder,SOURCE_VERSION + trace_builder = ArchiveBuilder() + context_receipts = archive.export(frames,trace_builder=trace_builder) + payload = {'version': SOURCE_PACKET_VERSION, 'encoding': SOURCE_PACKET_ENCODING, 'capture_epoch': capture.epoch, + 'capture_epoch_start_ns': capture.epoch_start_ns, 'capture_epoch_cutoff_ns': capture.epoch_cutoff_ns, + 'parent_epoch_receipt': capture.parent_epoch_receipt, 'source_segment_starts': tuple(capture.segments), 'qualified': False, + 'plant_lag_ns': None, 'source_windows': (), 'replay_prefix': windows[0], 'source_poses': tuple(asdict(row) for row in rows), + 'context_receipts': context_receipts, + 'actual_frames': tuple(trace_builder.record(frame) for frame in frames), 'actual_command_tape': tape, + 'original_replay_source': {'cp_bytes_hex': cp_bytes.hex(), 'cp_sha256': adapter.cp_sha256, + 'controller_revision': adapter.controller_revision, 'model_revision': engine.model_revision, + 'raw_mode': engine.raw_mode, 'mode': str(engine.factory.mode), + 'turn_assist': engine.factory.turn_assist, 'manual_baseline': engine.factory.manual_baseline, + 'manual_source_epoch': engine.factory.manual_source_epoch, 'checkpoints': checkpoints}, + 'counts': {'source_poses': len(rows), 'frames': len(frames), 'emissions': len(tape['events']), + 'carriers': len(tape['carriers']), 'windows': len(windows), 'span_ns': windows[-1][1] - windows[0][0]}, + 'retained_raw_counts': {'legacy': retained_legacy, 'pending': retained_pending, 'original': len(capture.rows)}, + 'limits': {'source_poses': MAX_RETAINED_SOURCE_POSES, 'shared_raw_rows': MAX_SOURCE_POSES, 'frames': MAX_PACKET_FRAMES, 'windows': MAX_PACKET_WINDOWS, + 'compressed_bytes': MAX_FIT_SNAPSHOT_BYTES, 'raw_bytes': MAX_FIT_RAW_BYTES, 'duration_ns': MAX_PACKET_NS}} + if trace_builder.rows: + payload['version'] = SOURCE_VERSION + payload['trace_archive'] = trace_builder.literal() + feedback_archive = getattr(adapter,'original_feedback_archive',None) + if trace_builder.rows and feedback_archive is None: + raise ValueError('normalized traces require explicit feedback source protocol') + if feedback_archive is not None: + payload['feedback_observations'] = feedback_archive.export(payload) + return pack_source_packet(payload) + + +def _source_primitives(value): + """Lossless known witness schema; classless primitive transport only. + + Context archive asdict preserves TorqueProjection NamedTuple. Its explicit + field dictionary is the existing reversible projection schema; CAN bytes, + Float64 values, tuple/list distinctions and checkpoint tuples remain exact. + No arbitrary class/tag/object serializer is accepted. + """ + from opendbc.car.steering_projection import TorqueProjection, projection_to_dict + kind = type(value) + if isinstance(value, TorqueProjection): + primitive = projection_to_dict(value) + if primitive is None: + raise ValueError('invalid original native projection schema') + return _source_primitives(primitive) + if value is None or kind in (bool,int,str,bytes): + return value + if kind is float: + if not math.isfinite(value): + raise ValueError('nonfinite original source packet') + return value + if kind in (tuple,list): + rows = (_source_primitives(child) for child in value) + return tuple(rows) if kind is tuple else list(rows) + if kind is dict and all(type(key) is str for key in value): + return {key:_source_primitives(child) for key,child in value.items()} + raise ValueError('original source packet requires known primitive/witness types') + + +from openpilot.starpilot.flm.normalized_trace_archive import SOURCE_VERSION as NORMALIZED_SOURCE_VERSION + + +def pack_source_packet(payload): + if (type(payload) is not dict or payload.get('version') not in (SOURCE_PACKET_VERSION, NORMALIZED_SOURCE_VERSION) or + payload.get('encoding') != SOURCE_PACKET_ENCODING or payload.get('qualified') is not False): + raise ValueError('explicit original source packet codec version required') + primitive = _source_primitives(payload) + if 'feedback_observations' in primitive: + from openpilot.starpilot.flm.primitive_source_memo import intern_primitive_graph + from openpilot.starpilot.flm.normalized_trace_archive import reserve_for_packet + reserve = reserve_for_packet(primitive) + primitive = intern_primitive_graph((primitive,bytes(reserve)))[0] if reserve else intern_primitive_graph(primitive) + return pack_primitive_snapshot(primitive) + + +def unpack_source_packet(wire): + payload = unpack_primitive_snapshot(wire) + if (type(payload) is not dict or payload.get('version') not in (SOURCE_PACKET_VERSION, NORMALIZED_SOURCE_VERSION) or + payload.get('encoding') != SOURCE_PACKET_ENCODING or payload.get('qualified') is not False): + raise ValueError('unsupported original source packet codec/version') + if 'feedback_observations' in payload: + from openpilot.starpilot.flm.primitive_source_memo import independent_primitive_graph + from openpilot.starpilot.flm.normalized_trace_archive import reserve_for_packet + return independent_primitive_graph(payload,reserved_bytes=reserve_for_packet(payload)) + return _source_primitives(payload) + + +def source_packet_cost(wire): + """Explicit source-only costs; outer payload+derived replay counts separately.""" + import zlib + unpack_source_packet(wire) + decoder = zlib.decompressobj() + remaining, raw_bytes = wire, 0 + while not decoder.eof: + chunk = decoder.decompress(remaining, 65536) + raw_bytes += len(chunk) + remaining = decoder.unconsumed_tail + if raw_bytes > MAX_FIT_RAW_BYTES or not chunk and not remaining and not decoder.eof: + raise ValueError('original source raw cost bound') + return {'primitive_raw_bytes':raw_bytes, + 'compressed_bytes':len(wire), 'base64_bytes':4*((len(wire)+2)//3), + 'compressed_cap':MAX_FIT_SNAPSHOT_BYTES, 'raw_cap':MAX_FIT_RAW_BYTES} + + +def associate_original_command(events, row, lag_ns): + """Frozen-lag child join at BOTH clock bounds, never a zero-lag plant claim. + + This pure association does no native replay/fit/admission. The child must + reject an ambiguous row/experiment wholesale under its frozen protocol. + """ + if not isinstance(row, OriginalSourcePose) or type(lag_ns) is not int or not 0 <= lag_ns <= 500_000_000: + raise ValueError('typed original pose and frozen plant lag required') + validate_pose(row, row.completed_ns) + if row.source_clock_domain == 'mono_boot_equivalent_interval_v1': + lower, upper = row.source_lower_ns - lag_ns, row.source_upper_ns - lag_ns + else: + lower = upper = row.mono_ns - lag_ns + if type(events) not in (tuple, list) or not 1 <= len(events) <= MAX_PACKET_FRAMES: + raise ValueError('bounded original command tape required') + previous = 0 + left = right = None + for event in events: + stamp = event['apply_ns'] if type(event) is dict else event.apply_ns + if type(stamp) is not int or stamp <= previous: + raise ValueError('original command tape clock order') + previous = stamp + if stamp <= lower: + left = event + if stamp <= upper: + right = event + if left is None or right is None: + raise ValueError('original command tape source interval unavailable') + stamp_left = left['apply_ns'] if type(left) is dict else left.apply_ns + stamp_right = right['apply_ns'] if type(right) is dict else right.apply_ns + if stamp_left != stamp_right or not 0 <= upper - stamp_right <= 30_000_000: + raise ValueError('ambiguous or stale original command at source interval bounds') + return left diff --git a/openpilot/starpilot/flm/physical_input_observation.py b/openpilot/starpilot/flm/physical_input_observation.py new file mode 100644 index 0000000000..378139e99d --- /dev/null +++ b/openpilot/starpilot/flm/physical_input_observation.py @@ -0,0 +1,374 @@ +"""Physical-only observed inputs. No profile, candidate or current authority. + +Records are immutable child-derived evidence/tamper bindings, NOT certificates +that a caller executed native verification. Wire callers cannot supply these as +proof: the actual child recomputes them from original replay/context receipts. +Performance tags never replace original bodies or hard source/native guards. +""" +from dataclasses import asdict, dataclass, replace, fields +import hashlib +import json +import math +import re +from openpilot.starpilot.flm.native_primitives import try_native as _try_native, FALLBACK as _NATIVE_FALLBACK + +PROTOCOL = 'verified-delivered-physical-observation-v1' +_LOWER_HEX = re.compile('[0-9a-f]*') + + +def _hash(value): + return type(value) is str and len(value) == 64 and _LOWER_HEX.fullmatch(value) is not None + + +def _typed_body(item): + """Original exact primitive transform; no metadata or permission shortcuts.""" + kind = type(item) + if item is None or kind in (bool, int, str): + return [kind.__name__, item] + if kind is float and math.isfinite(item): + return ['float64', item.hex()] + if kind is bytes: + return ['bytes', item.hex()] + if kind in (tuple, list): + return [kind.__name__, [_typed_body(child) for child in item]] + if kind is dict and all(type(key) is str for key in item): + return ['dict', [[key, _typed_body(item[key])] for key in sorted(item)]] + raise ValueError('physical exact original body primitive') + + +def _body_digest_original(value): + """Bit/type exact original primitives, including CAN bytes and signed zero.""" + return hashlib.sha256(json.dumps(_typed_body(value), separators=(',', ':'), allow_nan=False).encode()).hexdigest() + + +def _exact_primitive_body(value): + """Validate before serialization, matching original rejection traversal.""" + kind=type(value) + if value is None or kind in (bool,int,str,bytes): + return True + if kind is float: + return math.isfinite(value) + if kind in (tuple,list): + return all(_exact_primitive_body(child) for child in value) + if kind is dict and all(type(key) is str for key in value): + return all(_exact_primitive_body(value[key]) for key in sorted(value)) + return False + + +def _large_sequence(value, depth=0): + """Only size selection. Never an integrity, source or type-validation claim.""" + if type(value) in (tuple,list): + return len(value)>256 + if type(value) is dict and depth<3: + return any(_large_sequence(child,depth+1) for child in value.values()) + return False + + +def _typed_json_chunks(value): + """Byte-identical typed JSON, batching whole rows through the C encoder.""" + kind=type(value) + if kind in (tuple,list) and len(value)>256: + yield '["tuple",[' if kind is tuple else '["list",[' + for start in range(0,len(value),64): + if start: + yield ',' + batch=[_typed_body(child) for child in value[start:start+64]] + yield json.dumps(batch,separators=(',', ':'),allow_nan=False)[1:-1] + yield ']]' + elif kind is dict and _large_sequence(value): + yield '["dict",[' + for index,key in enumerate(sorted(value)): + if index: + yield ',' + yield '['+json.dumps(key,separators=(',', ':'),allow_nan=False)+',' + yield from _typed_json_chunks(value[key]) + yield ']' + yield ']]' + else: + yield json.dumps(_typed_body(value),separators=(',', ':'),allow_nan=False) + + +def _body_digest_python(value): + """Same exact digest; ordinary row bodies use one original C JSON encoding.""" + if not _large_sequence(value): + return _body_digest_original(value) + try: + supported=_exact_primitive_body(value) + except RecursionError: + return _body_digest_original(value) + if not supported: + return _body_digest_original(value) + digest=hashlib.sha256() + for chunk in _typed_json_chunks(value): + digest.update(chunk.encode()) + return digest.hexdigest() + + +class _UnsupportedPhysicalBody(Exception): + pass + + +def _physical_asdict(value, *, _trace_owner=None, **overrides): + from openpilot.starpilot.flm.normalized_trace_archive import TraceRef,record_body + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence + if type(value) is ControlEvidence and type(value.effect_trace) is TraceRef: + if _trace_owner is None: + raise ValueError('normalized physical record requires literal owner') + return {**record_body(value,_trace_owner),**overrides} + from openpilot.starpilot.flm.torque_emission_replay import Emission + from opendbc.car.steering_projection import TorqueProjection + registered = (ControlEvidence, AppliedEvidence, Emission, OriginalPerformanceTags, PhysicalDeliveredInput) + if type(value) not in registered or type(value) is ControlEvidence and value.effect_trace is not None: + return {**asdict(value), **overrides} + + def convert(item): + kind = type(item) + if item is None or kind in (bool, int, float, str, bytes): + return item + if kind is TorqueProjection: + return TorqueProjection(*(convert(child) for child in item)) + if kind is tuple: + return tuple(convert(child) for child in item) + if kind is list: + return [convert(child) for child in item] + if kind is dict: + return {convert(key): convert(child) for key, child in item.items()} + if kind in registered: + return {field.name: convert(getattr(item, field.name)) for field in fields(item)} + raise _UnsupportedPhysicalBody + + try: + result = {field.name: convert(getattr(value, field.name)) for field in fields(value)} + result.update(overrides) + return result + except _UnsupportedPhysicalBody: + return {**asdict(value), **overrides} + + +def _copy_callback_free(value): + from opendbc.car.steering_projection import TorqueProjection + kind = type(value) + if value is None or kind in (bool, int, float, str, bytes): + return True + if kind in (tuple, list, TorqueProjection): + return all(_copy_callback_free(child) for child in value) + if kind is dict: + return all(_copy_callback_free(key) and _copy_callback_free(child) for key, child in value.items()) + return False + + +def _applied_body_hashes(output, original_output, witness) -> tuple[str, str]: + from openpilot.starpilot.flm.fullrate_collector import AppliedEvidence + from openpilot.starpilot.flm.torque_emission_replay import projection + from opendbc.car.steering_projection import projection_to_dict, TorqueProjection + original_body = _physical_asdict(original_output) + original_body['projection'] = projection_to_dict(projection(original_output.projection)) + original = body_digest(original_body) + if (type(output) is AppliedEvidence and output is original_output and output.projection is witness and + type(witness) is TorqueProjection and all(_copy_callback_free(getattr(output, field.name)) for field in fields(output))): + return original, original + effective_body = _physical_asdict(output) + effective_body['projection'] = projection_to_dict(witness) + return original, body_digest(effective_body) + + +def _emission_body_hash(event): + from opendbc.car.steering_projection import projection_to_dict + body = _physical_asdict(event) + body['witness'] = projection_to_dict(event.witness) + return body_digest(body) + + +def body_digest(value) -> str: + result = _try_native('whole_body_digest', value) + return _body_digest_python(value) if result is _NATIVE_FALLBACK else result + + +@dataclass(frozen=True) +class OriginalPerformanceTags: + control_limited: bool + output_limited: bool + controller_saturated: bool + model_goal_wobble: bool + controller_chatter: bool + + +@dataclass(frozen=True) +class PhysicalDeliveredInput: + """Distinct from strict DeliveredCommand; has no permissive health defaults.""" + protocol: str + control_ns: int + state_ns: int + output_state_ns: int + apply_ns: int + command: float + source_receipt_ns: int + native_observed_ns: int + native_expires_ns: int + carrier_frame: int + carrier_observed_ns: int + encoded_witness_sha256: str + control_sha256: str + original_applied_sha256: str + effective_applied_sha256: str + actual_emission_sha256: str + identity: str + generation: int + source_epoch: str + baseline_revision: str + controller_revision_sha256: str + native_identity: str + native_required: bool + limiter_kind: str + tags: OriginalPerformanceTags + disposition: str + binding_digest: str + observation_digest: str + + +@dataclass(frozen=True) +class VerifiedPhysicalTape: + protocol: str + commands: tuple[PhysicalDeliveredInput, ...] + binding_digest: str + verified_tape_digest: str + verifier_source_sha256: str + + +def observation_digest(row): + value = _physical_asdict(row) + value.pop('observation_digest') + return body_digest(value) + + +def physical_tape_digest(binding, commands): + return body_digest({'protocol': PROTOCOL, 'binding': asdict(binding), + 'commands': [_physical_asdict(row) for row in commands]}) + + +def validate_input(row, binding, binding_digest): + """Pure structural/custody guard; not a native/source proof executor.""" + if type(row) is not PhysicalDeliveredInput or type(row.tags) is not OriginalPerformanceTags: + raise ValueError('explicit physical-only input type') + if row.protocol != PROTOCOL or any(type(value) is not bool for value in asdict(row.tags).values()): + raise ValueError('physical-only protocol and exact original boolean tags') + expected = 'observed_input_performance_tagged' if any(asdict(row.tags).values()) else 'observed_input_clean' + integer_fields = (row.control_ns, row.state_ns, row.output_state_ns, row.apply_ns, + row.source_receipt_ns, row.carrier_frame, row.carrier_observed_ns) + if (any(type(value) is not int or value <= 0 for value in integer_fields) or + not row.state_ns <= row.control_ns <= row.state_ns + 30_000_000 or + not row.state_ns <= row.output_state_ns <= row.apply_ns or + not row.control_ns <= row.apply_ns <= row.control_ns + 30_000_000 or + row.apply_ns - row.output_state_ns > 30_000_000 or + not row.source_receipt_ns <= row.control_ns <= row.source_receipt_ns + 500_000_000 or + not row.apply_ns <= row.carrier_observed_ns <= row.apply_ns + 100_000_000 or + type(row.command) not in (int, float) or not math.isfinite(row.command) or abs(row.command) > 1 or + type(row.generation) is not int or row.generation < 0 or + type(row.native_required) is not bool or row.native_required is not binding.native_required or + any(type(value) is not int for value in (row.native_observed_ns, row.native_expires_ns)) or + (not 0 < row.native_observed_ns <= row.control_ns <= row.apply_ns <= row.native_expires_ns or + row.native_expires_ns - row.native_observed_ns > 200_000_000 if row.native_required else + (row.native_observed_ns, row.native_expires_ns) != (0, 0)) or + (row.identity, row.source_epoch, row.baseline_revision, row.controller_revision_sha256, row.native_identity) != + (binding.identity, binding.source_epoch, binding.baseline_revision, binding.controller_revision, binding.native_identity) or + row.binding_digest != binding_digest or row.disposition != expected or + row.limiter_kind not in ('driver', 'measured', 'normalized_rate') or + any(not _hash(value) for value in (row.encoded_witness_sha256, row.control_sha256, + row.original_applied_sha256, row.effective_applied_sha256, row.actual_emission_sha256, + row.binding_digest, row.observation_digest)) or row.observation_digest != observation_digest(row)): + raise ValueError('physical original clocks/leases/identity/CAN/tag custody') + + +def from_original(control, output, engine, binding, *, original_output): + """Actual child boundary. Performance labels preserved; hard guards remain. + + Must be called after original_context_receipts.verify. Never accepts caller + callbacks, saved qualified flags, strict-record conversions or fake engines. + """ + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence + from openpilot.starpilot.flm.controller_replay import ControllerReplay + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current + from openpilot.starpilot.flm.torque_emission_replay import delivered, projection + from openpilot.starpilot.flm.runtime_adapter_registry import digest + from openpilot.starpilot.flm.plant_lag import SourceBinding, binding_digest + from opendbc.car.steering_projection import projection_to_dict + + if (type(control) is not ControlEvidence or type(output) is not AppliedEvidence or + type(original_output) is not AppliedEvidence or type(binding) is not SourceBinding or + type(engine) not in (ControllerReplay, EventControllerReplay) or + type(engine) is EventControllerReplay and type(engine.engine) is not ControllerReplay): + raise ValueError('original physical source/actual replay types required') + flags = (control.enabled, control.manual, control.active, control.valid, control.override, + control.fault, control.saturated, control.limited, control.model_wobble, control.controller_chatter, + control.native_required, control.native_allowed, output.valid, output.active, output.override, + output.fault, output.limited, output.native_allowed, original_output.native_allowed) + if (any(type(value) is not bool for value in flags) or engine.failed or not engine.raw_mode or + not control.enabled or control.manual or not control.active or not control.valid or + control.override or control.fault or any(control.applied_coordinates) or + not output.valid or not output.active or output.override or output.fault or not output.native_allowed or + output.domain != 'normalized_torque' or control.domain != output.domain or + output.control_ns != control.control_ns or not consumed_sources_current(control, output)): + raise ValueError('physical hard validity/override/fault/neutral/source-consumption guard') + stamps = (control.source_receipt_ns, control.native_observed_ns, control.native_expires_ns, + output.carrier_frame, output.carrier_observed_ns) + if (any(type(value) is not int for value in stamps) or + type(control.generation) is not int or control.generation < 0 or + not control.native_required and control.native_allowed or + not 0 < control.source_receipt_ns <= control.control_ns <= control.source_receipt_ns + 500_000_000 or + output.carrier_frame <= 0 or not output.apply_ns <= output.carrier_observed_ns <= output.apply_ns + 100_000_000 or + (not control.native_allowed or not 0 < control.native_observed_ns <= control.control_ns <= output.apply_ns <= control.native_expires_ns or + control.native_expires_ns - control.native_observed_ns > 200_000_000 if control.native_required else + (control.native_observed_ns, control.native_expires_ns) != (0, 0))): + raise ValueError('physical original source/native lease/carrier guard') + if output != original_output and (not control.native_required or + output != replace(original_output, native_allowed=True)): + raise ValueError('physical original applied body changed') + # verify_original_replay installs this map only after the actual original + # controller/native/context replay. A real class plus a truthy method is not + # by itself proof that this particular ControlEvidence was in that replay. + originals = getattr(engine, 'original_context_controls', None) + original_control = originals.get(control.control_ns) if type(originals) is dict else None + control_body_sha256 = body_digest(_physical_asdict(control,_trace_owner=engine)) + if (type(original_control) is not ControlEvidence or + original_control is not control and body_digest(_physical_asdict(original_control,_trace_owner=engine)) != control_body_sha256): + raise ValueError('physical verified original context owner body required') + if (control.native_required and not control.native_allowed or + engine.identity != binding.identity or engine.baseline_revision != binding.baseline_revision or + engine._recorded_cp_digest() != binding.cp_sha256 or + digest(control.controller_revision) != binding.controller_revision or + (control.identity, control.source_epoch, control.baseline_revision, control.native_identity) != + (binding.identity, binding.source_epoch, binding.baseline_revision, binding.native_identity)): + raise ValueError('physical original selected/native identity binding') + witness = projection(output.projection) + if (projection_to_dict(witness) is None or not witness.steer_requested or + witness.requested_torque != control.requested or witness.reported_torque != output.applied or + witness.family != str(engine.factory.cp.brand) or + witness.fingerprint != str(engine.factory.cp.carFingerprint) or witness.flags != int(engine.factory.cp.flags) or + not engine.applied_matches(control, output)): + raise ValueError('physical actual native/CAN/limiter-state proof') + # applied_matches just ran the actual decoder/encoder proof. Bind that exact + # archived event without executing a second limiter replay for every input. + event = engine.emissions.events.get(output.apply_ns) + if (event is None or (event.control_ns, event.state_ns, event.apply_ns) != + (control.control_ns, output.state_ns, output.apply_ns) or event.witness != witness or + event.limited is not output.limited or + event.native_required is not control.native_required or + (event.native_identity, event.native_observed_ns, event.native_expires_ns) != + (control.native_identity, control.native_observed_ns, control.native_expires_ns) or + not event.valid or not event.active or event.override or event.fault or not event.native_allowed): + raise ValueError('physical exact actual emission native owner') + tags = OriginalPerformanceTags(control.limited, original_output.limited, control.saturated, + control.model_wobble, control.controller_chatter) + row = PhysicalDeliveredInput(PROTOCOL, control.control_ns, control.state_ns, output.state_ns, + output.apply_ns, -delivered(output), control.source_receipt_ns, control.native_observed_ns, + control.native_expires_ns, output.carrier_frame, output.carrier_observed_ns, + body_digest(projection_to_dict(witness)), control_body_sha256, + *_applied_body_hashes(output, original_output, witness), + _emission_body_hash(event), control.identity, control.generation, control.source_epoch, + control.baseline_revision, digest(control.controller_revision), control.native_identity, control.native_required, + witness.limiter_kind, tags, 'observed_input_performance_tagged' if any(asdict(tags).values()) else + 'observed_input_clean', binding_digest(binding), '') + row = replace(row, observation_digest=observation_digest(row)) + validate_input(row, binding, binding_digest(binding)) + return row diff --git a/openpilot/starpilot/flm/pid_controller_replay.py b/openpilot/starpilot/flm/pid_controller_replay.py index 12afa96672..bb943c8b08 100644 --- a/openpilot/starpilot/flm/pid_controller_replay.py +++ b/openpilot/starpilot/flm/pid_controller_replay.py @@ -127,4 +127,5 @@ def pid_frame(evidence): desired_kinematic_accel=stage['setpoint'], healthy=evidence.valid and not evidence.fault and not evidence.override, manual=evidence.manual, limited=evidence.limited, applied_parameter_names=tuple(evidence.applied_parameter_names), effect_trace=evidence.effect_trace, - effect_prepare_ns=evidence.effect_prepare_ns, controller_domain='torque_pid') + effect_prepare_ns=evidence.effect_prepare_ns, controller_domain='torque_pid', + vehicle_model_anchor=evidence.vehicle_model_anchor, live_applied_coordinates=evidence.live_applied_coordinates) diff --git a/openpilot/starpilot/flm/plant_lag.py b/openpilot/starpilot/flm/plant_lag.py new file mode 100644 index 0000000000..55c8a6c7d4 --- /dev/null +++ b/openpilot/starpilot/flm/plant_lag.py @@ -0,0 +1,570 @@ +"""PRIVATE source-only candidate. Pure arithmetic over caller-verified evidence. + +This module DOES NOT verify CAN/native signatures, controller replay, profile +admission or persisted authority. The real child must recompute those before +constructing VerifiedCommandTape and again bind the result to its activation +and current-admission proofs. A qualified result is only an affine plant test. +No product imports, file I/O, controllers, executable callbacks or optimization +against heldout data. Protocol constants cannot be passed as tuning arguments. +""" +from bisect import bisect_right +from dataclasses import asdict, dataclass +import hashlib +import json +import math + +PROTOCOL = 'source-pose-plant-lag-v4-private' +PHYSICAL_PROTOCOL = 'source-pose-physical-observation-lag-v1-private' +LAGS_NS = tuple(range(0, 500_000_001, 10_000_000)) +JOIN_NS = 30_000_000 +MIN_EXCITATION = .05 +MAX_RMSE = .05 +MAX_COMMANDS = 12_000 +MAX_POSES = 8 * 240 +TRAIN_WINDOWS = 4 +NUM_WINDOWS = 8 +STRICT_SUPPORT = 'strict-heldout-domain-v1' +TRAINING_DOMAIN_ONLY = 'training-domain-only-heldout-v1' +LAG_STABILITY_NS = 20_000_000 +TIE_ABS_MSE = 1e-12 + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest() + + +def _hash(value): + return type(value) is str and len(value) == 64 and all(c in '0123456789abcdef' for c in value) + + +def _integer(value, *, zero=False): + return type(value) is int and (value >= 0 if zero else value > 0) + + +def _number(value): + return type(value) in (int, float) and math.isfinite(value) + + +@dataclass(frozen=True) +class SourceBinding: + # All hashes are independently computed by the actual child, never read as + # authority from parent saved healthy/normalized flags. + identity: str + cp_sha256: str + source_sha256: str + controller_revision: str + source_epoch: str + baseline_revision: str + native_identity: str + calibration_generation: str + vehicle_model_generation: str + session: str + source_clock_contract_sha256: str + native_required: bool + timestamp_convention: str = 'original-kf-bounded-source-clock-v3' + + +@dataclass(frozen=True) +class DeliveredCommand: + control_ns: int + state_ns: int + apply_ns: int + # Oriented coordinate MUST be -delivered(output) from original CAN verifier. + command: float + source_receipt_ns: int + native_observed_ns: int + native_expires_ns: int + carrier_frame: int + carrier_observed_ns: int + encoded_witness_sha256: str + control_sha256: str + applied_sha256: str + identity: str + source_epoch: str + baseline_revision: str + native_identity: str + native_required: bool = True + healthy: bool = True + neutral_coordinates: bool = True + native_allowed: bool = True + overridden: bool = False + saturated: bool = False + limited: bool = False + + +@dataclass(frozen=True) +class VerifiedCommandTape: + commands: tuple[DeliveredCommand, ...] + binding_digest: str + # Must equal command_tape_digest(binding, commands). This is tamper binding, + # not a cryptographic certificate that caller actually ran a verifier. + verified_tape_digest: str + verifier_source_sha256: str + verified_source_native_can: bool + + +@dataclass(frozen=True) +class SourcePose: + pose_id: str + mono_ns: int + received_ns: int + source_timestamp_ns: int + source_clock_domain: str + clock_offset_ns: int + clock_uncertainty_ns: int + source_lower_ns: int + source_upper_ns: int + clock_generation: int + speed_mps: float + calibrated_yaw_rate: float + calibrated_roll_rad: float + binding_digest: str + healthy: bool = True + native_allowed_at_receipt: bool = True + overridden: bool = False + fault: bool = False + + +@dataclass(frozen=True) +class CalibrationWindow: + # Caller recomputes original whole source-window boundaries independently. + block_id: str + original_start_ns: int + original_end_ns: int + pose_ids: tuple[str, ...] + source_window_sha256: str + + +@dataclass(frozen=True) +class Pair: + block_id: str + pose_id: str + command_index: int + command_ns: int + response_ns: int + command: float + response: float + + +@dataclass(frozen=True) +class LagDecision: + protocol: str + lag_ns: int + slope: float + intercept: float + command_low: float + command_high: float + train_block_ids: tuple[str, ...] + train_pose_ids: tuple[str, ...] + train_source_digest: str + training_pairs_digest: str + training_scores: tuple[tuple[int, float | None], ...] + training_leave_one_out_lags: tuple[int, ...] + # Contains NO heldout response primitives, scores or outcome. + digest: str + + +@dataclass(frozen=True) +class IdentificationResult: + status: str + reason: str + binding_digest: str + experiment_digest: str + decision: LagDecision | None + rows: tuple[Pair, ...] + heldout_blocks: tuple[tuple[str, float, int], ...] + model_receipt_digest: str | None + cutoff_ns: int | None + support_policy: str = STRICT_SUPPORT + + +def binding_digest(binding): + return digest(asdict(binding)) + + +def command_tape_digest(binding, commands): + return digest({'protocol': PROTOCOL, 'binding': asdict(binding), 'commands': [asdict(c) for c in commands]}) + + +def _validate(binding, tape, poses, windows): + if not isinstance(binding, SourceBinding) or not isinstance(tape, VerifiedCommandTape): + raise ValueError('explicit source binding and independently verified command tape required') + fields = asdict(binding) + if (any(not _hash(value) for key, value in fields.items() if key not in ('session', 'timestamp_convention', 'native_required', 'native_identity')) + or type(binding.native_required) is not bool + or (not _hash(binding.native_identity) if binding.native_required else binding.native_identity != '') + or type(binding.session) is not str or not binding.session + or binding.timestamp_convention != 'original-kf-bounded-source-clock-v3'): + raise ValueError('source/native/CP/calibration/model binding') + bound = binding_digest(binding) + if (type(tape.commands) is not tuple or not 1 <= len(tape.commands) <= MAX_COMMANDS + or tape.binding_digest != bound or not _hash(tape.verifier_source_sha256) + or tape.verified_source_native_can is not True + or tape.verified_tape_digest != command_tape_digest(binding, tape.commands)): + raise ValueError('exact independently verified tape binding/bounds') + previous = 0 + previous_carrier = 0 + for c in tape.commands: + if not isinstance(c, DeliveredCommand): + raise ValueError('actual delivered command type') + integers = (c.control_ns, c.state_ns, c.apply_ns, c.source_receipt_ns, + c.carrier_frame, c.carrier_observed_ns) + if (not all(_integer(v) for v in integers) or c.apply_ns <= previous or + not 0 < c.state_ns <= c.control_ns <= c.state_ns + 30_000_000 or + not c.control_ns <= c.apply_ns <= c.control_ns + 100_000_000 or + not 0 < c.source_receipt_ns <= c.control_ns <= c.source_receipt_ns + 500_000_000 or + c.native_required is not binding.native_required or + (not all(type(v) is int for v in (c.native_observed_ns,c.native_expires_ns))) or + (not 0 < c.native_observed_ns <= c.control_ns <= c.apply_ns <= c.native_expires_ns or + c.native_expires_ns-c.native_observed_ns > 200_000_000 if c.native_required else + (c.native_observed_ns,c.native_expires_ns) != (0,0)) or + not c.apply_ns <= c.carrier_observed_ns <= c.apply_ns + 100_000_000 or + previous_carrier and c.apply_ns - previous <= JOIN_NS and c.carrier_frame not in (previous_carrier, previous_carrier + 1) or + not _number(c.command) or abs(c.command) > 1 or + any(not _hash(v) for v in (c.encoded_witness_sha256, c.control_sha256, c.applied_sha256)) or + (c.identity, c.source_epoch, c.baseline_revision, c.native_identity) != + (binding.identity, binding.source_epoch, binding.baseline_revision, binding.native_identity) or + any(v is not True for v in (c.healthy, c.neutral_coordinates, c.native_allowed)) or + any(v is not False for v in (c.overridden, c.saturated, c.limited))): + raise ValueError('actual source/native/receipt/clock/neutral/unclipped command guard') + previous = c.apply_ns + previous_carrier = c.carrier_frame + if type(poses) is not tuple or not 1 <= len(poses) <= MAX_POSES or type(windows) is not tuple or len(windows) != NUM_WINDOWS: + raise ValueError('exact eight calibration windows / bounded original poses') + previous = 0 + pose_ids = set() + for p in poses: + if not isinstance(p, SourcePose): + raise ValueError('unpaired independent source pose type') + if (not _hash(p.pose_id) or p.pose_id in pose_ids or + not all(_integer(v) for v in (p.mono_ns, p.received_ns, p.source_timestamp_ns)) or + type(p.clock_offset_ns) is not int or not _integer(p.clock_uncertainty_ns, zero=True) or + p.clock_uncertainty_ns > 1_000_000 or p.source_timestamp_ns - p.clock_offset_ns != p.mono_ns or + p.source_clock_domain not in ('monotonic', 'boottime', 'unknown-equivalent-at-receipt', 'mono_boot_equivalent_interval_v1') or + p.source_clock_domain != 'boottime' and p.clock_offset_ns != 0 or + not all(_integer(v) for v in (p.source_lower_ns,p.source_upper_ns)) or + not _integer(p.clock_generation,zero=True) or + not p.source_lower_ns <= p.mono_ns <= p.source_upper_ns or + max(p.mono_ns-p.source_lower_ns,p.source_upper_ns-p.mono_ns) > p.clock_uncertainty_ns or + not p.mono_ns <= p.received_ns <= p.mono_ns + 100_000_000 or p.mono_ns <= previous or + not all(_number(v) for v in (p.speed_mps, p.calibrated_yaw_rate, p.calibrated_roll_rad)) or + p.speed_mps <= 0 or p.binding_digest != bound or + p.clock_generation != poses[0].clock_generation or + any(v is not True for v in (p.healthy, p.native_allowed_at_receipt)) or + any(v is not False for v in (p.overridden, p.fault))): + raise ValueError('immutable source pose/clock/native/calibration guard') + previous = p.mono_ns + pose_ids.add(p.pose_id) + if poses[-1].mono_ns - poses[0].mono_ns + LAGS_NS[-1] > 600_000_000_000: + raise ValueError('original maximum 600 second physical history including candidate lookback') + mapped = {p.pose_id: p for p in poses} + assigned = [] + block_ids = set() + previous = 0 + for w in windows: + if (not isinstance(w, CalibrationWindow) or not _hash(w.block_id) or w.block_id in block_ids or + not _hash(w.source_window_sha256) or not _integer(w.original_start_ns) or not _integer(w.original_end_ns) or + not previous < w.original_start_ns <= w.original_end_ns or type(w.pose_ids) is not tuple or + not 40 <= len(w.pose_ids) <= 240 or any(pid not in mapped for pid in w.pose_ids)): + raise ValueError('original fixed chronological whole-window declaration') + selected = [mapped[pid] for pid in w.pose_ids] + if (tuple(sorted(selected, key=lambda p: p.mono_ns)) != tuple(selected) or + any(not w.original_start_ns <= p.mono_ns <= w.original_end_ns for p in selected)): + raise ValueError('pose outside original declared source window/order') + assigned.extend(w.pose_ids) + block_ids.add(w.block_id) + previous = w.original_end_ns + if assigned != [p.pose_id for p in poses] or len(set(assigned)) != len(assigned): + raise ValueError('all original poses must be assigned exactly once without dropping/reordering') + return bound + + +def _validate_physical(binding, tape, poses, windows): + from openpilot.starpilot.flm.physical_input_observation import ( + PROTOCOL as OBSERVATION_PROTOCOL, VerifiedPhysicalTape, physical_tape_digest, validate_input, + ) + if type(binding) is not SourceBinding or type(tape) is not VerifiedPhysicalTape: + raise ValueError('explicit physical-only binding and tape required') + fields = asdict(binding) + if (any(not _hash(value) for key, value in fields.items() if key not in ('session', 'timestamp_convention', 'native_required', 'native_identity')) + or type(binding.native_required) is not bool + or (not _hash(binding.native_identity) if binding.native_required else binding.native_identity != '') + or type(binding.session) is not str or not binding.session + or binding.timestamp_convention != 'original-kf-bounded-source-clock-v3'): + raise ValueError('source/native/CP/calibration/model binding') + bound = binding_digest(binding) + if (tape.protocol != OBSERVATION_PROTOCOL or type(tape.commands) is not tuple or + not 1 <= len(tape.commands) <= MAX_COMMANDS or tape.binding_digest != bound or + not _hash(tape.verifier_source_sha256) or + tape.verified_tape_digest != physical_tape_digest(binding, tape.commands)): + raise ValueError('physical-only complete tape custody/bounds') + previous, previous_carrier = 0, 0 + for command in tape.commands: + validate_input(command, binding, bound) + if (command.generation != tape.commands[0].generation or command.apply_ns <= previous or + previous_carrier and command.apply_ns - previous <= JOIN_NS and + command.carrier_frame not in (previous_carrier, previous_carrier + 1)): + raise ValueError('physical original FIFO/carrier chronology') + previous, previous_carrier = command.apply_ns, command.carrier_frame + if type(poses) is not tuple or not 1 <= len(poses) <= MAX_POSES or type(windows) is not tuple or len(windows) != NUM_WINDOWS: + raise ValueError('exact eight calibration windows / bounded original poses') + previous = 0 + pose_ids = set() + for p in poses: + if not isinstance(p, SourcePose): + raise ValueError('unpaired independent source pose type') + if (not _hash(p.pose_id) or p.pose_id in pose_ids or + not all(_integer(v) for v in (p.mono_ns, p.received_ns, p.source_timestamp_ns)) or + type(p.clock_offset_ns) is not int or not _integer(p.clock_uncertainty_ns, zero=True) or + p.clock_uncertainty_ns > 1_000_000 or p.source_timestamp_ns - p.clock_offset_ns != p.mono_ns or + p.source_clock_domain not in ('monotonic', 'boottime', 'unknown-equivalent-at-receipt', 'mono_boot_equivalent_interval_v1') or + p.source_clock_domain != 'boottime' and p.clock_offset_ns != 0 or + not all(_integer(v) for v in (p.source_lower_ns,p.source_upper_ns)) or + not _integer(p.clock_generation,zero=True) or + not p.source_lower_ns <= p.mono_ns <= p.source_upper_ns or + max(p.mono_ns-p.source_lower_ns,p.source_upper_ns-p.mono_ns) > p.clock_uncertainty_ns or + not p.mono_ns <= p.received_ns <= p.mono_ns + 100_000_000 or p.mono_ns <= previous or + not all(_number(v) for v in (p.speed_mps, p.calibrated_yaw_rate, p.calibrated_roll_rad)) or + p.speed_mps <= 0 or p.binding_digest != bound or + p.clock_generation != poses[0].clock_generation or + any(v is not True for v in (p.healthy, p.native_allowed_at_receipt)) or + any(v is not False for v in (p.overridden, p.fault))): + raise ValueError('immutable source pose/clock/native/calibration guard') + previous = p.mono_ns + pose_ids.add(p.pose_id) + if poses[-1].mono_ns - poses[0].mono_ns + LAGS_NS[-1] > 600_000_000_000: + raise ValueError('original maximum 600 second physical history including candidate lookback') + mapped = {p.pose_id: p for p in poses} + assigned = [] + block_ids = set() + previous = 0 + for w in windows: + if (not isinstance(w, CalibrationWindow) or not _hash(w.block_id) or w.block_id in block_ids or + not _hash(w.source_window_sha256) or not _integer(w.original_start_ns) or not _integer(w.original_end_ns) or + not previous < w.original_start_ns <= w.original_end_ns or type(w.pose_ids) is not tuple or + not 40 <= len(w.pose_ids) <= 240 or any(pid not in mapped for pid in w.pose_ids)): + raise ValueError('original fixed chronological whole-window declaration') + selected = [mapped[pid] for pid in w.pose_ids] + if (tuple(sorted(selected, key=lambda p: p.mono_ns)) != tuple(selected) or + any(not w.original_start_ns <= p.mono_ns <= w.original_end_ns for p in selected)): + raise ValueError('pose outside original declared source window/order') + assigned.extend(w.pose_ids) + block_ids.add(w.block_id) + previous = w.original_end_ns + if assigned != [p.pose_id for p in poses] or len(set(assigned)) != len(assigned): + raise ValueError('all original poses must be assigned exactly once without dropping/reordering') + return bound + + +def _join_all(tape, poses, windows): + """Common support uses timestamps/command evidence only, never response y.""" + stamps = [c.apply_ns for c in tape.commands] + blocks = {pid: w.block_id for w in windows for pid in w.pose_ids} + joined = {} + invariants = {} if all(type(p) is SourcePose for p in poses) else None + for lag in LAGS_NS: + pairs = [] + previous_command = 0 + for pose_index, p in enumerate(poses): + _target = p.mono_ns - lag + index = bisect_right(stamps, p.source_lower_ns-lag) - 1 + upper_index = bisect_right(stamps, p.source_upper_ns-lag) - 1 + if index < 0 or index != upper_index or p.source_upper_ns-lag-stamps[index] > JOIN_NS or stamps[index] <= previous_command: + # Entire experiment rejects rather than drop this original row at a lag. + raise ValueError(f'common_command_support:lag={lag}:pose={p.pose_id}') + c = tape.commands[index] + # Lazy only after the SAME lag0 endpoint guard. Original max-lag + # continuity and measured force do not vary with candidate lag. Exact + # frozen SourcePose only; subclasses retain the original repeated path. + y = None if invariants is None else invariants.get(pose_index) + if y is None: + # A candidate may not bridge unknown/invalid history or original windows. + first = bisect_right(stamps, p.source_lower_ns - LAGS_NS[-1]) - 1 + last = bisect_right(stamps, p.source_upper_ns) - 1 + if (first < 0 or p.source_lower_ns - LAGS_NS[-1] - stamps[first] > JOIN_NS or last < first or + any(stamps[i + 1] - stamps[i] > JOIN_NS for i in range(first, last))): + raise ValueError(f'common_command_continuity:pose={p.pose_id}') + # Source pose conversion is independent of desired/controller signals. + y = p.speed_mps * p.calibrated_yaw_rate - 9.81 * math.sin(p.calibrated_roll_rad) + if not math.isfinite(y): + raise ValueError('finite measured force response') + if invariants is not None: + invariants[pose_index] = y + pairs.append(Pair(blocks[p.pose_id], p.pose_id, index, c.apply_ns, p.mono_ns, c.command, y)) + previous_command = c.apply_ns + joined[lag] = tuple(pairs) + return joined + + +def _ols(rows): + n = len(rows) + mx = sum(r.command for r in rows) / n + my = sum(r.response for r in rows) / n + variance = sum((r.command - mx)**2 for r in rows) / n + if variance < MIN_EXCITATION**2: + return None + slope = sum((r.command - mx) * (r.response - my) for r in rows) / (n * variance) + intercept = my - slope * mx + if not math.isfinite(slope) or slope <= 0 or not math.isfinite(intercept): + return None + return slope, intercept, min(r.command for r in rows), max(r.command for r in rows) + + +def _score(rows, model): + slope, intercept, _, _ = model + ids = tuple(dict.fromkeys(r.block_id for r in rows)) + return sum(sum((r.response - (slope * r.command + intercept))**2 for r in rows if r.block_id == bid) / + sum(r.block_id == bid for r in rows) for bid in ids) / len(ids) + + +def _direction_supported(rows): + for left in (True, False): + selected = [r for r in rows if (r.command > 0 if left else r.command < 0)] + if len(selected) < 80: + return False + counts = {bid: sum(r.block_id == bid for r in selected) for bid in {r.block_id for r in selected}} + if sum(n >= 20 for n in counts.values()) < 2: + return False + return True + + +def _pick(training, *, directions=True): + scores = [] + models = {} + for lag in LAGS_NS: + rows = training[lag] + model = _ols(rows) + if model is None or directions and not _direction_supported(rows): + scores.append((lag, None)) + else: + scores.append((lag, _score(rows, model))) + models[lag] = model + valid = sorted((score, lag) for lag, score in scores if score is not None) + if not valid: + return None, tuple(scores), 'training_excitation_slope_direction' + best_score, best = valid[0] + if len(valid) > 1 and valid[1][0] - best_score <= TIE_ABS_MSE: + return None, tuple(scores), 'training_lag_ambiguous' + return (best, models[best]), tuple(scores), '' + + +def heldout_supported(result): + """Keep every heldout error; scope use to the unchanged TRAINING domain. + + Out-of-domain rows remain in the archive and error score. They neither + widen the model's command range nor count toward its interpolation support. + """ + if len(result.heldout_blocks) != NUM_WINDOWS-TRAIN_WINDOWS or any( + error > MAX_RMSE for _, error, _ in result.heldout_blocks): + return False + if result.support_policy == STRICT_SUPPORT: + return not any(outside for _, _, outside in result.heldout_blocks) + if result.support_policy != TRAINING_DOMAIN_ONLY or result.decision is None: + return False + decision = result.decision + ids = tuple(bid for bid, _, _ in result.heldout_blocks) + if len(set(ids)) != len(ids) or set(ids).intersection(decision.train_block_ids): + return False + inside = [] + for bid, error, outside in result.heldout_blocks: + group = tuple(row for row in result.rows if row.block_id == bid) + supported = tuple(row for row in group if decision.command_low <= row.command <= decision.command_high) + if len(supported) < 40 or outside != len(group)-len(supported): + return False + # Score ALL original rows, including out-of-domain observations. No + # response-dependent trimming or favorable validation subset is allowed. + rmse = math.sqrt(sum((row.response-(decision.slope*row.command+decision.intercept))**2 + for row in group)/len(group)) + if error != rmse: + return False + inside.extend(supported) + return _direction_supported(tuple(inside)) + + +def identify_plant_lag(binding, tape, poses, windows): + """Legacy strict-input entrypoint; new physical-only types are rejected.""" + bound = _validate(binding, tape, poses, windows) + return _identify_validated(binding, tape, poses, windows, bound, protocol=PROTOCOL) + + +def identify_physical_plant_lag(binding, tape, poses, windows, *, support_policy=STRICT_SUPPORT): + """Physical-only observations; never profile, stage or current authority. + + The child must reconstruct original native/CAN/source proof before building + this tape. This pure entrypoint verifies custody and unchanged affine gates. + """ + bound = _validate_physical(binding, tape, poses, windows) + return _identify_validated(binding, tape, poses, windows, bound, protocol=PHYSICAL_PROTOCOL, support_policy=support_policy) + + +def _identify_validated(binding, tape, poses, windows, bound, *, protocol, support_policy=STRICT_SUPPORT): + """Shared original all51, train-only lag/affine, heldout/support arithmetic.""" + if support_policy not in (STRICT_SUPPORT, TRAINING_DOMAIN_ONLY) or (support_policy != STRICT_SUPPORT and protocol != PHYSICAL_PROTOCOL): + raise ValueError('explicit physical response support policy') + experiment = digest({'protocol': protocol, 'binding': asdict(binding), 'tape': tape.verified_tape_digest, + 'verifier': tape.verifier_source_sha256, 'poses': [asdict(p) for p in poses], + 'windows': [asdict(w) for w in windows], 'grid_ns': LAGS_NS}) + def unavailable(reason, *, decision=None, rows=(), heldout=(), cutoff=None): + return IdentificationResult('collecting' if decision is None else 'rejected', reason, + bound, experiment, decision, rows, heldout, None, cutoff) + try: + joined = _join_all(tape, poses, windows) + except ValueError as exc: + return unavailable(str(exc)) + train_ids = tuple(w.block_id for w in windows[:TRAIN_WINDOWS]) + training = {lag: tuple(r for r in rows if r.block_id in train_ids) for lag, rows in joined.items()} + picked, scores, reason = _pick(training) + if picked is None: + return unavailable(reason) + lag, model = picked + block_out = [] + for omitted in train_ids: + folds = {key: tuple(r for r in rows if r.block_id != omitted) for key, rows in training.items()} + fold, _, fold_reason = _pick(folds, directions=False) + if fold is None: + return unavailable('training_block_out_' + fold_reason) + block_out.append(fold[0]) + if any(abs(value - lag) > LAG_STABILITY_NS for value in block_out): + return unavailable('training_lag_block_out_unstable') + rows = joined[lag] + train_rows = training[lag] + train_pose_ids = tuple(r.pose_id for r in train_rows) + # Bind ONLY the training command data actually visible at/before training end. + # Heldout witness/response changes cannot alter the locked decision digest. + train_end = max(r.response_ns for r in train_rows) + train_source = digest({'binding': asdict(binding), + 'commands': [asdict(c) for c in tape.commands if c.apply_ns <= train_end], + 'poses': [asdict(p) for p in poses if p.pose_id in set(train_pose_ids)], + 'windows': [asdict(w) for w in windows[:TRAIN_WINDOWS]], + 'verifier_source': tape.verifier_source_sha256}) + slope, intercept, low, high = model + declaration = {'protocol': protocol, 'lag_ns': lag, 'slope': slope, 'intercept': intercept, + 'command_low': low, 'command_high': high, 'train_block_ids': train_ids, 'train_pose_ids': train_pose_ids, + 'train_source_digest': train_source, 'training_pairs_digest': digest([asdict(r) for r in train_rows]), + 'training_scores': scores, 'training_leave_one_out_lags': tuple(block_out)} + decision = LagDecision(**declaration, digest=digest(declaration)) + # No heldout response has been used by _pick, _ols, folds or declaration. + heldout = tuple(r for r in rows if r.block_id not in train_ids) + cutoff = max(r.response_ns for r in rows) + if not _direction_supported(heldout): + return unavailable('heldout_direction_support', decision=decision, rows=rows, cutoff=cutoff) + results = [] + for window in windows[TRAIN_WINDOWS:]: + group = tuple(r for r in heldout if r.block_id == window.block_id) + outside = sum(not low <= r.command <= high for r in group) + rmse = math.sqrt(sum((r.response - (slope * r.command + intercept))**2 for r in group) / len(group)) + results.append((window.block_id, rmse, outside)) + frozen_results = tuple(results) + if support_policy == STRICT_SUPPORT and any(outside for _, _, outside in results): + return unavailable('heldout_command_range', decision=decision, rows=rows, heldout=frozen_results, cutoff=cutoff) + if any(error > MAX_RMSE for _, error, _ in results): + return unavailable('heldout_rmse', decision=decision, rows=rows, heldout=frozen_results, cutoff=cutoff) + if support_policy == TRAINING_DOMAIN_ONLY: + proposed = IdentificationResult('physical_affine_test_passed', 'NO_PROFILE_NO_CURRENT_AUTHORITY', bound, + experiment, decision, rows, frozen_results, None, cutoff, support_policy) + if not heldout_supported(proposed): + return unavailable('heldout_training_domain_support', decision=decision, rows=rows, heldout=frozen_results, cutoff=cutoff) + model_digest = digest({'support_policy': support_policy, 'protocol': protocol, 'experiment': experiment, 'decision': decision.digest, + 'heldout': frozen_results, 'cutoff_ns': cutoff, 'binding': bound}) + return IdentificationResult('physical_affine_test_passed', 'NO_PROFILE_NO_CURRENT_AUTHORITY', bound, + experiment, decision, rows, frozen_results, model_digest, cutoff, support_policy) diff --git a/openpilot/starpilot/flm/primitive_source_memo.py b/openpilot/starpilot/flm/primitive_source_memo.py new file mode 100644 index 0000000000..c0eb8d9c9c --- /dev/null +++ b/openpilot/starpilot/flm/primitive_source_memo.py @@ -0,0 +1,180 @@ +"""Lossless primitive memo sharing for level1 transport, never evidence authority. + +Wire remains the existing classless primitive pickle. All dictionary/list/tuple +occurrences are independently rebuilt on decode. Identity is never a digest or +permission proof. Float keys use binary64 bits, never numeric equality. +""" +import math +import struct +from openpilot.starpilot.flm.native_primitives import try_native as _try_native, FALLBACK as _NATIVE_FALLBACK + +VERSION='lossless-source-primitive-memo-v1' +MAX_LOGICAL_BYTES=128*1024*1024 +MAX_DEPTH=128 + + +def _scan(value): + nodes=[] + sizes=[] + heights=[] + keys={} + original={} + active=set() + def visit(item,depth,recurse): + if depth>MAX_DEPTH: + raise ValueError('primitive memo depth bound') + kind=type(item) + if item is None: + key=('none',) + cost=1 + children=None + elif kind is bool: + key=('bool',item) + cost=2 + children=None + elif kind is int: + key=('int',item) + cost=9+(abs(item).bit_length()+7)//8 + children=None + elif kind is float: + if not math.isfinite(item): + raise ValueError('nonfinite primitive memo input') + key=('float',struct.pack('!d',item)) + cost=9 + children=None + elif kind in (str,bytes): + key=('str' if kind is str else 'bytes',item) + cost=9+len(item.encode('utf8','surrogatepass') if kind is str else item) + children=None + elif kind in (tuple,list,dict): + ident=id(item) + if ident in active: + raise ValueError('cyclic primitive memo input') + if ident in original: + index=original[ident] + if depth+heights[index]>MAX_DEPTH: + raise ValueError('primitive memo depth bound') + return index + if kind is dict and any(type(k) is not str for k in item): + raise ValueError('primitive memo dictionary key') + active.add(ident) + try: + children=tuple(recurse(child,depth+1,recurse) for child in (tuple(v for pair in item.items() for v in pair) if kind is dict else item)) + finally: + active.remove(ident) + cost=9+sum(sizes[index] for index in children) + key=('dict' if kind is dict else 'tuple' if kind is tuple else 'list',children) + else: + raise ValueError('primitive memo input type') + height=0 if children is None else 1+max((heights[index] for index in children),default=-1) + if depth+height>MAX_DEPTH: + raise ValueError('primitive memo depth bound') + if cost>MAX_LOGICAL_BYTES: + raise ValueError('primitive memo logical expansion bound') + index=keys.get(key) + if index is None: + if children is None: + out=item + elif kind is tuple: + out=tuple(nodes[i] for i in children) + elif kind is list: + out=[nodes[i] for i in children] + else: + out={nodes[children[i]]:nodes[children[i+1]] for i in range(0,len(children),2)} + index=len(nodes) + nodes.append(out) + sizes.append(cost) + heights.append(height) + keys[key]=index + if children is not None: + original[id(item)]=index + return index + index=visit(value,0,visit) + return nodes[index],sizes[index] + + +def intern_primitive_graph(value): + """Exact typed full-key equality; shared composites exist ONLY during encoding.""" + return _scan(value)[0] + + +def _logical_limits(value): + """Validation-only local immutable/owned-object memo; no canonical graph built.""" + memo={} + active=set() + def visit(item,depth,recurse): + if depth>MAX_DEPTH: + raise ValueError('primitive memo depth bound') + ident=id(item) + if ident in active: + raise ValueError('cyclic primitive memo input') + if ident in memo: + cost,height=memo[ident] + if depth+height>MAX_DEPTH: + raise ValueError('primitive memo depth bound') + return cost,height + kind=type(item) + height=0 + if item is None: + cost=1 + elif kind is bool: + cost=2 + elif kind is int: + cost=9+(abs(item).bit_length()+7)//8 + elif kind is float: + if not math.isfinite(item): + raise ValueError('nonfinite primitive memo input') + cost=9 + elif kind in (str,bytes): + cost=9+len(item.encode('utf8','surrogatepass') if kind is str else item) + elif kind in (tuple,list,dict): + if kind is dict and any(type(k) is not str for k in item): + raise ValueError('primitive memo dictionary key') + active.add(ident) + cost=9 + maximum=-1 + try: + children=(v for pair in item.items() for v in pair) if kind is dict else item + for child in children: + child_cost,child_height=recurse(child,depth+1,recurse) + cost+=child_cost + maximum=max(maximum,child_height) + if cost>MAX_LOGICAL_BYTES: + raise ValueError('primitive memo logical expansion bound') + finally: + active.remove(ident) + height=1+maximum + else: + raise ValueError('primitive memo input type') + if depth+height>MAX_DEPTH: + raise ValueError('primitive memo depth bound') + if cost>MAX_LOGICAL_BYTES: + raise ValueError('primitive memo logical expansion bound') + memo[ident]=(cost,height) + return cost,height + return visit(value,0,visit) + + +def _independent_primitive_graph_python(value): + """Validate logical expansion BEFORE allocating independently owned containers.""" + _logical_limits(value) + def copy(item,recurse): + kind=type(item) + if kind is dict: + return {key:recurse(child,recurse) for key,child in item.items()} + if kind is list: + return [recurse(child,recurse) for child in item] + if kind is tuple: + return tuple(recurse(child,recurse) for child in item) + return item + return copy(value,copy) + + +def independent_primitive_graph(value, *, reserved_bytes=0): + if type(reserved_bytes) is not int or not 0 <= reserved_bytes <= 512 * 1024: + raise ValueError('bounded normalized transient reservation') + if reserved_bytes: + # Declared bounded allocation padding, never encoded semantic trace data. + return independent_primitive_graph((value, bytes(reserved_bytes)))[0] + result = _try_native('independent_primitive_graph', value) + return _independent_primitive_graph_python(value) if result is _NATIVE_FALLBACK else result diff --git a/openpilot/starpilot/flm/producer_clock_equivalence.py b/openpilot/starpilot/flm/producer_clock_equivalence.py new file mode 100644 index 0000000000..2eb5499b6f --- /dev/null +++ b/openpilot/starpilot/flm/producer_clock_equivalence.py @@ -0,0 +1,124 @@ +"""Pinned mixed-producer closure and fresh bounded clock equivalence. + +Source/ABI provenance is not inspection of a vehicle's running kernel. This +optional source packet grants no native, plant, model or live authority. +""" +from dataclasses import dataclass +import hashlib +import json +import math +from pathlib import Path +import time + +MAX_CLOCK_UNCERTAINTY_NS = 1_000_000 +PRODUCER_SOURCE_HASHES = ( + ('openpilot', 'system/camerad/cameras/spectra.cc', '4fa26c17f0fb792a4ab6815657beccddee1d641faa2fd02e5aa7393f717a2f3a'), + ('openpilot', 'system/camerad/cameras/camera_qcom2.cc', 'b34c24c9984a0ac086f7b961cb7ff09d0501231da50aaad0e676bb3dc3f2edd9'), + ('openpilot', 'system/camerad/cameras/camera_common.cc', 'bd9a416a6ee21047b4deec6df04276d8717cd92fe14de43f3e9daff86b62bb8f'), + ('openpilot', 'selfdrive/modeld/modeld.py', '10c6fb78806c952d8061a28f16207a71c049ed68fd809aaccaad077545e1706d'), + ('openpilot', 'selfdrive/modeld/fill_model_msg.py', 'b3916c34d1073d0c26a7a8adc51e7b148f63a1ac68b388226170fd0f8b8b56b4'), + ('openpilot', 'system/sensord/sensord.py', '7c79ab07d775622a43001e5a9b9ccc9a7bafb065940185849b33eefc2fd26420'), + ('openpilot', 'system/sensord/sensors/lsm6ds3_gyro.py', '3e561f3d8572e1477c9b4b810c7570837d51e81c5672fe21ec67a68cae00dd92'), + ('openpilot', 'system/sensord/sensors/lsm6ds3_accel.py', 'a7db3f51a2645f9b712dd34c4df4be6fd03a22565b9e72078dba13d2d4c5f073'), + ('openpilot', 'selfdrive/locationd/locationd.py', '632ce44e7137d2f5fbadc420a7bafecf0bfa9558ea5d2eb5321273e322c524c5'), + ('openpilot', 'selfdrive/locationd/models/pose_kf.py', '05925942c847e00ff01f4eade113bf2ef938bc638e5c119942059901219a8901'), + ('msgq', 'visionipc/visionipc_server.cc', 'a4325a5171eacb931eb271d06a992d2f27216040df113d92099f62cdfa67d81a'), + ('msgq', 'visionipc/visionipc_client.cc', 'd142cd60eedabfc581e9770fcd6f0ca81b762ba9dbc608b54dd20a27a317a52d'), + ('msgq', 'visionipc/visionipc.h', 'e21425d41aea4e01a19ed17969f0e9ff6f76caeb8f6818860c76791e042b92b9'), + ('msgq', 'visionipc/visionipc_pyx.pyx', '346e7a53985132855954e62f4274984c2df39abe88ae2a3d050906327d9d0b83'), + ('rednose', 'helpers/kalmanfilter.py', '103e29e336d87ad23134495d5db4853a3f72c9677a8422d0697d70ec1ae1cd4e'), + ('rednose', 'helpers/ekf_sym.py', 'e56d50c5c6cd11f1ebc2b682e1b30e60fee89856a8225a30bdf4b2ce0a7b7dc0'), + ('rednose', 'helpers/ekf_sym.cc', 'cb9003888d56fd9da8b45402030c6821e83333c3146867e1759e44b2fda20771'), + ('rednose', 'helpers/ekf_sym_pyx.pyx', '039236aeb965d87a5ac7a880e40e8cb15ddd09685ada12b17bcc540681ed0b17'), + ('rednose', 'helpers/ekf_sym.h', 'f08854f4336900737ff2e9ff7a5ec930f245795d394e02468f0a4fd73a4a9068'), +) +KERNEL_ABI_SOURCE_PIN = ( + 'agnos-sdm845:4ee6b71b8ab9a461248985470726f5917943bc91', + 'V4L_EVENT_CAM_REQ_MGR_SOF_BOOT_TS:get_monotonic_boottime64', + '56c3d16e405a435d49ca6a814a95a698582b8127ad7c71d25846f89b90bca114', + '86e5c4212d6ca6fcae7eb1d2c763331ec100c930ac3ed5789a0ae4f3b73f2b70', + 'GPIO:ktime_get_real_ns->sensord-wall-minus-monotonic', +) +CLOSURE_BYTES = json.dumps((PRODUCER_SOURCE_HASHES, KERNEL_ABI_SOURCE_PIN), separators=(',', ':')).encode('ascii') +CLOSURE_SHA256 = hashlib.sha256(CLOSURE_BYTES).hexdigest() +CONVENTION = 'mono_boot_equivalent_interval_v1' + + +def pinned_default_contract(): + """Startup-only bounded installed producer closure reads, never per control/pose work.""" + from openpilot.starpilot.flm.original_source_capture import OriginalSourceClockContract + import importlib.util + roots = {} + if len(PRODUCER_SOURCE_HASHES) > 32: + raise ValueError('original producer closure bound') + for package, relative, expected in PRODUCER_SOURCE_HASHES: + if package not in roots: + spec = importlib.util.find_spec(package) + if spec is None or not spec.submodule_search_locations: + raise ValueError('installed original producer package source unavailable') + roots[package] = Path(next(iter(spec.submodule_search_locations))).resolve() + data = (roots[package] / relative).read_bytes() + if not 0 < len(data) <= 1024 * 1024 or hashlib.sha256(data).hexdigest() != expected: + raise ValueError('unsupported original mixed-producer source closure') + return OriginalSourceClockContract(CONVENTION, CLOSURE_SHA256, CLOSURE_SHA256, 'pinned-source-abi') + + +@dataclass(frozen=True) +class ClockEquivalence: + samples: tuple + mono_resolution_ns: int + boot_resolution_ns: int + offset_lower_ns: int + offset_upper_ns: int + uncertainty_ns: int + + def primitive(self): + return (self.samples, self.mono_resolution_ns, self.boot_resolution_ns, + self.offset_lower_ns, self.offset_upper_ns, self.uncertainty_ns) + + +def equivalence_from_samples(samples, mono_resolution_ns, boot_resolution_ns): + if (type(samples) is not tuple or len(samples) != 3 or + any(type(value) is not int or not 0 < value <= MAX_CLOCK_UNCERTAINTY_NS + for value in (mono_resolution_ns, boot_resolution_ns))): + raise ValueError('three original samples and explicit clock resolutions required') + intervals = [] + previous = 0 + quantization = mono_resolution_ns + boot_resolution_ns + for sample in samples: + if type(sample) is not tuple or len(sample) != 3 or any(type(value) is not int or value <= 0 for value in sample): + raise ValueError('original clock sample shape/types') + mono_before, boot, mono_after = sample + if not previous <= mono_before <= mono_after: + raise ValueError('original clock sample reversed/reordered') + previous = mono_after + intervals.append((boot - mono_after - quantization, boot - mono_before + quantization)) + lower = max(a for a, _ in intervals) + upper = min(b for _, b in intervals) + uncertainty = max(abs(lower), abs(upper)) + if not lower <= 0 <= upper or uncertainty > MAX_CLOCK_UNCERTAINTY_NS: + raise ValueError('mixed producer clocks not bounded-equivalent at original receipt') + return ClockEquivalence(samples, mono_resolution_ns, boot_resolution_ns, lower, upper, uncertainty) + + +def sample_equivalence(): + # Include clock_getres quantization; zero or omitted resolution is invalid. + mono_resolution = time.clock_getres(time.CLOCK_MONOTONIC) + boot_resolution = time.clock_getres(time.CLOCK_BOOTTIME) + if any(type(value) not in (int, float) or not math.isfinite(value) or value <= 0 for value in (mono_resolution, boot_resolution)): + raise ValueError('original clock resolution unavailable') + resolutions = tuple(max(1, math.ceil(value * 1e9)) for value in (mono_resolution, boot_resolution)) + samples = tuple((time.monotonic_ns(), time.clock_gettime_ns(time.CLOCK_BOOTTIME), time.monotonic_ns()) for _ in range(3)) + return equivalence_from_samples(samples, *resolutions) + + +def source_interval(raw_timestamp_ns, received_ns, now_ns, witness): + if not isinstance(witness, ClockEquivalence): + raise ValueError('typed original clock witness required') + checked = equivalence_from_samples(witness.samples, witness.mono_resolution_ns, witness.boot_resolution_ns) + if checked != witness or any(type(value) is not int for value in (raw_timestamp_ns, received_ns, now_ns)): + raise ValueError('changed original clock witness') + lower, upper = raw_timestamp_ns - witness.uncertainty_ns, raw_timestamp_ns + witness.uncertainty_ns + if not 0 < lower <= upper <= received_ns <= now_ns or now_ns - lower > 100_000_000: + raise ValueError('original source interval freshness/future') + return lower, upper diff --git a/openpilot/starpilot/flm/restore_activation.py b/openpilot/starpilot/flm/restore_activation.py index 3cffdc315b..027f9d30f7 100644 --- a/openpilot/starpilot/flm/restore_activation.py +++ b/openpilot/starpilot/flm/restore_activation.py @@ -8,7 +8,7 @@ import hashlib import json from openpilot.starpilot.flm.clipping_quality import ( - FIXED_MAP_IDENTIFICATION, RESTORE_ACTIVATION_VERSION, RESTORE_AGGREGATE_ACTIVATION_VERSION, RESTORE_ZERO_REFERENCE_VERSION, + FIXED_MAP_IDENTIFICATION, RESTORE_ACTIVATION_VERSION, RESTORE_AGGREGATE_ACTIVATION_VERSION, RESTORE_ZERO_REFERENCE_VERSION, RESTORE_FEEDBACK_VERSION, restore_policy, restore_quality, ) from openpilot.starpilot.flm.event_activation import EventActivationPlan, EXACT_EMISSION_ACTIVATION_VERSION, build_activation_plan @@ -40,7 +40,7 @@ class WholeMapRestorePlan(EventActivationPlan): restore_envelope_sha256: str def validate(self): - versions = (RESTORE_ACTIVATION_VERSION, RESTORE_AGGREGATE_ACTIVATION_VERSION, RESTORE_ZERO_REFERENCE_VERSION) + versions = (RESTORE_ACTIVATION_VERSION, RESTORE_AGGREGATE_ACTIVATION_VERSION, RESTORE_ZERO_REFERENCE_VERSION, RESTORE_FEEDBACK_VERSION) if (self.version not in versions or type(self.restore_envelope_sha256) is not str or len(self.restore_envelope_sha256) != 64 or any(c not in '0123456789abcdef' for c in self.restore_envelope_sha256)): raise ValueError('whole-map restore plan version/envelope') @@ -52,7 +52,7 @@ class WholeMapRestorePlan(EventActivationPlan): @property def clipping_quality(self): self.validate() - return restore_quality(6 if self.version == RESTORE_ZERO_REFERENCE_VERSION else 4) + return restore_quality(8 if self.version == RESTORE_FEEDBACK_VERSION else 6 if self.version == RESTORE_ZERO_REFERENCE_VERSION else 4) @property def identification_policy(self): @@ -77,7 +77,7 @@ def restore_plan_matches(plan, envelope): def build_restore_plan(engine, *, model, cutoff_ns, stage_events, physical_context, envelope, restore_version=4): - if type(restore_version) is not int or restore_version not in (4, 5, 6): + if type(restore_version) is not int or restore_version not in (4, 5, 6, 8): raise ValueError('explicit whole-map restore support version') policy = restore_policy(restore_version) digest = restore_envelope_digest(envelope) diff --git a/openpilot/starpilot/flm/runtime_adapter_registry.py b/openpilot/starpilot/flm/runtime_adapter_registry.py index b90e2a4e54..af336bd979 100644 --- a/openpilot/starpilot/flm/runtime_adapter_registry.py +++ b/openpilot/starpilot/flm/runtime_adapter_registry.py @@ -46,6 +46,8 @@ def implementation_revision(controller_type, interface_type): raise ValueError('implementation source bound') result.update(path.name.encode('utf-8')) result.update(hashlib.sha256(data).digest()) + from openpilot.starpilot.flm.native_primitives import native_identity + result.update(native_identity().encode('utf-8')) return REVISION + ':' + result.hexdigest() @@ -473,6 +475,20 @@ def vm_regime(values, anchor=None): return None +def control_vm_regime(command, anchor=None): + """Use genuine Controls anchor when delivered; preserve legacy empty shape.""" + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + supplied = command.vehicle_model_anchor if type(command) is ControlEvidence else () + if type(supplied) is not tuple or len(supplied) not in (0, 3): + return None + if supplied: + if any(type(value) is not float or not math.isfinite(value) or value == 0. for value in supplied): + return None + current = vm_regime(dict(command.replay_inputs), supplied) + return supplied if current == supplied else None + return vm_regime(dict(command.replay_inputs), anchor) + + def physical_observer(sm, cp, collector, now_ns, *, distinguish_pending=False): """Convert independently measured BOOT pose source into the evidence MONO clock.""" from openpilot.starpilot.flm.fullrate_collector import PhysicalEvidence @@ -544,7 +560,7 @@ def physical_observer(sm, cp, collector, now_ns, *, distinguish_pending=False): if ready and getattr(getattr(collector, 'adapter', None), 'command_domain', None) != 'ford_path_angle': command = ready[-1][0] anchor = getattr(collector, '_physical_vm_anchor', None) - regime = vm_regime(dict(command.replay_inputs), anchor) + regime = control_vm_regime(command, anchor) if regime is None: collector.split('vehicle_model_unavailable', reset=True) collector._physical_vm_anchor = None @@ -642,3 +658,248 @@ def physical_observer(sm, cp, collector, now_ns, *, distinguish_pending=False): return None except (AttributeError, KeyError, TypeError, ValueError, OverflowError): return reject() + + +def original_source_pose_observer(sm, cp, collector, now_ns, *, distinguish_pending=False): + """Capture original KF source with explicit producer clock provenance.""" + if getattr(collector, "original_source_capture", None) is None: + return None + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose, OriginalSourceClockContract + contract = getattr(collector, 'original_source_clock_contract', None) + if not isinstance(contract, OriginalSourceClockContract): + return None + from openpilot.selfdrive.locationd.helpers import PoseCalibrator, Pose + + def reject(): + collector._original_source_pose_queue = () + collector.reset_original_source_capture('original_source_unavailable') + return None + + services = ('deviceMotion', 'extrinsicsCalibration', 'carState', 'driverMonitoringState') + if not sm.all_checks(list(services)): + return reject() + try: + contract.validate() + equivalence = None + if contract.source_domain == 'mono_boot_equivalent_interval_v1': + from openpilot.starpilot.flm.producer_clock_equivalence import sample_equivalence, source_interval + try: + equivalence = sample_equivalence() + except (AttributeError, TypeError, ValueError, OverflowError, OSError): + collector.reset_original_source_capture('original_clock_equivalence_discontinuity', hard=True) + return reject() + # This is a fresh receipt witness, never a cached/restamped old proof. + collector._original_source_equivalence = equivalence + boot_mono_offset = (equivalence.offset_lower_ns + equivalence.offset_upper_ns) // 2 + clock_offset = 0 + uncertainty = equivalence.uncertainty_ns + mono_before, _, mono_after = equivalence.samples[-1] + else: + mono_before = time.monotonic_ns() + boot = time.clock_gettime_ns(time.CLOCK_BOOTTIME) + mono_after = time.monotonic_ns() + offset = boot - (mono_before + mono_after) // 2 + cache = getattr(collector, '_original_source_clock', None) + if cache is None: + collector._original_source_clock = offset + elif abs(offset - cache) > 1_000_000 + (mono_after - mono_before) // 2: + collector.reset_original_source_capture('physical_clock_discontinuity', hard=True) + return reject() + boot_mono_offset = collector._original_source_clock + if contract.source_domain == 'unknown-equivalent-at-receipt' and abs(boot_mono_offset) > (mono_after - mono_before) // 2: + return reject() + clock_offset = boot_mono_offset if contract.source_domain == 'boottime' else 0 + uncertainty = (mono_after - mono_before) // 2 + dm, calibration, cs = sm['deviceMotion'], sm['extrinsicsCalibration'], sm['carState'] + pose_ns = int(dm.timestamp) - clock_offset + if ( + not max(0, getattr(collector, '_original_source_revoked_before_ns', 0)) < pose_ns <= now_ns + or now_ns - pose_ns > 100_000_000 + or not dm.inputsOK + or not dm.sensorsOK + or not dm.posenetOK + or not dm.angularVelocityDevice.valid + or not dm.orientationNED.valid + or str(calibration.calStatus) != 'calibrated' + ): + return reject() + if equivalence is not None: + source_lower, source_upper = source_interval(int(dm.timestamp), int(sm.recv_time['deviceMotion'] * 1e9), now_ns, equivalence) + else: + source_lower = source_upper = pose_ns + state_ns = int(sm.logMonoTime['carState']) + if not 0 < state_ns <= now_ns or now_ns - state_ns > 100_000_000: + return reject() + from openpilot.starpilot.flm.calibration_lifecycle import calibrated_basis + generation = digest(tuple(float(v) for v in calibration.rpyCalib)) + estimate = generation + lifecycle = getattr(collector, '_calibration_generation', '') + if lifecycle: + if calibrated_basis(calibration) is None: + return reject() + generation = digest(('live_vehicle_frame_v2', lifecycle)) + cache = getattr(collector, '_original_source_calibrator', None) + if cache is None or cache[0] != estimate: + calibrator = PoseCalibrator() + calibrator.feed_extrinsics_calibration(calibration) + collector._original_source_calibrator = (estimate, calibrator) + calibrator = collector._original_source_calibrator[1] + if not calibrator.calib_valid: + return reject() + pose = calibrator.build_calibrated_pose(Pose.from_device_motion(dm)) + ready = getattr(collector, 'ready', ()) + history = {} + if ready and getattr(getattr(collector, 'adapter', None), 'command_domain', None) != 'ford_path_angle': + command = ready[-1][0] + anchor = getattr(collector, '_original_source_vm_anchor', None) + regime = control_vm_regime(command, anchor) + if regime is None: + collector.reset_original_source_capture('vehicle_model_unavailable') + collector._original_source_vm_anchor = None + return reject() + if anchor is not None and regime != anchor: + collector.reset_original_source_capture('vehicle_model_material_change', hard=True) + collector._original_source_vm_anchor = regime + return reject() + collector._original_source_vm_anchor = regime + native_epoch = command.native_identity + native_required = getattr(command, 'native_required', bool(native_epoch)) + if type(native_required) is not bool: + return reject() + if native_required and ( + type(native_epoch) is not str or len(native_epoch) != 64 or any(c not in '0123456789abcdef' for c in native_epoch) or + not getattr(command, 'native_allowed', False) or + not 0 < getattr(command, 'native_observed_ns', 0) <= getattr(command, 'control_ns', 0) <= + ready[-1][1].apply_ns <= getattr(command, 'native_expires_ns', 0) or + command.native_expires_ns-command.native_observed_ns > 200_000_000): + return reject() + ordinary_safety = () + ordinary_allowed = True + if not native_required: + # Ordinary supported controllers have no axis ACK/lease/identity. + # Reuse actual legacy Panda/CP/current permission guards unchanged. + if native_epoch != '' or getattr(command, 'native_observed_ns', 0) != 0 or getattr(command, 'native_expires_ns', 0) != 0: + return reject() + ordinary_allowed, _, _ = current_authority(sm, cp, now_ns=now_ns, required=False) + cp_sha = getattr(collector.adapter, 'cp_sha256', '') + if not ordinary_allowed or type(cp_sha) is not str or len(cp_sha) != 64 or any(c not in '0123456789abcdef' for c in cp_sha): + return reject() + configuration = ( + int(cp.alternativeExperience), + tuple((str(config.safetyModel), int(config.safetyParam)) for config in cp.safetyConfigs), + tuple((index, str(panda.safetyModel), int(panda.safetyParam), int(panda.alternativeExperience), + bool(panda.controlsAllowed), bool(panda.safetyRxChecksInvalid), bool(panda.heartbeatLost), + tuple(str(fault) for fault in panda.faults)) for index, panda in enumerate(sm['pandaStates'])), + ) + ordinary_safety = ('ordinary-panda-current-v1', cp_sha, int(sm.recv_time['pandaStates'] * 1e9), + int(sm.logMonoTime['pandaStates']), configuration) + native = bool( + ordinary_allowed and + getattr(collector, '_learning_current_native_allowed', False) and ready[-1][1].native_allowed and 0 <= now_ns - ready[-1][1].apply_ns <= 100_000_000 + ) + current_identity = (command.identity, command.generation, command.source_epoch, command.baseline_revision, + command.controller_revision, command.native_identity) + for command, _ in ready: + if (command.identity, command.generation, command.source_epoch, command.baseline_revision, + command.controller_revision, command.native_identity) != current_identity: + continue # State brackets cannot bridge a command/native epoch. + values = dict(command.replay_inputs) + if 'vEgo' in values and 'steeringAngleDeg' in values: + history[command.state_ns] = (float(values['vEgo']), float(values['steeringAngleDeg'])) + else: + return reject() + if not native: + return reject() + override = bool( + cs.steeringPressed + or cs.brakePressed + or cs.regenBraking + or any(str(button.type) == 'cancel' for button in cs.buttonEvents) + or sm['driverMonitoringState'].noResponseForceDecel + ) + fault = bool(cs.canTimeout or not cs.canValid or cs.steerFaultTemporary or cs.steerFaultPermanent or cs.accFaulted) + if override or fault: + return reject() + # Freeze converted primitives at their original pose receipt. A following + # CarState may provide the right interpolation bracket, but may not provide + # a replacement pose timestamp, transform, authority or source identity. + command = ready[-1][0] + def observation_context(): + return ( + getattr(collector, 'session', ''), + command.identity, command.generation, command.source_epoch, + command.baseline_revision, command.controller_revision, + generation, ('original_source_clock_generation', getattr(collector, '_original_source_clock_generation', 0)), + (contract.source_domain, contract.producer_revision, contract.proof_sha256, contract.scope), native_epoch, regime, + float(cp.steerRatio), float(cp.wheelbase), float(cp.centerToFront), float(cp.mass), + ) + context = observation_context() + queue = getattr(collector, '_original_source_pose_queue', ()) + old_context = getattr(collector, '_original_source_pose_context', None) + context_changed = old_context is not None and old_context != context + if old_context != context: + queue = () + if old_context is not None: + collector.reset_original_source_capture('original_source_context_changed', hard=True) + # Hard reset advances the clock generation. Bind the NEW epoch now; + # saving the pre-reset context would reset again on every healthy poll. + context = observation_context() + collector._original_source_pose_context = context + queue = tuple(value for value in queue if 0 <= now_ns - value.mono_ns <= 100_000_000) + if pose_ns <= getattr(collector, '_original_source_revoked_before_ns', 0): + return reject() + seen = getattr(collector, '_original_source_seen_pose', 0) + if pose_ns < seen: + return reject() + if pose_ns > seen: + collector._original_source_seen_pose = pose_ns + evidence = OriginalSourcePose( + mono_ns=pose_ns, received_ns=int(sm.recv_time['deviceMotion'] * 1e9), observed_ns=now_ns, + source_boot_ns=int(dm.timestamp) if contract.source_domain == 'boottime' else 0, clock_offset_ns=clock_offset, + source_timestamp_ns=int(dm.timestamp), source_clock_domain=contract.source_domain, + source_clock_contract=(contract.source_domain, contract.producer_revision, contract.proof_sha256, contract.scope), + boot_mono_offset_ns=boot_mono_offset, + clock_uncertainty_ns=uncertainty, + clock_equivalence=() if equivalence is None else equivalence.primitive(), + clock_generation=getattr(collector, '_original_source_clock_generation', 0), + source_lower_ns=source_lower, source_upper_ns=source_upper, + calibration=generation, calibration_estimate=estimate, context=context, + yaw_rate=float(pose.angular_velocity.xyz[2]), roll_rad=float(pose.orientation.xyz[0]), + native_identity=native_epoch, native_allowed=True, valid=True, + native_required=native_required, native_control_ns=getattr(command, 'control_ns', 0), + native_observed_ns=getattr(command, 'native_observed_ns', 0), + native_expires_ns=getattr(command, 'native_expires_ns', 0), ordinary_safety=ordinary_safety, + source_epoch=command.source_epoch, identity=command.identity, + baseline_revision=command.baseline_revision, controller_revision=command.controller_revision, + source_state_ns=state_ns, native_apply_ns=ready[-1][1].apply_ns, + angular_velocity_device=tuple(float(v) for v in (dm.angularVelocityDevice.x, dm.angularVelocityDevice.y, dm.angularVelocityDevice.z)), + orientation_ned=tuple(float(v) for v in (dm.orientationNED.x, dm.orientationNED.y, dm.orientationNED.z)), + calibration_rpy=tuple(float(v) for v in calibration.rpyCalib), + ) + queue = (*queue, evidence)[-4:] + collector._original_source_pose_queue = queue + for index, evidence in enumerate(queue): + before = [stamp for stamp in history if stamp <= (evidence.source_lower_ns or evidence.mono_ns)] + after = [stamp for stamp in history if stamp >= (evidence.source_upper_ns or evidence.mono_ns)] + if not before or not after: + continue + left, right = max(before), min(after) + if (evidence.source_upper_ns or evidence.mono_ns) - left > 100_000_000 or right - (evidence.source_lower_ns or evidence.mono_ns) > 100_000_000: + continue + weight = 0.0 if left == right else (evidence.mono_ns - left) / (right - left) + speed, steering_angle = tuple(a + (b - a) * weight for a, b in zip(history[left], history[right], strict=True)) + # Consume once, in source order; downstream command/response joins keep + # their original independent 30 ms actuation and 100 ms response limits. + collector._original_source_pose_queue = queue[index + 1:] + return replace(evidence, speed_mps=speed, steering_angle_deg=steering_angle, completed_ns=now_ns, + left_state_ns=left, right_state_ns=right, left_state=history[left], right_state=history[right]) + if distinguish_pending: + from openpilot.starpilot.flm.physical_observation import PendingPhysical + # Healthy but no independently bracketed new pose. Retain original + # clocks; this is NEVER a new physical sample or a current-fit grant. + waiting = queue[0] if queue else None + return PendingPhysical(pose_ns if waiting is None else waiting.mono_ns, + int(sm.recv_time['deviceMotion'] * 1e9) if waiting is None else waiting.received_ns, context_changed) + return None + except (AttributeError, KeyError, TypeError, ValueError, OverflowError): + return reject() diff --git a/openpilot/starpilot/flm/selected_effects.py b/openpilot/starpilot/flm/selected_effects.py index 45ec236aac..460279778b 100644 --- a/openpilot/starpilot/flm/selected_effects.py +++ b/openpilot/starpilot/flm/selected_effects.py @@ -289,9 +289,20 @@ class EffectReplaySource: if not isinstance(trial, EffectTrial): raise ValueError('effect trial required') self.source = trial.source() + self.source.allow_limited_profile = True self.original_effect_anchor = self.source.original_effect_anchor self.applied_coordinates = self.source.applied_coordinates + @classmethod + def from_coordinates(cls, coordinates, baseline_revision, *, accumulated): + if cls is not EffectReplaySource: + raise ValueError('exact numerical replay source required') + obj = cls.__new__(cls) + obj.source = CanonicalSurfaceEffects(coordinates, baseline_revision, accumulated=accumulated, allow_limited_profile=True) + obj.original_effect_anchor = obj.source.original_effect_anchor + obj.applied_coordinates = obj.source.applied_coordinates + return obj + def begin_frame(self, control_ns): self.source.begin_frame(control_ns) @@ -331,9 +342,30 @@ def manual(parent): return getattr(parent, 'flm_surface', None) is not None or getattr(policy, 'surface', None) is not None +def bounded_profile_on_limited_frame(parent): + # This is parameter shaping, not a native grant. The live owner must already + # hold current source/native permission; replay uses the same bounded math. + # Legacy residual owners and arbitrary callback objects remain inhibited. + from openpilot.starpilot.flm.live_trial_owner import TrackingTrialOwner + + owner = getattr(parent, 'automatic_flm_effects', None) + return ( + type(owner) in (TrackingTrialOwner, EffectReplaySource, EffectReplayRecordedSource) + and type(owner.source) is CanonicalSurfaceEffects + and owner.source.allow_limited_profile + ) + + def apply_selected_stage(parent, frame): source = getattr(parent, 'automatic_flm_effects', None) - if source is None or not frame.valid() or not frame.active or frame.driver_override or frame.limited or manual(parent): + if ( + source is None + or not frame.valid() + or not frame.active + or frame.driver_override + or (frame.limited and (frame.curvature_limited or not bounded_profile_on_limited_frame(parent))) + or manual(parent) + ): return frame.baseline_ff, frame.baseline_threshold try: ff, threshold = source.selected_stage(frame, original_anchor=source.original_effect_anchor) @@ -346,9 +378,16 @@ def apply_selected_stage(parent, frame): return frame.baseline_ff, frame.baseline_threshold -def apply_selected_friction(parent, cs, setpoint, baseline, *, active, limited): +def apply_selected_friction(parent, cs, setpoint, baseline, *, active, limited, curvature_limited=False): source = getattr(parent, 'automatic_flm_effects', None) - if source is None or not active or limited or cs.steeringPressed or manual(parent) or not finite(baseline): + if ( + source is None + or not active + or (limited and (curvature_limited or not bounded_profile_on_limited_frame(parent))) + or cs.steeringPressed + or manual(parent) + or not finite(baseline) + ): return baseline try: candidate = source.selected_friction(float(baseline), original_anchor=source.original_effect_anchor) @@ -359,9 +398,9 @@ def apply_selected_friction(parent, cs, setpoint, baseline, *, active, limited): return baseline -def apply_selected_deadzone(parent, cs, vm, baseline_accel, requested_curvature, *, active, limited): +def apply_selected_deadzone(parent, cs, vm, baseline_accel, requested_curvature, *, active, limited, curvature_limited=False): source = getattr(parent, 'automatic_flm_effects', None) - if source is None or not active or limited or cs.steeringPressed or manual(parent): + if source is None or not active or (limited and (curvature_limited or not bounded_profile_on_limited_frame(parent))) or cs.steeringPressed or manual(parent): return baseline_accel try: speed = float(cs.vEgo) @@ -375,9 +414,16 @@ def apply_selected_deadzone(parent, cs, vm, baseline_accel, requested_curvature, return baseline_accel -def apply_selected_output(parent, cs, vm, params, requested_curvature, result, *, active, limited): +def apply_selected_output(parent, cs, vm, params, requested_curvature, result, *, active, limited, curvature_limited=False): source = getattr(parent, 'automatic_flm_effects', None) - if source is None or not active or limited or cs.steeringPressed or manual(parent) or cs.vEgo < 0.1 * 0.44704: + if ( + source is None + or not active + or (limited and (curvature_limited or not bounded_profile_on_limited_frame(parent))) + or cs.steeringPressed + or manual(parent) + or cs.vEgo < 0.1 * 0.44704 + ): return result command, angle, log = result try: @@ -413,7 +459,7 @@ class EffectReplayRecordedSource(EffectReplaySource): envelope = trace.envelope values = dict(trace.applied) group = {n: values[n] for n in envelope.trial_names} - self.source = CanonicalSurfaceEffects(group, envelope.selected_baseline_revision, accumulated=values) + self.source = CanonicalSurfaceEffects(group, envelope.selected_baseline_revision, accumulated=values, allow_limited_profile=True) self.original_effect_anchor = self.source.original_effect_anchor self.applied_coordinates = self.source.applied_coordinates @@ -485,10 +531,11 @@ class EffectReplayTransition: # Per-transition geometry only: trace validation and exact step_full above # still run EVERY frame. Packed doubles preserve signed zero and full30. import struct + key = self.trial.selected_baseline_revision, self.names, struct.pack('!30d', *(self.applied[n] for n in NAMES)) if key != self.__source_key: group = {n: self.applied[n] for n in self.names} - self.__source = CanonicalSurfaceEffects(group, self.trial.selected_baseline_revision, accumulated=self.applied) + self.__source = CanonicalSurfaceEffects(group, self.trial.selected_baseline_revision, accumulated=self.applied, allow_limited_profile=True) self.__source_key = key source.source = self.__source # The ordinary _run -> begin_frame path still resets ALL call counters and diff --git a/openpilot/starpilot/flm/source_plant_fit.py b/openpilot/starpilot/flm/source_plant_fit.py new file mode 100644 index 0000000000..08ed5d18ea --- /dev/null +++ b/openpilot/starpilot/flm/source_plant_fit.py @@ -0,0 +1,526 @@ +from openpilot.starpilot.flm.normalized_trace_archive import guarded_archive +from openpilot.starpilot.flm.normalized_trace_archive import SOURCE_VERSION as NORMALIZED_SOURCE_VERSION,archive_of,record_body,stored_record +"""Child-owned independent source-pose plant experiment. + +No legacy RawPoint or external delay enters this module. The controller owner +must first replay the complete original inventory, including excluded context. +Only actual healthy native/CAN endpoints become the arithmetic command tape. +""" +from bisect import bisect_right +from dataclasses import asdict, dataclass +import hashlib +import math + +from openpilot.starpilot.flm import plant_lag as lag + +PROTOCOL = 'chronological-source-physical-observation-fit-v1' +CONTEXT_VERSION = 'original-source-physical-observation-context-v1' +BLOCK_NS = 4_000_000_000 +PURGE_NS = 600_000_000 +FIRST_SLOT_NS = 2_600_000_000 + + +@dataclass(frozen=True) +class TemporalPlantWindow: + """Chronological physical validation block, not an independent maneuver.""" + window_id: str + slot: int + checkpoint_ns: int + start_ns: int + end_ns: int # Inclusive integer convention for the pure helper. + crossing_ns: int + + +def _hash(value): + from openpilot.starpilot.flm.runtime_adapter_registry import digest + return digest(value) + + +@dataclass(frozen=True) +class PreparedPlant: + binding: lag.SourceBinding + identification: lag.IdentificationResult + context: tuple + model: object + source_poses: tuple + command_pairs: tuple + original_windows: tuple + + +def compact_receipt(result): + """Bounded worker report; full original rows stay inside the existing child. + + The original decision/model digests still bind the complete training IDs and + exact row inventory. This wire schema adds compact custody digests rather + than exceeding FitProcess's unchanged64KiB result limit. + """ + value = asdict(result) + rows = value.pop('rows') + value.update(receipt_schema='compact-source-plant-receipt-v1',row_count=len(rows),rows_sha256=lag.digest(rows)) + if value['decision'] is not None: + decision = value['decision'] + ids = decision.pop('train_pose_ids') + decision.update(train_pose_ids_sha256=lag.digest(ids),train_pose_count=len(ids)) + decision['compact_decision_sha256'] = lag.digest(decision) + return value + + +def _pose(value, states, binding, controls,outputs,engine): + """Recompute calibrated yaw/roll and interpolation from original primitives.""" + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose, validate_pose + from openpilot.common.transformations.orientation import rot_from_euler, euler_from_rot + import numpy as np + + def tuples(item): + return tuple(tuples(child) for child in item) if type(item) in (list,tuple) else item + row = dict(value) + for name in ('context', 'angular_velocity_device', 'orientation_ned', 'calibration_rpy', 'left_state', 'right_state', + 'source_clock_contract', 'clock_equivalence', 'ordinary_safety'): + row[name] = tuples(row[name]) + original = OriginalSourcePose(**row) + validate_pose(original, original.completed_ns) + native = controls.get(original.native_control_ns) + if (native is None or original.native_required is not binding.native_required or + (native.native_required,native.native_identity,native.native_observed_ns,native.native_expires_ns) != + (original.native_required,original.native_identity,original.native_observed_ns,original.native_expires_ns)): + raise ValueError('plant original pose native owner receipt') + if not original.native_required and original.ordinary_safety[1] != binding.cp_sha256: + raise ValueError('plant ordinary native exact CarParams') + applied = outputs.get((original.native_control_ns,original.native_apply_ns)) + if (applied is None or applied.apply_ns != original.native_apply_ns or + not native.enabled or not native.active or not native.valid or native.manual or native.override or native.fault or + not applied.valid or not applied.active or applied.override or applied.fault or not applied.native_allowed or + not engine.applied_matches(native,applied)): + raise ValueError('plant original pose receipt actual native/CAN') + if ((original.identity, original.source_epoch, original.baseline_revision, _hash(original.controller_revision), + original.native_identity, original.calibration, _hash((original.source_clock_contract,original.context[7]))) != + (binding.identity, binding.source_epoch, binding.baseline_revision, binding.controller_revision, + binding.native_identity, binding.calibration_generation, binding.source_clock_contract_sha256)): + raise ValueError('plant original pose source/native/calibration') + # Both speed brackets must occur in independently reconstructed original + # ControlEvidence, rather than a second copy of supplied pose floats. + if (states.get(original.left_state_ns) != original.left_state or + states.get(original.right_state_ns) != original.right_state): + raise ValueError('plant original state bracket custody') + device_from_calib = rot_from_euler(np.asarray(original.calibration_rpy)) + yaw = float((device_from_calib.T @ np.asarray(original.angular_velocity_device))[2]) + roll = float(euler_from_rot(rot_from_euler(np.asarray(original.orientation_ned)) @ device_from_calib)[0]) + if yaw != original.yaw_rate or roll != original.roll_rad or not math.isfinite(yaw + roll): + raise ValueError('plant original calibrated pose recomputation') + # The estimate identifies exact per-pose extrinsics; lifecycle remains stable. + if original.calibration_estimate != _hash(tuple(original.calibration_rpy)): + raise ValueError('plant original extrinsics estimate') + return lag.SourcePose(pose_id=_hash(asdict(original)),mono_ns=original.mono_ns,received_ns=original.received_ns, + source_timestamp_ns=original.source_timestamp_ns,source_clock_domain=original.source_clock_domain, + clock_offset_ns=original.clock_offset_ns,clock_uncertainty_ns=original.clock_uncertainty_ns, + source_lower_ns=original.source_lower_ns,source_upper_ns=original.source_upper_ns, + clock_generation=original.clock_generation,speed_mps=original.speed_mps, + calibrated_yaw_rate=yaw,calibrated_roll_rad=roll,binding_digest=lag.binding_digest(binding)) + + +def _command(control, output, engine, binding): + from openpilot.starpilot.flm.torque_eligibility import exact_unclipped_pair, consumed_sources_current + from openpilot.starpilot.flm.torque_emission_replay import delivered + from opendbc.car.steering_projection import projection_to_dict + + # Preserve the exact original guard order; label a bounded decline in this + # child-only phase instead of concealing which original endpoint failed. + rejected = ( + ('disabled',lambda:not control.enabled),('manual',lambda:control.manual), + ('inactive',lambda:not control.active),('invalid_control',lambda:not control.valid), + ('required_native_denied',lambda:control.native_required and not control.native_allowed), + ('override',lambda:control.override),('fault',lambda:control.fault), + ('saturated',lambda:control.saturated),('model_wobble',lambda:control.model_wobble), + ('controller_chatter',lambda:control.controller_chatter), + ('limited_control',lambda:control.limited and not exact_unclipped_pair(control,output)), + ('legacy_coordinates',lambda:any(control.applied_coordinates)), + ('command_clock',lambda:output.control_ns != control.control_ns), + ('invalid_output',lambda:not output.valid),('inactive_output',lambda:not output.active), + ('output_override',lambda:output.override),('output_fault',lambda:output.fault), + ('limited_output',lambda:output.limited),('output_native_denied',lambda:not output.native_allowed), + ('consumed_original_sources',lambda:not consumed_sources_current(control,output)), + ('actual_native_CAN',lambda:not engine.applied_matches(control,output)), + ) + for reason,predicate in rejected: + if predicate(): + raise ValueError('plant actual endpoint '+reason+' control_ns='+str(control.control_ns)+' apply_ns='+str(output.apply_ns)) + return lag.DeliveredCommand(control.control_ns, control.state_ns, output.apply_ns, -delivered(output), + control.source_receipt_ns, control.native_observed_ns, control.native_expires_ns, + output.carrier_frame, output.carrier_observed_ns, _hash(projection_to_dict(output.projection)), + _hash(record_body(control,engine)), _hash({**asdict(output), 'projection': projection_to_dict(output.projection)}), + control.identity, control.source_epoch, control.baseline_revision, control.native_identity,control.native_required) + + +def native_output(control, value): + """Same evidence-only native reconstruction as FullrateCollector._join_emission. + + Original raw output stays in the context packet and packet hash. A current + independently verified control lease supplies the native permission used by + the existing healthy EmissionHistory; a parent output flag alone cannot. + """ + from openpilot.starpilot.flm.torque_raw_fit import output_from_dict + return _native_output_from_original(control, output_from_dict(value)) + + + +def _native_output_from_original(control, output): + """Same native lease reconstruction from an already parsed original body. + + Private caller-local reuse only, no cached permission or packet authority. + """ + from dataclasses import replace + from openpilot.starpilot.flm.fullrate_collector import AppliedEvidence + if control.native_required: + if (not control.native_allowed or not control.native_identity or + not 0 < control.native_observed_ns <= control.control_ns <= output.apply_ns <= control.native_expires_ns or + control.native_expires_ns-control.native_observed_ns > 200_000_000): + return output # Context only; _command rejects if a pose selects it. + if type(output) is not AppliedEvidence or output.native_allowed is not True: + output = replace(output,native_allowed=True) + return output + + + +def _physical_observation(control, output, engine, binding, original_output): + """New physical-only type. Legacy _command/stage/current stay strict.""" + from openpilot.starpilot.flm.physical_input_observation import from_original + return from_original(control, output, engine, binding, original_output=original_output) + + +def _required_controls(controls, poses, *, stamp_field="control_ns"): + """Exact closed union of every all-lag lookback, in original control order.""" + merged = [] + for lower, upper in sorted((p.source_lower_ns-lag.LAGS_NS[-1]-2*lag.JOIN_NS, + p.source_upper_ns) for p in poses): + if merged and lower <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], upper)) + else: + merged.append((lower, upper)) + starts = tuple(lower for lower, _ in merged) + def included(stamp): + index = bisect_right(starts, stamp)-1 + return index >= 0 and stamp <= merged[index][1] + return tuple(control for control in controls.values() if included(getattr(control,stamp_field))) + + +BOOTSTRAP_SLOT_POLICY = 'interval-contained-bootstrap-slots-v1' +SOURCE_COMPLETE_SLOT_POLICY = 'source-complete-fixed-bootstrap-slots-v2' + + +def bootstrap_slot_rows(rows, start_ns, end_ns): + selected = [] + for row in rows: + if type(row) is not dict or type(row.get('mono_ns')) is not int: + raise ValueError('exact original bootstrap pose clock') + if not start_ns <= row['mono_ns'] <= end_ns: + continue + low, high = row.get('source_lower_ns'), row.get('source_upper_ns') + if type(low) is not int or type(high) is not int or not 0 < low <= row['mono_ns'] <= high: + raise ValueError('known original bootstrap source interval required') + if start_ns <= low and high <= end_ns: + selected.append(row) + return tuple(selected) + + +@guarded_archive(position=1) +def prepare(packet, engine, binding, *, slot_policy=None, support_policy=lag.STRICT_SUPPORT): + """Exactly one earliest octet; child verifies boundaries before reading y. + + `engine` is the complete original inventory replay returned by the actual + registered-controller/native archive verifier. A caller assertion is not a + substitute for that source/native/controller replay. + """ + from openpilot.starpilot.flm.original_context_receipts import verify as verify_context, reuse_verified_context + from openpilot.starpilot.flm.torque_raw_fit import output_from_dict + from openpilot.starpilot.flm.physical_response import PhysicalModel + from openpilot.starpilot.flm.physical_input_observation import ( + PROTOCOL as OBSERVATION_PROTOCOL, VerifiedPhysicalTape, physical_tape_digest, + ) + from openpilot.starpilot.flm.runtime_adapter_registry import canonical_physical_context + from openpilot.starpilot.flm.physical_input_observation import _physical_asdict + + if (packet.get('version') not in ('unpaired-original-source-v8',NORMALIZED_SOURCE_VERSION) or + packet.get('encoding') != 'primitive-snapshot-v1' or packet.get('qualified') is not False or + packet.get('plant_lag_ns') is not None or not isinstance(binding, lag.SourceBinding) or + len(engine.records) > 12000 or len(packet['source_poses']) > 3116): + raise ValueError('plant independent original packet/bounds') + if (packet.get('version') == NORMALIZED_SOURCE_VERSION) != (archive_of(engine) is not None): + raise ValueError('explicit normalized physical source owner') + if slot_policy not in (None, BOOTSTRAP_SLOT_POLICY, SOURCE_COMPLETE_SLOT_POLICY): + raise ValueError('explicit physical calibration slot policy') + frames = tuple(engine.records) + controls = reuse_verified_context(packet['context_receipts'],engine,frames) + if controls is None: + controls = verify_context(packet['context_receipts'],engine,frames) + q = frames[0].control_ns + if q not in dict(engine.checkpoints): + raise ValueError('plant original first checkpoint unavailable') + final_end = q+FIRST_SLOT_NS+7*(BLOCK_NS+PURGE_NS)+BLOCK_NS + crossings = tuple(frame for frame in frames if frame.control_ns >= final_end) + if not crossings: + return {'status':'collecting','reason':'chronological_plant_slots_not_sealed','profile':None,'validation':None} + # Seal each slot by its ORIGINAL crossing tick in the same continuous healthy + # prefix. No fault, buffer tail or source-window closure is called independence. + prefix = tuple(frame for frame in frames if frame.control_ns <= crossings[0].control_ns) + if any(not frame.healthy or frame.manual or + not controls[frame.control_ns].enabled or not controls[frame.control_ns].active or + not controls[frame.control_ns].valid or controls[frame.control_ns].manual or + controls[frame.control_ns].override or controls[frame.control_ns].fault for frame in prefix): + raise ValueError('plant chronological prefix contains invalid physical source context') + windows = [] + for slot in range(8): + start = q+FIRST_SLOT_NS+slot*(BLOCK_NS+PURGE_NS) + end = start+BLOCK_NS + crossing = next(frame.control_ns for frame in frames if frame.control_ns >= end) + original = tuple(_physical_asdict(controls[frame.control_ns],_trace_owner=engine) for frame in prefix if start <= frame.control_ns < end) + window_id = _hash((PROTOCOL,lag.binding_digest(binding),q,slot,start,end,original)) + if slot_policy is not None: + window_id = _hash((slot_policy,window_id)) + windows.append(TemporalPlantWindow(window_id,slot,q,start,end-1,crossing)) + outputs, original_outputs = {}, {} + for row in packet['context_receipts']['outputs']: + stamp = row['control_ns'] + control = controls[stamp] # Preserve original lookup-before-parse order. + original = output_from_dict(row) + key=(stamp,original.apply_ns) + outputs[key] = _native_output_from_original(control,original) + original_outputs[key] = original + states = {} + for control in controls.values(): + values = dict(control.replay_inputs) + state = (float(values['vEgo']), float(values['steeringAngleDeg'])) + if control.state_ns in states and states[control.state_ns] != state: + raise ValueError('plant conflicting original state bracket') + states[control.state_ns] = state + poses, selected = [], [] + # Predeclared exact eight slots; a short/long slot rejects this experiment. + # Never skip/top-up/slide a slot after command or response outcomes are known. + for window in windows: + raw = (bootstrap_slot_rows(packet['source_poses'], window.start_ns, window.end_ns) if slot_policy is not None else + tuple(row for row in packet['source_poses'] if window.start_ns <= row['mono_ns'] <= window.end_ns)) + if not 40 <= len(raw) <= 240: + return {'status':'collecting','reason':'chronological_plant_slot_pose_count', + 'profile':None,'validation':None,'slot':window.slot,'poseCount':len(raw)} + selected.append((window, raw)) + if len(selected) != 8: + return {'status': 'collecting', 'reason': 'independent_original_source_window_support', + 'profile': None, 'validation': None, 'temporalBlocks': len(selected)} + if slot_policy == SOURCE_COMPLETE_SLOT_POLICY: + # Preserve the FULL original packet, native reconstruction and fixed slots. + # Determine eligibility solely from original input metadata. The independent + # physical verifier below still has to admit EVERY command of EVERY + # retained all-lag lookback; parent-provided masks are never trusted here. + from types import SimpleNamespace + from openpilot.starpilot.flm.live_tracking_bootstrap_cohorts import source_complete_groups + from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current + original_events = tuple(engine.emissions.events.values()) + stamps = tuple(event.apply_ns for event in original_events) + source_known = [] + for event in original_events: + control = controls.get(event.control_ns) + output = outputs.get((event.control_ns,event.apply_ns)) + known = (control is not None and output is not None and event.valid and event.active and + not event.override and not event.fault and event.native_allowed and + control.valid and control.active and control.enabled and not control.manual and + not control.override and not control.fault and consumed_sources_current(control,output)) + source_known.append(0. if known else None) + # Only source clock intervals are exposed to selection, never response + # yaw/roll/force, tracking error, residual or a proposed tuning coordinate. + clocks = tuple(tuple(SimpleNamespace(mono_ns=row['mono_ns'],source_lower_ns=row['source_lower_ns'], + source_upper_ns=row['source_upper_ns']) for row in raw) for _,raw in selected) + eligible = source_complete_groups(stamps,source_known,clocks) + replacement = [] + for (window,raw), group in zip(selected,eligible,strict=True): + included = {row.mono_ns for row in group} + retained = tuple(row for row in raw if row['mono_ns'] in included) + if not 40 <= len(retained) <= 240: + return {'status':'collecting','reason':'source_complete_fixed_slot_count', + 'profile':None,'validation':None,'slot':window.slot,'poseCount':len(retained)} + replacement.append((window,retained)) + selected = replacement + declared = [] + for window, raw in selected: + block_poses = tuple(_pose(row, states, binding,controls,outputs,engine) for row in raw) + if any(p.source_lower_ns < window.start_ns or p.source_upper_ns > window.end_ns for p in block_poses): + raise ValueError('plant source interval crosses immutable temporal slot') + poses.extend(block_poses) + declared.append(lag.CalibrationWindow(_hash((PROTOCOL, window.window_id)), window.start_ns, + window.end_ns, tuple(p.pose_id for p in block_poses), _hash(asdict(window)))) + # Verify every original command in all-candidate lookbacks once. An invalid + # command is an unavailable experiment; never silently drop it from the tape. + # Controller ticks and Card applications are different clocks. Preserve + # every actual emission, including repeated CCs; a superseded CC that was + # never sent is controller context, not an invented physical plant input. + required = _required_controls(engine.emissions.events,poses,stamp_field="apply_ns") + if slot_policy == SOURCE_COMPLETE_SLOT_POLICY: + events = tuple(engine.emissions.events.values()) + stamps = tuple(event.apply_ns for event in events) + covered = set() + for pose in poses: + first = bisect_right(stamps,pose.source_lower_ns-lag.LAGS_NS[-1])-1 + last = bisect_right(stamps,pose.source_upper_ns)-1 + if first < 0 or last < first: + return {'status':'collecting','reason':'source_complete_lookback_missing','profile':None,'validation':None} + covered.update(range(first,last+1)) + required = tuple(event for index,event in enumerate(events) if index in covered) + + if not required or any((event.control_ns,event.apply_ns) not in outputs for event in required): + return {'status':'collecting','reason':'plant_original_application_receipt_missing','profile':None,'validation':None} + pairs = tuple((controls[event.control_ns], outputs[event.control_ns,event.apply_ns]) for event in required) + try: + commands = tuple(_physical_observation(control, output, engine, binding, + original_outputs[control.control_ns,output.apply_ns]) for control, output in pairs) + except ValueError as error: + return {'status':'collecting','reason':'plant_common_original_command_unavailable', + 'detail':str(error),'profile':None,'validation':None} + verifier_hash = hashlib.sha256(__loader__.get_data(__file__)).hexdigest() + tape = VerifiedPhysicalTape(OBSERVATION_PROTOCOL, commands, lag.binding_digest(binding), + physical_tape_digest(binding, commands), verifier_hash) + if support_policy != lag.STRICT_SUPPORT and slot_policy not in (BOOTSTRAP_SLOT_POLICY,SOURCE_COMPLETE_SLOT_POLICY): + raise ValueError('training-domain support is response-bootstrap only') + result = lag.identify_physical_plant_lag(binding, tape, tuple(poses), tuple(declared), support_policy=support_policy) + if result.status != 'physical_affine_test_passed': + return {'status': result.status, 'reason': result.reason, 'profile': None, 'validation': None, + 'plant_identification': compact_receipt(result)} + # Freeze this experiment context before child fitting. The exact packet binds + # every decision input, and the model itself carries the chosen lag. A parent + # cannot promote an old external-seed report into this context/version. + context = experiment_context(packet, binding) + canonical = canonical_physical_context(context) + # The new typed observation protocol passed the SAME affine, direction, + # excitation and ALL-row per-block .05 arithmetic. Bootstrap may retain + # an explicitly training-domain-only model; its range never expands to cover + # unseen inputs and every heldout block still needs in-domain support. + # Do not manufacture legacy + # PhysicalRow(healthy=True,saturated=False) from tagged originals merely to + # get past the legacy unclipped-input policy. A numeric plant model is NOT + # permission; its context binds this exact packet, tagged tape and decision. + decision = result.decision + if (decision.protocol != lag.PHYSICAL_PROTOCOL or decision.slope <= 0 or + not lag.heldout_supported(result) or + decision.train_block_ids != tuple(window.block_id for window in declared[:4]) or + tuple(block for block, _, _ in result.heldout_blocks) != tuple(window.block_id for window in declared[4:])): + raise ValueError('physical observation original affine receipt mismatch') + model = PhysicalModel(engine.identity, canonical, 'normalized_torque', decision.lag_ns, + decision.slope, decision.intercept, decision.command_low, decision.command_high, + decision.train_block_ids, tuple(window.block_id for window in declared[4:]), + max(error for _, error, _ in result.heldout_blocks)) + return PreparedPlant(binding, result, context, model, tuple(poses), pairs, tuple(window for window, _ in selected)) + + + +def experiment_context(packet, binding): + version = 'original-source-feedback-mean-context-v1' if 'feedback_observations' in packet else CONTEXT_VERSION + return (version, lag.binding_digest(binding), _hash(packet)) + + +def source_binding(packet, engine): + """Binding from exact selected factory/anchor and original observer context.""" + from openpilot.starpilot.flm.runtime_adapter_registry import car_params_digest + first = packet['source_poses'][0] + context = tuple(first['context']) + if len(context) != 15 or any(tuple(row['context']) != context for row in packet['source_poses']): + raise ValueError('plant stable original source observer context') + session, identity, generation, epoch, baseline, revision, calibration, clock_context, context_clock, native, regime, *geometry = context + cp = engine.factory.cp + if (identity != engine.identity or epoch != engine.objective.context_key or baseline != engine.baseline_revision or + revision != engine.original_anchor['controller_revision'] or geometry != + [float(cp.steerRatio), float(cp.wheelbase), float(cp.centerToFront), float(cp.mass)] or + first['identity'] != identity or first['source_epoch'] != epoch or first['native_identity'] != native): + raise ValueError('plant selected factory/source observer binding') + contract = tuple(first['source_clock_contract']) + if tuple(clock_context) != ('original_source_clock_generation', first['clock_generation']): + raise ValueError('plant original interval clock context mismatch') + if any(tuple(row['source_clock_contract']) != contract for row in packet['source_poses']): + raise ValueError('plant original source clock contract changed') + required = packet['context_receipts']['controls'][0]['native_required'] + if type(required) is not bool or any(row['native_required'] is not required for row in packet['context_receipts']['controls']): + raise ValueError('plant original native-required contract changed') + return lag.SourceBinding(identity, car_params_digest(cp), _hash(engine.original_anchor), _hash(revision), + epoch, baseline, native, calibration, _hash((generation, regime, geometry)), session, _hash((contract,clock_context)),required) + + +STAGE_MEMBERSHIP_PROTOCOL = 'original-completed-whole-source-membership-v1' + + +def completed_stage_windows(engine, cutoff_ns): + """Goal/source geometry only; no response, fit, clipping or loss selection. + + Whole original windows come from the actual verified replay. Incomplete + source prefixes/tails are not completed maneuvers. Their frames, poses and + native packets remain archived and in every counterfactual rollout. + """ + from openpilot.starpilot.flm.controller_replay import ControllerReplay + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.event_windows import whole_event_windows + if (type(engine) is not EventControllerReplay or type(engine.engine) is not ControllerReplay or + engine.failed or not engine.raw_mode or type(cutoff_ns) is not int or cutoff_ns <= 0 or + not 1 <= len(engine.records) <= 12000): + raise ValueError('stage membership requires actual original replay') + return tuple(window for window in whole_event_windows(engine) if window[1] > cutoff_ns) + + +def _stage_context_endpoint(control, output, engine): + """Excluded goal prefix/tail stays HARD context, never a fitting input. + + Original performance labels remain unchanged/archived. Manual, source, + clocks, native permission, override, fault and actual CAN are mandatory even + for an endpoint outside the completed goal windows. + """ + from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current + if (not control.enabled or control.manual or not control.active or not control.valid or + control.native_required and not control.native_allowed or control.override or control.fault or + any(control.applied_coordinates) or output.control_ns != control.control_ns or + not output.valid or not output.active or output.override or output.fault or not output.native_allowed or + not consumed_sources_current(control,output) or not engine.applied_matches(control,output)): + raise ValueError('plant excluded whole-source prefix hard endpoint control_ns='+str(control.control_ns)) + + +def stage_control_member(windows, control_ns): + """Pure membership predicate; it confers no source/native/application proof.""" + return any(start <= control_ns <= end for _,start,end,_ in windows) + + +def rejoin_stage(poses, commands, prepared, engine): + """Associate NEW stage poses with original delivered commands at locked lag.""" + from openpilot.starpilot.flm.torque_raw_fit import control_from_dict + from openpilot.starpilot.flm.torque_emission_replay import delivered + from opendbc.car.steering_projection import projection_to_dict + + if not isinstance(prepared, PreparedPlant): + raise ValueError('plant frozen stage plan required') + cutoff, delay = prepared.identification.cutoff_ns, prepared.model.lag_ns + def parsed_pair(c,o): + control = control_from_dict(c,trace_archive=archive_of(engine)) + return control,native_output(control,o) + ordered = tuple(parsed_pair(c,o) for c, o in commands) + stamps = tuple(output.apply_ns for _, output in ordered) + if any(right <= left for left, right in zip(stamps, stamps[1:], strict=False)): + raise ValueError('plant original stage command order') + windows = completed_stage_windows(engine,cutoff) + previous, previous_command, rows = cutoff, cutoff, [] + for pose in poses: + index = bisect_right(stamps, pose.source_lower_ns-delay)-1 + upper_index = bisect_right(stamps,pose.source_upper_ns-delay)-1 + if index < 0 or index != upper_index or pose.source_upper_ns-delay-stamps[index] > lag.JOIN_NS: + raise ValueError('plant original stage lag support') + control, output = ordered[index] + if min(control.control_ns, output.apply_ns, pose.mono_ns) <= cutoff or pose.mono_ns <= previous: + raise ValueError('plant stage strictly after physical cutoff') + previous = pose.mono_ns # Preserve ordering of EVERY original stage pose. + if not stage_control_member(windows,control.control_ns): + _stage_context_endpoint(control,output,engine) + continue # Original goal geometry only; no performance/y/native selection. + if output.apply_ns <= previous_command: + raise ValueError('plant stage strictly after physical cutoff') + # Invoke exactly the same source/native/output guards again for NEW joins. + # The binding used by _command is only carried metadata; actual verification + # is engine.applied_matches, not the earlier calibration tape boolean. + _command(control, output, engine, prepared.binding) + previous_command = output.apply_ns + rows.append({'control_ns': control.control_ns, 'apply_ns': output.apply_ns, 'mono_ns': pose.mono_ns, + 'control': stored_record(control), 'output': {**asdict(output), 'projection': projection_to_dict(output.projection)}, + 'physical_context': prepared.context, 'physical': {'applied_command': delivered(output), + 'lateral_accel': pose.speed_mps*pose.calibrated_yaw_rate-9.81*math.sin(pose.calibrated_roll_rad), + 'speed_mps': pose.speed_mps, 'response_ns': pose.mono_ns}}) + return tuple(rows) diff --git a/openpilot/starpilot/flm/source_plant_pipeline.py b/openpilot/starpilot/flm/source_plant_pipeline.py new file mode 100644 index 0000000000..7eaf811325 --- /dev/null +++ b/openpilot/starpilot/flm/source_plant_pipeline.py @@ -0,0 +1,297 @@ +"""Explicit optional producer/child integration for independent plant inputs. + +The separately versioned feedback/mean path requires complete original witness +and candidate feedback-regime proof. Legacy protocol semantics remain separate. +""" +from dataclasses import asdict, dataclass +from typing import TypedDict, cast + +from openpilot.starpilot.flm.source_plant_fit import PROTOCOL + + +class _EffectObjectiveFields(TypedDict): + group: str + names: tuple[str, ...] + anchor_digest: str + context_key: str + + +@dataclass(frozen=True) +class OriginalReplayTemplate: + """Unverified selected-factory descriptor, never a replay/model certificate. + + Fresh ControllerReplay is intentionally not wrapped in EventControllerReplay: + it has no original checkpoint yet and correctly starts failed. The archive + verifier owns checkpoint/record lifecycle and returns the actual verified ECR. + """ + engine: object + objective: object + original_anchor: dict + + def __getattr__(self, name): + return getattr(self.engine,name) + + +def replay_data(packet, *, objective, identity, baseline_revision): + original = packet['original_replay_source'] + frames = tuple(packet['actual_frames']) + if not frames or original['raw_mode'] is not True: + raise ValueError('plant original raw controller replay') + first, last = frames[0]['control_ns'], frames[-1]['control_ns'] + if tuple(packet['replay_prefix']) != (first,last): + raise ValueError('plant original complete replay prefix') + from openpilot.starpilot.flm.normalized_trace_archive import normalized_packet,reserve_for_packet + normalized = normalized_packet(packet) + reserve_for_packet(packet) + data = {'version':2 if normalized else 1,'raw_mode':True, + 'cp_bytes':bytes.fromhex(original['cp_bytes_hex']), 'mode':original['mode'], + 'turn_assist':original['turn_assist'], 'manual_baseline':tuple(original['manual_baseline']), + 'manual_source_epoch':original['manual_source_epoch'], 'objective':asdict(objective), + 'identity':identity, 'baseline_revision':baseline_revision,'model_revision':original['model_revision'], + 'windows':((first,last),),'checkpoints':tuple((stamp,revision,tuple(tuple(pair) for pair in values)) for stamp,revision,values in original['checkpoints']), + 'frames':frames,'emissions':packet['actual_command_tape']} + if normalized: + data['trace_archive'] = packet['trace_archive'] + return data + + +def _template(payload, factory): + from openpilot.starpilot.flm.controller_replay import ControllerReplay + from openpilot.starpilot.flm.event_objective import EffectObjective + from openpilot.starpilot.flm.full_surface_profile import envelope_from_dict + from openpilot.starpilot.flm.torque_adapter import Objective + legacy = Objective(**payload['objective']) + objective = EffectObjective(**cast(_EffectObjectiveFields, + {**payload['event_objective'],'names':tuple(payload['event_objective']['names'])})) + envelope = envelope_from_dict(payload['effect_envelope']) + envelope.validate() + if (envelope.trial_anchor != objective.anchor_digest or envelope.trial_names != objective.names or + envelope.context_key != objective.context_key or envelope.source_epoch != objective.context_key): + raise ValueError('plant original full anchor/objective') + engine = ControllerReplay(factory,legacy,identity=payload['identity'], + baseline_revision=envelope.selected_baseline_revision,model_revision=payload['source_packet']['original_replay_source']['model_revision']) + engine.raw_mode = True + return OriginalReplayTemplate(engine,objective,envelope.replay_anchor()), objective, envelope + + +def _stage_engine(original, cutoff, *, context_packet=None): + """Use genuine healthy-window checkpoints; no synthesized boundary state.""" + from openpilot.starpilot.flm.controller_replay import ControllerReplay, pack_primitive_snapshot + from openpilot.starpilot.flm.event_storage import source_windows + from types import SimpleNamespace + windows = source_windows(SimpleNamespace(records=original.records,replay_windows=(),_frame_context=original._frame_context)) + checkpoints = dict(original.checkpoints) + chosen = [] + for start,end in windows: + if end <= cutoff: + continue + # Preserve an original checkpoint BEFORE the cutoff when this same healthy + # interval crosses it. A 500ms true lag can select a genuinely later command + # before the next one-second checkpoint; dropping that original prefix + # would falsely remove its controller/CAN support. + before = tuple(stamp for stamp in checkpoints if start <= stamp <= cutoff) + candidates = tuple(stamp for stamp in checkpoints if start <= stamp <= end) + if candidates: + chosen.append((max(before) if before else min(candidates),end)) + if not chosen or len(chosen) > 24: + raise ValueError('plant later original healthy checkpoint unavailable') + records = tuple(frame for frame in original.records if any(a <= frame.control_ns <= b for a,b in chosen)) + # This is a subset of an already bounded complete inventory, never another + # extra budget of 12000 frames. It supplies original stage state only. + if len({frame.control_ns for frame in original.records} | {frame.control_ns for frame in records}) > 12000: + raise ValueError('plant union replay frame bound') + if context_packet is not None: + from openpilot.starpilot.flm.original_context_receipts import verified_replay_subset + view = verified_replay_subset(context_packet,original,tuple(chosen)) + if view is not None: + return view + from openpilot.starpilot.flm.normalized_trace_archive import archive_key + if archive_key(original) is not None: + raise ValueError('normalized stage requires complete verified original subset') + data = {'version':1,'raw_mode':True,'cp_bytes':original.factory.cp.as_builder().to_bytes(), + 'mode':str(original.factory.mode),'turn_assist':original.factory.turn_assist, + 'manual_baseline':original.factory.manual_baseline,'manual_source_epoch':original.factory.manual_source_epoch, + 'objective':asdict(original.engine.objective),'identity':original.identity, + 'baseline_revision':original.baseline_revision,'model_revision':original.engine.model_revision, + 'windows':tuple(chosen),'checkpoints':tuple((stamp,cp.revision,cp.values) for stamp,cp in original.checkpoints + if any(a <= stamp <= b for a,b in chosen)), + 'frames':tuple(asdict(frame) for frame in records),'emissions':original.emissions.snapshot(tuple(chosen))} + return ControllerReplay.from_snapshot(original.factory,original.engine.objective,pack_primitive_snapshot(data),rebuild=False) + + +def _feedback_summary(evidence,packet,protocol): + from openpilot.starpilot.flm.candidate_feedback import LEGACY_POLICY,PROOF_VERSION + body = {'protocol':protocol,'binding_digest':evidence.binding, + 'archive_digest':packet['feedback_observations']['digest'].hex(),'verified_digest':evidence.original.digest, + 'known_rows':len(evidence.original.rows),'context_rows':len(evidence.original.context_only)} + if evidence.candidate_policy != LEGACY_POLICY: + body.update(candidate_feedback_policy=evidence.candidate_policy,candidate_feedback_proof_version=PROOF_VERSION) + return body + + +def fit_snapshot(payload): + """Bounded FitProcess child entry; controller and CAN proofs precede OLS.""" + from openpilot.starpilot.flm.controller_replay import factory_from_snapshot + from openpilot.starpilot.flm.original_context_receipts import verify_original_replay + from openpilot.starpilot.flm.source_plant_fit import source_binding, experiment_context, prepare, _pose, rejoin_stage, native_output, compact_receipt + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.event_activation import build_activation_plan + from openpilot.starpilot.flm.event_windows import regroup_whole_events + from openpilot.starpilot.flm.clipping_quality import event_clipping_quality + from openpilot.starpilot.flm.event_raw_fit import fit_event_torque + from openpilot.starpilot.flm.plant_lag import LAGS_NS + + from openpilot.starpilot.flm.feedback_replay import PROTOCOL as FEEDBACK_PROTOCOL,EVENT_VERSION + feedback_job = payload.get('plant_protocol') == FEEDBACK_PROTOCOL + from openpilot.starpilot.flm.candidate_feedback import LEGACY_POLICY,policy + candidate_policy = policy(payload.get('candidate_feedback_policy',LEGACY_POLICY)) + if not feedback_job and 'candidate_feedback_policy' in payload: + raise ValueError('candidate feedback policy requires explicit source feedback protocol') + if feedback_job: + from openpilot.starpilot.flm.primitive_source_memo import independent_primitive_graph + from openpilot.starpilot.flm.normalized_trace_archive import reserve_for_packet + payload = independent_primitive_graph(payload,reserved_bytes=reserve_for_packet(payload['source_packet'])) + protocol = FEEDBACK_PROTOCOL if feedback_job else PROTOCOL + if (payload.get('plant_protocol') != protocol or payload.get('event_version') not in ((EVENT_VERSION,8) if feedback_job else (3,4,5,6)) or + payload.get('event_operation') not in ('trial','restore')): + raise ValueError('explicit source plant trial protocol') + packet = payload['source_packet'] + from openpilot.starpilot.flm.controller_replay import pack_primitive_snapshot,MAX_FIT_RAW_BYTES + from openpilot.starpilot.flm.torque_adapter import Objective + from openpilot.starpilot.flm.full_surface_profile import envelope_from_dict + import pickle + envelope = envelope_from_dict(payload['effect_envelope']) + replay = replay_data(packet,objective=Objective(**payload['objective']),identity=payload['identity'], + baseline_revision=envelope.selected_baseline_revision) + aggregate_raw = len(pickle.dumps(payload,protocol=5))+len(pickle.dumps(replay,protocol=5)) + if aggregate_raw > MAX_FIT_RAW_BYTES: + raise ValueError('plant aggregate original raw worker byte bound') + replay = pack_primitive_snapshot(replay) + factory,_ = factory_from_snapshot(replay) + template,objective,envelope = _template(payload,factory) + actual = verify_original_replay(packet['context_receipts'],template,replay) + binding = source_binding(packet,actual) + original_feedback = None + if feedback_job: + from openpilot.starpilot.flm.feedback_replay import verify_archive + from openpilot.starpilot.flm.event_mean_evidence import VERSION as MEAN_VERSION + operation = payload.get('event_operation') + if operation == 'trial': + if payload.get('event_version') != EVENT_VERSION or payload.get('event_mean_version') != MEAN_VERSION: + raise ValueError('explicit new feedback/mean trial version required') + elif operation == 'restore': + if payload.get('event_version') != 8: + raise ValueError('explicit new feedback restore version required') + event_clipping_quality(payload) + else: + raise ValueError('explicit original feedback operation') + original_feedback = verify_archive(packet,actual) + elif 'event_mean_version' in payload or 'feedback_observations' in packet: + raise ValueError('legacy source packet cannot implicitly adopt feedback policy') + # Key selection is original epoch metadata, never a fit/readiness counter. + from openpilot.starpilot.flm.learner_factory import collection_keys + from openpilot.starpilot.flm.source_plant_runtime import SCHEDULE_VERSION + schedule = payload.get('fit_schedule') + registered = tuple(collection_keys(next(iter(actual.original_context_controls.values())))) + if (type(schedule) is not dict or schedule.get('version') != SCHEDULE_VERSION or + type(packet.get('capture_epoch')) is not int or packet['capture_epoch'] < 0 or + schedule.get('epoch') != packet['capture_epoch'] or + schedule.get('start_ns') != packet['capture_epoch_start_ns'] or + schedule.get('cutoff_ns') != packet['capture_epoch_cutoff_ns'] or + not 0 < schedule['start_ns'] < schedule['cutoff_ns'] or + schedule['cutoff_ns']-schedule['start_ns'] > 100_000_000_000 or + tuple(tuple(key) for key in schedule.get('registered_keys',())) != registered or + tuple(schedule.get('objective_key',())) != registered[packet['capture_epoch'] % len(registered)] or + tuple(payload.get('objective_key',())) != tuple(schedule['objective_key']) or + any(not schedule['start_ns'] <= row['mono_ns'] < schedule['cutoff_ns'] for row in packet['source_poses'])): + raise ValueError('original source experiment objective binding') + if tuple(payload['physical_context']) != experiment_context(packet,binding): + raise ValueError('plant exact submitted source experiment context') + prepared = prepare(packet,actual,binding) + if type(prepared) is dict: + return {**prepared,'plant_protocol':protocol,'physical_context':payload['physical_context']} + later = _stage_engine(actual,prepared.identification.cutoff_ns,context_packet=packet['context_receipts']) + restoring = payload['event_operation'] == 'restore' + if (restoring and payload['event_version'] not in ((8,) if feedback_job else (4,5,6)) or + not restoring and payload['event_version'] != (EVENT_VERSION if feedback_job else 3)): + raise ValueError('plant exact trial/restore declaration') + engine = EventControllerReplay(later,objective,envelope.replay_anchor(), + restore_envelope=envelope if restoring else None) + controls = actual.original_context_controls + states = {} + for control in controls.values(): + values = dict(control.replay_inputs) + state = (float(values['vEgo']),float(values['steeringAngleDeg'])) + if control.state_ns in states and states[control.state_ns] != state: + raise ValueError('plant changed original state bracket') + states[control.state_ns] = state + raw_outputs = packet['context_receipts']['outputs'] + receipt_outputs = {row['control_ns']:native_output(controls[row['control_ns']],row) for row in raw_outputs} + poses = tuple(_pose(row,states,binding,controls,receipt_outputs,actual) for row in packet['source_poses'] if + row['mono_ns'] > prepared.identification.cutoff_ns+LAGS_NS[-1]) + outputs = packet['context_receipts']['outputs'] + from openpilot.starpilot.flm.normalized_trace_archive import stored_record + commands = tuple((stored_record(controls[row['control_ns']]),row) for row in outputs) + try: + rows = rejoin_stage(poses,commands,prepared,engine) + except ValueError as error: + # A verified physical fit is not a stage/application certificate. Preserve + # its bounded receipt while the independently strict stage gate declines. + return {'status':'collecting','reason':'original_later_stage_endpoint_unqualified', + 'detail':str(error)[:512],'profile':None,'validation':None,'plant_protocol':protocol, + 'physical_context':prepared.context,'plant_identification':compact_receipt(prepared.identification)} + packets = ({'identity':engine.identity,'samples':rows},) + events = regroup_whole_events(engine,packets,objective) + if len(events) < 8: + return {'status':'collecting','reason':'independent_source_later_whole_stage_support', + 'profile':None,'validation':None,'plant_protocol':protocol, + 'plant_identification':compact_receipt(prepared.identification),'stageBlocks':len(events)} + if restoring: + from openpilot.starpilot.flm.restore_activation import build_restore_plan + from openpilot.starpilot.flm.event_restore_fit import fit_event_restore + plan = build_restore_plan(engine,model=prepared.model,cutoff_ns=prepared.identification.cutoff_ns, + stage_events=events,physical_context=prepared.model.context,envelope=envelope,restore_version=payload['event_version']) + engine = EventControllerReplay(later,objective,envelope.replay_anchor(),restore_envelope=envelope,activation_plan=plan) + if feedback_job: + from openpilot.starpilot.flm.feedback_replay import attach_feedback_engine + attach_feedback_engine(engine,original_feedback,candidate_policy=candidate_policy) + result = fit_event_restore(payload,engine,objective,source_plant=(prepared,events)) + if feedback_job: + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + evidence = require_feedback_engine(engine) + result = {**result,'feedback_evidence':_feedback_summary(evidence,packet,protocol)} + return {**result,'plant_protocol':protocol,'physical_context':prepared.context, + 'plant_identification':compact_receipt(prepared.identification),'plant_binding':asdict(binding)} + plan = build_activation_plan(engine,model=prepared.model,cutoff_ns=prepared.identification.cutoff_ns, + stage_events=events,physical_context=prepared.model.context,quality_contract=event_clipping_quality(payload)) + engine = EventControllerReplay(later,objective,envelope.replay_anchor(),activation_plan=plan) + if feedback_job: + from openpilot.starpilot.flm.feedback_replay import attach_feedback_engine + attach_feedback_engine(engine,original_feedback,candidate_policy=candidate_policy) + engine.rebuild_causal_features() + # The existing caller chooses identifiable coordinates only from actual + # stage feature support. No calibration/heldout targets enter that choice. + from openpilot.starpilot.flm.event_fit_factory import _identified + selected = _identified(engine,events) + if not selected: + return {'status':'collecting','reason':'no_identifiable_canonical_coordinates','profile':None,'validation':None, + 'plant_protocol':protocol,'plant_identification':compact_receipt(prepared.identification)} + if selected != objective.names: + from openpilot.starpilot.flm.event_objective import EffectObjective + previous = engine + envelope = envelope.for_trial(selected) + objective = EffectObjective(objective.group,selected,envelope.trial_anchor,objective.context_key) + engine = EventControllerReplay(later,objective,envelope.replay_anchor(),activation_plan=plan) + if feedback_job: + attach_feedback_engine(engine,original_feedback,candidate_policy=candidate_policy) + if engine.reuse_subset_causal_features(previous) is not True: + engine.rebuild_causal_features() + payload = {**payload,'effect_envelope':asdict(envelope),'event_objective':asdict(objective), + 'prior':tuple(dict(envelope.prior_coordinates)[name] for name in selected)} + result = fit_event_torque(payload,engine,objective,source_plant=(prepared,events)) + if feedback_job: + from openpilot.starpilot.flm.feedback_replay import require_feedback_engine + evidence = require_feedback_engine(engine) + result = {**result,'feedback_evidence':_feedback_summary(evidence,packet,protocol)} + return {**result,'plant_protocol':protocol,'physical_context':prepared.context, + 'plant_identification':compact_receipt(prepared.identification),'plant_binding':asdict(binding)} diff --git a/openpilot/starpilot/flm/source_plant_runtime.py b/openpilot/starpilot/flm/source_plant_runtime.py new file mode 100644 index 0000000000..9d07dc4996 --- /dev/null +++ b/openpilot/starpilot/flm/source_plant_runtime.py @@ -0,0 +1,509 @@ +"""Optional learner plumbing; never imported by controller realtime loops.""" +from dataclasses import asdict +import base64 +import math + +from openpilot.starpilot.flm.source_plant_fit import PROTOCOL, CONTEXT_VERSION, source_binding, experiment_context + + +def enable(adapter, clock_contract, *, protocol=PROTOCOL): + from openpilot.starpilot.flm.original_source_capture import OriginalSourceClockContract + if (not getattr(adapter,'event_collection',False) or adapter.replay_version != 3 or + clock_contract is not None and not isinstance(clock_contract,OriginalSourceClockContract)): + raise ValueError('explicit registered event/source clock plant contract') + if clock_contract is not None: + clock_contract.validate() + from openpilot.starpilot.flm.feedback_replay import PROTOCOL as FEEDBACK_PROTOCOL + if protocol not in (PROTOCOL,FEEDBACK_PROTOCOL): + raise ValueError('explicit original plant protocol') + if protocol == FEEDBACK_PROTOCOL: + from openpilot.starpilot.flm.feedback_archive import OriginalFeedbackArchive + adapter.original_feedback_archive = OriginalFeedbackArchive() + adapter.source_plant_protocol = protocol + adapter.source_plant_clock = clock_contract + adapter.source_plant_receipt = None + adapter.source_plant_binding = None + adapter.source_plant_submitted_epoch = None + adapter.source_plant_submitted_contexts = {} + return adapter + + +def start_original_capture(collector): + """Source-pin/clock unavailability denies learning without killing producer.""" + adapter = collector.adapter + try: + collector.enable_original_source_capture(clock_contract=adapter.source_plant_clock) + except (OSError,ValueError): + adapter.source_plant_capture_error = 'original_source_clock_contract_unavailable' + adapter.current_physical_valid = False + adapter.last_physical_ns = collector.last_qualified_response = 0 + return False + adapter.source_plant_capture_error = None + return True + + +def observe(sm, cp, collector, now_ns, current_native): + from openpilot.starpilot.flm.runtime_adapter_registry import original_source_pose_observer + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose + from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current + from openpilot.starpilot.flm.torque_emission_replay import delivered + + adapter = collector.adapter + def revoke(): + adapter.current_physical_valid = False + adapter.last_physical_ns = collector.last_qualified_response = 0 + if not current_native: + revoke() + return + row = original_source_pose_observer(sm,cp,collector,now_ns,distinguish_pending=True) + from openpilot.starpilot.flm.physical_observation import PendingPhysical + if type(row) is PendingPhysical and row.fresh(now_ns) and not row.context_changed: + return # Keep the ORIGINAL prior receipt; do not renew its timestamp. + if not isinstance(row,OriginalSourcePose): + revoke() + return + revoke() + collector.original_source_pose(row,now_ns) + live = getattr(collector, 'live_tracking', None) + live_lag = live.optimizer.lag_ns if live is not None else getattr(collector, 'live_response_lag_ns', None) + collector.live_response_pair = None + if type(live_lag) is int and 0 <= live_lag <= 500_000_000: + lower = [(c,o) for c,o in collector.ready if o.apply_ns <= row.source_lower_ns-live_lag] + upper = [(c,o) for c,o in collector.ready if o.apply_ns <= row.source_upper_ns-live_lag] + if lower and upper and lower[-1][1].apply_ns == upper[-1][1].apply_ns: + collector.live_response_pair = (*lower[-1], row) + if live is not None: + live.observe_original_pose(row, collector, now_ns=now_ns, current_native=current_native) + model, binding = adapter.model, adapter.source_plant_binding + if model is None or binding is None or adapter.source_plant_receipt is None: + return + from openpilot.starpilot.flm.runtime_adapter_registry import digest + if (row.identity != binding['identity'] or row.source_epoch != binding['source_epoch'] or + row.baseline_revision != binding['baseline_revision'] or row.calibration != binding['calibration_generation'] or + row.native_identity != binding['native_identity'] or digest((row.source_clock_contract,row.context[7])) != binding['source_clock_contract_sha256'] or + digest(row.controller_revision) != binding['controller_revision'] or + digest((row.context[2],row.context[10],list(row.context[11:]))) != binding['vehicle_model_generation'] or + model.context != digest(collector.physical_context) or + not 0 < row.source_lower_ns <= row.source_upper_ns <= now_ns or now_ns-row.source_lower_ns > 100_000_000): + return + lower = tuple((control,output) for control,output in collector.ready if output.apply_ns <= row.source_lower_ns-model.lag_ns) + upper = tuple((control,output) for control,output in collector.ready if output.apply_ns <= row.source_upper_ns-model.lag_ns) + if not lower or not upper or lower[-1][1].apply_ns != upper[-1][1].apply_ns: + return + control,output = lower[-1] + if (row.source_upper_ns-model.lag_ns-output.apply_ns > 30_000_000 or + not consumed_sources_current(control,output) or not adapter.replay.applied_matches(control,output) or + not output.native_allowed or output.limited or output.override or output.fault or not output.valid or + control.native_required and not control.native_allowed or control.override or control.fault or not control.valid or control.manual or + not control.enabled or not control.active or control.saturated or control.model_wobble or control.controller_chatter or + min(control.control_ns,output.apply_ns,row.mono_ns) <= adapter.model_cutoff_ns): + return + command = -delivered(output) + measured = row.speed_mps*row.yaw_rate-9.81*math.sin(row.roll_rad) + bounds = adapter.model_speed_bounds + valid = (bounds is not None and bounds[0] <= row.speed_mps <= bounds[1] and + model.command_low <= command <= model.command_high and + abs(model.slope*command+model.intercept-measured) <= max(.01,3*model.heldout_rmse)) + if valid: + adapter.current_physical_valid = True + adapter.last_physical_ns = collector.last_qualified_response = row.mono_ns + + +MIN_NORMAL_JOB_AGE_NS = 60_000_000_000 # Fixed before data/fit outcomes; source epoch is100s. +NORMAL_JOB_ATTEMPT_SPACING_NS = 80_000_000_000 + + +def _current_fit_pair_reason(collector, now_ns, *, physical_binding=None): + """Cheap latest-pair availability only; no history search or fit authority. + + The child/current validators remain independent. A known-unqualified latest + endpoint must not consume this fixed experiment's one fitting attempt. + """ + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence, MAX_AGE_NS + from openpilot.starpilot.flm.torque_eligibility import consumed_sources_current, exact_unclipped_pair + try: + control, output = collector.ready[-1] + if (type(control) is not ControlEvidence or type(output) is not AppliedEvidence or + type(now_ns) is not int or now_ns <= 0): + return 'invalid_current_pair' + if not output.apply_ns <= now_ns <= output.apply_ns+MAX_AGE_NS or not consumed_sources_current(control,output) or output.control_ns != control.control_ns: + return 'current_pair_not_fresh' + if (control.identity != collector.adapter.identity or control.baseline_revision != collector.adapter.baseline_revision or + not 0 < control.source_receipt_ns <= control.control_ns <= now_ns <= control.source_receipt_ns+500_000_000): + return 'current_pair_source_mismatch' + observed_clip = False + if physical_binding is not None and output.limited: + from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + from openpilot.starpilot.flm.live_tracking_response import verified_clipped_input + if type(physical_binding) is LiveTrialBinding: + observed_clip = verified_clipped_input(collector.adapter.replay, control, output, physical_binding) + if (not control.enabled or control.manual or not control.active or not control.valid or control.override or control.fault or + control.saturated or control.model_wobble or control.controller_chatter or + control.limited and not exact_unclipped_pair(control,output) and not observed_clip): + return 'current_control_unqualified' + if (not output.valid or not output.active or output.override or output.fault or output.limited and not observed_clip or not output.native_allowed): + return 'current_output_unqualified' + if control.native_required and (not control.native_allowed or not control.native_identity or + not 0 < control.native_observed_ns <= control.control_ns <= output.apply_ns <= now_ns <= control.native_expires_ns or + control.native_expires_ns-control.native_observed_ns > 200_000_000): + return 'current_native_unqualified' + if not control.native_required and (control.native_allowed or control.native_identity or + (control.native_observed_ns,control.native_expires_ns) != (0,0)): + return 'current_native_unqualified' + return None + except (AttributeError,IndexError,KeyError,TypeError,ValueError,OverflowError): + return 'invalid_current_pair' + + +def snapshot(collector,now_ns): + adapter,capture = collector.adapter,collector.original_source_capture + if capture is None: + adapter.snapshot_readiness = {'reason':'original_source_capture_unavailable'} + return None + if (adapter.envelope is None or not collector.ready or not capture.rows or + adapter.source_plant_submitted_epoch == capture.epoch or + collector.last_fit and now_ns-collector.last_fit < NORMAL_JOB_ATTEMPT_SPACING_NS): + return None + current_decline = _current_fit_pair_reason(collector,now_ns) + if current_decline is not None: + adapter.snapshot_readiness = {'reason':current_decline,'childQualification':'not_yet_evaluated'} + return None + checkpoints = tuple(stamp for stamp,_ in adapter.replay.checkpoints if stamp <= capture.rows[0].mono_ns) + if not checkpoints: + adapter.snapshot_readiness = {'reason':'plant_original_prefix_checkpoint_unavailable'} + return None + start = max(checkpoints) + if now_ns-start < MIN_NORMAL_JOB_AGE_NS: + adapter.snapshot_readiness = {'reason':'plant_fixed_cohort_still_collecting', + 'elapsedNs':now_ns-start,'requiredNs':MIN_NORMAL_JOB_AGE_NS} + return None + # Full original geometry retains the fixed attempt spacing. Its counts + # are availability only; the child independently reconstructs every join. + geometry_context = (capture.epoch,getattr(capture,'epoch_start_ns',0), + getattr(capture,'epoch_cutoff_ns',0),getattr(collector,'context',None), + getattr(adapter.envelope,'prior_full_profile_sha256',None)) + if getattr(collector,'source_plant_geometry_context',None) == geometry_context: + last_check = getattr(collector,'source_plant_geometry_check_ns',0) + if last_check and now_ns-last_check < 1_000_000_000: + return None + collector.source_plant_geometry_context = geometry_context + collector.source_plant_geometry_check_ns = now_ns + records = tuple(frame for frame in adapter.replay.records if frame.control_ns >= start) + if not records or len(records) > 12000: + adapter.snapshot_readiness = {'reason':'plant_original_prefix_frame_budget'} + return None + from openpilot.starpilot.flm.event_storage import source_windows + from openpilot.starpilot.flm.event_windows import whole_event_windows + from openpilot.starpilot.flm.source_plant_fit import FIRST_SLOT_NS,BLOCK_NS,PURGE_NS + from types import SimpleNamespace + view = SimpleNamespace(records=records,replay_windows=(),_frame_context=adapter.replay._frame_context) + view.replay_windows = source_windows(view) + cutoff = start+FIRST_SLOT_NS+7*(BLOCK_NS+PURGE_NS)+BLOCK_NS + count,pending,last_window = 0,0,None + complete_goal_windows = 0 + for window,first,last,_ in whole_event_windows(view): + if first <= cutoff: + continue + if window != last_window: + pending = 0 + last_window = window + points = sum(first <= row.mono_ns <= last for row in capture.rows) + if points > 240: + pending = 0 + continue + # The child requires nine comparable subtle cycles (or three large + # maneuvers), independently of eight statistical storage bundles. Wait + # for enough complete original geometry before consuming this epoch's + # sole attempt. This is availability only: error/excitation/quality and + # train/heldout lineage remain fresh child qualifications. + if points >= 4: + complete_goal_windows += 1 + if pending+points > 240: + pending = 0 + pending += points + if pending >= 40: + count += 1 + pending = 0 + if count < 8 or complete_goal_windows < 9: + adapter.snapshot_readiness = {'reason':'plant_later_original_goal_geometry', + 'candidateBundles':count,'requiredBundles':8,'completeGoalWindows':complete_goal_windows, + 'requiredCompleteGoalWindows':9,'childQualification':'not_yet_evaluated'} + return None + # Geometry availability is not a submitted/failed experiment. Only a ready + # export attempt starts the existing fit cooldown; later original geometry + # in this fixed epoch may complete without waiting beyond its100s cutoff. + collector.last_fit = now_ns + collector.snapshot_attempts += 1 + try: + wire = adapter.original_source_packet(collector,((start,records[-1].control_ns),),now_ns) + return snapshot_from_packet(collector,wire,now_ns) + except (AttributeError,KeyError,TypeError,ValueError,OverflowError) as error: + adapter.snapshot_readiness = {'reason':'plant_snapshot_unavailable','detail':str(error)} + return None + + +SCHEDULE_VERSION = 'original-source-experiment-objective-v1' + + +def fit_schedule(collector): + """Fixed original experiment ordinal, independent of readiness/fit outcomes.""" + from openpilot.starpilot.flm.learner_factory import collection_keys + capture = collector.original_source_capture + if capture is None or not collector.ready or not capture.rows or not capture.epoch_start_ns: + return None + keys = tuple(collection_keys(collector.ready[-1][0])) + if (type(capture.epoch) is not int or capture.epoch < 0 or + not 0 < capture.epoch_start_ns < capture.epoch_cutoff_ns or + capture.epoch_cutoff_ns-capture.epoch_start_ns > 100_000_000_000 or + not 1 <= len(keys) <= 12 or len(set(keys)) != len(keys)): + raise ValueError('original source experiment objective registry') + return {'version':SCHEDULE_VERSION,'epoch':capture.epoch, + 'start_ns':capture.epoch_start_ns,'cutoff_ns':capture.epoch_cutoff_ns, + 'registered_keys':keys,'objective_key':keys[capture.epoch % len(keys)]} + + +def fit_key(collector): + schedule = fit_schedule(collector) + return None if schedule is None else schedule['objective_key'] + + +def submission_key(snapshot,collector,capture_key): + if getattr(collector.adapter,'source_plant_protocol',None) is None: + return capture_key + key = tuple(snapshot.get('objective_key',())) + if key != fit_key(collector): + raise ValueError('original source experiment submitted objective registry') + return key + + +def snapshot_from_packet(collector,packet_wire,now_ns): + """Select the original experiment's key only around the job transport.""" + schedule = fit_schedule(collector) + if schedule is None: + return None + adapter = collector.adapter + capture_key = adapter.key + adapter.select_objective(schedule['objective_key']) + try: + return _snapshot_from_packet(collector,packet_wire,now_ns,schedule) + finally: + adapter.select_objective(capture_key) + + +def _snapshot_from_packet(collector,packet_wire,now_ns,schedule,*,response_only=False): + """Common lossless worker transport for an already exported ORIGINAL packet. + + This does no fit-readiness or admission declaration. The normal parent calls + it only after its predeclared readiness/cadence gates. + """ + from openpilot.starpilot.flm.controller_replay import pack_primitive_snapshot,MAX_FIT_RAW_BYTES + from openpilot.starpilot.flm.source_plant_pipeline import replay_data + import pickle + adapter,capture = collector.adapter,collector.original_source_capture + if type(response_only) is not bool: + raise ValueError('explicit response-only transport purpose') + try: + from openpilot.starpilot.flm.original_source_capture import ( + unpack_source_packet, SOURCE_PACKET_VERSION, SOURCE_PACKET_ENCODING, NORMALIZED_SOURCE_VERSION, + ) + if response_only: + # This parent owns a freshly encoded, read-only transport graph. Do not + # expand every shared container just to intern it again. The response + # worker still makes its independent copy and verifies ALL original data. + from openpilot.starpilot.flm.controller_replay import unpack_primitive_snapshot + from openpilot.starpilot.flm.primitive_source_memo import _logical_limits + from openpilot.starpilot.flm.normalized_trace_archive import reserve_for_packet + packet = unpack_primitive_snapshot(packet_wire) + if (type(packet) is not dict or packet.get('version') not in (SOURCE_PACKET_VERSION, NORMALIZED_SOURCE_VERSION) or + packet.get('encoding') != SOURCE_PACKET_ENCODING or packet.get('qualified') is not False): + raise ValueError('unsupported original source packet codec/version') + _logical_limits((packet, bytes(reserve_for_packet(packet)))) + context = None # Computed independently by the worker; never admission. + else: + packet = unpack_source_packet(packet_wire) + # Trial/restore retains its exact pending experiment binding unchanged. + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + template = EventControllerReplay(adapter.replay,adapter.event_objective,adapter.envelope.replay_anchor()) + binding = source_binding(packet,template) + context = experiment_context(packet,binding) + envelope = adapter.envelope + from openpilot.starpilot.flm.feedback_replay import PROTOCOL as FEEDBACK_PROTOCOL,EVENT_VERSION + feedback_job = adapter.source_plant_protocol == FEEDBACK_PROTOCOL + operation,version = 'trial',EVENT_VERSION if feedback_job else 3 + if adapter.restore_envelope is not None and not response_only: + if feedback_job and adapter.restore_version != 8 or not feedback_job and adapter.restore_version == 8: + raise ValueError('feedback restore requires explicit version8') + envelope = adapter.envelope.for_trial(adapter.event_objective.names).restore_from_history(adapter.optimizer_prior) + operation,version = 'restore',adapter.restore_version + from openpilot.starpilot.flm.clipping_quality import identification_for_quality,restore_policy,FIXED_MAP_IDENTIFICATION + payload = {'version':1,'family':adapter.command_family,'identity':adapter.identity, + 'plant_protocol':adapter.source_plant_protocol,'source_packet':packet,'physical_context':context, + 'event_version':version,'event_operation':operation,'event_objective':asdict(adapter.event_objective), + 'effect_envelope':asdict(envelope),'objective':asdict(adapter.objective), + 'event_clipping_quality':adapter.clipping_quality, + 'event_identification_policy':identification_for_quality(adapter.clipping_quality) if operation=='trial' else FIXED_MAP_IDENTIFICATION, + 'now_ns':now_ns,'sequence':collector.sequence+1, + 'prior':tuple(dict(envelope.prior_coordinates)[name] for name in adapter.event_objective.names), + 'objective_key':adapter.key,'fit_schedule':schedule} + if feedback_job: + from openpilot.starpilot.flm.event_mean_evidence import VERSION as MEAN_VERSION + from openpilot.starpilot.flm.feedback_archive import archive_binding + feedback_wire = packet['feedback_observations'] + from openpilot.starpilot.flm.candidate_feedback import CLOSED_POLICY + payload['candidate_feedback_policy'] = CLOSED_POLICY + if operation == 'trial': + payload['event_mean_version'] = MEAN_VERSION + feedback_submission = None if response_only else (archive_binding(packet),feedback_wire['digest'].hex(),MEAN_VERSION,CLOSED_POLICY) + else: + from openpilot.starpilot.flm.clipping_quality import RESTORE_FEEDBACK_STATISTICS + payload['event_restore_statistics'] = RESTORE_FEEDBACK_STATISTICS + feedback_submission = (archive_binding(packet),feedback_wire['digest'].hex(),RESTORE_FEEDBACK_STATISTICS,CLOSED_POLICY) + else: + feedback_submission = None + if operation == 'restore': + payload['event_restore_policy'] = restore_policy(version) + from opendbc.car.steering_projection import projection_to_dict + if not response_only: + # Trial/restore jobs require their current endpoints. Response-only + # bootstrap independently verifies its frozen original experiment and + # never issues a profile or current permission; its worker uses neither. + control,output = collector.ready[-1] + from openpilot.starpilot.flm.normalized_trace_archive import normalized_packet,current_record + payload['control'] = current_record(control,packet['trace_archive']) if normalized_packet(packet) else asdict(control) + payload['output'] = {**asdict(output),'projection':projection_to_dict(output.projection)} + replay = replay_data(packet,objective=adapter.objective,identity=adapter.identity,baseline_revision=adapter.baseline_revision) + payload_raw_bytes = len(pickle.dumps(payload,protocol=5)) + replay_raw_bytes = len(pickle.dumps(replay,protocol=5)) + if payload_raw_bytes+replay_raw_bytes > MAX_FIT_RAW_BYTES: + raise ValueError('plant aggregate original raw worker byte bound') + if feedback_job: + from openpilot.starpilot.flm.primitive_source_memo import intern_primitive_graph + from openpilot.starpilot.flm.normalized_trace_archive import reserve_for_packet + reserve = reserve_for_packet(packet) + if response_only: + # Source packet sharing is already canonical. Preserve it and validate + # the SAME logical expansion/depth/reservation caps without rebuilding. + _logical_limits((payload,bytes(reserve))) + else: + payload = intern_primitive_graph((payload,bytes(reserve)))[0] if reserve else intern_primitive_graph(payload) + packed = pack_primitive_snapshot(payload) # Existing raw/IPC caps unchanged. + adapter.source_plant_export_cost = {'sourceWireBytes':len(packet_wire),'payloadRawBytes':payload_raw_bytes, + 'derivedReplayRawBytes':replay_raw_bytes,'aggregateRawBytes':payload_raw_bytes+replay_raw_bytes,'workerCompressedBytes':len(packed)} + wrapper = {'version':1,'family':adapter.command_family,'snapshot_b64':base64.b64encode(packed).decode('ascii'), + 'objective_key':tuple(adapter.key)} + if len(wrapper['snapshot_b64']) > 6*1024*1024: + raise ValueError('plant existing fit wrapper bound') + except (AttributeError,KeyError,TypeError,ValueError,OverflowError) as error: + adapter.snapshot_readiness = {'reason':'plant_snapshot_unavailable','detail':str(error)} + return None + # Pending calibration is not the active model context. A failed/new attempt + # must not switch off an independently fresh previously accepted correction. + if response_only: + collector.sequence += 1 + return wrapper + import weakref + adapter.source_plant_collector = weakref.ref(collector) + adapter.source_plant_pending_context = context + collector.sequence += 1 + adapter.source_plant_pending_epoch = capture.epoch + adapter.source_plant_pending_feedback = feedback_submission + adapter.snapshot_readiness = {'reason':'snapshot_ready','plantProtocol':adapter.source_plant_protocol,'childQualification':'not_yet_evaluated'} + return wrapper + + +def submitted(collector): + adapter = collector.adapter + adapter.source_plant_submitted_epoch = adapter.source_plant_pending_epoch + contexts = adapter.source_plant_submitted_contexts + contexts[adapter.source_plant_pending_context] = adapter.source_plant_pending_feedback or True + while len(contexts) > 14: # Same bound as the existing cached profile registry. + contexts.pop(next(iter(contexts))) + + +def install(adapter, report, physical_context, *, commit=False): + from openpilot.starpilot.flm.runtime_adapter_registry import digest + protocol = adapter.source_plant_protocol + if report.get('plant_protocol') != protocol or tuple(report.get('physical_context',())) != tuple(physical_context): + return False + expected_context = ('original-source-feedback-mean-context-v1' if protocol == + 'chronological-source-feedback-mean-fit-v1' else CONTEXT_VERSION) + if not physical_context or physical_context[0] != expected_context: + return False + if tuple(physical_context) not in adapter.source_plant_submitted_contexts: + return False + from openpilot.starpilot.flm.feedback_replay import PROTOCOL as FEEDBACK_PROTOCOL,EVENT_VERSION + if protocol == FEEDBACK_PROTOCOL: + from openpilot.starpilot.flm.event_mean_evidence import VERSION as MEAN_VERSION + offered = report.get('feedback_evidence') + submitted = adapter.source_plant_submitted_contexts[tuple(physical_context)] + basis = report.get('event_mean_basis') + from openpilot.starpilot.flm.clipping_quality import RESTORE_FEEDBACK_STATISTICS + restoring = report.get('event_version') == 8 + marker = RESTORE_FEEDBACK_STATISTICS if restoring else MEAN_VERSION + from openpilot.starpilot.flm.candidate_feedback import CLOSED_POLICY,PROOF_VERSION,report_policy_matches + closed = type(submitted) is tuple and len(submitted) == 4 and submitted[3] == CLOSED_POLICY + expected_fields = {'protocol','binding_digest','archive_digest','verified_digest','known_rows','context_rows'} + if closed: + expected_fields |= {'candidate_feedback_policy','candidate_feedback_proof_version'} + if (type(submitted) is not tuple or len(submitted) not in (3,4) or len(submitted) == 4 and not closed or + report.get('event_version') not in (EVENT_VERSION,8) or + not report_policy_matches(submitted,offered) or type(offered) is not dict or set(offered) != expected_fields or + ((offered['binding_digest'],offered['archive_digest'],marker,CLOSED_POLICY) if closed else + (offered['binding_digest'],offered['archive_digest'],marker)) != submitted or + closed and (offered['candidate_feedback_policy'] != CLOSED_POLICY or + offered['candidate_feedback_proof_version'] != PROOF_VERSION) or + offered['protocol'] != FEEDBACK_PROTOCOL or type(offered['verified_digest']) is not str or len(offered['verified_digest']) != 64 or + type(offered['known_rows']) is not int or not 1 <= offered['known_rows'] <= 12000 or + type(offered['context_rows']) is not int or not 0 <= offered['context_rows'] <= 1 or + (restoring and (adapter.restore_version != 8 or report.get('event_restore_statistics') != marker or basis is not None) or + not restoring and (type(basis) is not dict or basis.get('version') != MEAN_VERSION))): + return False + elif 'feedback_evidence' in report or 'event_mean_basis' in report: + return False + receipt = report.get('plant_identification') + binding = report.get('plant_binding') + if (type(receipt) is not dict or type(binding) is not dict or + receipt.get('receipt_schema') != 'compact-source-plant-receipt-v1' or + receipt.get('status') != 'physical_affine_test_passed' or not receipt.get('model_receipt_digest') or + receipt.get('binding_digest') != physical_context[1] or digest(binding) != physical_context[1] or + receipt.get('cutoff_ns') != report.get('model_cutoff_ns') or + receipt['decision']['lag_ns'] != report['model']['lag_ns'] or + report['model']['context'] != digest(physical_context)): + return False + from openpilot.starpilot.flm import plant_lag + decision = receipt['decision'] + declaration = {key:value for key,value in decision.items() if key != 'compact_decision_sha256'} + expected_receipt = plant_lag.digest({'protocol':plant_lag.PHYSICAL_PROTOCOL,'experiment':receipt['experiment_digest'], + 'decision':decision['digest'],'heldout':receipt['heldout_blocks'],'cutoff_ns':receipt['cutoff_ns'], + 'binding':receipt['binding_digest']}) + if (decision.get('protocol') != plant_lag.PHYSICAL_PROTOCOL or decision.get('compact_decision_sha256') != plant_lag.digest(declaration) or + receipt['model_receipt_digest'] != expected_receipt or + tuple(row[0] for row in decision['training_scores']) != plant_lag.LAGS_NS or + any(not 0 <= error <= .05 or outside != 0 for _,error,outside in receipt['heldout_blocks']) or + any(report['model'][name] != decision[name] for name in ('slope','intercept','command_low','command_high'))): + return False + if commit: + collector = adapter.source_plant_collector() + if collector is None: + return False + adapter.source_plant_receipt = receipt + adapter.source_plant_binding = binding + collector.physical_context = tuple(physical_context) + return True + + +def installed_report_context_current(adapter,report,physical_context): + """Recognize only the exact report installed while awaiting fresh admission. + + The original submission token stays immutable. Installation switches active + model context, so that same pending report may survive to the next ORIGINAL + pose without pretending its submission used the new context. + """ + from openpilot.starpilot.flm.runtime_adapter_registry import canonical_physical_context + context = tuple(report.get('physical_context',())) + return (getattr(adapter,'source_plant_protocol',None) in (PROTOCOL, + 'chronological-source-feedback-mean-fit-v1') and + report.get('plant_protocol') == adapter.source_plant_protocol and context == physical_context and + adapter.model is not None and adapter.model.context == canonical_physical_context(context) and + adapter.source_plant_receipt is not None and adapter.source_plant_receipt == report.get('plant_identification') and + adapter.source_plant_binding == report.get('plant_binding')) diff --git a/openpilot/starpilot/flm/steerlearnerd.py b/openpilot/starpilot/flm/steerlearnerd.py index 296eaf373a..7a56a01391 100644 --- a/openpilot/starpilot/flm/steerlearnerd.py +++ b/openpilot/starpilot/flm/steerlearnerd.py @@ -16,28 +16,74 @@ from functools import partial from opendbc.car.structs import car from openpilot.cereal import messaging from openpilot.common.params import Params -from openpilot.starpilot.flm.fullrate_collector import FullrateCollector, drain +from openpilot.starpilot.flm.fullrate_collector import FullrateCollector, drain, ControlEvidence, AppliedEvidence from openpilot.starpilot.flm.learning_wire import read_control, read_applied, result_message, write_source_receipt from openpilot.starpilot.flm.runtime_adapter_registry import SOURCE_LEASE_NS +class _SourceDelivery: + """Consume an actual incidental SubMaster delivery before polling again. + + No SubMaster clock, updated/health flag or body is written. Final native + refresh remains in place; its already-received input is not skipped by the + next blocking poll. At most one pending latest delivery is retained. + """ + def __init__(self): + self.pending = False + self.pose_header = 0 + + def refresh(self, sm): + sm.update(0) + self.pending = self.pending or any(sm.updated.values()) + if sm.updated['deviceMotion']: + self.pose_header = int(sm.logMonoTime['deviceMotion']) + + def next(self, sm): + if not self.pending: + sm.update() + header = self.pose_header or (int(sm.logMonoTime['deviceMotion']) if sm.updated['deviceMotion'] else 0) + self.pending = False + self.pose_header = 0 + return header + + MAX_OBJECTIVES = 12 MAX_CACHED_PROFILES = 14 # Six FF cells and four independently qualified centre speed bands. -def production_adapter_for_evidence(cp, evidence, key, *, source): +def production_adapter_for_evidence(cp, evidence, key, *, source, source_plant_clock=None, source_plant_protocol=None): """Adopt the qualified event contract only in the optional live producer. Legacy/reference callers retain version2. The live event producer explicitly - selects the separate fixed-map version6 restore declaration, never FD shadow. + selects fresh feedback trial7 and whole-map restore8, never FD shadow. """ from openpilot.starpilot.flm.clipping_quality import ZERO_REFERENCE_QUALITY from openpilot.starpilot.flm.runtime_adapter_registry import SourceState, adapter_for_evidence if not isinstance(source, SourceState) or not source.enabled or source.manual: return None - return adapter_for_evidence(cp, evidence, key, source=source, - event_replay_version=3, event_clipping_quality=ZERO_REFERENCE_QUALITY, event_restore_version=6) + from openpilot.starpilot.flm.feedback_replay import PROTOCOL as FEEDBACK_PROTOCOL + if source_plant_protocol is None: + source_plant_protocol = FEEDBACK_PROTOCOL + adapter = adapter_for_evidence(cp, evidence, key, source=source, + event_replay_version=3, event_clipping_quality=ZERO_REFERENCE_QUALITY, + event_restore_version=8 if source_plant_protocol == FEEDBACK_PROTOCOL else 6) + if source_plant_clock is not None or source_plant_protocol is not None: + from openpilot.starpilot.flm.source_plant_runtime import enable + from openpilot.starpilot.flm.source_plant_fit import PROTOCOL + from openpilot.starpilot.flm.feedback_replay import PROTOCOL as FEEDBACK_PROTOCOL + if source_plant_protocol not in (None,PROTOCOL,FEEDBACK_PROTOCOL): + raise ValueError('unknown independent plant producer protocol') + if adapter is not None and getattr(adapter,'event_collection',False): + return enable(adapter,source_plant_clock,protocol=source_plant_protocol) + return adapter + + +def _collect_source_control(collector, prepared, sm, cp, now_ns): + """An arriving record cannot inherit the previous record's health verdict.""" + from openpilot.starpilot.flm.native_health import current_native_health + collector._learning_current_native_allowed = current_native_health(sm, cp, prepared, now_ns) + return collector.control(prepared, now_ns) def _source_config_binding(source): @@ -86,6 +132,169 @@ def _pause_historical_collector(collector, reason): replay.cached_validation = replay.cached_key = None +def tracking_message(source, *, now_ns, current_native, offer=None, restore=None): + """Serialize existing owned evidence; never renew a trial or physical lease.""" + from openpilot.starpilot.flm.live_trial_wire import write_tracking_source, write_trial, write_restore + message = messaging.new_message('starpilotSteeringTracking') + message.valid = True + write_tracking_source(message, source) + current = bool(current_native is True and _source_config_known(source, now_ns) and source.enabled and not source.manual) + if current and offer is not None and offer.observed_ns <= now_ns <= offer.expires_ns: + write_trial(message, offer) + elif current and restore is not None and restore.observed_ns <= now_ns <= restore.expires_ns: + write_restore(message, restore) + return message + + +class _LiveHistoryRestore: + """Historical target only; fresh measured response and actual owner ack gate reuse.""" + def __init__(self, priors): + self.priors = tuple(priors) + self.response_recovery_generation = 0 + self.blocked_seed = None + self.seed = self.prior = self.requalifier = self.offer = None + self.restored = False + self.last_pose_ns = self.last_publication_pose_ns = 0 + + def reset(self): + self.seed = self.prior = self.requalifier = self.offer = None + self.restored = False + self.last_pose_ns = self.last_publication_pose_ns = 0 + + def observe_response_recovery(self, actor): + generation = getattr(actor, 'response_recovery_generation', 0) + if type(generation) is not int or generation <= self.response_recovery_generation: + return False + self.response_recovery_generation = generation + # Quarantine ONLY this drive/source attempt; retain the immutable store. + # reset() deliberately preserves this block across collector interruptions. + self.blocked_seed = actor.seed if actor.seed is not None else self.seed + self.reset() + actor.hold_restore(False) + return True + + @staticmethod + def _pair(collector, pose, lag_ns) -> tuple[ControlEvidence, AppliedEvidence] | None: + lower = tuple(pair for pair in collector.ready if pair[1].apply_ns <= pose.source_lower_ns-lag_ns) + upper = tuple(pair for pair in collector.ready if pair[1].apply_ns <= pose.source_upper_ns-lag_ns) + if not lower or not upper or lower[-1][1].apply_ns != upper[-1][1].apply_ns: + return None + return lower[-1] + + def _retry_local_scope(self, actor, collector, seed): + """A NEW fresh local experiment; never expand an issued certificate. + + Only an unfinished historical restore may retry on input-domain departure. + Residual failures, changed contexts/native denial and live trials retain + their original fail-closed/quarantine paths. No monitor rows, command, + lease, target acknowledgement or current permission cross this boundary. + """ + from openpilot.starpilot.flm.live_tracking_requalification import RetainedResponseRequalifier, VERSION + if (self.restored or not actor.restore_pending or self.prior is None or self.requalifier is None or + actor.ingress is None or actor.certificate is None or actor.seed != seed or + actor.ingress.optimizer.offer_state is not None or + actor.ingress.monitor.reason != 'response_command_outside_identified_range' or + actor.certificate.provenance != VERSION or + actor.certificate.parent_model_revision != self.prior.certificate.model_revision): + return False + actor.interrupt('retained_local_input_scope_left', context_changed=True) + actor.ingress = actor.certificate = actor.application = actor.initial_basis = None + collector.live_tracking = None + self.offer = None + self.requalifier = RetainedResponseRequalifier(self.prior.certificate,self.prior.response_basis,seed) + capture = collector.original_source_capture + if capture is not None and capture.rows: + self.last_pose_ns = max(self.last_pose_ns,capture.rows[-1].mono_ns) + actor.hold_restore(True) + return True + + def advance(self, actor, collector, source, *, now_ns, current_native, native_key, hardware_inventory): + if collector is None or source is None or not collector.ready or current_native is not True: + if self.requalifier is not None: + self.requalifier.interrupt('retained_current_source_or_native_unavailable') + return None + from openpilot.starpilot.flm.live_trial_store import LiveConfirmedPrior, stable_binding, scope + try: + seed = actor.binding_for(collector, source) + if self.blocked_seed is not None: + if seed == self.blocked_seed: + self.seed = seed + actor.hold_restore(False) + return None + self.blocked_seed = None # Actual source binding changed, not a clock. + if seed != self.seed: + self.reset() + actor.hold_restore(False) + self.seed = seed + matches = tuple(prior for prior in self.priors if type(prior) is LiveConfirmedPrior and + prior.historical_only is True and stable_binding(prior.binding) == stable_binding(seed)) + if len(matches) == 1: + self.prior = matches[0] + actor.hold_restore(True) + if self.prior is None: + return None + self._retry_local_scope(actor, collector, seed) + if actor.ingress is None and self.prior.response_basis is not None: + from openpilot.starpilot.flm.live_tracking_requalification import RetainedResponseRequalifier + if self.requalifier is None: + self.requalifier = RetainedResponseRequalifier(self.prior.certificate, self.prior.response_basis, seed) + capture = collector.original_source_capture + if capture is not None and capture.rows: + pose = capture.rows[-1] + if pose.mono_ns > self.last_pose_ns: + self.last_pose_ns = pose.mono_ns + pair = self._pair(collector, pose, self.prior.certificate.lag_ns) + if pair is not None: + accepted = self.requalifier.observe(*pair, pose, now_ns=now_ns, current_native=current_native, + native_key=native_key, hardware_inventory=hardware_inventory) + if accepted and self.requalifier.persistent_residual_failure(now_ns): + # No adopted ingress exists yet. This fresh physical failure + # quarantines ONLY the current historical restore attempt. + self.blocked_seed = seed + self.reset() + actor.hold_restore(False) + return None + if self.requalifier.reason == 'measured_calibration_vm_or_hardware_changed': + # Keep the historical store, but do not retry an incompatible + # basis forever or suppress genuine fresh calibration. + self.prior = self.requalifier = self.offer = None + actor.hold_restore(False) + return None + qualified = self.requalifier.qualified(now_ns) + if qualified is not None: + certificate, _ = qualified + # A fresh ingress collects its OWN residual window before restore + # publication; no retained monitor rows or permission are copied. + actor.adopt_response(certificate, collector) + if actor.ingress is None: + return None # Missing/incompatible basis honestly needs fresh bootstrap. + witness = actor.ingress.monitor.witness(now_ns) + if witness is None: + return None + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + raw = validate_response_witness(witness, actor.certificate.binding, now_ns) + if raw.last_pose_ns <= self.last_publication_pose_ns: + return self.offer + self.last_publication_pose_ns = raw.last_pose_ns + from openpilot.starpilot.flm.live_trial_contract import LiveRestoreOffer, RESTORE_VERSION, MAX_LEASE_NS + actor.ingress.optimizer.publication_sequence += 1 + self.offer = LiveRestoreOffer(RESTORE_VERSION, actor.certificate.binding, scope(self.prior.binding), + self.prior.trial_id, self.prior.target, self.prior.target_sha256, + actor.ingress.optimizer.publication_sequence, now_ns, min(now_ns+MAX_LEASE_NS, raw.valid_until_ns), witness).validate() + return self.offer + except (AttributeError, IndexError, KeyError, TypeError, ValueError, OverflowError): + # An incompatible historical model never gets a fabricated current basis. + self.requalifier = None + return None + + def acknowledge(self, actor, application, *, now_ns): + if self.offer is None or not actor.observe_restored(application, self.offer, now_ns=now_ns): + return False + self.restored = True + actor.hold_restore(False) + return True + + def run( CP, sm, @@ -105,6 +314,9 @@ def run( clock=time.monotonic_ns, reporter=None, publication_source_reader: Callable[[int], SourceState | None] | None = None, + live_actor=None, + live_persistence=None, + confirmed_history=(), ): from openpilot.starpilot.flm.native_health import current_native_health @@ -116,12 +328,19 @@ def run( raw_context = active_key = pending_key = pending_token = None cached_reports = {} prior_attached = {} - previous_publish = publication_sequence = 0 + next_publish_ns = publication_sequence = 0 unavailable_reason = 'no_current_controller_evidence' retained_event = pending_event = None + live_offer = None + previous_tracking_pose_header = 0 + saved_live_targets = set() + live_history = _LiveHistoryRestore(confirmed_history) + import gc + gc_initially_enabled = gc.isenabled() + trial_gc_deferred = False def clear(reason): - nonlocal collector, raw_context, active_key, pending_key, pending_token, unavailable_reason, retained_event, pending_event + nonlocal collector, raw_context, active_key, pending_key, pending_token, unavailable_reason, retained_event, pending_event, live_offer unavailable_reason = reason retained_event = pending_event = None if collector is not None: @@ -132,16 +351,27 @@ def run( cached_reports.clear() prior_attached.clear() raw_context = active_key = pending_key = pending_token = None - fitter.cancel(reason) + if fitter is not None: + fitter.cancel(reason) + if live_actor is not None: + live_actor.interrupt(reason, context_changed=True) + live_offer = None + live_history.reset() def uncertain(reason): - nonlocal unavailable_reason, retained_event, active_key, pending_key, pending_token, pending_event + nonlocal unavailable_reason, retained_event, active_key, pending_key, pending_token, pending_event, live_offer unavailable_reason, retained_event, pending_event = reason, None, None if collector is not None: _pause_historical_collector(collector, reason) cached_reports.clear() active_key = pending_key = pending_token = None - fitter.cancel(reason) + if fitter is not None: + fitter.cancel(reason) + if live_actor is not None: + live_actor.interrupt(reason) + live_offer = None + + source_delivery = _SourceDelivery() def source_current(*, read=False, publication=False): nonlocal source, source_rejected, source_cp_sha256, last_accepted_source @@ -150,10 +380,10 @@ def run( return False, now if read or source is None or not source.observed_ns <= now <= source.expires_ns: if not read: - sm.update(0) + source_delivery.refresh(sm) now = clock() if publication: - sm.update(0) + source_delivery.refresh(sm) now = clock() reader = publication_source_reader if publication else source_reader source = cast(Callable[[int], SourceState | None], reader)(now) @@ -200,16 +430,41 @@ def run( sink = Sink() try: while True: + state = (live_actor.ingress.optimizer.offer_state + if live_actor is not None and live_actor.ingress is not None else None) + defer_gc = bool(gc_initially_enabled and state is not None and state['phase'] == 'trial' and + not state.get('accepted', False) and clock() <= state['deadline_ns']) + if defer_gc and not trial_gc_deferred: + # This separate optional process must receive the real 100Hz tape during + # the existing bounded trial. Reference counting continues normally; + # cyclic collection resumes on rollback, completion or the same deadline. + gc.disable() + trial_gc_deferred = True + elif trial_gc_deferred and not defer_gc: + gc.enable() + trial_gc_deferred = False # Parent handles bounded primitive ingest and live receipts. Physical - # fitting, stage normalization and full replay run in ONE separate child. - sm.update() + # calibration and native sensitivity execute in bounded separate children. + delivered_pose_header = source_delivery.next(sm) source_rejected = False now = clock() + # This is an ORIGINAL header from the real poll or an already consumed + # final refresh. Never wait for another pose while this one is pending. + tracking_pose_header = delivered_pose_header if live_actor is not None else 0 + if not 0 < tracking_pose_header <= now: + tracking_pose_header = 0 source_valid, now = source_current(read=True) if source_valid and collector is None: unavailable_reason = 'no_current_controller_evidence' - def control(event, current, stamp): + if collector is not None: + # Geometry arrives at100Hz independently of the actual native emission + # cadence. This Boolean is current owner health, never a fabricated CAN. + previous = collector.ready[-1][0] if collector.ready else None + collector._learning_current_native_allowed = bool(source_valid and previous is not None and + current_native_health(sm, CP, previous, now)) + + def control(event, current, stamp, offered=live_offer): nonlocal collector, raw_context, active_key, unavailable_reason valid_source, received_now = source_current() if not valid_source: @@ -261,6 +516,12 @@ def run( if getattr(adapter, 'event_collection', False): from openpilot.starpilot.flm.event_collector import EventFullrateCollector collector = EventFullrateCollector(adapter) + if getattr(adapter,'source_plant_protocol',None) is not None: + from openpilot.starpilot.flm.source_plant_runtime import start_original_capture + if not start_original_capture(collector): + unavailable_reason = adapter.source_plant_capture_error + collector = None + return else: collector = FullrateCollector(adapter) if hasattr(collector.adapter, 'bind_source'): @@ -269,17 +530,40 @@ def run( collector.adapter.select_objective(active_key) # This records primitive actual frames/checkpoints ONLY. No selected # controller update, baseline probes or numerical fitting at100Hz. + # Feedback describes the original controller consumption, including + # context-only frames that cannot become fitting samples. Preserve its + # exact decoded wire evidence before stage eligibility can decline it. + if getattr(collector.adapter,'original_feedback_archive',None) is not None: + from openpilot.starpilot.flm.feedback_archive import record_companion + record_companion(collector,event,received_now,control=evidence) prepared = collector.adapter.prepare_control(evidence) if prepared is None: collector.split('actual_stage_capture_unqualified') else: - collector.control(prepared, received_now) + # Re-evaluate current native admission against THIS arriving control. + # The pre-drain reference can have expired while fresh controls were + # queued; inheriting its Boolean rejects the fresh batch as well. + # No old receipt or deadline is renewed by this current check. + _collect_source_control(collector, prepared, sm, CP, received_now) + if live_actor is not None and offered is not None: + from openpilot.starpilot.flm.live_trial_wire import read_application + application = read_application(event.starpilotLateralState, now_ns=received_now, offer=offered) + if application is not None and current_native_health(sm, CP, evidence, received_now): + live_actor.observe_application(application, now_ns=received_now) + elif live_actor is not None and live_history.offer is not None: + from openpilot.starpilot.flm.live_trial_wire import read_restore_application + application = read_restore_application(event.starpilotLateralState, now_ns=received_now, offer=live_history.offer) + if application is not None and current_native_health(sm, CP, evidence, received_now): + live_history.acknowledge(live_actor, application, now_ns=received_now) def output(event, current, stamp): valid_source, received_now = source_current() if not valid_source: return if collector is not None: + if getattr(collector.adapter,'original_feedback_archive',None) is not None: + from openpilot.starpilot.flm.feedback_archive import record_companion + record_companion(collector,event,received_now) if event.which() == 'starpilotCarState' and event.starpilotCarState.appliedEvidence.version == 0: collector.split('no_current_applied_evidence') return @@ -292,23 +576,27 @@ def run( if source_valid and collector is not None and collector.ready: current_native = current_native_health(sm, CP, collector.ready[-1][0], now) collector._learning_current_native_allowed = current_native - physical = physical_observer(sm, CP, collector, now) - from openpilot.starpilot.flm.physical_observation import PendingPhysical - if not current_native or physical is None: - collector.split('physical_or_native_witness_unavailable') - elif type(physical) is PendingPhysical: - if not physical.fresh(now): - collector.split('physical_or_native_witness_unavailable') - elif physical.context_changed: - # Discard prior current admission on a lineage/transform boundary; - # waiting for new-context data must not restart its own queue. - collector.last_qualified_response = 0 - collector.adapter.current_physical_valid = False - collector.adapter.last_physical_ns = 0 - # Pending data is not physical evidence: no timestamp, model, - # last_qualified_response or current admission is renewed here. + if getattr(collector.adapter,'source_plant_protocol',None) is not None: + from openpilot.starpilot.flm.source_plant_runtime import observe + observe(sm,CP,collector,now,current_native) else: - collector.physical(physical, now) + physical = physical_observer(sm, CP, collector, now) + from openpilot.starpilot.flm.physical_observation import PendingPhysical + if not current_native or physical is None: + collector.split('physical_or_native_witness_unavailable') + elif type(physical) is PendingPhysical: + if not physical.fresh(now): + collector.split('physical_or_native_witness_unavailable') + elif physical.context_changed: + # Discard prior current admission on a lineage/transform boundary; + # waiting for new-context data must not restart its own queue. + collector.last_qualified_response = 0 + collector.adapter.current_physical_valid = False + collector.adapter.last_physical_ns = 0 + # Pending data is not physical evidence: no timestamp, model, + # last_qualified_response or current admission is renewed here. + else: + collector.physical(physical, now) source_valid, now = source_current() current_native = bool(source_valid and collector is not None and collector.ready and @@ -316,59 +604,142 @@ def run( token = ((raw_context, pending_key, collector.physical_context, collector.adapter.envelope.prior_full_profile_sha256) if getattr(collector.adapter, 'event_collection', False) else (raw_context, pending_key, collector.physical_context)) \ if collector is not None and collector.ready and current_native else None - report = fitter.poll(token=token, now_ns=now) - if report is not None: - decoded = decode_fit_result(report) - if source_valid and token is not None and decoded is not None and collector is not None and token == pending_token: - if reporter is not None: - reporter.fitted(decoded[0], pending_key) - if len(cached_reports) < MAX_CACHED_PROFILES or pending_key in cached_reports: - cached_reports[pending_key] = (report, decoded, token) - if getattr(collector.adapter, 'event_collection', False) and decoded[0].profile is not None: - # A completed proposal owns its fitting objective independently - # from the rotating capture selector. This is not live admission. - pending_event = (cached_reports[pending_key], pending_key) - pending_key = pending_token = None - if fitter.child is None: - pending_key = pending_token = None - if collector is not None and current_native and active_key is not None: - collector.adapter.select_objective(active_key) - if collector.context is not None: - scope = (collector.context[0], active_key, source.reset) - cached_prior = prior_attached.get(scope) - if cached_prior is None or cached_prior[0] != tuple(collector.adapter.parameters()): - persistence.attach_prior(collector, identity=collector.context[0], source_epoch=source.source_epoch, reset=source.reset) - prior_attached[scope] = (tuple(collector.adapter.parameters()), collector.prior_coefficients) - else: - collector.restore_prior(cached_prior[1]) - snapshot = collector.job_snapshot(now) - if snapshot is not None: - token = ((raw_context, active_key, collector.physical_context, collector.adapter.envelope.prior_full_profile_sha256) - if getattr(collector.adapter, 'event_collection', False) else (raw_context, active_key, collector.physical_context)) - valid_source, now = source_current() - current_native = bool(valid_source and collector is not None and collector.ready and - current_native_health(sm, CP, collector.ready[-1][0], now)) - if valid_source and current_native and collector is not None and fitter.submit(snapshot, token=token, now_ns=now): - if reporter is not None: - reporter.fit_attempts += 1 - collector.mark_job_submitted(now) - pending_key, pending_token = active_key, token + if live_actor is None: + report = fitter.poll(token=token, now_ns=now) + if report is not None: + decoded = decode_fit_result(report) + if source_valid and token is not None and decoded is not None and collector is not None and token == pending_token: + if reporter is not None: + reporter.fitted(decoded[0], pending_key) + if len(cached_reports) < MAX_CACHED_PROFILES or pending_key in cached_reports: + cached_reports[pending_key] = (report, decoded, token) + if getattr(collector.adapter, 'event_collection', False) and decoded[0].profile is not None: + # A completed proposal owns its fitting objective independently + # from the rotating capture selector. This is not live admission. + pending_event = (cached_reports[pending_key], pending_key) + pending_key = pending_token = None + if fitter.child is None: + pending_key = pending_token = None + if collector is not None and current_native and active_key is not None: + collector.adapter.select_objective(active_key) + if collector.context is not None: + scope = (collector.context[0], active_key, source.reset) + cached_prior = prior_attached.get(scope) + if cached_prior is None or cached_prior[0] != tuple(collector.adapter.parameters()): + persistence.attach_prior(collector, identity=collector.context[0], source_epoch=source.source_epoch, reset=source.reset) + prior_attached[scope] = (tuple(collector.adapter.parameters()), collector.prior_coefficients) + else: + collector.restore_prior(cached_prior[1]) + snapshot = collector.job_snapshot(now) + if snapshot is not None: + if getattr(collector.adapter,'source_plant_protocol',None) is not None: + from openpilot.starpilot.flm.source_plant_runtime import submission_key + submitted_key = submission_key(snapshot,collector,active_key) + else: + submitted_key = active_key + token = ((raw_context, submitted_key, collector.physical_context, collector.adapter.envelope.prior_full_profile_sha256) + if getattr(collector.adapter, 'event_collection', False) else (raw_context, submitted_key, collector.physical_context)) + valid_source, now = source_current() + current_native = bool(valid_source and collector is not None and collector.ready and + current_native_health(sm, CP, collector.ready[-1][0], now)) + if valid_source and current_native and collector is not None and fitter.submit(snapshot, token=token, now_ns=now): + if reporter is not None: + reporter.fit_attempts += 1 + collector.mark_job_submitted(now) + pending_key, pending_token = submitted_key, token source_valid, now = source_current() current_native = bool(source_valid and collector is not None and collector.ready and current_native_health(sm, CP, collector.ready[-1][0], now)) - if now - previous_publish >= 250_000_000: - previous_publish = now + if live_actor is not None: + from openpilot.starpilot.flm.runtime_adapter_registry import current_authority, native_identity_key + native_key = hardware_inventory = () + if current_native and collector is not None and collector.ready: + control_evidence = collector.ready[-1][0] + if control_evidence.native_required: + allowed, actual_native, _ = current_authority(sm, CP, now_ns=now, required=True) + if allowed: + native_key = native_identity_key(actual_native) + else: + hardware_inventory = tuple((index, str(panda.pandaType)) for index, panda in enumerate(sm['pandaStates'])) + prior_restore_offer = live_history.offer + if live_offer is None: + live_history.advance(live_actor, collector, source if source_valid else None, now_ns=now, + current_native=current_native, native_key=native_key, hardware_inventory=hardware_inventory) + latest = live_actor.tick(collector, source if source_valid else None, now_ns=now, + current_native=current_native, native_key=native_key, hardware_inventory=hardware_inventory, + allow_bootstrap=live_history.requalifier is None) + if live_history.observe_response_recovery(live_actor): + # Clear cached trial/restore publication in this SAME parent tick. + # No old history or accepted target is silently promoted afterward. + live_offer = latest = None + if latest is not None: + live_offer = latest + live_history.offer = None + # Publish when a genuine qualified physical witness becomes available, + # including when its raw camera header arrived in an earlier loop. + # Both owners already gate offers on a strictly newer physical pose. + # Waiting for another raw header can spend an additional camera period + # from the original witness lease. Never restamp or renew an old offer. + new_raw_pose = tracking_pose_header > previous_tracking_pose_header + if new_raw_pose: + previous_tracking_pose_header = tracking_pose_header + new_restore = live_history.offer is not None and live_history.offer is not prior_restore_offer + # The transport/source status has its own20Hz cadence. Waiting for a + # new admitted offer can starve service health and prevent fresh raw + # collection from recovering after a withdrawal. A cached offer keeps + # its ORIGINAL observed time, sequence, witness and expiry; the writer + # omits it when expired or current permission is absent. + publish_tracking = latest is not None or new_restore or new_raw_pose + if publish_tracking: + message = tracking_message(source, now_ns=now, current_native=current_native, + offer=live_offer, restore=live_history.offer) + if (live_offer is not None and int(message.starpilotSteeringTracking.trial.version) == 1 and + hasattr(live_actor, 'mark_publication')): + live_actor.mark_publication(live_offer) + pm.send('starpilotSteeringTracking', message) + if live_persistence is not None: + inputs = live_actor.persistence_inputs(now) + if inputs is not None: + confirmed_offer, application, certificate = inputs + saved_key = (confirmed_offer.binding, confirmed_offer.trial_id, confirmed_offer.target_sha256) + if saved_key not in saved_live_targets: + def save_authorized(expected_certificate=certificate): + valid, check_now = source_current(read=True, publication=True) + return bool(valid and collector is not None and collector.ready and + current_native_health(sm, CP, collector.ready[-1][0], check_now) and + live_actor.certificate is expected_certificate) + if live_persistence.save_confirmed(confirmed_offer, application, certificate, + now_ns=now, current_binding=lambda: live_actor.certificate.binding if live_actor.certificate is not None else None, + authorized=save_authorized, response_basis=live_actor.initial_basis): + saved_live_targets.add(saved_key) + if len(saved_live_targets) > 64: + saved_live_targets.pop() + + if now >= next_publish_ns: + if next_publish_ns == 0: + next_publish_ns = now publication_sequence += 1 if publication_source_reader is not None: source_valid, now = source_current(read=True, publication=True) - if source_valid and collector is not None and active_key is not None and raw_context is not None: + if live_actor is not None: + message = result_message(None, 'live-tracking', publication_sequence, now) + value = message.autoSteeringLearning.learning + value.status, value.reason = 'collecting', live_actor.reason + elif source_valid and collector is not None and active_key is not None and raw_context is not None: cached = cached_reports.get(active_key) collector.fit_result = collector.fit_validation = None if getattr(collector.adapter, 'event_collection', False): if pending_event is not None: candidate, candidate_key = pending_event - expected = (raw_context, candidate_key, collector.physical_context, + token_context = collector.physical_context + if getattr(collector.adapter,'source_plant_protocol',None) is not None: + from openpilot.starpilot.flm.source_plant_runtime import installed_report_context_current + if installed_report_context_current(collector.adapter,candidate[0],collector.physical_context): + # Keep exact original submission custody while the installed + # proposal awaits a genuinely later physical/current proof. + token_context = candidate[2][2] + expected = (raw_context, candidate_key, token_context, collector.adapter.envelope.prior_full_profile_sha256) if candidate[2] != expected: pending_event = None @@ -432,10 +803,24 @@ def run( if reporter is not None: cached = cached_reports.get(active_key) reporter.publish(now_ns=now, message=message, collector=collector, source=source, objective=active_key, - pending=fitter.child is not None, native_current=current_native, + pending=live_actor.pending if live_actor is not None else fitter.child is not None, native_current=current_native, fit_result=None if cached is None else cached[1][0]) + # Preserve the original phase instead of adding poll jitter each time. + # Work/read delays skip missed slots; one actual loop sends at most once. + # This clock is scheduling only: source receipts and500ms leases above + # keep their actual current read times, never a deadline/backdated time. + publication_complete_ns = max(now, clock()) + next_publish_ns += ((publication_complete_ns-next_publish_ns)//250_000_000+1)*250_000_000 finally: - fitter.close('learner_parent_exit') + if trial_gc_deferred: + gc.enable() + reaped = True + if live_actor is not None: + reaped = live_actor.close('learner_parent_exit') is not False + if fitter is not None: + reaped = fitter.close('learner_parent_exit') is not False and reaped + if not reaped: + raise RuntimeError('learner child shutdown reaping unproved') def main(): @@ -451,7 +836,9 @@ def main(): ) from openpilot.starpilot.flm.learner_factory import decode_fit_result from openpilot.starpilot.flm.persistence_module import PersistenceModule - from openpilot.starpilot.flm.fit_process import FitProcess + from openpilot.starpilot.flm.live_tracking_actor import LiveTrackingActor + from openpilot.starpilot.flm.live_tracking_probe_process import ProbeProcess + from openpilot.starpilot.flm.live_trial_store import LiveTrialPersistence, load_confirmed_history try: if hasattr(os, 'sched_setscheduler'): @@ -477,10 +864,10 @@ def main(): 'onroadEvents', 'aolSafetyWire', ] - sm = messaging.SubMaster(services, poll='deviceMotion', ignore_alive=['aolSafetyWire'], ignore_valid=['aolSafetyWire'], ignore_avg_freq=['aolSafetyWire']) + sm = messaging.SubMaster(services, poll='carState', ignore_alive=['aolSafetyWire'], ignore_valid=['aolSafetyWire'], ignore_avg_freq=['aolSafetyWire']) control_socket = messaging.sub_sock('starpilotLateralState', conflate=False) applied_socket = messaging.sub_sock('starpilotCarState', conflate=False) - pm = messaging.PubMaster(['autoSteeringLearning']) + pm = messaging.PubMaster(['autoSteeringLearning', 'starpilotSteeringTracking']) cp_sha256 = car_params_digest(CP) source_cache = [None] vehicle_session = uuid.uuid4().hex @@ -533,24 +920,38 @@ def main(): return production_adapter_for_evidence(cp, evidence, key, source=source_reader(time.monotonic_ns())) persistence = PersistenceModule(params, current_sources) - run( - CP, - sm, - control_socket, - applied_socket, - pm, - current_adapter, - partial(physical_observer, distinguish_pending=True), - source_reader, - collection_keys, - objective_key, - decode_fit_result, - persistence, - FitProcess(), - adapter_for_ford, - reporter=reporter, - publication_source_reader=publication_source_reader, - ) + import gc + gc_was_enabled = gc.isenabled() + if gc_was_enabled: + # Reclaim startup cycles; temporary visitors release their own graphs. + gc.collect() + # Keep standard GC enabled; recursive temporary visitors release explicitly. + try: + run( + CP, + sm, + control_socket, + applied_socket, + pm, + current_adapter, + partial(physical_observer, distinguish_pending=True), + source_reader, + collection_keys, + objective_key, + decode_fit_result, + persistence, + None, + adapter_for_ford, + reporter=reporter, + publication_source_reader=publication_source_reader, + live_actor=LiveTrackingActor(probe_process=ProbeProcess(),source_tracking=True), + live_persistence=LiveTrialPersistence(params), + confirmed_history=load_confirmed_history(params), + ) + finally: + # run() closes/reaps optional children before returning or propagating. + if gc_was_enabled: + gc.enable() if __name__ == '__main__': diff --git a/openpilot/starpilot/flm/tests/_probe_lifecycle_fixture.py b/openpilot/starpilot/flm/tests/_probe_lifecycle_fixture.py new file mode 100644 index 0000000000..b63669027e --- /dev/null +++ b/openpilot/starpilot/flm/tests/_probe_lifecycle_fixture.py @@ -0,0 +1,21 @@ +"""Root-only lifecycle fixture: no controller, native or sensitivity proof.""" +import os +import resource +import time +from unittest.mock import patch + +from openpilot.starpilot.flm import live_tracking_probe_worker as worker + + +def report(job, *, now_ns): + if job.get('stall'): + time.sleep(10.) + return {'cpu': resource.getrlimit(resource.RLIMIT_CPU)[0], + 'as': resource.getrlimit(resource.RLIMIT_AS)[0], 'nice': os.nice(0), + 'scheduler': os.sched_getscheduler(0) if hasattr(os, 'sched_getscheduler') else None, + 'parent': os.getppid(), 'real_now': now_ns} + + +if __name__ == '__main__': + with patch.object(worker, "evaluate_job", report): + raise SystemExit(worker.main()) diff --git a/openpilot/starpilot/flm/tests/test_calibration_first_binding.py b/openpilot/starpilot/flm/tests/test_calibration_first_binding.py new file mode 100644 index 0000000000..0c25f65fc6 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_calibration_first_binding.py @@ -0,0 +1,205 @@ +"""Original Controls, real SubMaster rate history and first identity binding.""" +from contextlib import contextmanager +from unittest import mock + +import pytest +from opendbc.car.car_helpers import interfaces +from opendbc.car.toyota.values import CAR +from openpilot.cereal import messaging +from openpilot.starpilot.flm.tests.test_live_calibration_paths import BASE, live_frame +from openpilot.starpilot.flm.tests.test_registered_controls_global import controls_for + + +@contextmanager +def calibration_receipts(controls, *, first=0, second=5, period=25, blocks_before_105=5): + original = controls.sm.update_msgs + sent = set() + + def deliver(now, events): + rows = [event for event in events if event.which() != 'extrinsicsCalibration'] + tick = round((now * 1e9 - BASE) / 10_000_000) + due = tick == first or tick >= second and (tick-second) % period == 0 + if now >= BASE / 1e9 and due and tick not in sent: + event = messaging.new_message('extrinsicsCalibration', valid=True, logMonoTime=BASE+tick*10_000_000) + event.extrinsicsCalibration.calStatus = 'calibrated' + event.extrinsicsCalibration.validBlocks = blocks_before_105 if tick < 105 else 5 + event.extrinsicsCalibration.rpyCalib = (0., 0., 0.) + event.extrinsicsCalibration.rpyCalibSpread = (0., 0., 0.) + rows.append(event.as_reader()) + sent.add(tick) + return original(now, rows) + + with mock.patch.object(controls.sm, 'update_msgs', side_effect=deliver): + yield original + + +def no_source_before_85(event): + if int(event.logMonoTime) < BASE+850_000_000: + event.autoSteeringLearning.learning.sourceReceipt.version = 0 + + +def frame(controls, tick, generation='a'*32, **kwargs): + return live_frame(controls, tick, generation, blocks=5, + publish=tick % 25 == 10, wire_mutator=no_source_before_85, **kwargs) + + +def test_actual_nonzero_first_rate_failure_defers_binding_then_fresh_source_admits(): + cp = interfaces[CAR.TOYOTA_RAV4_TSS2].get_non_essential_params(CAR.TOYOTA_RAV4_TSS2) + with controls_for(cp, 'standard', True) as controls, calibration_receipts(controls): + original_tune = controls.LaC.torque_params.to_dict() + for tick in range(121): + frame(controls, tick) + lifecycle = controls.global_learning._calibration_lifecycle + if tick == 85: + frequency = controls.sm.freq_tracker['extrinsicsCalibration'] + assert frequency.avg_dt.count == frequency.recent_avg_dt.count == 4 + assert frequency.avg_dt.get_average() == pytest.approx(.2) + assert not controls.sm.freq_ok['extrinsicsCalibration'] + # OLD captures and retires here; NEW keeps no admitted identity. + assert 'a'*32 not in lifecycle.rejected + assert lifecycle.generation == '' and not lifecycle.current + assert not controls.global_learning.eligible + if 85 <= tick < 105: + assert not controls.global_learning.eligible + assert lifecycle.last_invalid_ns == BASE+tick*10_000_000 + if tick == 105: + assert controls.sm.freq_ok['extrinsicsCalibration'] + assert not controls.global_learning.eligible # Receipt85 predates last invalid104. + assert lifecycle.generation == 'a'*32 and 'a'*32 not in lifecycle.rejected + assert controls.global_learning.eligible + assert controls.LaC.torque_params.to_dict() == original_tune + + +def test_actual_zero_interval_defers_without_admission_or_frequency_flag_writes(): + cp = interfaces[CAR.TOYOTA_RAV4_TSS2].get_non_essential_params(CAR.TOYOTA_RAV4_TSS2) + with controls_for(cp, 'standard', True) as controls, calibration_receipts(controls, first=80, second=105): + for tick in range(121): + frame(controls, tick) + lifecycle = controls.global_learning._calibration_lifecycle + if tick == 85: + frequency = controls.sm.freq_tracker['extrinsicsCalibration'] + assert frequency.avg_dt.count == frequency.recent_avg_dt.count == 0 + assert not controls.sm.freq_ok['extrinsicsCalibration'] + assert lifecycle.generation == '' and 'a'*32 not in lifecycle.rejected + assert not lifecycle.current and not controls.global_learning.eligible + assert controls.global_learning.eligible + + +@pytest.mark.parametrize('failure', ['invalid', 'stale', 'future', 'invalid_wire', 'regression']) +def test_pending_true_calibration_fault_retires_and_same_generation_cannot_return(failure): + cp = interfaces[CAR.TOYOTA_RAV4_TSS2].get_non_essential_params(CAR.TOYOTA_RAV4_TSS2) + with controls_for(cp, 'standard', True) as controls, calibration_receipts(controls) as original_delivery: + for tick in range(86): + frame(controls, tick) + runtime, sm = controls.global_learning, controls.sm + assert runtime._calibration_pending == ('a'*32, 5) + now = BASE+860_000_000 + header = now-500_000_001 if failure == 'stale' else now+1 if failure == 'future' else now + event = messaging.new_message('extrinsicsCalibration', valid=failure != 'invalid_wire', logMonoTime=header) + event.extrinsicsCalibration.calStatus = 'invalid' if failure == 'invalid' else 'calibrated' + event.extrinsicsCalibration.validBlocks = 4 if failure == 'regression' else 5 + event.extrinsicsCalibration.rpyCalib = (0., 0., 0.) + event.extrinsicsCalibration.rpyCalibSpread = (0., 0., 0.) + original_delivery(now/1e9, [event.as_reader()]) + runtime.prepare(sm, now_ns=now, active=True, healthy=False) + assert 'a'*32 in runtime._calibration_lifecycle.rejected + assert not runtime._calibration_lifecycle.current and not runtime.eligible + for tick in range(87, 146): + frame(controls, tick) + assert 'a'*32 in runtime._calibration_lifecycle.rejected + assert not runtime.eligible + + +def test_pending_semantically_valid_block_regression_retires_generation(): + cp = interfaces[CAR.TOYOTA_RAV4_TSS2].get_non_essential_params(CAR.TOYOTA_RAV4_TSS2) + with controls_for(cp, 'standard', True) as controls, calibration_receipts(controls) as original_delivery: + for tick in range(86): + frame(controls, tick) + runtime, sm = controls.global_learning, controls.sm + event = messaging.new_message('extrinsicsCalibration', valid=True, logMonoTime=BASE+860_000_000) + event.extrinsicsCalibration = sm['extrinsicsCalibration'] + event.extrinsicsCalibration.validBlocks = 6 + original_delivery((BASE+860_000_000)/1e9, [event.as_reader()]) + runtime.prepare(sm, now_ns=BASE+860_000_000, active=True, healthy=False) + assert runtime._calibration_pending == ('a'*32, 6) + event.logMonoTime = BASE+870_000_000 + event.extrinsicsCalibration.validBlocks = 5 + original_delivery((BASE+870_000_000)/1e9, [event.as_reader()]) + runtime.prepare(sm, now_ns=BASE+870_000_000, active=True, healthy=False) + assert 'a'*32 in runtime._calibration_lifecycle.rejected + assert runtime._calibration_pending is None and not runtime.eligible + for tick in range(88, 146): + frame(controls, tick) + assert 'a'*32 in runtime._calibration_lifecycle.rejected + assert not runtime.eligible + + + +def test_pending_generation_replacement_retires_previous_and_reappearance_cannot_admit(): + cp = interfaces[CAR.TOYOTA_RAV4_TSS2].get_non_essential_params(CAR.TOYOTA_RAV4_TSS2) + with controls_for(cp, 'standard', True) as controls, calibration_receipts(controls, period=10): + for tick in range(136): + generation = 'a'*32 if tick < 110 or tick >= 135 else 'b'*32 + frame(controls, tick, generation) + runtime = controls.global_learning + if tick == 85: + assert runtime._calibration_pending == ('a'*32, 5) + if tick == 110: + assert runtime._calibration_pending == ('b'*32, 5) + assert 'a'*32 in runtime._calibration_lifecycle.rejected + assert not runtime.eligible + assert 'a'*32 in runtime._calibration_lifecycle.rejected + assert 'b'*32 in runtime._calibration_lifecycle.rejected + assert runtime._calibration_pending is None + + +def test_pending_generation_history_cannot_bypass_original_exhaustion_bound(): + cp = interfaces[CAR.TOYOTA_RAV4_TSS2].get_non_essential_params(CAR.TOYOTA_RAV4_TSS2) + with controls_for(cp, 'standard', True) as controls, calibration_receipts(controls, period=10): + for tick in range(936): + generation = f'{max(1, (tick-85)//25+1):032x}' + frame(controls, tick, generation) + lifecycle = controls.global_learning._calibration_lifecycle + assert len(lifecycle.rejected) <= 32 + assert not lifecycle.current and not controls.global_learning.eligible + assert lifecycle.exhausted + assert controls.global_learning._calibration_pending is None + + +@pytest.mark.parametrize('missing_tick', [100, 105]) +def test_missing_source_preserves_regression_guard_before_or_at_rate_recovery(missing_tick): + cp = interfaces[CAR.TOYOTA_RAV4_TSS2].get_non_essential_params(CAR.TOYOTA_RAV4_TSS2) + with controls_for(cp, 'standard', True) as controls, calibration_receipts(controls, blocks_before_105=6): + for tick in range(171): + if tick == missing_tick: + def missing_source(event): + event.autoSteeringLearning.learning.sourceReceipt.version = 0 + live_frame(controls, tick, 'a'*32, blocks=5, publish=True, wire_mutator=missing_source) + else: + frame(controls, tick) + runtime = controls.global_learning + if tick == 85: + assert runtime._calibration_pending == ('a'*32, 6) + if tick == 100 and missing_tick == 100: + # Missing source during the unqualified rate phase conservatively binds + # the pending token, then original observe(False) retires it immediately. + assert runtime._calibration_lifecycle.generation == 'a'*32 + assert not runtime._calibration_lifecycle.current + assert runtime._calibration_lifecycle.last_invalid_ns == BASE+1_000_000_000 + assert runtime._calibration_pending is None + if tick == 105: + assert controls.sm.freq_ok['extrinsicsCalibration'] + assert runtime._calibration_lifecycle.current + if missing_tick == 100: + assert runtime._calibration_lifecycle.generation == 'a'*32 + assert runtime._calibration_pending is None + else: + # A source gap first seen with genuinely current raw calibration + # keeps the saved6 blocks; later same-token5 cannot erase history. + assert runtime._calibration_lifecycle.generation == '' + assert runtime._calibration_pending == ('a'*32, 6) + if tick >= 100 and missing_tick == 100: + assert 'a'*32 in runtime._calibration_lifecycle.rejected + assert not runtime.eligible + assert 'a'*32 in runtime._calibration_lifecycle.rejected + assert runtime._calibration_pending is None diff --git a/openpilot/starpilot/flm/tests/test_candidate_feedback.py b/openpilot/starpilot/flm/tests/test_candidate_feedback.py new file mode 100644 index 0000000000..aacd7a0580 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_candidate_feedback.py @@ -0,0 +1,314 @@ +"""Feedback chronology units plus actual registered controller/native paths. + +Mocked original-tape seals in caller units grant NO original/current admission. +Fresh actual owner capture remains required for complete qualification. +""" +from types import SimpleNamespace +import pickle + +import pytest + +from openpilot.starpilot.flm.candidate_feedback import CandidateFeedbackLoop,CLOSED_POLICY,LEGACY_POLICY,policy +from openpilot.starpilot.flm.feedback_observation import f32 +from openpilot.starpilot.flm.torque_emission_replay import EmissionHistory + + +def arithmetic_fixture(monkeypatch, *, context=False, two_windows=False): + from openpilot.starpilot.flm import feedback_replay + stamps = (100,110,120,130) if not two_windows else (100,110,200,210) + rows = [] + for index,stamp in enumerate(stamps): + previous = stamps[index-1] if index else 0 + row = SimpleNamespace(control_ns=stamp,prior_publish_ns=previous,publication_ns=stamp-1, + recv_frame=index+1,receipt_ns=stamp,co_body_sha256='1'*64, + reported_torque=f32(.18199999630451202),requested_torque=f32(.17206627130508423), + lat_active=True,input_limited=False,published_limited=False) + pub = SimpleNamespace(output_control_ns=stamp-10,output_apply_ns=stamp-9, + publication_ns=stamp-1,output_carrier_sequence=index+1,native_identity='2'*64, + applied_sha256='3'*64,reported_torque=row.reported_torque) + rows.append(SimpleNamespace(consumption=row,publication=pub)) + context_rows = (rows[0].consumption,) if context else () + if two_windows: + windows = (((100,110),SimpleNamespace(rows=tuple(rows[:2]),digest='4'*64),100), + ((200,210),SimpleNamespace(rows=tuple(rows[2:]),digest='5'*64),200)) + else: + windows = (((100,130),SimpleNamespace(rows=tuple(rows[1:] if context else rows),digest='4'*64),110 if context else 100),) + evidence = SimpleNamespace(candidate_policy=CLOSED_POLICY,engine_input_digest='a'*64, + original=SimpleNamespace(context_only=context_rows,digest='b'*64),windows=windows) + engine = SimpleNamespace(emissions=EmissionHistory.__new__(EmissionHistory),verified_feedback_tape=evidence) + monkeypatch.setattr(feedback_replay,'require_feedback_engine',lambda owner:evidence) + frames = tuple(SimpleNamespace(control_ns=stamp,inputs=(('active',1.),('safety_limited',0.),('untouched',-0.))) for stamp in stamps) + return engine,rows,frames + + +@pytest.mark.parametrize('candidate_request,reported',[(.17249134182929993,.18266665935516357),(.17164121568202972,.18199999630451202)]) +def test_demonstrated_threshold_crossing_feeds_next_update_not_current(monkeypatch,candidate_request,reported): + engine,rows,frames = arithmetic_fixture(monkeypatch) + reports = {} + loop = CandidateFeedbackLoop(engine,reports,(.01,),native_policy='strict') + requests = {} + previous = False + observed_inputs = [] + for _index,(joined,frame) in enumerate(zip(rows,frames,strict=True)): + reports[joined.publication.output_apply_ns] = f32(reported) + before = loop.before(frame) + assert dict(before)['safety_limited'] == float(previous) + observed_inputs.append(previous) + assert pickle.dumps(tuple(pair for pair in before if pair[0]!='safety_limited')) == pickle.dumps( + tuple(pair for pair in frame.inputs if pair[0]!='safety_limited')) + requests[frame.control_ns] = f32(candidate_request) + loop.after(frame.control_ns,candidate_request) + consumed = joined.publication.reported_torque if joined.publication.output_control_ns < 100 else f32(reported) + previous = abs(f32(candidate_request)-consumed) > .01 + assert any(observed_inputs) + proof = loop.finish(requests,reports) + assert proof.row_count == 4 and proof.native_policy == 'strict' + assert proof.original_digest == engine.verified_feedback_tape.original.digest + + +def test_missing_consumed_native_report_never_uses_nearest_or_physical_value(monkeypatch): + engine,rows,frames = arithmetic_fixture(monkeypatch) + reports = {} + loop = CandidateFeedbackLoop(engine,reports,(.01,),native_policy='strict') + loop.before(frames[0]) + loop.after(frames[0].control_ns,.17249134182929993) + reports[rows[2].publication.output_apply_ns] = 0. # Wrong apply coordinate is not a fallback. + loop.before(frames[1]) + with pytest.raises(ValueError,match='not yet produced'): + loop.after(frames[1].control_ns,.17249134182929993) + + +def test_complete_finish_rejects_changed_native_result_or_omitted_frame(monkeypatch): + engine,rows,frames = arithmetic_fixture(monkeypatch) + reports,requests = {},{} + loop = CandidateFeedbackLoop(engine,reports,(.01,),native_policy='native-shadow-identification') + for joined,frame in zip(rows,frames,strict=True): + reports[joined.publication.output_apply_ns] = f32(.18266665935516357) + loop.before(frame) + requests[frame.control_ns]=f32(.17249134182929993) + loop.after(frame.control_ns,requests[frame.control_ns]) + reports[rows[-1].publication.output_apply_ns] = f32(.17) + with pytest.raises(ValueError,match='changed'): + loop.finish(requests,reports) + fresh = CandidateFeedbackLoop(engine,{},(.01,),native_policy='strict') + with pytest.raises(ValueError,match='omitted'): + fresh.finish({},fresh.reported) + + +def test_context_prefix_and_window_checkpoints_use_real_original_prior(monkeypatch): + engine,rows,frames = arithmetic_fixture(monkeypatch,context=True) + reports,requests = {},{} + loop = CandidateFeedbackLoop(engine,reports,(.01,),native_policy='strict') + for index,(joined,frame) in enumerate(zip(rows,frames,strict=True)): + reports[joined.publication.output_apply_ns] = f32(.18266665935516357) + inputs=loop.before(frame) + request=joined.consumption.requested_torque if index==0 else f32(.17249134182929993) + requests[frame.control_ns]=request + loop.after(frame.control_ns,request) + assert dict(inputs)['safety_limited'] == (0. if index<3 else 1.) + assert loop.finish(requests,reports).row_count == 4 + engine,rows,frames = arithmetic_fixture(monkeypatch,two_windows=True) + reports,requests = {},{} + loop = CandidateFeedbackLoop(engine,reports,(.01,),native_policy='strict') + for index,(joined,frame) in enumerate(zip(rows,frames,strict=True)): + reports[joined.publication.output_apply_ns] = f32(.18266665935516357) + assert dict(loop.before(frame))['safety_limited'] == (0. if index in (0,2) else 1.) + requests[frame.control_ns]=f32(.17164121568202972) + loop.after(frame.control_ns,requests[frame.control_ns]) + assert loop.finish(requests,reports).row_count == 4 + + +def test_original_baseline_and_pre_activation_remain_exact(monkeypatch): + engine,rows,frames = arithmetic_fixture(monkeypatch,context=True) + reports={} + loop=CandidateFeedbackLoop(engine,reports,(0.,),native_policy='strict',original_baseline=True) + loop.before(frames[0]) + with pytest.raises(ValueError,match='baseline or preactivation'): + loop.after(frames[0].control_ns,.17249134182929993) + for unknown in (None,'made-up',1): + with pytest.raises(ValueError,match='policy'): + policy(unknown) + assert policy(LEGACY_POLICY)==LEGACY_POLICY + + +def actual_caller_fixture(domain): + """Actual controller/CAN numeric unit; original source seal remains mocked.""" + from openpilot.starpilot.flm.controller_replay import ControllerReplay,ReplayFrame,SelectedControllerFactory,_run,capture_snapshot,final_command,frame_input + from openpilot.starpilot.flm.pid_controller_replay import PidControllerFactory + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + from openpilot.starpilot.flm.event_activation import EventActivationPlan,ZERO_REFERENCE_ACTIVATION_VERSION + from openpilot.starpilot.flm.event_objective import EffectObjective + from openpilot.starpilot.flm.torque_adapter import Objective + from openpilot.starpilot.flm.torque_emission_replay import Emission + from openpilot.starpilot.flm.tests.test_full_surface_profile import base + from openpilot.starpilot.flm.effect_trace import FullEffectTrace + from openpilot.starpilot.flm.selected_effects import EffectReplayRecordedSource + from openpilot.starpilot.flm.tests.test_event_lazy_native import TOYOTA,HYUNDAI,HONDA,make_interface,install_observer,settle,control,apply,BASE_NS + platform={'torque':TOYOTA.TOYOTA_COROLLA_TSS2,'starpilot':HYUNDAI.HYUNDAI_IONIQ_6,'pid':HONDA.HONDA_CIVIC}[domain] + ci=make_interface(platform) + assert install_observer(ci) + settled=settle(ci,control(0.)) + factory=PidControllerFactory(ci.CP,'native') if domain=='pid' else SelectedControllerFactory(ci.CP,'starpilot' if domain=='starpilot' else 'standard') + objective=Objective('feedforward','high','steady') + original=ControllerReplay(factory,objective,identity='f'*64,baseline_revision='a'*64) + actual,vm=factory.create() + original.raw_mode=True + first=BASE_NS+ci.CC.frame*10_000_000 + original.checkpoint(first,capture_snapshot(actual,vm)) + previous_flag=False + previous_report=f32(settled[-1][0].torque) + prior_stamp=BASE_NS+settled[-1][2].frame*10_000_000 + previous_event=SimpleNamespace(control_ns=prior_stamp,apply_ns=prior_stamp+1_000_000) + frames,joins,contexts=[],[],[] + envelope=base() + for tick in range(60): + stamp=first+tick*10_000_000 + inputs=frame_input(ci.CS.out,vm,SimpleNamespace(angleOffsetDeg=0.,roll=0.),active=True, + safety_limited=previous_flag,desired_curvature=.00003,curvature_limited=False,lat_delay=.2) + inputs+=(('control_frame',float(tick+1)),('raw_model_curvature',.00003)) + tuning=() if domain=='pid' else (('effective_factor',float(actual.torque_params.latAccelFactor)), + ('offset',float(actual.torque_params.latAccelOffset)),('friction',float(actual.torque_params.friction)), + ('pid_pos_limit',float(actual.pid.pos_limit)),('pid_neg_limit',float(actual.pid.neg_limit))) + trace=FullEffectTrace(1,envelope,stamp,stamp-10_000_000,True, + envelope.coordinates,envelope.coordinates,envelope.coordinates,envelope.full_profile_sha256).validate() + value,_=_run(actual,vm,inputs,objective,(0.,0.),tuning,actual_stage_residual=(0.,0.), + effect_trial=EffectReplayRecordedSource(trace)) + request=f32(final_command(value)) + flag=bool(abs(request-previous_report)>.01) + row=SimpleNamespace(control_ns=stamp,prior_publish_ns=stamp-10_000_000 if tick else 0, + publication_ns=stamp-1,recv_frame=tick+1,receipt_ns=stamp,co_body_sha256='1'*64, + reported_torque=previous_report,requested_torque=request,lat_active=True, + input_limited=previous_flag,published_limited=flag) + if previous_event is None: + contexts.append(row) + else: + pub=SimpleNamespace(output_control_ns=previous_event.control_ns,output_apply_ns=previous_event.apply_ns, + publication_ns=stamp-1,output_carrier_sequence=tick,native_identity='2'*64,applied_sha256='3'*64, + reported_torque=previous_report) + joins.append(SimpleNamespace(consumption=row,publication=pub)) + frame=ReplayFrame(stamp,inputs,request,(0.,0.),frame_sequence=tick+1,actual_lac_command=value, + limited=flag,torque_values=tuning,actual_stage_residual=(0.,0.), + effect_trace=trace,effect_prepare_ns=trace.control_ns, + controller_domain='torque_pid' if domain=='pid' else 'torque') + frames.append(frame) + output,packets,witness=apply(ci,control(value)) + if witness is not None: + assert (witness.can_address,witness.can_data,witness.can_bus) in packets + clipped=witness.unlimited_raw is None or witness.unlimited_raw!=witness.final_raw + event=Emission(stamp,stamp,stamp+1_000_000,witness,'f'*64,'e'*64,'a'*64, + True,True,False,False,clipped,False,False,0,0,'') + original.emissions.events[event.apply_ns]=event + original.emissions.by_control[event.control_ns]=event + previous_event=event + previous_report=f32(output.torque) + previous_flag=flag + assert not contexts + original.records.extend(frames) + bounds=(frames[0].control_ns,frames[-1].control_ns) + original.replay_windows=(bounds,) + selected=EffectObjective('ff.left',envelope.trial_names,envelope.trial_anchor,envelope.context_key) + activation=frames[30].control_ns + prepare=frames[30].effect_prepare_ns + assert prepare==frames[30].effect_trace.control_ns==activation + plan=EventActivationPlan(ZERO_REFERENCE_ACTIVATION_VERSION,original.identity,original.baseline_revision, + selected.context_key,'b'*64,tuple('train-'+str(i) for i in range(4)),tuple('held-'+str(i) for i in range(4)), + activation-10_000_000,tuple('stage-'+str(i) for i in range(9)),((bounds[0],bounds[1],activation,prepare),)).validate() + engine=EventControllerReplay(original,selected,envelope.replay_anchor(),activation_plan=plan) + evidence=SimpleNamespace(candidate_policy=CLOSED_POLICY,engine_input_digest='a'*64, + binding='c'*64,original=SimpleNamespace(context_only=tuple(contexts),digest='b'*64), + windows=((bounds,SimpleNamespace(rows=tuple(joins),digest='d'*64),activation),)) + engine.verified_feedback_tape=evidence + return engine,evidence,tuple(joins),tuple(contexts) + + +@pytest.mark.parametrize('domain',['torque','starpilot','pid']) +@pytest.mark.parametrize('coordinates',[(0.,),(.00001,)]) +@pytest.mark.parametrize('shadow',[False,True]) +def test_actual_selected_controller_native_interleave_and_next_input(domain,coordinates,shadow,monkeypatch): + from openpilot.starpilot.flm import feedback_replay,event_controller_replay + engine,evidence,joins,contexts=actual_caller_fixture(domain) + monkeypatch.setattr(feedback_replay,'require_feedback_engine',lambda owner:evidence) + updates=[] + native_sink=[] + original_run=event_controller_replay._run + original_native=engine.emissions.rollout + def update(*args,**kwargs): + result=original_run(*args,**kwargs) + updates.append((args[2],f32(result[0]))) + return result + def native(*args,**kwargs): + native_sink.append(kwargs['feedback_reported']) + return original_native(*args,**kwargs) + monkeypatch.setattr(event_controller_replay,'_run',update) + monkeypatch.setattr(engine.emissions,'rollout',native) + kwargs={'_identification_token':engine._EventControllerReplay__identification_token} if shadow else {} + requests,encoded,_=engine._rollout_uncached(coordinates,**kwargs) + assert len(updates)==len(engine.records)==60 + reports=native_sink[0] + associations={joined.consumption.control_ns:joined for joined in joins} + context={row.control_ns:row for row in contexts} + activation=evidence.windows[0][2] + previous=False + for frame,(inputs,request) in zip(engine.records,updates,strict=True): + assert dict(inputs)['safety_limited']==float(previous) + assert pickle.dumps(tuple(pair for pair in inputs if pair[0]!='safety_limited'))==pickle.dumps( + tuple(pair for pair in frame.inputs if pair[0]!='safety_limited')) + if frame.control_ns in context: + row=context[frame.control_ns] + report=row.reported_torque + else: + joined=associations[frame.control_ns] + row=joined.consumption + pub=joined.publication + report=pub.reported_torque if pub.output_control_ns.01) + if frame.control_ns= 39 * SECOND else 0. + records.append(SimpleNamespace(control_ns=START + elapsed, frame_sequence=tick + 1, + desired_physical_accel=goal)) + capture = SimpleNamespace(epoch=0, epoch_start_ns=START, epoch_cutoff_ns=START + MAX_CAPTURE_SPAN_NS, + rows=tuple(SimpleNamespace(mono_ns=stamp) for stamp in range(START, now + 1, SECOND // pose_hz))) + exported = [] + + def original_source_packet(collector, windows, observed_ns): + exported.append((windows, observed_ns)) + # A parent-selection fixture must never fabricate a qualified child packet. + raise ValueError('parent-only export boundary reached') + + adapter = SimpleNamespace(envelope=object(), source_plant_submitted_epoch=-1, identity=control.identity, + baseline_revision=control.baseline_revision, snapshot_readiness=None, + replay=SimpleNamespace(records=tuple(records), checkpoints=((START, object()),), _frame_context=lambda frame: True), + original_source_packet=original_source_packet) + collector = SimpleNamespace(adapter=adapter, original_source_capture=capture, ready=[(control, output)], + last_fit=last_attempt, snapshot_attempts=0) + assert runtime._current_fit_pair_reason(collector, now) is None + return collector, now, exported + + +def test_predeclared_constants_and_plant_design_remain_separate(): + assert runtime.MIN_NORMAL_JOB_AGE_NS == 60 * SECOND + assert runtime.NORMAL_JOB_ATTEMPT_SPACING_NS == 80 * SECOND + assert MAX_CAPTURE_SPAN_NS == 100 * SECOND + assert FIRST_SLOT_NS + 7 * (BLOCK_NS + PURGE_NS) + BLOCK_NS == 38_800_000_000 + + +@pytest.mark.parametrize('age_ns,attempt', [(60 * SECOND - 1, False), (60 * SECOND, True)]) +def test_complete_parent_geometry_first_attempt_boundary(age_ns, attempt): + collector, now, exported = parent_fixture(age_ns) + assert runtime.snapshot(collector, now) is None + assert bool(exported) is attempt + assert collector.snapshot_attempts == int(attempt) + assert collector.last_fit == (now if attempt else 0) + if attempt: + assert exported == [(((START, now - age_ns % 10_000_000),), now)] + assert collector.adapter.snapshot_readiness['reason'] == 'plant_snapshot_unavailable' + else: + assert collector.adapter.snapshot_readiness['reason'] == 'plant_fixed_cohort_still_collecting' + + +@pytest.mark.parametrize('pose_hz,goals', [(20, False), (10, True)]) +def test_incomplete_geometry_never_consumes_attempt_or_cooldown(pose_hz, goals): + collector, now, exported = parent_fixture(60 * SECOND, pose_hz=pose_hz, with_goals=goals) + assert runtime.snapshot(collector, now) is None + assert exported == [] + assert collector.snapshot_attempts == collector.last_fit == 0 + assert collector.adapter.snapshot_readiness['reason'] == 'plant_later_original_goal_geometry' + assert collector.adapter.snapshot_readiness['candidateBundles'] < 8 + + +@pytest.mark.parametrize('elapsed,attempt', [(80 * SECOND - 1, False), (80 * SECOND, True)]) +def test_actual_attempt_spacing_boundary(elapsed, attempt): + collector, now, exported = parent_fixture(90 * SECOND) + collector.last_fit = now - elapsed + previous = collector.last_fit + assert runtime.snapshot(collector, now) is None + assert bool(exported) is attempt + assert collector.snapshot_attempts == int(attempt) + assert collector.last_fit == (now if attempt else previous) + + +def test_submitted_epoch_still_forbids_another_attempt(): + collector, now, exported = parent_fixture(90 * SECOND) + collector.adapter.source_plant_submitted_epoch = collector.original_source_capture.epoch + assert runtime.snapshot(collector, now) is None + assert not exported and collector.snapshot_attempts == 0 + + +def test_real_capture_keeps_fixed_hundred_second_epoch_and_no_reselection(): + capture = OriginalSourceCapture() + receipt = {'epoch_id': 'first', 'start_ns': START, 'cutoff_ns': START + 100 * SECOND, + 'identity': '1' * 64, 'source_epoch': '2' * 64, 'baseline_revision': 'baseline'} + capture.begin_epoch(receipt) + ordinal = capture.epoch + with pytest.raises(ValueError, match='cannot reselect'): + capture.begin_epoch({**receipt, 'epoch_id': 'overlap', 'start_ns': START + 60 * SECOND, + 'cutoff_ns': START + 160 * SECOND}) + assert capture.epoch == ordinal and capture.epoch_cutoff_ns == START + 100 * SECOND + capture.begin_epoch({**receipt, 'epoch_id': 'second', 'start_ns': START + 100 * SECOND, + 'cutoff_ns': START + 200 * SECOND}) + assert capture.epoch == ordinal + 1 + assert capture.epoch_cutoff_ns - capture.epoch_start_ns == 100 * SECOND diff --git a/openpilot/starpilot/flm/tests/test_event_admission_clock.py b/openpilot/starpilot/flm/tests/test_event_admission_clock.py index 7f9f91304e..9877354ebf 100644 --- a/openpilot/starpilot/flm/tests/test_event_admission_clock.py +++ b/openpilot/starpilot/flm/tests/test_event_admission_clock.py @@ -46,6 +46,10 @@ class ClockedBoundaryAdapter(BoundaryAdapter, EventCollectionAdapter): raw_collection = False decisions = None + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.original_context_receipts = None + def physical_observable(self, row=None, *, value=None): return super().physical_observable(row if value is None else value) diff --git a/openpilot/starpilot/flm/tests/test_event_bounded_ridge.py b/openpilot/starpilot/flm/tests/test_event_bounded_ridge.py new file mode 100644 index 0000000000..f9a86c37b1 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_event_bounded_ridge.py @@ -0,0 +1,111 @@ +"""Bounded train proposal; no native permission or heldout-selected direction.""" +from dataclasses import asdict, replace +from unittest import mock + +import pytest + +from openpilot.starpilot.flm.adaptive_profile import Parameter, evaluate, loss +from openpilot.starpilot.flm.event_bounded_ridge import bounded_ridge +from openpilot.starpilot.flm.event_evaluate import evaluate_event +from openpilot.starpilot.flm.tests.test_event_native_backtrack import problem + + +def parameter(low=-1., high=1., max_step=1.): + return Parameter('test', 'test', low, high, max_step, .005) + + +def test_big_error_legacy_rejects_but_event_computes_actual_bounded_improvement(): + args, kwargs = problem() + blocks = tuple(replace(block, samples=tuple(replace(row, residual_target=10.) for row in block.samples)) for block in args[2]) + before = tuple(asdict(p) for p in args[1]), blocks, asdict(kwargs['effect_envelope']) + legacy = evaluate(args[0], args[1], blocks, **{k: kwargs[k] for k in ('now_ns', 'required_cohorts', 'response_revision', 'prior')}) + assert legacy.reason == 'coordinate_envelope' and legacy.profile is None + native = mock.Mock(return_value=True) + result = evaluate_event(args[0], args[1], blocks, **kwargs, native_feasible=native) + assert result.profile is not None and result.profile.coefficients == (.005,) + native.assert_called_once_with((.005,)) + held = tuple(row for block in blocks[4:] for row in block.samples) + assert result.profile.candidate_loss == loss(held, (.005,)) < result.profile.baseline_loss == loss(held, (0.,)) + assert (tuple(asdict(p) for p in args[1]), blocks, asdict(kwargs['effect_envelope'])) == before + assert result.reason == 'event_tiny_trial_pending_actual_replay' # Not installed/admitted/current authority. + + +def test_box_solver_reoptimizes_coupled_free_coordinate_and_satisfies_kkt(): + matrix, rhs = ((2., 1.), (1., 2.)), (2., 0.) + result = bounded_ridge(matrix, rhs, (parameter(), parameter()), (0., 0.), .5) + assert result == (.5, -.25) + gradient = tuple(sum(a*b for a, b in zip(row, result, strict=True))-r for row, r in zip(matrix, rhs, strict=True)) + assert gradient[0] < 0 and gradient[1] == 0 # Upper active / free KKT. + assert result != (.5, -.5) # Componentwise clamp of unconstrained (4/3,-2/3). + + +@pytest.mark.parametrize('sign', (-1., 1.)) +def test_actual_domain_max_step_and_nonzero_prior_intersection(sign): + p = parameter(-.02, .02, .012) + result = bounded_ridge(((1.,),), (100.*sign,), (p,), (.01*sign,), .005) + assert result == (.012*sign,) + assert abs(result[0]-.01*sign) <= .005 and abs(result[0]) <= p.max_step + p = parameter(0., .003, .25) + assert bounded_ridge(((1.,),), (100.,), (p,), (0.,), .005) == (.003,) + + +def test_train_direction_is_identical_on_adversarial_heldout_and_no_native_call(): + args, kwargs = problem() + train_big = tuple(replace(b, samples=tuple(replace(r, residual_target=10.) for r in b.samples)) for b in args[2]) + bad = tuple(replace(b, samples=tuple(replace(r, residual_target=-10.) for r in b.samples)) if i >= 4 else b + for i, b in enumerate(train_big)) + from openpilot.starpilot.flm import event_bounded_ridge + original = event_bounded_ridge.bounded_ridge + selected = [] + def observe(*a): + result = original(*a) + selected.append(result) + return result + native = mock.Mock(return_value=True) + with mock.patch.object(event_bounded_ridge, 'bounded_ridge', side_effect=observe): + good = evaluate_event(args[0], args[1], train_big, **kwargs) + rejected = evaluate_event(args[0], args[1], bad, **kwargs, native_feasible=native) + assert selected == [(.005,), (.005,)] and good.profile is not None + assert rejected.profile is None and rejected.reason == 'heldout_cohort_regression_or_noise' + native.assert_not_called() + + +@pytest.mark.parametrize('defect', ('limited', 'duplicate', 'unexcited', 'confounded', 'wrong_prior', 'nan')) +def test_original_evidence_excitation_and_contract_gates_remain(defect): + args, kwargs = problem() + parameters, blocks = args[1], args[2] + if defect in ('limited', 'nan', 'unexcited'): + rows = tuple(replace(row, **({'native_allowed': False} if defect == 'limited' else + {'features': (float('nan'),)} if defect == 'nan' else {'features': (0.,)})) + for row in blocks[0].samples) + blocks = (replace(blocks[0], samples=rows), *blocks[1:]) + if defect == 'unexcited': + blocks = tuple(replace(b, samples=tuple(replace(r, features=(0.,)) for r in b.samples)) for b in blocks) + elif defect == 'duplicate': + blocks = (blocks[0], replace(blocks[1], block_id=blocks[0].block_id), *blocks[2:]) + elif defect == 'wrong_prior': + kwargs = {**kwargs, 'prior': (True,)} + else: + # Excitation floor remains mandatory even when a constrained optimum exists. + parameters = tuple(replace(p, excitation_floor=2.) for p in parameters) + native = mock.Mock(return_value=True) + result = evaluate_event(args[0], parameters, blocks, **kwargs, native_feasible=native) + assert result.profile is None + native.assert_not_called() + + +@pytest.mark.parametrize('matrix,rhs,centre,step', ( + (((float('nan'),),), (1.,), (0.,), .005), + (((1.,),), (float('inf'),), (0.,), .005), + (((1.,),), (1.,), (True,), .005), + (((1.,),), (1.,), (0.,), True), + (((1.,),), (1.,), (0.,), -.005), +)) +def test_nonfinite_and_boolean_solver_inputs_fail_closed(matrix, rhs, centre, step): + with pytest.raises(ValueError): + bounded_ridge(matrix, rhs, (parameter(),), centre, step) + + +def test_all_four_dimensions_and_fixed_training_only_bounded_domain(): + matrix = tuple(tuple(2. if i == j else 0. for j in range(4)) for i in range(4)) + assert bounded_ridge(matrix, (10., -10., .002, -.002), (parameter(),)*4, (0.,)*4, .005) == (.005, -.005, .001, -.001) diff --git a/openpilot/starpilot/flm/tests/test_event_fd_reference_reuse.py b/openpilot/starpilot/flm/tests/test_event_fd_reference_reuse.py new file mode 100644 index 0000000000..a5811dd461 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_event_fd_reference_reuse.py @@ -0,0 +1,141 @@ +"""Exact FD baseline reuse, actual registered controllers/native projection. + +No fitted result/current permission is supplied. Native/reference parity and +owned-source mutation misses are verified independently from FD arithmetic. +""" +from dataclasses import replace +import pickle +from unittest import mock + +import pytest + +from openpilot.starpilot.flm.canonical_surface_adapter import GROUPS +from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay +from openpilot.starpilot.flm.event_objective import EffectObjective +from openpilot.starpilot.flm.tests.test_event_controller_prefix import fixture + + +def engine(domain='torque'): + original=fixture(domain) + envelope=original.records[0].effect_trace.envelope.for_trial(GROUPS['deadband.high']) + objective=EffectObjective('deadband.high',envelope.trial_names,envelope.trial_anchor,envelope.context_key) + value=EventControllerReplay(original.engine,objective,envelope.replay_anchor(),activation_plan=original.activation_plan) + assert all(parameter.low==0. for parameter in value.objective.parameters) + return value + + +def run(value, *, reuse): + traces=[] + original=value._rollout_uncached + def execute(coordinates,**kwargs): + result=original(coordinates,**kwargs) + traces.append((pickle.dumps(tuple(coordinates),protocol=5), + kwargs.get('_identification_token') is not None,pickle.dumps(result,protocol=5))) + return result + with mock.patch.object(value,'_rollout_uncached',side_effect=execute), \ + mock.patch.object(value.emissions,'decode',wraps=value.emissions.decode) as decode: + if reuse: + count=value.rebuild_causal_features() + else: + with mock.patch.object(value,'_strict_fd_reference',return_value=None): + count=value.rebuild_causal_features() + decoded=decode.call_count + return count,pickle.dumps(value.projected,protocol=5),traces,decoded + + +@pytest.mark.parametrize('domain',('torque','starpilot','pid')) +def test_actual_boundary_fd_matrix_raw_requests_saturation_and_native_outputs_exact(domain): + fast=engine(domain) + slow=EventControllerReplay(fast.engine,fast.objective,fast.original_anchor,activation_plan=fast.activation_plan) + expected=run(slow,reuse=False) + actual=run(fast,reuse=True) + assert actual[:2]==expected[:2] + columns=len(fast.objective.names) + assert len(expected[2])==1+2*columns + assert len(actual[2])==1+columns + baseline_bits=pickle.dumps(fast.reference,protocol=5) + baseline_trace=next(trace[2] for trace in actual[2] if trace[0]==baseline_bits) + for coordinates,shadow,result in expected[2]: + if coordinates==baseline_bits: + # Strict successful baseline equals every omitted shadow reference side. + assert result==baseline_trace + else: + assert (coordinates,shadow,result) in actual[2] + assert actual[3]0 # Every unique rollout still decodes native history. + + +def certificate(value): + value._rollout(value.reference) + saved=value._EventControllerReplay__strict_rollouts + assert saved is not None and 'baseline' in saved[1] + return saved,saved[1]['baseline'],pickle.dumps(value.reference,protocol=5) + + +def test_success_is_readonly_copy_and_not_shadow_or_live_permission(): + value=engine() + proof=certificate(value) + first=value._strict_fd_reference(value.reference,proof) + assert first is not None + expected=dict(first) + first[next(iter(first))]+=1. + assert value._strict_fd_reference(value.reference,proof)==expected + assert value._strict_fd_reference((0,0.),proof) is None # Type exact, not numeric equality. + assert value._strict_fd_reference((-0.,0.),proof) is None # Signed zero is not the recorded bits. + assert value._strict_fd_reference((.01,0.),proof) is None + assert value._strict_fd_reference(value.reference,None) is None + + +@pytest.mark.parametrize('change',('cp','native','record','checkpoint','prior','plan','unknown_key','lost_success')) +def test_every_owned_source_change_denies_reference_reuse(change): + value=engine() + proof=certificate(value) + if change=='cp': + cp=value.factory.cp.as_builder() + cp.steerRatio+=.01 + value.factory.cp=cp.as_reader() + elif change=='native': + stamp,event=next(iter(value.emissions.events.items())) + value.emissions.events[stamp]=replace(event,witness=event.witness._replace(final_raw=event.witness.final_raw+1)) + elif change=='record': + row=value.records[0] + value.records[0]=replace(row,inputs=row.inputs+(('extra_input',1.),)) + elif change=='checkpoint': + stamp,state=value.checkpoints[0] + value.checkpoints[0]=(stamp,replace(state,values=state.values+(('unknown',1.),))) + elif change=='prior': + value.prior_full['ff_gain_right']=.001 + elif change=='plan': + value.activation_plan=replace(value.activation_plan,physical_context='c'*64) + elif change=='unknown_key': + with mock.patch.object(value,'_causal_input_key',return_value=None): + assert value._strict_fd_reference(value.reference,proof) is None + return + else: + value._EventControllerReplay__strict_rollouts=None + assert value._strict_fd_reference(value.reference,proof) is None + + +def test_missing_success_falls_back_to_original_shadow_rollouts(): + value=engine() + original=value._rollout + calls=[] + def rollout(coordinates,**kwargs): + result=original(coordinates,**kwargs) + calls.append((tuple(coordinates),kwargs.get('_identification_token') is not None)) + if len(calls)==1: + # Remove only the successful reuse record; never mint a replacement proof. + value._EventControllerReplay__strict_rollouts=None + return result + with mock.patch.object(value,'_rollout',side_effect=rollout): + value.rebuild_causal_features() + assert len(calls)==1+2*len(value.objective.names) + assert any(coords==value.reference and shadow for coords,shadow in calls) + + +def test_failed_strict_baseline_never_enters_fd_or_creates_success(): + value=engine() + with mock.patch.object(value,'_rollout',side_effect=ValueError('strict baseline refused')) as rollout: + with pytest.raises(ValueError,match='strict baseline refused'): + value.rebuild_causal_features() + assert rollout.call_count==1 + assert value._EventControllerReplay__strict_rollouts is None diff --git a/openpilot/starpilot/flm/tests/test_fd_proof_local_validation.py b/openpilot/starpilot/flm/tests/test_fd_proof_local_validation.py new file mode 100644 index 0000000000..6cf0af625f --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_fd_proof_local_validation.py @@ -0,0 +1,83 @@ +"""Actual controller/native numerical checks; original seals remain unit-mocked. + +Root's authentic owner/component run is required for source admission and gain. +""" +from types import MethodType +from dataclasses import replace +import pytest + +from openpilot.starpilot.flm.tests.test_closed_fd_reference_reuse import closed_engine, certificate + + +def actual_key_engine(monkeypatch, domain='torque'): + from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay + engine, evidence = closed_engine(monkeypatch, domain) + # Use actual full-body causal keys, not the imported fixture's constant key. + monkeypatch.setattr(engine, '_causal_input_key', MethodType(EventControllerReplay._causal_input_key, engine)) + from openpilot.starpilot.flm import feedback_replay + monkeypatch.setattr(feedback_replay, 'require_feedback_engine', lambda owner: owner.__dict__['verified_feedback_tape']) + return engine, evidence + + +@pytest.mark.parametrize('domain', ['torque', 'starpilot', 'pid']) +def test_actual_strict_reference_retains_two_full_checks_and_removes_only_middle_require(monkeypatch, domain): + from openpilot.starpilot.flm import feedback_replay, candidate_feedback + engine, evidence = actual_key_engine(monkeypatch, domain) + cert, result = certificate(engine) + calls = [] + monkeypatch.setattr(feedback_replay, 'require_feedback_engine', lambda owner: (calls.append(owner), evidence)[1]) + key = engine._causal_input_key + keys = [] + monkeypatch.setattr(engine, '_causal_input_key', lambda epsilon: (keys.append(epsilon), key(epsilon))[1]) + actual = engine._strict_fd_reference(engine.reference, cert) + assert actual == result[1] and actual is not result[1] + assert keys == [.01, .01] + assert calls == [engine, engine] + proof, expected = cert[1][2] + calls.clear() + assert candidate_feedback.strict_reference_proof_key(proof, engine, engine.reference, len(result[0])) == expected + assert calls == [engine] # Public API still owns its fresh complete require. + + +@pytest.mark.parametrize('change', ['factory_cp', 'native_cp', 'records', 'evidence', 'policy', 'native_events']) +def test_full_post_key_still_declines_mutation_during_proof_projection(monkeypatch, change): + from openpilot.starpilot.flm import candidate_feedback + engine, evidence = actual_key_engine(monkeypatch) + cert, _ = certificate(engine) + original = candidate_feedback._strict_reference_fields + + def mutated(*args): + value = original(*args) + if change in ('factory_cp', 'native_cp'): + owner = engine.factory if change == 'factory_cp' else engine.emissions + builder = owner.cp.as_builder() + builder.steerRatio += .125 + owner.cp = builder.as_reader() + elif change == 'records': + engine.records[0] = replace(engine.records[0], desired_physical_accel=1.25) + elif change == 'evidence': + engine.verified_feedback_tape = replace(evidence, binding='f' * 64) + elif change == 'policy': + object.__setattr__(evidence, 'candidate_policy', candidate_feedback.LEGACY_POLICY) + else: + first = next(iter(engine.emissions.events)) + engine.emissions.events[first] = replace(engine.emissions.events[first], state_ns=1) + return value + + monkeypatch.setattr(candidate_feedback, '_strict_reference_fields', mutated) + assert engine._strict_fd_reference(engine.reference, cert) is None + + +def test_private_projection_is_not_a_wire_or_unsealed_proof_minter(monkeypatch): + from openpilot.starpilot.flm import candidate_feedback + from openpilot.starpilot.flm.feedback_replay import FeedbackReplayEvidence + engine, evidence = actual_key_engine(monkeypatch) + cert, result = certificate(engine) + proof, expected = cert[1][2] + fn = candidate_feedback._strict_reference_fields + assert fn(proof, evidence, engine.reference, len(result[0])) == expected + assert fn(replace(proof, _seal=object()), evidence, engine.reference, len(result[0])) is None + assert fn(proof, replace(evidence, _seal=object()), engine.reference, len(result[0])) is None + wire = FeedbackReplayEvidence(evidence.binding, evidence.original, evidence.windows, + evidence.engine_input_digest, object(), evidence.candidate_policy) + assert fn(proof, wire, engine.reference, len(result[0])) is None diff --git a/openpilot/starpilot/flm/tests/test_feedback_validation_only.py b/openpilot/starpilot/flm/tests/test_feedback_validation_only.py new file mode 100644 index 0000000000..21e51d2f88 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_feedback_validation_only.py @@ -0,0 +1,150 @@ +from collections.abc import Callable +from typing import cast, NoReturn +from dataclasses import fields +from types import SimpleNamespace + +import pytest + +from openpilot.starpilot.flm import feedback_observation as f +from openpilot.starpilot.flm import feedback_replay as replay + + +def record(kind, **overrides): + defaults: dict[object, object] = {int: 1, float: 0.25, bool: True, str: 'a' * 64} + values = {field.name: defaults[field.type] for field in fields(kind)} + values.update(overrides) + return kind(**values) + + +def tape(*, session='a' * 64): + rows = tuple(f.FeedbackJoin(record(f.CoConsumption, control_ns=stamp), record(f.CoPublication, session=session)) + for stamp in (100, 200)) + binding = 'a' * 64 + parent = 'b' * 64 + digest = f._tape_digest(rows, (), binding, parent) + return f.VerifiedFeedbackTape(f.PROTOCOL, rows, binding, digest, (), parent, f._VERIFIED) + + +def test_closed_validation_uses_one_fresh_key_and_no_descendant(monkeypatch): + value = tape() + calls = [] + original = f._tape_digest_key + monkeypatch.setattr(f, '_tape_digest_key', lambda *args: (calls.append(args), original(*args))[1]) + monkeypatch.setattr(f, '_mint_subset', lambda *args: cast(Callable[[str], NoReturn], pytest.fail)('discarded descendant mint')) + assert f.validate_subset_verified(value, (100, 200), binding_digest=value.binding_digest) is None + assert len(calls) == 1 + assert f.validate_subset_verified(value, (100, 200), binding_digest=value.binding_digest) is None + assert len(calls) == 2 + object.__setattr__(value.rows[0].consumption, 'requested_torque', 0.75) + with pytest.raises(ValueError, match='owned verified feedback tape/exact window required'): + f.validate_subset_verified(value, (100, 200), binding_digest=value.binding_digest) + assert len(calls) == 3 + + +def test_public_mint_preserves_parent_rows_digest(): + value = tape() + subset = f.subset_verified(value, (100,), binding_digest=value.binding_digest) + assert subset.rows == value.rows[:1] + assert subset.parent_digest == value.digest + assert subset.digest == f._tape_digest(value.rows[:1], (), value.binding_digest, value.digest) + assert subset._seal is f._VERIFIED + + +@pytest.mark.parametrize('controls', [(), [], (999,), (100, 999), (200, 100)]) +def test_validation_preserves_public_guard_errors_and_causes(controls): + outcomes = [] + for fn in (f.subset_verified, f.validate_subset_verified): + value = tape() + try: + fn(value, controls, binding_digest=value.binding_digest) + except Exception as error: + outcomes.append((type(error), str(error), type(error.__cause__), str(error.__cause__))) + assert len(outcomes) == 2 + assert outcomes[0] == outcomes[1] + + +@pytest.mark.parametrize('session', ['a' * 129, '\ud800']) +def test_unknown_key_domain_retains_mint(monkeypatch, session): + value = tape(session=session) + if f._tape_digest_key(value.rows, (), value.binding_digest, value.parent_digest) is not None: + # A supported surrogate remains a safe exact immutable scalar. + assert session == '\ud800' + return + calls = [] + original = f._mint_subset + monkeypatch.setattr(f, '_mint_subset', lambda *args: (calls.append(args), original(*args))[1]) + f.validate_subset_verified(value, (100, 200), binding_digest=value.binding_digest) + assert len(calls) == 1 + + +def test_custom_control_equality_keeps_original_mint_after_callback(monkeypatch): + value = tape() + class Stamp: + def __eq__(self, other): + object.__setattr__(value.rows[0].consumption, 'requested_torque', 0.5) + return other == 100 + calls = [] + original = f._mint_subset + monkeypatch.setattr(f, '_mint_subset', lambda *args: (calls.append(args), original(*args))[1]) + f.validate_subset_verified(value, (Stamp(), 200), binding_digest=value.binding_digest) + assert len(calls) == 1 + assert value.rows[0].consumption.requested_torque == 0.5 + + +def test_actual_require_caller_keeps_evidence_and_activation_checks(monkeypatch): + value = tape() + evidence = replay.FeedbackReplayEvidence(value.binding_digest, value, (((100, 200), value, 150),), 'owned', replay._SEAL) + engine = SimpleNamespace(verified_feedback_tape=evidence, records=(SimpleNamespace(control_ns=100), SimpleNamespace(control_ns=200)), + replay_windows=((100, 200),), activation_plan=SimpleNamespace(activation_for=lambda *args: (150,))) + def inputs(owner, *, candidate_policy): + assert owner is engine + assert candidate_policy == replay.LEGACY_POLICY + return 'owned' + monkeypatch.setattr(replay, '_inputs', inputs) + monkeypatch.setattr(f, '_mint_subset', lambda *args: cast(Callable[[str], NoReturn], pytest.fail)('require discarded descendant mint')) + assert replay.require_feedback_engine(engine) is evidence + assert replay.owned_feedback_key(engine)[-1] == (((100, 200), value.digest, 150),) + engine.activation_plan.activation_for = lambda *args: (151,) + with pytest.raises(ValueError, match='feedback activation changed'): + replay.require_feedback_engine(engine) + + +def old_hex(value, size=64): + return type(value) is str and len(value) == size and all(c in '0123456789abcdef' for c in value) + + +@pytest.mark.parametrize('size', [-1, 0, 1, 64, True, None, 1.0, '1']) +def test_hex_exact_equivalence_full_byte_alphabet_and_sizes(size): + for value in ('', '0' * 64, 'f' * 64, 'a' * 63 + 'F', *[chr(index) for index in range(256)], + '\u0100', '\uff10', '\U0001f642', '\ud800', 'a\n', 'a\x00'): + assert f._hex(value, size) == old_hex(value, size) + + +def test_hex_str_subclass_and_nonstring_short_circuit(): + class Text(str): + def __len__(self): + raise AssertionError('subclass must short circuit') + for value in (Text('a'), None, b'a', 1, True, [], {}): + assert f._hex(value, 1) is False + assert old_hex(value, 1) is False + + +def test_hex_custom_size_comparison_order_and_errors(): + class Size: + def __init__(self, result=False, raises=False): + self.result, self.raises, self.calls = result, raises, [] + def __eq__(self, value): + self.calls.append(value) + if self.raises: + raise ValueError('size callback') + return self.result + for text in ('a', 'z', ''): + for result in (True, False): + left, right = Size(result), Size(result) + assert f._hex(text, left) == old_hex(text, right) + assert left.calls == right.calls == [len(text)] + for operation in (f._hex, old_hex): + size = Size(raises=True) + with pytest.raises(ValueError, match='size callback'): + operation('a', size) + assert size.calls == [1] diff --git a/openpilot/starpilot/flm/tests/test_live_bootstrap_readiness.py b/openpilot/starpilot/flm/tests/test_live_bootstrap_readiness.py new file mode 100644 index 0000000000..a10313deb2 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_bootstrap_readiness.py @@ -0,0 +1,168 @@ +"""Reporting preserves original selection/caps; synthetic availability only.""" +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from openpilot.starpilot.flm import live_tracking_bootstrap_cohorts as cohorts +from openpilot.starpilot.flm.live_tracking_bootstrap_producer import response_snapshot, _epoch_checkpoints +from openpilot.starpilot.flm.controller_replay import Snapshot +from openpilot.starpilot.flm.tests.test_live_tracking_bootstrap_cohorts import source_data + + +def test_missing_raw_capture_still_declines_and_now_reports_its_actual_counts(): + binding = source_data(seconds=1)[0] + collector = SimpleNamespace(adapter=SimpleNamespace(envelope=None), original_source_capture=None, ready=()) + assert response_snapshot(collector, binding, 1) is None + assert collector._live_bootstrap_readiness == { + 'reason': 'raw_prerequisites_unavailable', 'observedMonoTime': 1, + 'rawPoses': 0, 'readyPairs': 0, 'captureEpoch': None, 'captureStartNs': None, 'captureCutoffNs': None} + + +@pytest.mark.parametrize('bad_until,expected_none', [(12, True), (0, False)]) +def test_diagnostics_do_not_change_original_cohort_decision(bad_until, expected_none): + binding, q, frames, poses, events, controls, checkpoints = source_data(bad_until=bad_until, seconds=41) + arguments = {'binding': binding, 'epoch': 1, 'epoch_start_ns': q, 'cutoff_ns': q+100_000_000_000, + 'now_ns': q+40_000_000_000, 'frames': frames, 'poses': poses, 'emissions': events, + 'controls': controls, 'checkpoints': checkpoints} + original = cohorts.select_first_cohort(**arguments) + diagnostics = {} + reported = cohorts.select_first_cohort(**arguments, diagnostics=diagnostics) + assert reported == original + assert (reported is None) is expected_none + assert diagnostics['requiredCohortNs'] == 38_800_000_000 + assert diagnostics['examinedAnchors'] > 0 + assert diagnostics['reason'] == ('fixed_cohort_frame_health_or_contiguity_unavailable' if expected_none else + 'source_cohort_selected_not_child_qualified') + + +def test_original_checkpoint_count_bound_still_raises_and_reports_literal_reason(): + diagnostics = {} + checkpoints = tuple((index+1, Snapshot('reporting-only', ())) for index in range(129)) + with pytest.raises(ValueError, match='genuine bounded checkpoint metadata'): + cohorts.source_quality_keys((), (), (), checkpoints, (), diagnostics=diagnostics) + assert diagnostics['reason'] == 'checkpoint_count_exceeds_128_bound' + + +def test_original_pose_support_decline_is_reported_without_response_inspection(): + binding, q, frames, poses, events, controls, _ = source_data(bad_until=0, seconds=41) + diagnostics = {} + assert cohorts.quality_for_start(q, frames=frames, poses=poses[::3], emissions=events, + controls=controls, binding=binding, now_ns=q+40_000_000_000, diagnostics=diagnostics) is None + assert diagnostics['reason'] == 'fixed_slot_pose_count_unavailable' + + +def test_throttle_keeps_current_counts_and_labels_previous_selection_historical(): + binding = source_data(seconds=1)[0] + previous = {'reason': 'fixed_cohort_clock_incomplete', 'observedMonoTime': 1, + 'rawPoses': 2, 'readyPairs': 1} + capture = SimpleNamespace(rows=(1, 2, 3), epoch=7, epoch_start_ns=1, epoch_cutoff_ns=100) + collector = SimpleNamespace(adapter=SimpleNamespace(envelope=object()), original_source_capture=capture, + ready=(1, 2), _live_bootstrap_selection_epoch=7, _live_bootstrap_selection_next_ns=100, + _live_bootstrap_last_selection=previous) + with patch('openpilot.starpilot.flm.source_plant_runtime._current_fit_pair_reason', return_value=None): + assert response_snapshot(collector, binding, 50) is None + report = collector._live_bootstrap_readiness + assert report['observedMonoTime'] == 50 + assert report['rawPoses'] == 3 + assert report['readyPairs'] == 2 + assert report['captureEpoch'] == 7 + assert report['reason'] == 'selection_throttled' + assert report['throttled'] is True + assert report['previousSelection'] == previous + assert report['previousSelection'] is not previous + + +def test_unready_clock_avoids_archive_serialization_but_ready_and_bad_metadata_keep_full_checks(): + binding, q, frames, poses, events, controls, checkpoints = source_data(bad_until=0, seconds=41) + + def make(elapsed, checkpoint_override=None): + now = q+int(elapsed*1e9) + capture = SimpleNamespace(rows=tuple(row for row in poses if row.completed_ns <= now), + epoch=1, epoch_start_ns=poses[0].mono_ns, epoch_cutoff_ns=q+100_000_000_000) + replay = SimpleNamespace(checkpoints=checkpoint_override if checkpoint_override is not None else + tuple(row for row in checkpoints if row[0] <= now), records=frames, + emissions=SimpleNamespace(events={row.apply_ns: row for row in events})) + adapter = SimpleNamespace(envelope=object(), replay=replay, + original_context_receipts=SimpleNamespace(controls={row.control_ns: row for row in controls})) + return SimpleNamespace(adapter=adapter, original_source_capture=capture, ready=(1,)), now + + # Unready is measured from original source clocks, never fit error/response. + with patch('openpilot.starpilot.flm.source_plant_runtime._current_fit_pair_reason', return_value=None): + with patch.object(cohorts, 'source_quality_keys', side_effect=AssertionError('full quality reached')) as full: + for elapsed in (1., 20., 38.79): + collector, now = make(elapsed) + assert response_snapshot(collector, binding, now) is None + assert collector._live_bootstrap_readiness['reason'] == 'fixed_cohort_clock_incomplete' + assert full.call_count == 0 + collector, now = make(39.) + with pytest.raises(AssertionError, match='full quality reached'): + response_snapshot(collector, binding, now) + assert full.call_count == 1 + # A bounded-clock preflight must not hide the original bad metadata guard. + collector, now = make(1., ((q, Snapshot('bad-bound', ())),)+tuple((poses[0].mono_ns+i*1_000_000, Snapshot('bad-bound', ())) for i in range(1, 129))) + with pytest.raises(ValueError, match='genuine bounded checkpoint metadata'): + response_snapshot(collector, binding, now) + + +def test_old_history_bound_failure_becomes_exact_original_epoch_selection(): + binding, q, frames, poses, events, controls, epoch = source_data(seconds=61) + cutoff = q+100_000_000_000 + # Old recorder history is structurally genuine metadata but irrelevant to this epoch. + old = tuple((stamp, Snapshot('synthetic-availability-only', ())) for stamp in range(1, 130)) + assert old[-1][0] < q + retained = old+epoch + with pytest.raises(ValueError, match='genuine bounded checkpoint metadata'): + cohorts.source_quality_keys(frames, poses, events, retained, controls) + scoped = _epoch_checkpoints(retained, poses, cutoff) + assert scoped == epoch + assert cohorts.source_quality_keys(frames, poses, events, scoped, controls) == ( + cohorts.source_quality_keys(frames, poses, events, epoch, controls)) + arguments = {'binding': binding, 'epoch': 1, 'epoch_start_ns': q, 'cutoff_ns': cutoff, + 'now_ns': q+60_000_000_000, 'frames': frames, 'poses': poses, 'emissions': events, 'controls': controls} + expected = cohorts.select_first_cohort(checkpoints=epoch, **arguments) + actual = cohorts.select_first_cohort(checkpoints=scoped, **arguments) + assert actual == expected + assert actual is not None + assert actual['start_ns'] == q+12_000_000_000 + assert actual['origin_checkpoint_ns'] == q + assert actual['pose_counts'] == (80,)*8 + # Every candidate anchor is retained, including later original valid alternatives. + assert tuple(stamp for stamp, _ in scoped) == tuple(stamp for stamp, _ in epoch) + changed_response = tuple(replace(row, yaw_rate=999., roll_rad=-1.) for row in poses) + assert _epoch_checkpoints(retained, changed_response, cutoff) == scoped + + +def test_129_checkpoints_inside_epoch_still_fail_original_cap(): + _, q, frames, poses, events, controls, _ = source_data(seconds=1) + snapshot = Snapshot('synthetic-availability-only', ()) + checkpoints = ((q, snapshot),)+tuple((poses[0].mono_ns+i*1_000_000, snapshot) for i in range(1, 129)) + scoped = _epoch_checkpoints(checkpoints, poses, q+100_000_000_000) + assert scoped == checkpoints + assert len(scoped) == 129 + with pytest.raises(ValueError, match='genuine bounded checkpoint metadata'): + cohorts.source_quality_keys(frames, poses, events, scoped, controls) + + +def test_scope_retains_preceding_anchor_and_discards_only_outside_epoch(): + _, q, _, poses, _, _, checkpoints = source_data(seconds=1) + cutoff = q+100_000_000_000 + retained = checkpoints+((cutoff, checkpoints[0][1]), (cutoff+1, checkpoints[0][1])) + assert _epoch_checkpoints(retained, poses, cutoff) == checkpoints + assert _epoch_checkpoints(((1, checkpoints[0][1]),)+retained, poses, cutoff) == checkpoints + + +@pytest.mark.parametrize('bad', ['order', 'type']) +def test_bad_metadata_is_rejected_even_outside_epoch(bad): + _, q, _, poses, _, _, checkpoints = source_data(seconds=1) + outside = ((2, checkpoints[0][1]), (1, checkpoints[0][1])) if bad == 'order' else ((1, object()),) + with pytest.raises(ValueError): + _epoch_checkpoints(outside+checkpoints, poses, q+100_000_000_000) + + +def test_missing_preceding_anchor_cannot_select_any_cohort(): + binding, q, frames, poses, events, controls, checkpoints = source_data(seconds=41) + scoped = _epoch_checkpoints(checkpoints[1:], poses, q+100_000_000_000) + assert scoped == () + assert cohorts.select_first_cohort(checkpoints=scoped, frames=frames, poses=poses, emissions=events, controls=controls, + binding=binding, epoch=1, epoch_start_ns=q, cutoff_ns=q+100_000_000_000, now_ns=q+40_000_000_000) is None diff --git a/openpilot/starpilot/flm/tests/test_live_daemon_process.py b/openpilot/starpilot/flm/tests/test_live_daemon_process.py new file mode 100644 index 0000000000..5660f43f40 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_daemon_process.py @@ -0,0 +1,49 @@ +"""Actual stalled child exercises nonblocking IPC/cleanup, not calibration proof.""" + +import os +import subprocess +import sys +from openpilot.starpilot.flm import fit_process +from openpilot.starpilot.flm.live_tracking_bootstrap_process import LiveBootstrapProcess +from openpilot.starpilot.flm.live_tracking_bootstrap_worker import PURPOSE + + +def test_actual_backpressured_child_never_waits_in_poll_and_shutdown_reaps(monkeypatch): + real_popen = subprocess.Popen + commands = [] + + def stalled_child(command, **kwargs): + commands.append(command) + return real_popen([sys.executable, '-B', '-c', 'import time; time.sleep(60)'], **kwargs) + + monkeypatch.setenv('STARPILOT_LIVE_BOOTSTRAP_PURPOSE', PURPOSE) + monkeypatch.setattr(fit_process.subprocess, 'Popen', stalled_child) + process = LiveBootstrapProcess() + child = None + wait_calls = [] + try: + assert process.submit({'bounded_unit_payload': 'x' * 200_000}, token=('unit-context',), now_ns=1_000_000_000) + child = process.child + assert child is not None and child.stdin is not None and child.stdout is not None + assert commands[0][-1] == 'openpilot.starpilot.flm.live_tracking_bootstrap_worker' + assert not os.get_blocking(child.stdin.fileno()) and not os.get_blocking(child.stdout.fileno()) + real_wait = child.wait + + def wait_at_shutdown(*args, **kwargs): + wait_calls.append((args, kwargs)) + return real_wait(*args, **kwargs) + + monkeypatch.setattr(child, 'wait', wait_at_shutdown) + backpressured = False + for tick in range(64): + previous = process.position + assert process.poll(token=('unit-context',), now_ns=1_010_000_000 + tick) is None + assert not wait_calls and child.poll() is None + if process.position == previous: + backpressured = True + break + assert backpressured and process.position < len(process.pending) + finally: + assert process.close('unit_parent_shutdown') + assert child is not None and child.poll() is not None and process.child is None + assert len(wait_calls) == 1 diff --git a/openpilot/starpilot/flm/tests/test_live_tracking.py b/openpilot/starpilot/flm/tests/test_live_tracking.py new file mode 100644 index 0000000000..9bf2d43b29 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking.py @@ -0,0 +1,419 @@ +"""Numerical state-machine fixtures, NOT actual pose/model/native admission. + +Real controller probes, route joins, owner application and bootstrap remain +independent qualification. Synthetic certificate contents test the protocol. +""" + +from dataclasses import replace +import hashlib +import pytest + +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding +from openpilot.starpilot.flm.live_tracking import LiveTrackingOptimizer, TrackingEpisode, HookSensitivity +from openpilot.starpilot.flm.live_tracking_response import ResponseCertificate, ResponseWitness, VERSION, _revision, validate_certificate + + +def fixture_certificate(): + binding = LiveTrialBinding( + 'a' * 64, 'b' * 64, 'actual-controller-registry-v2:' + 'c' * 64, 'd' * 64, 'e' * 64, 'reset-1', 'f' * 64, 'a' * 64, '', '0' * 64, 'b' * 64, False + ) + raw = ResponseCertificate( + VERSION, + binding, + '0' * 64, + 170_000_000, + 3.0, + 0.0, + -0.3, + 0.3, + 0.001, + ('train0', 'train1', 'train2', 'train3'), + ('hold0', 'hold1', 'hold2', 'hold3'), + 'c' * 64, + 'd' * 64, + 1, + ) + revision = _revision(raw) + return validate_certificate(replace(raw, model_revision=revision, binding=replace(binding, physical_model_revision=revision))) + + +def optimizer(domain='torque'): + cert = fixture_certificate() + return LiveTrackingOptimizer(cert.binding, tuple((name, 0.0) for name in NAMES), controller_domain=domain, response_certificate=cert) + + +def refresh(owner, now): + raw = ResponseWitness(owner.certificate, owner.binding, now - 180_000_000, now - 180_000_000, now, now - 350_000_000, 8, 0.001, now + 100_000_000) + assert owner.response_current(raw, now_ns=now) + + +def episode(owner, index, name, *, kind='large', profile=None, error=0.2, reversal=0.1, native=4, effect=0.01, start=None): + end = (index + 1) * 1_000_000_000 if start is None else start + 500_000_000 + begin = end - 500_000_000 + direction = -1 if kind == 'large' and name.endswith('_right') else 1 + return TrackingEpisode( + owner.binding, + profile or owner._hash(owner.accepted), + hashlib.sha256(str(index).encode()).hexdigest(), + begin, + end, + kind, + direction, + 20.0, + 21.0, + 0.5 if kind == 'large' else 0.1, + error, + reversal, + 20, + 10_000_000, + (HookSensitivity(name, 0.05, 1.0, native, 20, effect, native_gradient_complete=True),), + (float(direction),) * 8, + native_gradient_complete=True, + ) + + +def ready(owner, name='ff_gain_left', *, native=4, effect=0.01): + kind = 'subtle' if name.startswith('center_') or name == 'highway_center_taper_max' else 'large' + count = 9 if kind == 'subtle' else 3 + discovery_ids = set() + comparison_ids = set() + for index in range(2 * count): + row = episode(owner, index, name, kind=kind, native=native, effect=effect) + refresh(owner, row.end_ns) + owner.observe_episode(row, now_ns=row.end_ns) + (discovery_ids if index < count else comparison_ids).add(row.episode_id) + if index == count - 1: + # Geometry/probes select a candidate, never an immediate live offer. + assert owner.offer_state is None + if owner.offer_state is not None: + assert {row.episode_id for row in owner.before} == comparison_ids + assert discovery_ids.isdisjoint(row.episode_id for row in owner.before) + return owner.offer_state, row.end_ns + + +def acknowledge(owner, offer, *, phase=None, stamp=None): + stamp = stamp or offer.observed_ns + 10_000_000 + refresh(owner, stamp) + return owner.acknowledge_application( + trial_id=offer.trial_id, + target_sequence=offer.target_sequence, + phase=phase or offer.phase, + target_sha256=offer.target_sha256, + applied_full30=offer.target, + control_ns=stamp, + binding=offer.binding, + now_ns=stamp, + status='applied', + confirmed_target_sha256=offer.target_sha256 if (phase or offer.phase) == 'confirmed' else '', + ) + + +@pytest.mark.parametrize('name', NAMES) +def test_all_thirty_torque_hooks_are_available_without_model_specific_tune(name): + owner = optimizer() + state, now = ready(owner, name) + assert state is not None and state['names'] == (name,) + offer = owner.offer(now) + assert offer is not None and offer.phase == 'trial' + assert len(offer.baseline) == len(offer.target) == 30 + changes = [(key, value - dict(offer.baseline)[key]) for key, value in offer.target if value != dict(offer.baseline)[key]] + assert len(changes) == 1 and changes[0][0] == name and 0 < changes[0][1] <= 0.005 + + +@pytest.mark.parametrize( + 'name', ('turn_in_threshold_reduction_left', 'turn_in_threshold_reduction_right', 'unwind_threshold_increase_left', 'unwind_threshold_increase_right') +) +def test_pid_does_not_invent_four_unavailable_friction_hooks(name): + owner = optimizer('torque_pid') + state, _ = ready(owner, name) + assert state is None + + +@pytest.mark.parametrize('native,effect', [(0, 0.01), (1, 0.01), (4, 0.0)]) +def test_invisible_native_or_noise_trial_is_not_offered(native, effect): + owner = optimizer() + state, now = ready(owner, native=native, effect=effect) + assert state is None and owner.offer(now) is None + + +def test_future_independent_improvement_confirms_only_after_actual_target_ack(): + owner = optimizer() + _, now = ready(owner) + offer = owner.offer(now) + assert offer is not None + assert owner.accepted['ff_gain_left'] == 0.0 + # A future point without real application acknowledgement cannot count. + row = episode(owner, 10, 'ff_gain_left', profile=offer.target_sha256, error=0.001, start=now + 300_000_000) + refresh(owner, row.end_ns) + assert not owner.observe_episode(row, now_ns=row.end_ns) + assert not owner.future + assert acknowledge(owner, offer, stamp=now + 20_000_000) + for index in range(3): + row = episode(owner, 20 + index, 'ff_gain_left', profile=offer.target_sha256, error=0.001, start=now + 1_000_000_000 + index * 1_000_000_000) + refresh(owner, row.end_ns) + assert owner.observe_episode(row, now_ns=row.end_ns) + confirmed = owner.offer(row.end_ns) + assert confirmed is not None and confirmed.phase == 'confirmed' + assert owner.accepted['ff_gain_left'] == 0.0 # Future decision is not owner application/persistence. + assert acknowledge(owner, confirmed) + assert owner.accepted['ff_gain_left'] == dict(offer.target)['ff_gain_left'] + # Confirmed lease renewal preserves immutable trial baseline/target identity. + renewed = owner.offer(row.end_ns + 20_000_000) + assert renewed is not None and renewed.baseline == offer.baseline and renewed.target == offer.target + assert renewed.target_sequence == offer.target_sequence and renewed.sequence > offer.sequence + + +def test_worse_oscillation_rolls_back_without_persisting_trial(): + owner = optimizer() + _, now = ready(owner) + offer = owner.offer(now) + assert acknowledge(owner, offer) + row = episode(owner, 30, 'ff_gain_left', profile=offer.target_sha256, reversal=2.0, start=now + 1_000_000_000) + refresh(owner, row.end_ns) + assert not owner.observe_episode(row, now_ns=row.end_ns) + rollback = owner.offer(row.end_ns) + assert rollback is not None and rollback.phase == 'rollback' and rollback.target == rollback.baseline + assert owner.accepted['ff_gain_left'] == 0.0 + + +def test_original_deadline_and_quota_cannot_be_renewed(): + owner = optimizer() + _, now = ready(owner) + first = owner.offer(now) + refresh(owner, now + 50_000_000) + second = owner.offer(now + 50_000_000) + assert second.deadline_ns == first.deadline_ns == first.started_ns + 120_000_000_000 + assert second.target_sequence == first.target_sequence and second.sequence > first.sequence + assert owner.offer_state['required_episodes'] == 3 + refresh(owner, first.deadline_ns + 1) + expired = owner.offer(first.deadline_ns + 1) + assert expired is not None and expired.phase == 'rollback' + + +def test_context_or_pose_freshness_withdraws_trial_instead_of_renewing_old_pose(): + owner = optimizer() + _, now = ready(owner) + assert owner.offer(now) is not None + assert owner.offer(now + 100_000_001) is None + assert owner.offer_state['phase'] == 'rollback' + assert owner.accepted['ff_gain_left'] == 0.0 + + +def test_duplicate_and_custom_episode_identity_cannot_supply_independence(): + owner = optimizer() + row = episode(owner, 0, 'ff_gain_left') + refresh(owner, row.end_ns) + owner.observe_episode(row, now_ns=row.end_ns) + assert not owner.observe_episode(row, now_ns=row.end_ns) + assert len(owner.baseline) == 1 + with pytest.raises(ValueError): + replace(row, episode_id=object()).validate() + + +@pytest.mark.parametrize('future_error,expected_worse', ((0.21, True), (0.2, False), (0.19, False))) +def test_independent_closed_loop_loss_changes_next_direction_without_reusing_scores(future_error, expected_worse): + owner = optimizer() + _, now = ready(owner) + offer = owner.offer(now) + assert offer is not None and acknowledge(owner, offer, stamp=now + 20_000_000) + original_before = {row.episode_id for row in owner.before} + future_ids = set() + for index in range(3): + row = episode(owner, 200 + index, 'ff_gain_left', profile=offer.target_sha256, error=future_error, start=now + 1_000_000_000 + index * 1_000_000_000) + future_ids.add(row.episode_id) + refresh(owner, row.end_ns) + owner.observe_episode(row, now_ns=row.end_ns) + assert bool(owner.closed_loop_worse) is expected_worse + terminal = owner.offer(row.end_ns) + assert terminal is not None + if future_error < 0.2: + assert terminal.phase == 'confirmed' + return + assert terminal.phase == 'rollback' and owner.accepted['ff_gain_left'] == 0.0 + assert acknowledge(owner, terminal, stamp=row.end_ns + 10_000_000) + for index in range(6): + row = episode(owner, 300 + index, 'ff_gain_left', start=now + 40_000_000_000 + index * 1_000_000_000) + refresh(owner, row.end_ns) + owner.observe_episode(row, now_ns=row.end_ns) + next_offer = owner.offer(row.end_ns) + assert next_offer is not None and next_offer.phase == 'trial' + delta = dict(next_offer.target)['ff_gain_left'] + assert (-0.005 <= delta < 0) if expected_worse else (0 < delta <= 0.005) + assert next_offer.target_sequence == offer.target_sequence + 1 + assert owner.offer_state['alpha'] == 0.05 / (2 * 3) + assert {row.episode_id for row in owner.before}.isdisjoint(original_before | future_ids) + + +@pytest.mark.parametrize('outcome', ('improve', 'null', 'worse', 'deadline')) +def test_small_error_followup_looks_are_predeclared_and_split_original_budget(outcome): + owner = optimizer() + state, now = ready(owner, 'center_taper_max') + assert state is not None and state['look_quotas'] == (9, 18) + deadline, alpha = state['deadline_ns'], state['alpha'] + assert alpha == 0.05 / (1 * 2) + assert sum(alpha / len(state['look_quotas']) for _ in state['look_quotas']) == alpha + offer = owner.offer(now) + assert offer is not None and acknowledge(owner, offer, stamp=now + 20_000_000) + first_error = 0.202 if outcome == 'worse' else 0.2 if outcome == 'null' else 0.1995 + for index in range(9): + row = episode( + owner, 500 + index, 'center_taper_max', kind='subtle', profile=offer.target_sha256, error=first_error, start=now + 1_000_000_000 + index * 1_000_000_000 + ) + refresh(owner, row.end_ns) + owner.observe_episode(row, now_ns=row.end_ns) + if outcome == 'worse': + assert owner.offer_state['phase'] == 'rollback' + return + assert owner.offer_state['phase'] == 'trial' and len(owner.future) == 9 + assert owner.offer_state['required_episodes'] == 18 + assert owner.offer_state['look_quotas'] == (9, 18) + assert owner.offer_state['deadline_ns'] == deadline and owner.offer_state['alpha'] == alpha + assert owner.accepted['center_taper_max'] == 0.0 + if outcome == 'deadline': + refresh(owner, deadline + 1) + terminal = owner.offer(deadline + 1) + assert terminal is not None and terminal.phase == 'rollback' and terminal.deadline_ns == deadline + return + for index in range(9, 18): + row = episode( + owner, + 500 + index, + 'center_taper_max', + kind='subtle', + profile=offer.target_sha256, + error=0.198 if outcome == 'improve' else 0.2, + start=now + 1_000_000_000 + index * 1_000_000_000, + ) + refresh(owner, row.end_ns) + owner.observe_episode(row, now_ns=row.end_ns) + assert owner.offer_state['phase'] == ('confirmed' if outcome == 'improve' else 'rollback') + assert owner.offer_state['deadline_ns'] == deadline and owner.offer_state['alpha'] == alpha + assert owner.accepted['center_taper_max'] == 0.0 # Confirmation still needs actual owner ACK. + + +@pytest.mark.parametrize('future_rates,expected', [((1.4, 0.46, 0.46), 'confirmed'), ((1.4, 1.4, 1.4), 'rollback')]) +def test_oscillation_uses_baseline_envelope_then_independent_batch_limit(future_rates, expected): + owner = optimizer() + _, now = ready(owner) + # Same average/range relationship demonstrated by original native-cycle + # episodes: a single future turn is inside existing baseline variation. + owner.before = tuple(replace(row, reversal_hz=rate) for row, rate in zip(owner.before, (0.46, 0.46, 2.32), strict=True)) + offer = owner.offer(now) + assert acknowledge(owner, offer) + for index, rate in enumerate(future_rates): + row = episode(owner, 40 + index, 'ff_gain_left', profile=offer.target_sha256, error=0.001, reversal=rate, start=now + 1_000_000_000 + index * 1_000_000_000) + refresh(owner, row.end_ns) + owner.observe_episode(row, now_ns=row.end_ns) + if index == 0: + assert owner.offer_state['phase'] == 'trial' + assert owner.offer_state['phase'] == expected + if expected == 'rollback': + assert owner.reason == 'trial_oscillation_batch_worse' + assert owner.accepted['ff_gain_left'] == 0.0 + + +@pytest.mark.parametrize('discovery,expected', [(None, False), ({'recovery': False}, False), ({'recovery': True}, True)]) +def test_recovery_comparison_keeps_its_required_fresh_native_probes(discovery, expected): + from openpilot.starpilot.flm.live_tracking import discovery_needs_probes + + assert discovery_needs_probes(discovery) is expected + + +@pytest.mark.parametrize('recovery', (False, True)) +def test_actual_probe_coordinator_only_suspends_work_for_measurement_only_discovery(recovery): + from types import SimpleNamespace + from openpilot.starpilot.flm.live_tracking_actor import LiveTrackingActor + + calls = [] + probe = SimpleNamespace(child=None, poll_probe=lambda **kwargs: None, cancel=lambda reason: calls.append(('cancel_probe', reason))) + actor = LiveTrackingActor(probe_process=probe) + owner = optimizer() + owner.discovery = {'recovery': recovery} + actor.ingress = SimpleNamespace(optimizer=owner) + actor.certificate = owner.certificate + actor.probe_preparation = SimpleNamespace( + pid=None, poll=lambda **kwargs: calls.append(('poll_preparation', kwargs)) or None, cancel=lambda reason: calls.append(('cancel_preparation', reason)) + ) + actor._probe_tick(SimpleNamespace(live_response_pair=None), None, now_ns=1) + assert any(name == 'poll_preparation' for name, _ in calls) is recovery + assert any(name == 'cancel_probe' for name, _ in calls) is not recovery + + +@pytest.mark.parametrize('recovery', (False, True)) +def test_actual_ingress_retains_required_native_endpoint_work_during_recovery(recovery): + from types import SimpleNamespace + from openpilot.starpilot.flm.live_tracking_ingress import LiveTrackingIngress + + # Scheduling seam only: no source/native/controller admission is asserted. + obj = object.__new__(LiveTrackingIngress) + obj.last_pose_ns = 0 + obj.binding = fixture_certificate().binding + obj.replay = None + obj.scoring_enabled = True + obj.optimizer = SimpleNamespace(discovery={'recovery': recovery}, offer_state=None, response_current=lambda *a, **kw: True, offer=lambda *a, **kw: None) + obj.monitor = SimpleNamespace(observe=lambda *a, **kw: True, witness=lambda *a: object()) + released = [] + obj.episodes = SimpleNamespace( + defer_unprobed=False, release_measured_only=lambda: released.append(True) or True, observe_pose=lambda *a, **kw: None, drain=lambda **kw: () + ) + obj.observe_pose(SimpleNamespace(mono_ns=1), None, None, now_ns=1, current_native=False) + assert obj.episodes.defer_unprobed is recovery + assert bool(released) is not recovery + + +def test_fast_confidence_preserves_original_student_bounds(): + from openpilot.starpilot.flm.live_tracking import _critical, _student_survival + + def original(df, alpha): + low, high = 0.0, 1.0 + while _student_survival(high, df) > alpha and high < 1e8: + high *= 2.0 + if _student_survival(high, df) > alpha: + raise ValueError('Student confidence outside numerical range') + for _ in range(64): + middle = (low + high) / 2.0 + if _student_survival(middle, df) > alpha: + low = middle + else: + high = middle + return high + + for sequence in (0, 1, 2, 5, 20, 100): + for looks in (1, 2): + alpha = 0.05 / ((sequence + 1) * (sequence + 2)) / looks + for df in range(1, 48): + expected, actual = original(df, alpha), _critical(df, alpha) + assert actual >= expected + assert actual - expected <= 5e-11 * max(1.0, expected) + assert _student_survival(actual, df) <= alpha + + +def test_shared_planning_grid_preserves_original_allocation_loops_and_is_bounded(): + import math + from openpilot.starpilot.flm.live_tracking import _comparison_thresholds, _critical, _welch_df, _look_quotas + + _comparison_thresholds.cache_clear() + for variance, inflation, feasible, minimum, alpha in ((0.001, 1.0, 19, 9, 0.025), (0.02, 1.4, 15, 3, 0.008333333333333333)): + grid = _comparison_thresholds(variance, inflation, feasible, minimum, alpha) + for expected in (0.0002, 0.002, 0.01, 0.03, 0.1, 0.3, 1.0): + old, new = [], [] + for before in range(minimum, 25): + left = variance * inflation / before + left_df = max(1, math.floor(before / inflation) - 1) + for future in range(minimum, feasible + 1): + right = variance * inflation / future + right_df = max(1, math.floor(future / inflation) - 1) + critical = _critical(_welch_df(left, left_df, right, right_df), alpha / len(_look_quotas(future, feasible))) + if expected > (critical + 0.8416212335729143) * math.sqrt(left + right): + old.append((future, before + future, before)) + break + for before, futures in grid: + for future, threshold in futures: + if expected > threshold: + new.append((future, before + future, before)) + break + assert old == new + assert _comparison_thresholds(variance, inflation, feasible, minimum, alpha) is grid + info = _comparison_thresholds.cache_info() + assert info.maxsize == 32 and info.currsize == 2 and info.hits == 2 diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_actor.py b/openpilot/starpilot/flm/tests/test_live_tracking_actor.py new file mode 100644 index 0000000000..77ca27b22d --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_actor.py @@ -0,0 +1,111 @@ +"""Optional-child lifecycle only; no controller/native admission is mocked here.""" +import pytest + +from openpilot.starpilot.flm.live_tracking_actor import LiveTrackingActor + + +class Process: + def __init__(self, closed=True, error=None): + self.closed, self.error, self.calls = closed, error, [] + self.child = None + + def close(self, reason): + self.calls.append(reason) + if self.error is not None: + raise self.error + return self.closed + + +@pytest.mark.parametrize('first_closed', (False, True)) +def test_shutdown_reaps_both_optional_workers(first_closed): + bootstrap, probe = Process(first_closed), Process() + actor = LiveTrackingActor(bootstrap, probe_process=probe) + assert actor.close('shutdown') is first_closed + assert bootstrap.calls == probe.calls == ['shutdown'] + + +def test_unexpected_shutdown_failure_still_attempts_other_reaper(): + failure = RuntimeError('bootstrap lifecycle failed') + bootstrap, probe = Process(error=failure), Process() + actor = LiveTrackingActor(bootstrap, probe_process=probe) + with pytest.raises(RuntimeError, match='bootstrap lifecycle failed'): + actor.close('shutdown') + assert bootstrap.calls == probe.calls == ['shutdown'] + + +class PollProcess(Process): + def __init__(self, *, reap_after=0): + super().__init__() + self.child = object() if reap_after else None + self.reap_after = reap_after + self.polls, self.submissions = [], [] + + def poll(self, *, token, now_ns): + self.polls.append((token, now_ns)) + if self.reap_after: + self.reap_after -= 1 + if not self.reap_after: + self.child = None + return None + + def submit(self, snapshot, *, token, now_ns): + self.submissions.append((snapshot, token, now_ns)) + return True + + +def test_changed_source_waits_for_actual_old_probe_reap_before_bootstrap(monkeypatch): + from dataclasses import replace + from types import SimpleNamespace + from openpilot.starpilot.flm.tests.test_live_tracking_rollout import _optimizer + from openpilot.starpilot.flm import live_tracking_bootstrap_producer + old = _optimizer().binding + new = replace(old, source_epoch='9'*64) + bootstrap, probe = PollProcess(), PollProcess(reap_after=3) + actor = LiveTrackingActor(bootstrap, probe_process=probe) + actor.seed = old + monkeypatch.setattr(actor, 'binding_for', lambda collector, source: new) + collector = SimpleNamespace(ready=[None], original_source_capture=SimpleNamespace(epoch=1), live_tracking=None) + monkeypatch.setattr(live_tracking_bootstrap_producer, 'response_snapshot', lambda *args: {'lifecycle_test_only': True}) + assert actor.tick(collector, object(), now_ns=10, current_native=True) is None + assert probe.child is not None and not bootstrap.submissions + assert actor.tick(collector, object(), now_ns=20, current_native=True) is None + assert probe.child is None + assert len(bootstrap.submissions) == 1 + assert all(token is None for token, _ in probe.polls) + + +def test_invalid_binding_continues_nonblocking_reaping_without_new_job(monkeypatch): + from types import SimpleNamespace + bootstrap, probe = PollProcess(), PollProcess(reap_after=4) + actor = LiveTrackingActor(bootstrap, probe_process=probe) + def invalid(collector, source): + raise ValueError('current binding unavailable') + monkeypatch.setattr(actor, 'binding_for', invalid) + collector = SimpleNamespace(ready=[None]) + for now in (10, 20, 30): + assert actor.tick(collector, object(), now_ns=now, current_native=True) is None + assert probe.child is None and not bootstrap.submissions + assert all(token is None for token, _ in bootstrap.polls+probe.polls) + assert not bootstrap.calls and not probe.calls # No blocking close in tick. + + + +def test_probe_transport_splits_only_work_not_genuine_endpoint_support(monkeypatch): + from openpilot.starpilot.flm import live_tracking_probe_process + process = PollProcess() + process.submit_probe = process.submit + actor = LiveTrackingActor(probe_process=process) + jobs = [({'endpoint': index}, index) for index in range(4)] + actor.probe_queue = list(jobs) + actor.probe_queue_key = ((None, 'a'*64, 'b'*64), 1, 'ff.left', ('ff_gain_left',)) + def bounded_batch(rows, *, queued_ns): + if len(rows) > 2: + raise ValueError('numerical batch byte bound') + return {'rows': rows, 'queued_ns': queued_ns} + monkeypatch.setattr(live_tracking_probe_process, 'make_probe_batch', bounded_batch) + assert actor._submit_probe_queue('token', 100) + assert actor.probe_queue == jobs[2:] + assert len(process.submissions[0][0]['rows']) == 2 + assert actor._submit_probe_queue('token', 200) + assert actor.probe_queue == [] and actor.probe_queue_key is None + assert [row['endpoint'] for submission, _, _ in process.submissions for row in submission['rows']] == list(range(4)) diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_bootstrap.py b/openpilot/starpilot/flm/tests/test_live_tracking_bootstrap.py new file mode 100644 index 0000000000..efefc9ef1e --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_bootstrap.py @@ -0,0 +1,96 @@ +"""Bootstrap parent selection/isolated purpose; no fabricated fit/admission.""" +from types import SimpleNamespace + +import pytest + +from openpilot.starpilot.flm.source_plant_fit import bootstrap_slot_rows + + +def row(stamp, lower=None, upper=None): + return {'mono_ns': stamp, 'source_lower_ns': stamp if lower is None else lower, + 'source_upper_ns': stamp if upper is None else upper} + + +def test_clock_interval_selection_is_predeclared_not_response_selected(): + rows = [row(100, 99, 101), row(101, 100, 102), row(150), row(199, 198, 200), row(200, 199, 201)] + rows[1]['measured_error'] = 999. + rows[2]['measured_error'] = -999. + assert bootstrap_slot_rows(rows, 100, 200) == tuple(rows[1:4]) + rows[1]['measured_error'], rows[2]['measured_error'] = -1., 1. + assert bootstrap_slot_rows(rows, 100, 200) == tuple(rows[1:4]) + + +@pytest.mark.parametrize('low,high', [(None, 101), (True, 101), (102, 101), (100, float('nan'))]) +def test_unknown_or_invalid_clock_interval_rejects_not_silently_dropped(low, high): + value = row(101) + value['source_lower_ns'], value['source_upper_ns'] = low, high + with pytest.raises(ValueError, match='known original'): + bootstrap_slot_rows([value], 100, 200) + + +def test_minimum_must_be_counted_after_clock_only_filter(): + rows = [row(100, 99, 100), *(row(stamp) for stamp in range(101, 140))] + assert len(rows) == 40 and len(bootstrap_slot_rows(rows, 100, 200)) == 39 + + +@pytest.mark.parametrize('age,export', [(38_819_999_999, False), (38_820_000_000, True)]) +def test_actual_parent_export_available_after_eight_slots_without_later_fit_geometry(age, export): + from collections import OrderedDict + from dataclasses import replace + from openpilot.starpilot.flm.tests.test_live_tracking_bootstrap_cohorts import source_data + from openpilot.starpilot.flm.tests.test_live_tracking_requalification import synthetic_pair + from openpilot.starpilot.flm.live_tracking_bootstrap_producer import response_snapshot + binding,q,frames,poses,events,controls,checkpoints=source_data(bad_until=0,seconds=40) + now=q+age + frames=tuple(row for row in frames if row.control_ns<=now) + poses=tuple(row for row in poses if row.completed_ns<=now) + events=tuple(row for row in events if row.apply_ns<=now) + controls=tuple(row for row in controls if row.control_ns<=now) + checkpoints=tuple(row for row in checkpoints if row[0]<=now) + exported=[] + def original_source_packet(*args): + exported.append(args) + # Parent availability only: NEVER fabricate a qualified child packet. + raise ValueError('parent-only export boundary reached') + seed=SimpleNamespace(_binding=binding,original=SimpleNamespace(lag_ns=170_000_000)) + c,o,_=synthetic_pair(seed,0) + c=replace(c,control_ns=now-1_000_000,state_ns=now-2_000_000,source_receipt_ns=now-2_000_000, + native_required=False,native_allowed=False) + o=replace(o,control_ns=c.control_ns,state_ns=c.state_ns,apply_ns=now-500_000) + # These are explicit scheduling-only metadata. Any attempted child/native + # admission fails at the original_source_packet boundary above. + feedback=SimpleNamespace(failed=False,poison_through_ns=0,failures={}, + consumptions={frame.control_ns:object()for frame in frames}) + adapter=SimpleNamespace(envelope=object(),key=('availability-only',),identity=binding.identity, + baseline_revision=binding.baseline_revision,original_feedback_archive=feedback, + original_context_receipts=SimpleNamespace(controls=OrderedDict((row.control_ns,row)for row in controls)), + replay=SimpleNamespace(records=frames,checkpoints=checkpoints, + emissions=SimpleNamespace(events=OrderedDict((row.apply_ns,row)for row in events))), + original_source_packet=original_source_packet) + capture=SimpleNamespace(epoch=1,epoch_start_ns=q,epoch_cutoff_ns=q+100_000_000_000,rows=poses) + collector=SimpleNamespace(adapter=adapter,original_source_capture=capture,ready=[(c,o)],last_fit=0,snapshot_attempts=0) + # The actual fixture's20Hz physical clock is170ms after its control clock; + # the first real pose crossing38.8s is38.82s, not an invented38.8s pose. + if export: + with pytest.raises(ValueError,match='parent-only export boundary'): + response_snapshot(collector,binding,now) + else: + assert response_snapshot(collector,binding,now) is None + assert bool(exported) is export + assert collector.last_fit==collector.snapshot_attempts==0 + + +def test_legacy_process_policy_is_unchanged_and_bootstrap_is_explicit_diagnostic(monkeypatch): + from openpilot.starpilot.flm.fit_process import FitProcess + from openpilot.starpilot.flm.fit_worker import MAX_CPU + from openpilot.starpilot.flm.live_tracking_bootstrap_process import LiveBootstrapProcess + from openpilot.starpilot.flm.live_tracking_bootstrap_worker import MAX_CPU as BOOTSTRAP_CPU, PURPOSE + assert MAX_CPU == 20 and FitProcess.wall_ns == 30_000_000_000 + assert FitProcess.interval_ns == 60_000_000_000 + assert FitProcess.worker_module == 'openpilot.starpilot.flm.fit_worker' + assert BOOTSTRAP_CPU == 180 and LiveBootstrapProcess.wall_ns == 300_000_000_000 + assert PURPOSE == 'live-response-bootstrap-diagnostic-v1' + monkeypatch.delenv('STARPILOT_LIVE_BOOTSTRAP_PURPOSE', raising=False) + process = LiveBootstrapProcess() + assert not process.submit({}, token=None, now_ns=1) + assert process.child is None and process.reason == 'production_bootstrap_budget_not_yet_qualified' diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_bootstrap_cohorts.py b/openpilot/starpilot/flm/tests/test_live_tracking_bootstrap_cohorts.py new file mode 100644 index 0000000000..1e0fe7213d --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_bootstrap_cohorts.py @@ -0,0 +1,364 @@ +"""Outcome-blind availability tests, not original/native calibration proof.""" + +from dataclasses import replace +from types import SimpleNamespace +import math + +from openpilot.starpilot.flm import live_tracking_bootstrap_cohorts as cohorts +from openpilot.starpilot.flm.controller_replay import ReplayFrame, Snapshot +from openpilot.starpilot.flm.torque_emission_replay import Emission +from openpilot.starpilot.flm.tests.test_live_tracking_requalification import certificate, synthetic_pair + + +def source_data(*, bad_until=12, seconds=61): + cert = certificate() + owner = SimpleNamespace(_binding=cert.binding, original=cert) + first, _, _ = synthetic_pair(owner, 0) + q = first.control_ns + controls, frames, events, poses = [], [], [], [] + for i in range(seconds * 100 + 1): + elapsed = i / 100 + # Command-only periodic excitation, exactly equal extrema in train/holdout. + value = 0.15 * math.sin(2 * math.pi * elapsed / 4.0) + c, output, pose = synthetic_pair(owner, i, command=value) + stamp = q + i * 10_000_000 + good = elapsed >= bad_until + c = replace(c, control_ns=stamp, state_ns=stamp - 1_000_000, source_receipt_ns=stamp - 1_000_000, active=good) + controls.append(c) + frames.append(ReplayFrame(stamp, (), c.requested, (0.0, 0.0), healthy=good, frame_sequence=i)) + events.append( + Emission( + stamp, + stamp - 1_000_000, + stamp, + output.projection, + cert.binding.identity, + cert.binding.source_epoch, + cert.binding.baseline_revision, + True, + good, + False, + False, + False, + True, + False, + 0, + 0, + '', + ) + ) + if i % 5 == 0: + mono = stamp + cert.lag_ns + poses.append( + replace( + pose, + mono_ns=mono, + source_lower_ns=mono, + source_upper_ns=mono, + source_timestamp_ns=mono, + completed_ns=mono, + observed_ns=mono, + speed_mps=20.0 if good else 0.0, + ) + ) + checkpoints = tuple((q + i * 1_000_000_000, Snapshot('synthetic-availability-only', ())) for i in range(seconds + 1)) + return cert.binding, q, tuple(frames), tuple(poses), tuple(events), tuple(controls), checkpoints + + +def choose(data, elapsed, *, cache=None): + b, q, frames, poses, events, controls, checkpoints = data + now = q + int(elapsed * 1e9) + return cohorts.select_first_cohort( + binding=b, + epoch=1, + epoch_start_ns=q, + cutoff_ns=q + 100_000_000_000, + now_ns=now, + cache=cache, + frames=tuple(row for row in frames if row.control_ns <= now), + poses=tuple(row for row in poses if row.completed_ns <= now), + emissions=tuple(row for row in events if row.apply_ns <= now), + controls=tuple(row for row in controls if row.control_ns <= now), + checkpoints=tuple(row for row in checkpoints if row[0] <= now), + ) + + +def test_initial_inactive_standstill_does_not_strand_later_good_same_epoch(): + data = source_data() + assert choose(data, 40) is None + selected = choose(data, 60) + assert selected is not None + assert selected['start_ns'] == data[1] + 12_000_000_000 + assert selected['epoch'] == 1 + assert selected['pose_counts'] == (80,) * 8 + + +def test_response_values_never_choose_or_change_cohort(): + data = source_data() + expected = choose(data, 60) + changed = list(data) + changed[3] = tuple(replace(p, yaw_rate=999.0, roll_rad=-1.0) for p in data[3]) + assert choose(tuple(changed), 60) == expected + + +def test_examined_anchors_are_not_rescanned_after_source_watermark(monkeypatch): + data = source_data() + cache = {'examined': {}, 'selected': None} + calls = [] + original = cohorts.quality_for_start + + def counted(start, **kwargs): + calls.append(start) + return original(start, **kwargs) + + monkeypatch.setattr(cohorts, 'quality_for_start', counted) + assert choose(data, 40, cache=cache) is None + first_count = len(calls) + assert choose(data, 40, cache=cache) is None + assert len(calls) == first_count + selected = choose(data, 60, cache=cache) + assert selected is not None + assert len(calls) == len(set(calls)) + assert choose(data, 61, cache=cache) == selected + + +def test_source_value_mutation_invalidates_cache_but_append_retains_it(): + keys = ((b'a',), (b'b',)) + cache = cohorts.new_schedule_cache(None, b'epoch', keys) + cache['examined'][1] = False + appended = cohorts.new_schedule_cache(cache, b'epoch', ((b'a', b'next'), (b'b',))) + assert appended['examined'] == {1: False} + assert not cohorts.new_schedule_cache(appended, b'epoch', ((b'changed',), (b'b',)))['examined'] + assert not cohorts.new_schedule_cache({'token': b'epoch'}, b'epoch', keys)['examined'] + + +def test_interval_filter_is_source_only_and_minimum_applies_after_filter(): + data = source_data(bad_until=0) + b, q, frames, poses, events, controls, _ = data + selected = choose(data, 40) + assert selected is not None + lower = q + 2_600_000_000 + + def quality(rows): + return cohorts.quality_for_start(q, frames=frames, poses=rows, emissions=events, controls=controls, binding=b, now_ns=q + 61_000_000_000) + + inside = next(p for p in poses if p.mono_ns >= lower) + partial = replace(inside, source_lower_ns=lower - 1) + assert quality(tuple(partial if p is inside else p for p in poses))['pose_counts'][0] == 79 + unknown = replace(inside, source_lower_ns=0) + assert quality(tuple(unknown if p is inside else p for p in poses)) is None + reduced = tuple(p for p in poses if not lower <= p.mono_ns < lower + 2_050_000_000) + assert quality(reduced) is None + + +def test_one_real_submission_per_epoch_even_after_response_failure(monkeypatch): + from openpilot.starpilot.flm.live_tracking_actor import LiveTrackingActor + from openpilot.starpilot.flm import live_tracking_bootstrap_producer + from openpilot.starpilot.flm.tests.test_live_tracking_actor import PollProcess + + b = certificate().binding + process = PollProcess() + actor = LiveTrackingActor(process) + monkeypatch.setattr(actor, 'binding_for', lambda collector, source: b) + collector = SimpleNamespace(ready=[None], original_source_capture=SimpleNamespace(epoch=1), live_tracking=None) + calls = [] + + def snapshot(*args): + calls.append(1) + return {'availability_scheduling_only': True} + + monkeypatch.setattr(live_tracking_bootstrap_producer, 'response_snapshot', snapshot) + + # Snapshot preparation is now asynchronous. Exercise the same epoch latch + # across its separate completion; no source/native admission is mocked. + class SnapshotPreparation: + pid = None + epoch = token = None + reason = 'snapshot_submitted' + pending = None + + def poll(self, *, token, now_ns): + result, self.pending = self.pending, None + self.pid = None + return result + + def submit(self, collector, seed, *, token, now_ns): + self.epoch, self.token, self.pid = collector.original_source_capture.epoch, token, 1 + self.pending = {'snapshot': live_tracking_bootstrap_producer.response_snapshot(collector, seed, now_ns)} + + actor.snapshot_process = SnapshotPreparation() + monkeypatch.setattr(actor, '_same_bootstrap_source', lambda source, now_ns: True) + actor.tick(collector, object(), now_ns=10, current_native=True) + assert len(process.submissions) == 0 + actor.tick(collector, object(), now_ns=20, current_native=True) + assert len(process.submissions) == 1 + # A real child outcome cannot authorize another heldout/cohort search. + monkeypatch.setattr(process, 'poll', lambda **kwargs: {'production_admissible': True, 'status': 'collecting', 'reason': 'heldout_rmse'}) + actor.tick(collector, object(), now_ns=30, current_native=True) + actor.tick(collector, object(), now_ns=40, current_native=True) + assert len(calls) == len(process.submissions) == 1 + collector.original_source_capture.epoch = 2 + monkeypatch.setattr(process, 'poll', lambda **kwargs: None) + actor.tick(collector, object(), now_ns=50, current_native=True) + actor.tick(collector, object(), now_ns=60, current_native=True) + assert len(process.submissions) == 2 + + +def test_known_controller_quality_flags_cannot_consume_early_cohort(): + data = source_data(bad_until=0) + for name in ('saturated', 'model_wobble', 'controller_chatter'): + changed = list(data) + changed[5] = tuple(replace(c, **{name: c.control_ns < data[1] + 12_000_000_000}) for c in data[5]) + assert choose(tuple(changed), 40) is None + assert choose(tuple(changed), 60)['start_ns'] == data[1] + 12_000_000_000 + + +def test_unknown_limit_is_not_reclassified_as_encoder_quantization(): + data = source_data(bad_until=0) + changed = list(data) + changed[5] = tuple(replace(c, limited=True) for c in data[5]) + # No registered raw stage/curvature hint: exact_unclipped_pair fails closed. + assert choose(tuple(changed), 60) is None + + +def test_proven_exact_encoder_quantization_hint_keeps_command_support(): + data = source_data(bad_until=0) + changed = list(data) + changed[5] = tuple( + replace(c, limited=True, stage_schema_revision='torque-selected-stage-snapshot-v1', replay_inputs=c.replay_inputs + (('curvature_limited', 0.0),)) + for c in data[5] + ) + assert choose(tuple(changed), 40) is not None + + +def test_delayed_native_or_context_receipt_does_not_seal_failed_cohort(): + data = source_data(bad_until=0) + q = data[1] + # At39s the fixed38.8s cohort is geometrically complete. Later actual + # native/control receipts may still be queued on their independent topics. + for index in (4, 5): + changed = list(data) + changed[index] = tuple(row for row in data[index] if row.control_ns < q + 38_800_000_000) + cache = {'examined': {}, 'selected': None} + assert choose(tuple(changed), 39, cache=cache) is None + assert q not in cache['examined'] + assert choose(data, 40, cache=cache) is not None + assert cache['examined'][q] is True + + +def test_all_lag_lookback_rejects_gap_even_when_every_pose_has_lag_zero_endpoint(): + data = list(source_data(bad_until=0)) + q = data[1] + # Lag-zero20Hz poses fall either side of this40ms application gap. Their + # individual ages remain <=30ms, but the common all-lag tape is incomplete. + data[4] = tuple(row for row in data[4] if not q + 3_030_000_000 <= row.apply_ns <= q + 3_050_000_000) + b, q, frames, poses, events, controls, _ = data + diagnostics = {} + assert ( + cohorts.quality_for_start(q, frames=frames, poses=poses, emissions=events, controls=controls, binding=b, now_ns=q + 40_000_000_000, diagnostics=diagnostics) + is None + ) + assert diagnostics['reason'] == 'fixed_cohort_common_command_source_unavailable' + selected = choose(tuple(data), 40) + assert selected is not None and selected['start_ns'] == q + 1_000_000_000 + changed = list(data) + changed[3] = tuple(replace(row, yaw_rate=-999.0, roll_rad=1.0) for row in data[3]) + assert choose(tuple(changed), 40) == selected + + +def test_any_good_lag_cannot_hide_a_different_lag_source_interval_crossing(): + data = list(source_data(bad_until=0)) + q = data[1] + point = q + 3_020_000_000 + data[4] = tuple(replace(row, apply_ns=point + 1) if row.apply_ns == point else row for row in data[4]) + data[3] = tuple( + replace(row, source_lower_ns=point, source_upper_ns=point + 2, completed_ns=point + 2, observed_ns=point + 2) if row.mono_ns == point else row + for row in data[3] + ) + b, q, frames, poses, events, controls, _ = data + diagnostics = {} + assert ( + cohorts.quality_for_start(q, frames=frames, poses=poses, emissions=events, controls=controls, binding=b, now_ns=q + 40_000_000_000, diagnostics=diagnostics) + is None + ) + assert diagnostics['reason'] == 'fixed_cohort_common_command_source_unavailable' + selected = choose(tuple(data), 40) + assert selected is not None and selected['start_ns'] == q + 1_000_000_000 + + +def test_distinct_original_applications_may_repeat_one_consumed_control(): + from openpilot.starpilot.flm.live_tracking_bootstrap_producer import _epoch_rows + import inspect + + ordering = {'order_name': 'apply_ns'} if 'order_name' in inspect.signature(_epoch_rows).parameters else {} + data = source_data(bad_until=0) + b, q, frames, poses, events, controls, checkpoints = data + first = events[0] + extra = replace(first, apply_ns=first.apply_ns + 1) + repeated = (first, extra, *events[1:]) + assert _epoch_rows(repeated, Emission, 'control_ns', q, q + 100_000_000_000, **ordering) == repeated + keys = cohorts.source_quality_keys(frames, poses, repeated, checkpoints, controls) + assert len(keys[2]) == len(repeated) + # Scheduling cannot merge, replace, restamp or renew either real application. + assert repeated[0] is first and repeated[1] is extra + assert first.control_ns == extra.control_ns and first.apply_ns != extra.apply_ns + + +def test_repeated_control_support_still_rejects_application_or_control_regression(): + import pytest + from openpilot.starpilot.flm.live_tracking_bootstrap_producer import _epoch_rows + import inspect + + ordering = {'order_name': 'apply_ns'} if 'order_name' in inspect.signature(_epoch_rows).parameters else {} + data = source_data(bad_until=0) + b, q, frames, poses, events, controls, checkpoints = data + first, second = events[:2] + for changed in ((first, replace(second, apply_ns=first.apply_ns)), (first, replace(second, control_ns=first.control_ns - 1))): + with pytest.raises(ValueError): + _epoch_rows(changed, Emission, 'control_ns', q, q + 100_000_000_000, **ordering) + with pytest.raises(ValueError): + cohorts.source_quality_keys(frames, poses, changed, checkpoints, controls) + # Genuine controller ticks themselves still require strict unique clocks. + with pytest.raises(ValueError): + _epoch_rows((frames[0], frames[0]), ReplayFrame, 'control_ns', q, q + 100_000_000_000) + + +def test_source_complete_slots_exclude_unknown_lookbacks_before_any_response_is_read(): + data = source_data(bad_until=0) + b, q, frames, poses, events, controls, _ = data + # One actually unknown consumed command; surrounding genuine applications + # remain in the FULL source inventory. Exclude its entire500ms lookback. + bad = q + 5_000_000_000 + controls = tuple(replace(c, state_ns=c.control_ns - 40_000_000, source_receipt_ns=c.control_ns - 40_000_000) if c.control_ns == bad else c for c in controls) + events = tuple(replace(e, state_ns=e.control_ns - 40_000_000) if e.control_ns == bad else e for e in events) + kwargs = {'frames': frames, 'poses': poses, 'emissions': events, 'controls': controls, 'binding': b, 'now_ns': q + 61_000_000_000} + assert cohorts.quality_for_start(q, **kwargs) is None + selected = cohorts.quality_for_start(q, **kwargs, source_complete=True) + assert selected is not None + assert selected['start_ns'] == q + assert all(40 <= count <= 240 for count in selected['pose_counts']) + retained = tuple(t for group in selected['source_pose_ns'] for t in group) + assert not any(bad <= t <= bad + 500_000_000 for t in retained) + assert len(retained) < sum( + 1 for p in poses if any(q + 2_600_000_000 + k * 4_600_000_000 <= p.mono_ns < q + 6_600_000_000 + k * 4_600_000_000 for k in range(8)) + ) + changed = tuple(replace(p, yaw_rate=999.0, roll_rad=-1.0) for p in poses) + assert cohorts.quality_for_start(q, **dict(kwargs, poses=changed), source_complete=True) == selected + + +def test_source_complete_slots_do_not_top_up_an_underfilled_original_slot(): + data = source_data(bad_until=0) + b, q, frames, poses, events, controls, _ = data + bad_start, bad_end = q + 3_000_000_000, q + 5_600_000_000 + controls = tuple( + replace(c, state_ns=c.control_ns - 40_000_000, source_receipt_ns=c.control_ns - 40_000_000) if bad_start <= c.control_ns <= bad_end else c for c in controls + ) + events = tuple(replace(e, state_ns=e.control_ns - 40_000_000) if bad_start <= e.control_ns <= bad_end else e for e in events) + diagnostics = {} + assert ( + cohorts.quality_for_start( + q, frames=frames, poses=poses, emissions=events, controls=controls, binding=b, now_ns=q + 61_000_000_000, source_complete=True, diagnostics=diagnostics + ) + is None + ) + assert diagnostics['reason'] == 'fixed_slots_source_complete_pose_count_unavailable' diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_comparison.py b/openpilot/starpilot/flm/tests/test_live_tracking_comparison.py new file mode 100644 index 0000000000..8802ea9d67 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_comparison.py @@ -0,0 +1,82 @@ +"""Uncensored score-baseline and physical-drift seams, synthetic data only.""" +from dataclasses import replace +from types import SimpleNamespace + +from openpilot.starpilot.flm.tests.test_live_tracking_rollout import _optimizer, _refresh, _episode, NS +from openpilot.starpilot.flm.live_tracking_response import ResponseMonitor +from openpilot.starpilot.flm.tests.test_live_tracking_requalification import certificate, synthetic_pair + + +def test_error_selected_discovery_is_not_reused_as_comparison_baseline(): + owner = _optimizer() + index = 0 + while owner.discovery is None and index < 24: + index += 1 + now = index*8*NS + _refresh(owner, now) + owner.observe_episode(_episode(owner, index, now, owner.accepted, .6, .35, 22.), now_ns=now) + assert owner.discovery is not None + assert owner.offer_state is None + discovery = dict(owner.discovery) + comparison = [] + for _ in range(discovery['quota']): + index += 1 + now = index*8*NS + _refresh(owner, now) + # Every newly requested comparison episode is retained, even perfect0. + row = _episode(owner, index, now, owner.accepted, .6, 0., 22.) + comparison.append(row) + owner.observe_episode(row, now_ns=now) + if owner.offer_state is not None: + assert owner.before == tuple(comparison) + assert all(row.error_rms == 0. for row in owner.before) + else: + assert owner.reason == 'independent_baseline_trial_power_insufficient' + assert owner.discovery is None + + +def test_overlapping_comparison_does_not_count_twice(): + owner = _optimizer() + index = 0 + while owner.discovery is None and index < 24: + index += 1 + now = index*8*NS + _refresh(owner, now) + owner.observe_episode(_episode(owner, index, now, owner.accepted, .6, .35, 22.), now_ns=now) + assert owner.discovery is not None + index += 1 + now = index*8*NS + _refresh(owner, now) + first = _episode(owner, index, now, owner.accepted, .6, .35, 22.) + owner.observe_episode(first, now_ns=now) + overlap = replace(first, episode_id='1'*64, start_ns=first.start_ns+1, end_ns=first.end_ns+1) + assert not owner.observe_episode(overlap, now_ns=now+1) + assert owner.comparison == [first] + + +def test_constant_command_point12_physical_shift_cannot_refresh_response_witness(): + cert = certificate() + source = SimpleNamespace(_binding=cert.binding, original=cert) + monitor = ResponseMonitor(cert) + for index in range(8): + control, output, pose = synthetic_pair(source, index, command=.1) + assert monitor.observe(control, output, pose, now_ns=pose.mono_ns, binding=cert.binding, current_native=True) + assert monitor.witness(pose.mono_ns) is not None + for index in range(8, 16): + control, output, pose = synthetic_pair(source, index, command=.1) + shifted_yaw = pose.yaw_rate+.12/pose.speed_mps + pose = replace(pose, yaw_rate=shifted_yaw, angular_velocity_device=(0., 0., shifted_yaw)) + monitor.observe(control, output, pose, now_ns=pose.mono_ns, binding=cert.binding, current_native=True) + assert monitor.witness(pose.mono_ns) is None + + +def test_changed_actual_commands_explain_point12_shift_without_proving_trial_benefit(): + cert = certificate() + source = SimpleNamespace(_binding=cert.binding, original=cert) + monitor = ResponseMonitor(cert) + for index in range(16): + control, output, pose = synthetic_pair(source, index, command=.1 if index < 8 else .14) + assert monitor.observe(control, output, pose, now_ns=pose.mono_ns, binding=cert.binding, current_native=True) + assert monitor.witness(pose.mono_ns) is not None + # A response witness licenses the numerical model, never causal attribution + # of a tracking improvement to the offered knob rather than another input. diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_daemon.py b/openpilot/starpilot/flm/tests/test_live_tracking_daemon.py new file mode 100644 index 0000000000..3190b333aa --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_daemon.py @@ -0,0 +1,252 @@ +"""Actual daemon loop and cereal publication; fake actor does not model permission.""" + +from dataclasses import replace +from types import SimpleNamespace +import pytest +from collections.abc import Callable +from typing import NoReturn, cast +from openpilot.starpilot.flm import steerlearnerd +from openpilot.starpilot.flm.runtime_adapter_registry import SourceState +from openpilot.starpilot.flm.tests.test_live_trial_owner import offer + +BASE = 10_000_000_000 + + +def source(now): + return SourceState(True, False, 'a' * 64, 'b' * 64, now, now + 500_000_000, 'c' * 64, (0.0, 0.0, 0.0), 'd' * 32, 'e' * 64) + + +class NoLegacyFitter: + def __init__(self): + self.closed = False + + def poll(self, **kwargs): + cast(Callable[[str], NoReturn], pytest.fail)('live daemon must never poll legacy full FD fitter') + + def submit(self, *args, **kwargs): + cast(Callable[[str], NoReturn], pytest.fail)('live daemon must never submit legacy full FD fitter') + + @property + def child(self): + cast(Callable[[str], NoReturn], pytest.fail)('live daemon must never inspect legacy fitter child') + + def close(self, reason): + self.closed = True + + +class RecordingActor: + reason = 'production_bootstrap_budget_not_yet_qualified' + ingress = None + pending = False + certificate = None + + def __init__(self): + self.calls = [] + self.closed = False + + def tick(self, collector, current_source, **kwargs): + self.calls.append((collector, current_source, kwargs)) + return None + + def persistence_inputs(self, now_ns): + return None + + def close(self, reason): + self.closed = True + + +class LoopComplete(Exception): + pass + + +@pytest.mark.parametrize('shutdown_reaped', (True, False)) +def test_actual_daemon_publishes_separate20_and4_without_legacy_fit(monkeypatch, shutdown_reaped): + now = [BASE - 50_000_000] + + class SM: + updated = {'deviceMotion': False} + logMonoTime = {'deviceMotion': 0} + + def update(self, timeout=None): + now[0] += 50_000_000 + if now[0] > BASE + 1_000_000_000: + raise LoopComplete + self.updated['deviceMotion'] = True + self.logMonoTime['deviceMotion'] = now[0] + + sent = [] + pm = SimpleNamespace(send=lambda topic, message: sent.append((now[0], topic, message.as_reader()))) + actor, fitter = RecordingActor(), NoLegacyFitter() + original_close = actor.close + + def actor_close(reason): + original_close(reason) + return shutdown_reaped + + monkeypatch.setattr(actor, 'close', actor_close) + # No control/native is invented: this exercises actual collecting publications. + monkeypatch.setattr(steerlearnerd, 'drain', lambda *args: None) + with pytest.raises(LoopComplete if shutdown_reaped else RuntimeError): + steerlearnerd.run( + SimpleNamespace(), + SM(), + object(), + object(), + pm, + None, + None, + lambda stamp: source(stamp), + None, + None, + None, + None, + fitter, + None, + clock=lambda: now[0], + live_actor=actor, + ) + tracking = [(stamp, msg) for stamp, topic, msg in sent if topic == 'starpilotSteeringTracking'] + legacy = [(stamp, msg) for stamp, topic, msg in sent if topic == 'autoSteeringLearning'] + assert len(tracking) == 21 and len(legacy) == 5 + assert all(b[0] - a[0] == 50_000_000 for a, b in zip(tracking, tracking[1:], strict=False)) + assert all(b[0] - a[0] == 250_000_000 for a, b in zip(legacy, legacy[1:], strict=False)) + assert all(msg.starpilotSteeringTracking.version == 1 and msg.starpilotSteeringTracking.trial.version == 0 for _, msg in tracking) + assert all(msg.autoSteeringLearning.learning.status == 'collecting' for _, msg in legacy) + assert all(call[0] is None and call[2]['current_native'] is False for call in actor.calls) + assert actor.closed and fitter.closed + + +def test_tracking_serialization_never_renews_existing_proof_or_fabricates_native(): + pending = offer(BASE) + initial = steerlearnerd.tracking_message(source(BASE), now_ns=BASE, current_native=True, offer=pending) + renewed = steerlearnerd.tracking_message(source(BASE + 50_000_000), now_ns=BASE + 50_000_000, current_native=True, offer=pending) + for message in (initial, renewed): + row = message.starpilotSteeringTracking.trial + assert row.observedMonoTime == pending.observed_ns and row.validUntilMonoTime == pending.expires_ns + assert row.deadlineMonoTime == pending.deadline_ns and row.sequence == pending.sequence + expired = steerlearnerd.tracking_message(source(BASE + 110_000_000), now_ns=BASE + 110_000_000, current_native=True, offer=pending) + denied = steerlearnerd.tracking_message(source(BASE), now_ns=BASE, current_native=False, offer=pending) + manual = steerlearnerd.tracking_message(replace(source(BASE), manual=True), now_ns=BASE, current_native=True, offer=pending) + assert all(message.starpilotSteeringTracking.trial.version == 0 for message in (expired, denied, manual)) + + +def _run_poll_script(monkeypatch, rows, *, expire_during_drain=False): + now = [BASE] + + class SM: + def __init__(self): + self.updated = {'deviceMotion': False} + self.logMonoTime = {'deviceMotion': 0} + self.valid = {'deviceMotion': False} + self.index = self.zero_polls = 0 + + def update(self, timeout=None): + if timeout == 0: + self.zero_polls += 1 + self.updated['deviceMotion'] = False + return + if self.index == len(rows): + raise LoopComplete + stamp, header, valid = rows[self.index] + self.index += 1 + now[0] = stamp + self.updated['deviceMotion'] = header is not None + if header is not None: + self.logMonoTime['deviceMotion'] = header + self.valid['deviceMotion'] = valid + + sm = SM() + actor, fitter = RecordingActor(), NoLegacyFitter() + sent = [] + pm = SimpleNamespace(send=lambda topic, message: sent.append((now[0], topic, message.as_reader()))) + expired = False + + def drain(*args): + nonlocal expired + if expire_during_drain and not expired: + # Trigger the REAL source_current expiry path and its update(0). No + # physical/native/controller evidence is injected into this loop test. + now[0] += 600_000_000 + expired = True + + monkeypatch.setattr(steerlearnerd, 'drain', drain) + with pytest.raises(LoopComplete): + steerlearnerd.run(SimpleNamespace(), sm, object(), object(), pm, None, None, + lambda stamp: source(stamp), None, None, None, None, fitter, None, + clock=lambda: now[0], live_actor=actor) + assert actor.closed and fitter.closed + assert all(call[0] is None and call[2]['current_native'] is False for call in actor.calls) + return sent, sm + + +def _tracking_frequency(sent): + from openpilot.cereal.messaging import FrequencyTracker + tracker = FrequencyTracker(20., 100., False) # Actual Controls receiver: selfdriveState100Hz poll. + for stamp, topic, _message in sent: + if topic == 'starpilotSteeringTracking': + tracker.record_recv_time(stamp/1e9) + return tracker + + +def test_actual_run_tracking_survives_alternating_poll_jitter(monkeypatch): + # Existing reset-to-now >=50ms gate drops every49ms interval, yielding10Hz. + # These are21 genuine20Hz source deliveries with realistic poll jitter. + offsets = [0] + for index in range(20): + offsets.append(offsets[-1]+(49_000_000 if index % 2 == 0 else 51_000_000)) + rows = tuple((BASE+offset, BASE+offset, True) for offset in offsets) + sent, _sm = _run_poll_script(monkeypatch, rows) + tracking = [(stamp, message) for stamp, topic, message in sent if topic == 'starpilotSteeringTracking'] + assert [stamp for stamp, _ in tracking] == [stamp for stamp, _header, _valid in rows] + assert _tracking_frequency(sent).valid + assert all(message.valid and message.starpilotSteeringTracking.version == 1 and + message.starpilotSteeringTracking.trial.version == 0 for _, message in tracking) + + +def test_actual_run_tracking_does_not_pad_duplicate_regressed_future_or_missing_input(monkeypatch): + rows = ( + (BASE, BASE, True), + (BASE+49_000_000, BASE+49_000_000, True), + (BASE+100_000_000, BASE+100_000_000, True), + (BASE+149_000_000, BASE+100_000_000, True), # Actual duplicate header. + (BASE+200_000_000, None, False), # Poll timeout, no new source delivery. + (BASE+250_000_000, BASE+90_000_000, True), # Regressed source header. + (BASE+300_000_000, BASE+350_000_000, True), # Future source header. + (BASE+350_000_000, 0, False), + (BASE+400_000_000, BASE+400_000_000, False), # Genuine invalid raw packet: neutral status remains observable. + (BASE+449_000_000, BASE+449_000_000, True), + ) + sent, _sm = _run_poll_script(monkeypatch, rows) + tracking = [(stamp, message) for stamp, topic, message in sent if topic == 'starpilotSteeringTracking'] + assert [stamp for stamp, _ in tracking] == [BASE, BASE+49_000_000, BASE+100_000_000, BASE+400_000_000, BASE+449_000_000] + assert not _tracking_frequency(sent).valid # Lost data must remain visible to ordinary receiver checks. + assert all(message.starpilotSteeringTracking.trial.version == 0 for _, message in tracking) + + +def test_actual_run_captures_polled_header_before_source_current_zero_poll(monkeypatch): + sent, sm = _run_poll_script(monkeypatch, ((BASE, BASE, True),), expire_during_drain=True) + assert sm.zero_polls >= 1 and sm.updated['deviceMotion'] is False + tracking = [(stamp, message) for stamp, topic, message in sent if topic == 'starpilotSteeringTracking'] + assert len(tracking) == 1 and tracking[0][0] == BASE+600_000_000 + assert tracking[0][1].starpilotSteeringTracking.trial.version == 0 + + +def test_actual_nonlive_run_does_not_read_tracking_poll_metadata(monkeypatch): + now = [BASE-50_000_000] + + class SM: + def update(self, timeout=None): + now[0] += 50_000_000 + if now[0] > BASE+100_000_000: + raise LoopComplete + + # This pre-existing run API does not have a live actor or a tracking service. + # Minimal SM intentionally lacks tracking metadata; the legacy path is intact. + sent = [] + pm = SimpleNamespace(send=lambda topic, message: sent.append(topic)) + monkeypatch.setattr(steerlearnerd, 'drain', lambda *args: None) + fitter = SimpleNamespace(poll=lambda **kwargs: None, close=lambda reason: True, child=None) + with pytest.raises(LoopComplete): + steerlearnerd.run(SimpleNamespace(), SM(), object(), object(), pm, None, None, + lambda stamp: source(stamp), None, None, None, None, fitter, None, clock=lambda: now[0]) + assert sent == ['autoSteeringLearning'] diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_emission_cadence.py b/openpilot/starpilot/flm/tests/test_live_tracking_emission_cadence.py new file mode 100644 index 0000000000..acee762965 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_emission_cadence.py @@ -0,0 +1,99 @@ +"""Real GM/Toyota/Hyundai encoder cadence with separate fullrate geometry. + +This invokes actual CarInterface/CarController/CAN encoding and decodes every +observed native packet. Source pose/model/current-native are EXPLICIT numerical +unit seams; this does not claim Panda permission or an actual road experiment. +""" +from dataclasses import replace +import copy +from types import SimpleNamespace + +import pytest + +from opendbc.car import Bus, structs +from opendbc.car.car_helpers import interfaces +from opendbc.car.gm.values import CAR as GM +from opendbc.car.toyota.values import CAR as TOYOTA +from opendbc.car.hyundai.values import CAR as HYUNDAI +from opendbc.car.steering_projection import TorqueProjectionObserver +from openpilot.starpilot.flm.torque_emission_replay import Emission, EmissionDecoder +from openpilot.starpilot.flm.tests.test_live_tracking_episodes import ( + BASE, DT, owner, pair, numeric_pose, numerical_join_validation_only, +) + + +@pytest.mark.parametrize('fingerprint,step', [ + (GM.CHEVROLET_BOLT_EUV, 3), (TOYOTA.TOYOTA_RAV4_TSS2, 1), (HYUNDAI.HYUNDAI_IONIQ_6, 1), +]) +def test_actual_native_cadence_never_thins_100hz_geometry_or_invents_20hz_poses(monkeypatch, fingerprint, step): + interface_type = interfaces[fingerprint] + cp = interface_type.get_non_essential_params(fingerprint) + interface = interface_type(cp) + if fingerprint == GM.CHEVROLET_BOLT_EUV: + # Same parser-owned forwarding metadata assignment as GM CarState.update. + # This initializes the unrelated camera status frame; it asserts no CAN + # health, native permission or received physical vehicle traffic. + interface.CS.pscm_status = copy.copy(interface.can_parsers[Bus.pt].vl['PSCMStatus']) + # Retain current GM EPS tune: STEER_STEP=3, approximately33Hz. The + # controller's old50Hz comment is not an observed transmission contract. + assert interface.CC.params.STEER_STEP == 3 + state = structs.CarState.new_message(canValid=True, canTimeout=False, vEgo=20., vEgoRaw=20., + steeringAngleDeg=0., steeringRateDeg=0., steeringTorque=0., standstill=False) + state.cruiseState.enabled = state.cruiseState.available = True + state.cruiseState.speed = 20. + interface.CS.out = state.as_reader() + observer = TorqueProjectionObserver(cp) + interface.CC.learning_projection_observer = observer + command = structs.CarControl.new_message(enabled=True, latActive=True, longActive=False) + command.actuators.torque = 0. + decoder = EmissionDecoder(cp) + builder = owner() + numerical_join_validation_only(monkeypatch, builder) + actual_emissions = [] + emitted_frames = [] + pose_count = 0 + for index in range(150): + stamp = BASE+index*DT + geometry = .1 if index < 25 or 75 <= index < 100 else -.1 + control, _, profile = pair(builder, index, geometry) + assert builder.observe_source_control(control, profile_sha256=profile, now_ns=stamp, current_native=True) + actuators, packets = interface.apply(command.as_reader(), stamp) + projection = observer.current + if projection is not None: + assert projection.steer_requested + assert any(packet[0] == projection.can_address and packet[1] == projection.can_data and packet[2] == projection.can_bus + for packet in packets) + from openpilot.starpilot.flm.fullrate_collector import AppliedEvidence + applied = AppliedEvidence(stamp, stamp, stamp, 'normalized_torque', actuators.torque, + True, True, False, False, projection.unlimited_raw != projection.final_raw, True, projection) + emission = Emission(stamp, stamp, stamp, projection, control.identity, control.source_epoch, + control.baseline_revision, True, True, False, False, applied.limited, True, False, 0, 0, '') + decoder.check(emission) # Real packet/parser equality, not a patched native verdict. + assert builder.observe_control(control, applied, profile_sha256=profile, now_ns=stamp, current_native=True) + actual_emissions.append((control, applied)) + emitted_frames.append(projection.frame) + if index % 5 == 0: + target = stamp-builder.certificate.lag_ns + eligible = [(c, o) for c, o in actual_emissions if o.apply_ns <= target] + if eligible: + c, o = eligible[-1] + pose = numeric_pose(builder, c, yaw=0.) + pose = replace(pose, mono_ns=stamp, received_ns=stamp, observed_ns=stamp, completed_ns=stamp, + source_timestamp_ns=stamp, source_lower_ns=stamp, source_upper_ns=stamp) + witness = SimpleNamespace(certificate=builder.certificate, last_control_ns=c.control_ns, + last_apply_ns=o.apply_ns, last_pose_ns=stamp) + assert builder.observe_pose(pose, c, o, witness=witness, now_ns=stamp, current_native=True) + pose_count += 1 + assert len(builder.ticks) == 150 + expected_frames = tuple(range(step, 150, step)) if step > 1 else tuple(range(150)) + assert tuple(emitted_frames) == expected_frames + assert len(actual_emissions) == len(expected_frames) + assert sum(tick.output is not None for tick in builder.ticks.values()) == len(actual_emissions) + assert pose_count == 26 and len(builder.endpoints) == pose_count + completed = builder.drain() + assert len(completed) == 1 and completed[0].kind == 'subtle' + assert completed[0].start_ns == BASE+25*DT and completed[0].end_ns == BASE+100*DT-1 + assert completed[0].control_max_gap_ns == DT + assert completed[0].endpoints < len(actual_emissions) + if step > 1: + assert len(actual_emissions) < len(builder.ticks) diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_episodes.py b/openpilot/starpilot/flm/tests/test_live_tracking_episodes.py new file mode 100644 index 0000000000..2e238b4827 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_episodes.py @@ -0,0 +1,767 @@ +"""Adapter geometry/join units; synthetic witnesses are NOT native authority. + +The public control path uses real typed evidence. The finalization units inject +numerical independent endpoints explicitly; real source pose/native admission +and collector integration require root's separate actual caller qualification. +""" + +from dataclasses import replace +import math +from types import SimpleNamespace + +import pytest +from collections.abc import Callable +from typing import cast + +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, full_profile_sha256 +from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding +from openpilot.starpilot.flm.live_tracking_response import ResponseCertificate, VERSION, _revision, physical_calibration_generation +from openpilot.starpilot.flm.live_tracking_episodes import TrackingEpisodeBuilder, _Endpoint + +BASE = 1_000_000_000 +DT = 10_000_000 + + +def certificate(): + binding = LiveTrialBinding('a' * 64, 'b' * 64, 'controller-current', 'd' * 64, 'e' * 64, '1' * 64, 'f' * 32, 'a' * 64, '', '0' * 64, 'b' * 64, False) + value = ResponseCertificate( + VERSION, binding, '0' * 64, 170_000_000, 3.0, 0.0, -0.3, 0.3, 0.005, ('a0', 'a1', 'a2', 'a3'), ('b0', 'b1', 'b2', 'b3'), 'c' * 64, 'd' * 64, 1 + ) + revision = _revision(value) + return replace(value, model_revision=revision, binding=replace(binding, physical_model_revision=revision)) + + +def owner(): + cert = certificate() + return TrackingEpisodeBuilder(cert.binding, cert) + + +def pair(builder, index, geometry=0.1, *, coordinates=None): + stamp = BASE + index * DT + values = tuple(0.0 for _ in NAMES) if coordinates is None else coordinates + b = builder.binding + control = ControlEvidence( + b.identity, + 1, + stamp, + stamp, + 'normalized_torque', + 'stage', + 'response', + 0.0, + (), + 0.0, + True, + False, + True, + True, + False, + False, + False, + False, + False, + False, + stage_schema_revision='torque-selected-stage-snapshot-v1', + stage_values=(('setpoint', geometry),), + applied_coordinates=(0.0, 0.0), + live_applied_coordinates=() if coordinates is None else values, + source_epoch=b.source_epoch, + source_receipt_ns=stamp, + baseline_revision=b.baseline_revision, + controller_revision=b.controller_revision, + replay_inputs=(('desired_curvature', geometry / 400.0), ('vEgo', 20.0), ('roll', 0.0), ('curvature_limited', 0.0), ('control_frame', float(index))), + ) + output = AppliedEvidence(stamp, stamp, stamp, 'normalized_torque', 0.0, True, True, False, False, False, False) + return control, output, full_profile_sha256(b.baseline_revision, dict(zip(NAMES, values, strict=True))) + + +def tick(builder, index, geometry, **changes): + c, o, profile = pair(builder, index, geometry) + c = replace(c, **changes) + return builder.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + + +def test_subtle_requires_two_reversals_after_entry_not_a_storage_chunk(): + b = owner() + for index in range(100): + assert tick(b, index, 0.1 if index < 25 or index >= 75 else -0.1) + assert not b.pending # Only one later reversal, not a complete cycle. + for index in range(100, 126): + assert tick(b, index, -0.1) + assert len(b.pending) == 1 + start, end, kind, direction, _ = b.pending[0] + assert (start, end, kind, direction) == (BASE + 25 * DT, BASE + 100 * DT - 1, 'subtle', -1) + assert not b.drain() # Neither time nor geometry invents physical endpoints. + + +def test_large_neutral_requires_half_second_and_retains_true_direction(): + b = owner() + for index in range(80): + assert tick(b, index, 0.5 if index < 20 else 0.0) + assert len(b.pending) == 1 + assert b.pending[0][:4] == (BASE, BASE + 70 * DT, 'large', 1) + + +@pytest.mark.parametrize('field', ['manual', 'override', 'fault', 'saturated', 'model_wobble']) +def test_interruption_cannot_complete_an_open_episode(field): + b = owner() + for index in range(40): + assert tick(b, index, 0.5) + assert not tick(b, 40, 0.5, **{field: True}) + assert not b.pending and not b.ticks and b.start == 0 + for index in range(41, 100): + assert tick(b, index, 0.0) + assert not b.pending and not b.drain() + + +def test_actual_clipping_is_excluded_but_async_hint_requires_original_unclipped_proof(monkeypatch): + b = owner() + assert not tick(b, 0, 0.1, limited=True) + monkeypatch.setattr('openpilot.starpilot.flm.live_tracking_episodes.exact_unclipped_pair', lambda c, o: True) + assert tick(b, 1, 0.1, limited=True) # Unit branch only, not a forged native proof. + c, o, profile = pair(b, 2) + assert not b.observe_control(c, replace(o, limited=True), profile_sha256=profile, now_ns=c.control_ns, current_native=True) + + +def test_control_gap_and_profile_change_discard_partial_lineage(): + b = owner() + assert tick(b, 0, 0.5) + assert tick(b, 20, 0.5) + assert b.start == BASE + 20 * DT and not b.pending + c, o, profile = pair(b, 21, 0.5, coordinates=(0.001,) + (0.0,) * 29) + assert b.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + assert b.start == c.control_ns and not b.pending + + +def completed_builder(*, count=8, speed=20.0, error=0.0): + b = owner() + for index in range(80): + assert tick(b, index, 0.5 if index < 20 else 0.0) + # Numerical finalization fixtures only; no pose-validation/native assertion. + for index in range(count): + stamp = BASE + index * 5 * DT + b.endpoints.append(_Endpoint(stamp, stamp, stamp + b.certificate.lag_ns, speed if index else 20.0, 0.5, error, 0.5 - error)) + b.watermark = BASE + 79 * DT + return b + + +def test_four_independent_endpoints_minimum_and_good_future_zero_error_is_kept(): + b = completed_builder(count=3) + b._finish() + assert not b.drain() + b = completed_builder(count=4, error=0.0) + b._finish() + (episode,) = b.drain() + assert episode.endpoints == 4 and episode.error_rms == 0.0 + assert episode.control_max_gap_ns == DT and episode.sensitivity == () + assert episode.kind == 'large' and episode.direction == 1 + + +def test_speed_spread_and_pose_watermark_cannot_be_faked_by_wall_time(): + b = completed_builder(speed=24.0) + b._finish() + assert not b.drain() + b = completed_builder() + b.watermark = 0 + b._finish() + assert not b.drain() and len(b.pending) == 1 + + +def test_physical_probe_aggregates_have_signed_derivative_and_independent_native_support(monkeypatch): + monkeypatch.setattr('openpilot.starpilot.flm.live_tracking_probe.validate_probe_result', lambda value, binding: value) + b = completed_builder(count=4, error=0.2) + name = NAMES[0] + for index, point in enumerate(b.endpoints): + row = SimpleNamespace(name=name, physical_derivative=-2.0, native_separated=index % 2 == 0, max_step_physical_delta=0.01, status='separated') + b.probes[point.control_ns] = SimpleNamespace( + rows=(row,), control_ns=point.control_ns, apply_ns=point.apply_ns, profile_sha256=b.pending[0][4], model_revision=b.certificate.model_revision + ) + b._finish() + (episode,) = b.drain() + (sensitivity,) = episode.sensitivity + assert sensitivity.name == name + assert math.isclose(sensitivity.error_feature_sum, -1.6) + assert sensitivity.feature_energy == 16.0 and sensitivity.native_separated == 2 + assert sensitivity.native_endpoints == 4 and sensitivity.max_step_physical_rms == 0.01 + + +def test_same_stamp_different_control_is_not_accepted_as_fresh_input(): + b = owner() + assert tick(b, 0, 0.1) + assert not tick(b, 0, 0.1) # exact duplicate doesn't refresh + assert len(b.ticks) == 1 + assert not tick(b, 0, 0.2) # same stamp mutation invalidates lineage + assert not b.ticks + + +def numeric_pose(builder, control, *, yaw=0.01): + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose + + stamp = control.control_ns + builder.certificate.lag_ns + return OriginalSourcePose( + stamp, + stamp, + 0, + 0, + 0, + physical_calibration_generation(builder.binding), + '0' * 64, + ('synthetic-numeric-join-only',), + yaw, + 0.0, + '', + True, + True, + builder.binding.source_epoch, + builder.binding.identity, + builder.binding.baseline_revision, + builder.binding.controller_revision, + control.control_ns, + control.control_ns, + observed_ns=stamp, + completed_ns=stamp, + native_required=False, + source_lower_ns=stamp, + source_upper_ns=stamp, + source_timestamp_ns=stamp, + speed_mps=20.0, + ) + + +def numerical_join_validation_only(monkeypatch, builder): + # These two seams isolate adapter stamp/context checks. They do NOT claim + # native/source-clock admission; root actual source joins remain required. + monkeypatch.setattr('openpilot.starpilot.flm.live_tracking_episodes.validate_pose', lambda pose, now: None) + monkeypatch.setattr('openpilot.starpilot.flm.live_tracking_episodes.validate_response_witness', lambda witness, binding, now: witness) + + +def test_pose_uses_same_frame_force_target_and_independent_real_stamp_without_upsampling(monkeypatch): + b = owner() + numerical_join_validation_only(monkeypatch, b) + for index in range(30): + assert tick(b, index, 0.1) + for index in (0, 5, 10, 15): + c, o, _ = pair(b, index, 0.1) + pose = numeric_pose(b, c) + assert b.observe_pose( + pose, + c, + o, + witness=SimpleNamespace(certificate=b.certificate, last_control_ns=c.control_ns, last_apply_ns=o.apply_ns, last_pose_ns=pose.mono_ns), + now_ns=pose.mono_ns, + current_native=True, + ) + assert not b.observe_pose( + pose, + c, + o, + witness=SimpleNamespace(certificate=b.certificate, last_control_ns=c.control_ns, last_apply_ns=o.apply_ns, last_pose_ns=pose.mono_ns), + now_ns=pose.mono_ns, + current_native=True, + ) + assert len(b.endpoints) == 4 # 30 control ticks are not30 physical samples. + assert all(math.isclose(point.error, -0.1) for point in b.endpoints) + assert b.endpoints[0].pose_ns == BASE + b.certificate.lag_ns + + +@pytest.mark.parametrize('change', ['lag', 'calibration', 'controller', 'unseen_control', 'stale', 'native']) +def test_invalid_pose_association_resets_partial_window(monkeypatch, change): + b = owner() + numerical_join_validation_only(monkeypatch, b) + for index in range(20): + assert tick(b, index, 0.5) + c, o, _ = pair(b, 5, 0.5) + pose = numeric_pose(b, c) + now, current = pose.mono_ns, True + if change == 'lag': + pose = replace(pose, source_lower_ns=pose.source_lower_ns - 1) + elif change == 'calibration': + pose = replace(pose, calibration='9' * 64) + elif change == 'controller': + pose = replace(pose, controller_revision='other') + elif change == 'unseen_control': + c, o, _ = pair(b, 100, 0.5) + pose = numeric_pose(b, c) + now = pose.mono_ns + elif change == 'stale': + now += 100_000_001 + elif change == 'native': + current = False + assert not b.observe_pose( + pose, + c, + o, + witness=SimpleNamespace(certificate=b.certificate, last_control_ns=c.control_ns, last_apply_ns=o.apply_ns, last_pose_ns=pose.mono_ns), + now_ns=now, + current_native=current, + ) + assert not b.ticks and not b.pending and not b.endpoints + + +def test_plot_contains_only_last_completed_actual_endpoint_stamps_and_values(): + b = completed_builder(count=4, error=0.2) + b._finish() + (episode,) = b.drain() + plot = b.tracking_plot() + assert plot['episodeId'] == episode.episode_id + assert len(plot['points']) == 4 and plot['lagNs'] == b.certificate.lag_ns + for row, original in zip(plot['points'], b.endpoints, strict=True): + assert (row['controlMonoTime'], row['applyMonoTime'], row['poseMonoTime']) == (original.control_ns, original.apply_ns, original.pose_ns) + assert (row['desiredForceAccel'], row['measuredForceAccel'], row['errorForceAccel']) == (0.5, 0.3, 0.2) + plot['points'].clear() + assert len(b.tracking_plot()['points']) == 4 + b.interrupt('driver') + assert b.tracking_plot() is None + + +def test_missing_owner_frame_is_not_a_continuous_whole_episode(): + b = owner() + assert tick(b, 0, 0.5) + c, o, profile = pair(b, 1, 0.5) + values = tuple((key, 3.0 if key == 'control_frame' else value) for key, value in c.replay_inputs) + c = replace(c, replay_inputs=values) + assert b.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + assert b.start == c.control_ns and not b.pending + assert len(b.ticks) == 1 + + +def test_goal_shape_uses_original_fullrate_force_in_eight_real_time_bins(): + b = completed_builder(count=4, error=0.2) + b._finish() + (episode,) = b.drain() + ticks = [tick for stamp, tick in b.ticks.items() if episode.start_ns <= stamp <= episode.end_ns] + bins = [[] for _ in range(8)] + for tick in ticks: + index = min(7, (tick.control.control_ns - episode.start_ns) * 8 // (episode.end_ns - episode.start_ns + 1)) + bins[index].append(tick.goal) + assert len(episode.goal_shape) == 8 + assert episode.goal_shape == tuple(sum(values) / len(values) / episode.goal_rms for values in bins) + assert episode.goal_shape[0] == 1.0 and episode.goal_shape[-1] == 0.0 + + +def test_missing_real_goal_bin_never_interpolates_a_shape(): + b = completed_builder(count=4) + start, end, *_ = b.pending[0] + # Numerical finalization-only corrupted-inventory negative; no source proof. + for stamp in tuple(b.ticks): + if start + 100_000_000 <= stamp < start + 200_000_000: + del b.ticks[stamp] + b._finish() + assert not b.drain() + + +def test_actual_neutral_legacy_two_space_is_not_mistaken_for_missing_full30(): + b = owner() + c, o, profile = pair(b, 0) + assert c.applied_coordinates == (0.0, 0.0) and c.live_applied_coordinates == () + assert b.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + + +def test_explicit_actual_live_map_not_pending_offer_supplies_nonzero_full30(): + b = owner() + values = (0.001,) + (0.0,) * 29 + c, o, profile = pair(b, 0, coordinates=values) + assert b.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + b = owner() + c, o, _ = pair(b, 0) + assert not b.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + + +def test_old_nonneutral_selected_residual_is_not_unseen_full30(): + b = owner() + c, o, profile = pair(b, 0) + c = replace(c, applied_coordinates=(0.001, 0.0)) + assert not b.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + + +def test_existing_full_effect_trace_uses_actual_map_and_original_binding(): + from openpilot.starpilot.flm.full_surface_profile import FullSurfaceEnvelope + from openpilot.starpilot.flm.effect_trace import FullEffectTrace, VERSION as TRACE_VERSION + + b = owner() + c, o, profile = pair(b, 0) + env = FullSurfaceEnvelope.initial( + cp_sha256=b.binding.cp_sha256, + controller_revision=b.binding.controller_revision, + source_epoch=b.binding.source_epoch, + reset_generation=b.binding.reset_generation, + selected_baseline_revision=b.binding.baseline_revision, + persistent_source_epoch=b.binding.persistent_source_epoch, + trial_names=(NAMES[0],), + ) + trace = FullEffectTrace(TRACE_VERSION, env, c.control_ns, 0, True, env.coordinates, env.coordinates, env.coordinates, profile) + c = replace( + c, + effect_trace=trace, + effect_prepare_ns=c.control_ns, + applied_profile_sequence=1, + applied_profile_observed_ns=c.control_ns, + applied_profile_expires_ns=c.control_ns + 500_000_000, + ) + assert b.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + b = owner() + assert not b.observe_control(replace(c, live_applied_coordinates=(0.0,) * 30), o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + + +@pytest.mark.parametrize('emit_every', [1, 2]) +def test_fullrate_geometry_is_independent_of_native_emission_cadence(emit_every): + b = owner() + for index in range(130): + geometry = 0.1 if index < 25 or 75 <= index < 100 else -0.1 + c, o, profile = pair(b, index, geometry) + assert b.observe_source_control(c, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + if index % emit_every == 0: + assert b.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + assert len(b.ticks) == 130 and len(b.pending) == 1 + assert sum(tick.output is not None for tick in b.ticks.values()) == (130 + emit_every - 1) // emit_every + assert b.pending[0][:4] == (BASE + 25 * DT, BASE + 100 * DT - 1, 'subtle', -1) + assert not b.endpoints # Native outputs are not physical samples. + + +def test_non_emitted_source_tick_can_never_be_a_physical_endpoint(monkeypatch): + b = owner() + numerical_join_validation_only(monkeypatch, b) + c, o, profile = pair(b, 0, 0.5) + assert b.observe_source_control(c, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + assert b.ticks[c.control_ns].output is None + pose = numeric_pose(b, c) + witness = SimpleNamespace(certificate=b.certificate, last_control_ns=c.control_ns, last_apply_ns=o.apply_ns, last_pose_ns=pose.mono_ns) + assert not b.observe_pose(pose, c, o, witness=witness, now_ns=pose.mono_ns, current_native=True) + assert not b.endpoints + + +def test_delayed_real_native_join_does_not_discard_newer_source_geometry(): + b = owner() + for index in range(4): + c, o, profile = pair(b, index, 0.5) + assert b.observe_source_control(c, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + c, o, profile = pair(b, 1, 0.5) + assert b.observe_control(c, o, profile_sha256=profile, now_ns=BASE + 3 * DT, current_native=True) + assert len(b.ticks) == 4 and b.start == BASE + assert b.ticks[c.control_ns].output == o + + +def test_actual_output_clipping_and_controller_saturation_interrupt_all_cadences(): + b = owner() + for index in range(4): + c, o, profile = pair(b, index, 0.5) + assert b.observe_source_control(c, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + c, o, profile = pair(b, 2, 0.5) + assert not b.observe_control(c, replace(o, limited=True), profile_sha256=profile, now_ns=BASE + 3 * DT, current_native=True) + assert not b.ticks + c, o, profile = pair(b, 4, 0.5) + assert not b.observe_source_control(replace(c, saturated=True), profile_sha256=profile, now_ns=c.control_ns, current_native=True) + assert not b.ticks + + +def test_configured_turn_assist_is_source_context_not_a_false_clipping_flag(): + b = owner() + assert tick(b, 0, 0.1, turn_assist=True) + c, o, profile = pair(b, 1, 0.1) + c = replace(c, turn_assist=True) + assert not b.observe_control(c, replace(o, limited=True), profile_sha256=profile, now_ns=c.control_ns, current_native=True) + assert b.last_input_status == 'invalid' and not b.ticks + + +def test_pre_boundary_lagged_pose_does_not_deadlock_new_profile_history(monkeypatch): + b = owner() + numerical_join_validation_only(monkeypatch, b) + old_c, old_o, _ = pair(b, 0, 0.1) + assert tick(b, 0, 0.1) + c, o, profile = pair(b, 1, 0.1, coordinates=(0.001,) + (0.0,) * 29) + assert b.observe_control(c, o, profile_sha256=profile, now_ns=c.control_ns, current_native=True) + before = tuple(b.ticks.items()) + pose = numeric_pose(b, old_c) + witness = SimpleNamespace(certificate=b.certificate, last_control_ns=old_c.control_ns, last_apply_ns=old_o.apply_ns, last_pose_ns=pose.mono_ns) + assert not b.observe_pose(pose, old_c, old_o, witness=witness, now_ns=pose.mono_ns, current_native=True) + assert b.last_input_status == 'ignored' and tuple(b.ticks.items()) == before + assert not b.endpoints and b.watermark == 0 and b.last_pose_ns == 0 + assert not b.observe_control(old_c, old_o, profile_sha256=profile, now_ns=old_o.apply_ns, current_native=True) + assert b.last_input_status == 'ignored' and tuple(b.ticks.items()) == before + + +def requested_completed_builder(monkeypatch): + # Pure numerical support-seal seam; no child/native ownership is claimed. + monkeypatch.setattr('openpilot.starpilot.flm.live_tracking_probe.validate_probe_result', lambda value, binding: value) + from openpilot.starpilot.flm.canonical_surface_adapter import GROUPS + + b = completed_builder(count=4, error=0.2) + name = NAMES[0] + group = next(group for group, names in GROUPS.items() if name in names) + queued = 1_000_000_000_000 + for point in b.endpoints: + assert b.request_probe(point.control_ns, group=group, names=(name,), queued_ns=queued) + b._finish() + assert not b.drain(now_ns=queued) and len(b.completed) == 1 + return b, group, name, queued + + +def numerical_probe(b, point, group, name): + return SimpleNamespace( + binding=b.binding, + control_ns=point.control_ns, + apply_ns=point.apply_ns, + profile_sha256=next(iter(b.completed.values()))['episode'].profile_sha256, + model_revision=b.certificate.model_revision, + group=group, + rows=(SimpleNamespace(name=name, physical_derivative=-2.0, native_separated=True, max_step_physical_delta=0.01, status='separated'),), + ) + + +def test_late_four_endpoint_support_keeps_exact_episode_once_without_pose_renewal(monkeypatch): + b, group, name, queued = requested_completed_builder(monkeypatch) + saved = next(iter(b.completed.values())) + episode_id = saved['episode'].episode_id + points = saved['points'] + clock = (b.last_pose_ns, b.watermark) + b.ticks.clear() # Original control history may age out; exact completed endpoints remain. + for index, point in enumerate(points): + assert b.observe_probe(numerical_probe(b, point, group, name), now_ns=queued + index + 1) + if index < 3: + assert not b.drain(now_ns=queued + index + 1) + (episode,) = b.drain(now_ns=queued + 5) + assert episode.episode_id == episode_id and episode.sensitivity[0].native_endpoints == 4 + assert (b.last_pose_ns, b.watermark) == clock + assert not b.drain(now_ns=queued + 30_000_000_001) + assert not b.observe_probe(numerical_probe_from_saved(b, saved, points[0], group, name), now_ns=queued + 6) + + +def numerical_probe_from_saved(b, saved, point, group, name): + return SimpleNamespace( + binding=b.binding, + control_ns=point.control_ns, + apply_ns=point.apply_ns, + profile_sha256=saved['episode'].profile_sha256, + model_revision=b.certificate.model_revision, + group=group, + rows=(SimpleNamespace(name=name, physical_derivative=-2.0, native_separated=True, max_step_physical_delta=0.01, status='separated'),), + ) + + +def test_probe_deadline_is_first_request_fixed_and_partial_support_remains_partial(monkeypatch): + b, group, name, queued = requested_completed_builder(monkeypatch) + saved = next(iter(b.completed.values())) + point = saved['points'][0] + assert b.request_probe(point.control_ns, group=group, names=(name,), queued_ns=queued + 29_000_000_000) + assert saved['deadline'] == queued + 30_000_000_000 + assert b.observe_probe(numerical_probe(b, point, group, name), now_ns=queued + 1) + assert not b.drain(now_ns=queued + 29_999_999_999) + (episode,) = b.drain(now_ns=queued + 30_000_000_000) + assert episode.sensitivity[0].native_endpoints == 1 # Never pretend four. + assert b.reason == 'whole_episode_probe_deadline_partial_support' + assert not b.request_probe(point.control_ns, group=group, names=(name,), queued_ns=queued + 30_000_000_001) + assert not b.drain(now_ns=queued + 60_000_000_000) + + +@pytest.mark.parametrize('change', ['apply', 'profile', 'model', 'group', 'names', 'late']) +def test_late_probe_cannot_change_geometry_or_cross_numeric_context(monkeypatch, change): + b, group, name, queued = requested_completed_builder(monkeypatch) + saved = next(iter(b.completed.values())) + result = numerical_probe(b, saved['points'][0], group, name) + now = queued + 1 + if change == 'apply': + result.apply_ns += 1 + elif change == 'profile': + result.profile_sha256 = '9' * 64 + elif change == 'model': + result.model_revision = '9' * 64 + elif change == 'group': + result.group = 'not-the-requested-group' + elif change == 'names': + result.rows = () + else: + now = queued + 30_000_000_001 + assert not b.observe_probe(result, now_ns=now) + assert not saved['probes'] and len(b.completed) == 1 + + +def test_interruption_discards_retained_geometry_and_cannot_reemit_old_episode(monkeypatch): + b, group, name, queued = requested_completed_builder(monkeypatch) + saved = next(iter(b.completed.values())) + result = numerical_probe(b, saved['points'][0], group, name) + b.interrupt('actual_source_changed') + assert not b.completed and not b.probe_requests + assert not b.observe_probe(result, now_ns=queued + 1) + assert not b.drain(now_ns=queued + 30_000_000_000) + + +def test_unimplemented_steady_kind_and_constructor_policy_are_explicitly_rejected(): + b = completed_builder(count=4, error=0.0) + b._finish() + (episode,) = b.drain() + with pytest.raises(ValueError, match='complete real tracking episode required'): + replace(episode, kind='steady', direction=0).validate() + cert = certificate() + with pytest.raises(TypeError): + cast(Callable[..., TrackingEpisodeBuilder], TrackingEpisodeBuilder)(cert.binding, cert, enable_steady=True) + + +def test_supported_cyclic_geometry_keeps_perfect_future_tracking_without_error_floor(): + b = owner() + for index in range(150): + geometry = 0.1 if index < 25 or 75 <= index < 100 else -0.1 + assert tick(b, index, geometry) + # Numerical physical endpoints only; actual GM/Toyota/HKG native cadence + # remains exercised separately without changing its assertions or verdicts. + for index in range(25, 100, 5): + goal = 0.1 if index >= 75 else -0.1 + stamp = BASE + index * DT + b.endpoints.append(_Endpoint(stamp, stamp, stamp + b.certificate.lag_ns, 20.0, goal, 0.0, goal)) + b.watermark = BASE + 149 * DT + b._finish() + (episode,) = b.drain() + assert episode.kind == 'subtle' and episode.direction == -1 + assert episode.start_ns == BASE + 25 * DT and episode.end_ns == BASE + 100 * DT - 1 + assert episode.error_rms == 0.0 and episode.reversal_hz == 0.0 and episode.endpoints == 15 + assert episode.sensitivity == () and episode.control_max_gap_ns == DT + assert not b.drain() + + +@pytest.mark.parametrize('bad_reference', (None, 'missing', 'stale', 'future_only', 'profile', 'changed_goal', 'epoch', 'invalid')) +def test_tracking_reference_matches_physical_response_time_without_retiming_command(monkeypatch, bad_reference): + b = owner() + numerical_join_validation_only(monkeypatch, b) + for index in range(41): + assert tick(b, index, 0.1 if index < 17 else 0.3) + c, o, _ = pair(b, 0, 0.1) + pose = numeric_pose(b, c, yaw=0.01) + reference_stamp = BASE + 17 * DT + if bad_reference == 'missing': + for stamp in list(b.ticks): + if stamp > BASE: + del b.ticks[stamp] + elif bad_reference == 'stale': + for stamp in list(b.ticks): + if stamp > BASE + 13 * DT: + del b.ticks[stamp] + elif bad_reference == 'future_only': + for stamp in list(b.ticks): + if BASE < stamp <= pose.mono_ns: + del b.ticks[stamp] + elif bad_reference == 'profile': + b.ticks[reference_stamp] = replace(b.ticks[reference_stamp], profile='9' * 64) + elif bad_reference == 'changed_goal': + b.ticks[reference_stamp] = replace(b.ticks[reference_stamp], goal=0.9) + elif bad_reference == 'epoch': + b.ticks[reference_stamp] = replace(b.ticks[reference_stamp], control=replace(b.ticks[reference_stamp].control, source_epoch='9' * 64)) + elif bad_reference == 'invalid': + b.ticks[reference_stamp] = replace(b.ticks[reference_stamp], control=replace(b.ticks[reference_stamp].control, valid=False)) + result = b.observe_pose( + pose, + c, + o, + witness=SimpleNamespace(certificate=b.certificate, last_control_ns=c.control_ns, last_apply_ns=o.apply_ns, last_pose_ns=pose.mono_ns), + now_ns=pose.mono_ns, + current_native=True, + ) + if bad_reference is not None: + assert result is False and not b.endpoints + else: + assert result is True and len(b.endpoints) == 1 + point = b.endpoints[0] + assert point.control_ns == c.control_ns and point.apply_ns == o.apply_ns + assert point.pose_ns == pose.mono_ns and point.reference_ns == reference_stamp + assert math.isclose(point.goal, 0.3) and math.isclose(point.measured, 0.2) and math.isclose(point.error, 0.1) + assert not math.isclose(point.error, 0.1 - point.measured) + + +def test_comparison_releases_real_measurements_before_optional_probe_deadline_without_claiming_gradients(monkeypatch): + b, group, name, queued = requested_completed_builder(monkeypatch) + row = next(iter(b.completed.values())) + original = row['episode'] + deadline = row['deadline'] + assert queued + 1 < deadline and not b.drain(now_ns=queued + 1) + assert b.release_measured_only() + measured = b.drain(now_ns=queued + 1) + assert len(measured) == 1 + current = measured[0] + for field in ( + 'binding', + 'profile_sha256', + 'episode_id', + 'start_ns', + 'end_ns', + 'kind', + 'direction', + 'goal_rms', + 'error_rms', + 'endpoints', + 'control_max_gap_ns', + 'goal_shape', + ): + assert getattr(current, field) == getattr(original, field) + assert current.sensitivity == () and current.native_gradient_complete is False + assert row['deadline'] == deadline + assert not b.completed and not b.probe_requests + assert b.closed_through_ns == original.end_ns + assert b.release_measured_only() and not b.drain(now_ns=queued + 2) + assert not b.request_probe(row['points'][0].control_ns, group=group, names=(name,), queued_ns=queued + 2) + + +def test_measured_only_release_rejects_overflow_instead_of_dropping_or_fabricating_episodes(monkeypatch): + b, _, _, queued = requested_completed_builder(monkeypatch) + row = next(iter(b.completed.values())) + # Explicit malformed-backlog injection; never source/native evidence. + b.ready.extend([row['episode']] * b.ready.maxlen) + assert not b.release_measured_only() + assert b.reason == 'measured_episode_backlog' + assert not b.ready and not b.completed and not b.probe_requests + + +def test_tracking_goal_uses_original_actual_controller_target_not_future_feedforward_request(): + from openpilot.starpilot.flm.live_tracking_episodes import tracking_reference_force + + b = owner() + c, _, _ = pair(b, 0, geometry=-0.1) + # Actual delayed setpoint is negative while future model demand has already + # become positive. All ORIGINAL times remain unchanged. + c = replace(c, replay_inputs=tuple((key, 0.3 / 400.0 if key == 'desired_curvature' else 0.02 if key == 'roll' else value) for key, value in c.replay_inputs)) + expected = -0.1 - 9.81 * math.sin(0.02) + assert tracking_reference_force(c) == pytest.approx(expected) + assert tracking_reference_force(c) != pytest.approx(0.3 - 9.81 * math.sin(0.02)) + assert c.control_ns == BASE and dict(c.stage_values)['setpoint'] == -0.1 + + +def test_actual_standard_torque_controller_reports_a_different_target_from_its_future_request(): + from opendbc.car.toyota.values import CAR + from opendbc.car.car_helpers import interfaces + from opendbc.car import structs + from openpilot.starpilot.flm.controller_replay import SelectedControllerFactory + from openpilot.starpilot.flm.live_tracking_episodes import tracking_reference_force + + cp = interfaces[CAR.TOYOTA_RAV4_TSS2].get_non_essential_params(CAR.TOYOTA_RAV4_TSS2) + controller, vm = SelectedControllerFactory(cp, 'standard').create() + cs = structs.CarState.new_message(vEgo=20.0, steeringAngleDeg=0.0, steeringPressed=False, steeringRateDeg=0.0, standstill=False) + params = SimpleNamespace(angleOffsetDeg=0.0, roll=0.0) + for index in range(40): + curvature = (index + 1) * 0.00001 + _, _, log = controller.update(True, cs, vm, params, False, curvature, False, 0.2) + assert abs(log.desiredLateralAccel - curvature * cs.vEgo**2) > 0.03 + c, _, _ = pair(owner(), 0, geometry=float(log.desiredLateralAccel)) + c = replace(c, replay_inputs=tuple((key, curvature if key == 'desired_curvature' else value) for key, value in c.replay_inputs)) + assert tracking_reference_force(c) == pytest.approx(float(log.desiredLateralAccel)) + + +@pytest.mark.parametrize( + 'errors,expected', + [ + ((-0.03, -0.02, 0.003, -0.002, 0.004, -0.003, 0.02, 0.03), 0.5), + ((0.004, -0.004, 0.003, -0.003, 0.002, -0.002), 0.0), + ((0.03, -0.03, 0.03, -0.03, 0.03), 2.0), + ], +) +def test_weave_counter_uses_actionable_excursions_not_reference_zero_jitter(errors, expected): + from openpilot.starpilot.flm.live_tracking_episodes import tracking_reversal_rate + + assert tracking_reversal_rate(errors, 2.0, 0.0) == expected + + +def test_weave_counter_retains_raw_error_and_respects_identified_measurement_floor(): + from openpilot.starpilot.flm.live_tracking_episodes import tracking_reversal_rate + + errors = (0.04, -0.04, 0.04) + original = tuple(errors) + assert tracking_reversal_rate(errors, 2.0, 0.02) == 0.0 + assert tracking_reversal_rate((0.08, -0.08, 0.08), 2.0, 0.02) == 1.0 + assert errors == original diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_power_history.py b/openpilot/starpilot/flm/tests/test_live_tracking_power_history.py new file mode 100644 index 0000000000..d6a20cbaa5 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_power_history.py @@ -0,0 +1,108 @@ +"""Planning-only numerical regressions; no controller/native authority proof.""" +from dataclasses import replace +import math + +import pytest + +from openpilot.starpilot.flm import live_tracking as tracking +from openpilot.starpilot.flm.tests.test_live_tracking_rollout import NS, HOOK, _optimizer, _episode, _refresh, _ack + + +def setup_plan(): + owner = _optimizer() + rows = tuple(_episode(owner, index, (1000+index*8)*NS, owner.accepted, .6, .35, 22.) for index in range(20, 28)) + target = dict(owner.accepted) + target[HOOK] = .005 + reference = tuple(_episode(owner, index, (1000+index*8)*NS, owner.accepted, .6, score, 22.) + for index, score in enumerate((.31, .31, .34, .34, .37, .37, .34, .34, .31, .31, .34, .34), 1)) + return owner, rows, target, reference + + +def test_past_comparable_noise_can_only_increase_forecast_quota_or_require_more_baseline(): + owner, rows, target, reference = setup_plan() + original = owner._trial_plan(rows, .025, target) + assert original is not None + owner.planning_history.extend(reference) + forecast = owner._trial_plan(rows, .025, target) + assert forecast is None or forecast[0] > original[0] + # Planning never replaces or populates the actual independent score baseline. + assert owner.before == () and owner.future == [] + + +def test_larger_future_variance_cannot_admit_an_underpowered_trial_by_raising_welch_degrees_of_freedom(): + owner = _optimizer() + rows = tuple(_episode(owner, index, (1000+index*8)*NS, owner.accepted, .6, score, 22.) + for index, score in enumerate((.34, .35, .36), 20)) + target = dict(owner.accepted) + target[HOOK] = .005 + reference = tuple(_episode(owner, index, (1000+index*8)*NS, + owner.accepted, .6, score, 22.) for index, score in + enumerate((.35-math.sqrt(.00014), .35, .35+math.sqrt(.00014)), 1)) + variance, inflation, before_df = tracking._noise(rows, owner.rmse) + historical_variance, historical_inflation, _ = tracking._noise(reference, owner.rmse) + cadence = rows[1].end_ns-rows[0].end_ns + feasible = min(tracking.MAX_EPISODES, + math.floor((tracking.TRIAL_TIMEOUT_NS-NS-owner.lag_ns)/cadence)) + left = variance*inflation/len(rows) + + def nominal_rhs(quota, future_variance, future_inflation): + right = future_variance*future_inflation/quota + df = tracking._welch_df(left, before_df, right, max(1, math.floor(quota/future_inflation)-1)) + return (tracking._critical(df, .025)+.8416212335729143)*math.sqrt(left+right) + + # Inspect EVERY feasible original quota, including quota6 (which an earlier + # hand calculation missed). Construct the strict counterexample interval + # from actual numerical helpers rather than guessing the original quota. + original_best = min(nominal_rhs(q, variance, inflation) for q in range(3, feasible+1)) + naive_noisier = nominal_rhs(9, max(variance, historical_variance), max(inflation, historical_inflation)) + assert naive_noisier < original_best + expected = (original_best+naive_noisier)/2 + assert naive_noisier < expected < original_best + assert owner._trial_plan(rows, expected, target) is None + owner.planning_history.extend(reference) + # Without the original-critical envelope, quota9 becomes admissible solely + # because its larger forecast variance raises Welch df and lowers t. + assert owner._trial_plan(rows, expected, target) is None + + +@pytest.mark.parametrize('mismatch', ('binding', 'profile', 'shape', 'age')) +def test_forecast_never_borrows_different_binding_profile_shape_or_stale_history(mismatch): + owner, rows, target, reference = setup_plan() + original = owner._trial_plan(rows, .025, target) + if mismatch == 'binding': + reference = tuple(replace(row, binding=replace(row.binding, reset_generation='other-drive')) for row in reference) + elif mismatch == 'profile': + reference = tuple(replace(row, profile_sha256='0'*64) for row in reference) + elif mismatch == 'shape': + reference = tuple(replace(row, goal_shape=(2.,)*8) for row in reference) + else: + reference = tuple(replace(row, start_ns=row.start_ns-900*NS, end_ns=row.end_ns-900*NS) for row in reference) + owner.planning_history.extend(reference) + assert owner._trial_plan(rows, .025, target) == original + + +def test_history_keeps_zero_error_clean_geometry_but_not_trial_future_outcomes(): + owner = _optimizer() + zero = _episode(owner, 0, 8*NS, owner.accepted, .6, 0., 22.) + _refresh(owner, zero.end_ns) + owner.observe_episode(zero, now_ns=zero.end_ns) + assert list(owner.planning_history) == [zero] + owner.interrupt('new-physical-context', context_changed=True) + assert not owner.planning_history + for index in range(1, 49): + now = (8+index*8)*NS + _refresh(owner, now) + row = _episode(owner, index, now, owner.accepted, .6, .35, 22.) + owner.observe_episode(row, now_ns=now) + if owner.offer_state is not None: + break + offer = owner.offer(now) + assert offer is not None and offer.phase == 'trial' + _ack(owner, offer, now+NS) + history = tuple(owner.planning_history) + now += 8*NS + _refresh(owner, now) + future = _episode(owner, 100, now, dict(offer.target), .6, .325, 22.) + owner.observe_episode(future, now_ns=now) + assert tuple(owner.planning_history) == history + assert all(row.profile_sha256 == owner._hash(owner.accepted) for row in history) diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_probe.py b/openpilot/starpilot/flm/tests/test_live_tracking_probe.py new file mode 100644 index 0000000000..9afc561735 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_probe.py @@ -0,0 +1,197 @@ +"""Numerical/shape tests only; fixtures never establish real source/native authority. + +Root must additionally call probe_endpoint on actual authenticated checkpoint, +controller and emitted-CAN records. No fitted response or permission is mocked +as though these unit tests were that qualification. +""" +from dataclasses import replace +import math + +import pytest +from openpilot.starpilot.flm import live_tracking_probe as probe +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, GROUPS, canonical_names +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding, MAX_STEP +from openpilot.starpilot.flm.pid_stages import SUPPORTED + + +def binding(): + return LiveTrialBinding('1'*64, '2'*64, 'controller', '3'*64, '4'*64, + 'reset', 'calibration', 'vehicle', '', '5'*64, '', False).validate() + + +def numerical_fixture(domain='torque', group='ff.left'): + current = binding() + names = canonical_names(tuple(name for name in GROUPS[group] if domain == 'torque' or name in SUPPORTED)) + rows = tuple(probe._physical_row(name, 0., 0., MAX_STEP, .1, .1, .11, 2., 'separated') for name in names) + value = probe.ProbeResult(current, 100, 110, '6'*64, current.physical_model_revision, rows, domain, group, None) + object.__setattr__(value, '_seal', (probe._OWNER, probe._probe_key(value))) + return value + + +def test_oriented_delivered_torque_and_measured_slope_have_correct_units_and_sign(): + row = probe._physical_row(NAMES[0], 0., -.005, .005, .105, .1, .11, 2., 'separated') + assert row.physical_derivative == pytest.approx(-2.) + assert row.max_step_physical_delta == pytest.approx(.01) + assert row.max_step_physical_delta == abs(row.physical_derivative)*MAX_STEP + # The old unit ambiguity would overstate visibility by 1/.005 = 200. + assert row.max_step_physical_delta != abs(row.physical_derivative) + reversed_delivery = probe._physical_row(NAMES[0], 0., -.005, .005, .105, .11, .1, 2., 'separated') + assert reversed_delivery.physical_derivative == pytest.approx(2.) + + +def test_one_sided_original_bound_uses_actual_secant_width(): + row = probe._physical_row(NAMES[0], .25, .245, .25, .1, .1, .11, 2., 'separated') + assert row.physical_derivative == pytest.approx(-4.) + assert row.max_step_physical_delta == pytest.approx(.02) + + +@pytest.mark.parametrize('status', ['quantized', 'native_unavailable']) +def test_can_quantization_or_unavailable_native_support_cannot_fabricate_feature(status): + row = probe._physical_row(NAMES[0], 0., -.005, .005, .1, .1, .1, 2., status) + assert row.physical_derivative == row.max_step_physical_delta == 0. + assert row.native_separated is False + + +@pytest.mark.parametrize('domain,count', [('torque', 30), ('torque_pid', 26)]) +def test_full_domain_coverage_is_over_honest_independent_small_groups(domain, count): + covered = set() + for group, names in GROUPS.items(): + if domain == 'torque_pid' and not any(name in SUPPORTED for name in names): + continue + value = numerical_fixture(domain, group) + assert probe.validate_probe_result(value, value.binding) is value + assert 1 <= len(value.rows) <= 4 + covered.update(row.name for row in value.rows) + assert len(covered) == count + + +def test_partial_duplicate_or_forged_wire_result_never_validates(): + value = numerical_fixture() + for changed in [replace(value, _seal=None), replace(value, rows=value.rows[:-1]), + replace(value, rows=(value.rows[0],)*4), replace(value, model_revision='7'*64)]: + with pytest.raises(ValueError): + probe.validate_probe_result(changed) + + +def test_mutated_row_and_binding_fail_fresh_content_check(): + value = numerical_fixture() + object.__setattr__(value.rows[0], 'physical_derivative', value.rows[0].physical_derivative*2) + object.__setattr__(value.rows[0], 'max_step_physical_delta', value.rows[0].max_step_physical_delta*2) + with pytest.raises(ValueError, match='contents changed'): + probe.validate_probe_result(value) + value = numerical_fixture() + object.__setattr__(value.binding, 'vehicle_model_generation', 'changed') + with pytest.raises(ValueError, match='contents changed'): + probe.validate_probe_result(value) + + +@pytest.mark.parametrize('field,value', [('physical_derivative', math.nan), ('baseline_coordinate', True), + ('plus_coordinate', .006), ('native_separated', 1), ('status', 'unknown')]) +def test_invalid_per_coordinate_domain_rejected(field, value): + result = numerical_fixture() + row = replace(result.rows[0], **{field: value}) + altered = replace(result, rows=(row, *result.rows[1:])) + with pytest.raises(ValueError): + probe.validate_probe_result(altered) + + +def test_unknown_rows_and_source_objects_do_not_call_hooks_or_reducers(): + class Trap: + def __getattr__(self, name): + raise AssertionError('callback') + def __reduce__(self): + raise AssertionError('reducer') + result = numerical_fixture() + with pytest.raises(ValueError): + probe.validate_probe_result(replace(result, rows=(Trap(),))) + with pytest.raises(ValueError): + probe._primitive(Trap()) + + +def test_foreign_factory_rejected_before_constructor_or_snapshot_work(): + class Factory: + def create(self): + raise AssertionError('foreign constructor') + with pytest.raises(ValueError, match='actual selected'): + probe.probe_endpoint(factory=Factory(), control=None, output=None, accepted_full30=(), binding=None, + response_witness=None, source=None, now_ns=1, current_native=True, + current_vehicle_model_anchor=(1., 1., 1.), replay=None, group='ff.left', names=GROUPS['ff.left'], prefix_controls=()) + + +@pytest.mark.parametrize('broken', ['no_checkpoint', 'missing_tick', 'missing_endpoint', 'over100']) +def test_prefix_holds_instead_of_inventing_missing_controller_history(broken): + from collections import deque + from types import SimpleNamespace + from openpilot.starpilot.flm.controller_replay import ControllerReplay, ReplayFrame, Snapshot + from openpilot.starpilot.flm.torque_adapter import Objective + from openpilot.starpilot.flm.torque_emission_replay import EmissionHistory + # Invalid-only shells: any attempted actual construction/decoding is a test + # failure. These cannot satisfy the successful probe authority path. + current = binding() + factory = object() + replay = object.__new__(ControllerReplay) + replay.factory, replay.failed, replay.raw_mode = factory, False, True + replay.identity, replay.baseline_revision = current.identity, current.baseline_revision + replay.objective = Objective('feedforward', 'low', 'steady') + replay.emissions = object.__new__(EmissionHistory) + replay.checkpoints = deque(((100, Snapshot('invalid-test-only', (('value', 1.),))),)) + replay.records = deque(ReplayFrame(100+i*10, (), 0., (), frame_sequence=i+1) for i in range(3)) + endpoint = 120 + if broken == 'no_checkpoint': + replay.checkpoints.clear() + elif broken == 'missing_tick': + replay.records = deque((replay.records[0], replay.records[-1])) + elif broken == 'missing_endpoint': + replay.records.pop() + else: + replay.records = deque(ReplayFrame(100+i, (), 0., (), frame_sequence=i+1) for i in range(101)) + endpoint = 200 + with pytest.raises(ValueError, match='checkpoint|prefix'): + probe._prefix_before(replay, SimpleNamespace(control_ns=endpoint), SimpleNamespace(apply_ns=endpoint), current, factory) + + + +def test_actual_consumed_map_is_replayed_without_forging_legacy_trace(): + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + current = binding() + # Exact production dataclass shape; other fields are irrelevant to this + # numerical transport helper and do not impersonate successful probe authority. + control = object.__new__(ControlEvidence) + object.__setattr__(control, "effect_trace", None) + live = tuple(.001 if index == 0 else 0. for index in range(len(NAMES))) + object.__setattr__(control, "live_applied_coordinates", live) + rows = tuple(zip(NAMES, live, strict=True)) + actual, has_source = probe._consumed_coordinates(control, current) + assert actual == rows and has_source + assert control.effect_trace is None + changed = tuple(.002 if index == 0 else value for index, value in enumerate(live)) + object.__setattr__(control, "live_applied_coordinates", changed) + assert probe._consumed_coordinates(control, current) == (tuple(zip(NAMES, changed, strict=True)), True) + for malformed in (rows, live[:-1], tuple(True for _ in NAMES), tuple(math.nan for _ in NAMES), list(live)): + object.__setattr__(control, "live_applied_coordinates", malformed) + with pytest.raises(ValueError): + probe._consumed_coordinates(control, current) + object.__setattr__(control, "live_applied_coordinates", ()) + assert probe._consumed_coordinates(control, current)[1] is False + + +@pytest.mark.parametrize("pid", [False, True]) +def test_actual_frame_transport_preserves_live_map_and_owner_anchor(pid): + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + from openpilot.starpilot.flm.learner_factory import _frame + from openpilot.starpilot.flm.original_context_receipts import _same_original_frame + control = object.__new__(ControlEvidence) + values = {'control_ns': 100, 'replay_inputs': (("control_frame", 1), ("desired_curvature", .001), + ("vEgo", 20.), ("roll", 0.)), 'requested': .1, 'applied_coordinates': (), 'valid': True, 'fault': False, + 'override': False, 'manual': False, 'limited': False, 'applied_parameter_names': (), 'effect_trace': None, 'effect_prepare_ns': 0, + 'vehicle_model_anchor': (15., 1., 1.), 'live_applied_coordinates': tuple(.001 if i == 0 else 0. for i in range(30)), + 'stage_schema_revision': "pid-selected-stage-snapshot-v2" if pid else "torque-raw", + 'stage_values': tuple((name, 0.) for name in ("selected_command", "effective_factor", "offset", + "friction", "pid_pos_limit", "pid_neg_limit", "actual_delta_ff", "actual_delta_threshold", "setpoint"))} + for name, value in values.items(): + object.__setattr__(control, name, value) + frame = _frame(control) + assert frame.live_applied_coordinates == control.live_applied_coordinates + assert frame.vehicle_model_anchor == control.vehicle_model_anchor + assert not _same_original_frame(frame, replace(frame, live_applied_coordinates=(0.,)*30)) + assert not _same_original_frame(frame, replace(frame, vehicle_model_anchor=(16., 1., 1.))) diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_probe_process.py b/openpilot/starpilot/flm/tests/test_live_tracking_probe_process.py new file mode 100644 index 0000000000..e5d4e4c862 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_probe_process.py @@ -0,0 +1,337 @@ +"""Wire/lifecycle/unit tests; no fixture claims actual CAN/plant qualification.""" +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from openpilot.starpilot.flm.live_tracking_probe_process import ProbeProcess, decode_probe_job, VERSION + + +@pytest.mark.parametrize('prefix', [(), (None,)*101]) +def test_bounded_probe_export_rejects_invented_or_unbounded_recorder(prefix): + from openpilot.starpilot.flm.live_tracking_probe_process import probe_replay_snapshot + with pytest.raises(ValueError, match='genuine bounded'): + probe_replay_snapshot(SimpleNamespace(), prefix) + + +@pytest.mark.parametrize('foreign', ['tuning_leaf', 'frame_object', 'frame_clock', 'checkpoint_clock', 'container']) +def test_exporter_rejects_callback_objects_before_selection_or_deepcopy(foreign): + from collections import OrderedDict, deque + from openpilot.starpilot.flm.controller_replay import ControllerReplay, SelectedControllerFactory, ReplayFrame, Snapshot + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence + from openpilot.starpilot.flm.torque_emission_replay import EmissionHistory + from openpilot.starpilot.flm.live_tracking_probe_process import probe_replay_snapshot + calls = [] + + class Trap: + def __deepcopy__(self, memo): + calls.append('deepcopy') + return self + def __le__(self, other): + calls.append('compare') + return True + def __ge__(self, other): + calls.append('compare') + return True + @property + def control_ns(self): + calls.append('clock') + return 100 + + class ForeignDeque(deque): + def __reversed__(self): + calls.append('reverse') + return super().__reversed__() + + # Invalid-only exact-class shells. No controller, source/native lease or + # successful probe is manufactured, and no constructor may be reached. + snapshot = Snapshot('invalid-only', (('x', 1.),)) + control = ControlEvidence(identity='invalid-only', generation=1, control_ns=100, state_ns=100, + domain='normalized_torque', stage_revision='invalid-only', response_revision='invalid-only', requested=0., + features=(), baseline_prediction=0., enabled=True, manual=False, active=True, valid=True, + override=False, fault=False, saturated=False, limited=False, model_wobble=False, controller_chatter=False, + replay_snapshot=snapshot.values) + replay = object.__new__(ControllerReplay) + replay.factory = object.__new__(SelectedControllerFactory) + replay.failed, replay.raw_mode = False, True + replay.emissions = object.__new__(EmissionHistory) + replay.emissions.events, replay.emissions.carriers = OrderedDict(), OrderedDict() + replay.checkpoints = deque(((100, snapshot),)) + replay.records = deque((ReplayFrame(100, (), 0., ()),)) + object.__setattr__(replay.records[0], 'torque_values', (('x', Trap()),)) + if foreign == 'frame_object': + replay.records = deque((Trap(),)) + elif foreign == 'frame_clock': + object.__setattr__(replay.records[0], 'control_ns', Trap()) + elif foreign == 'checkpoint_clock': + replay.checkpoints = deque(((Trap(), snapshot),)) + elif foreign == 'container': + replay.records = ForeignDeque(replay.records) + with pytest.raises(ValueError): + probe_replay_snapshot(replay, (control,)) + assert calls == [] + + +@pytest.mark.parametrize('field', ['mode', 'policy']) +def test_factory_metadata_rejects_foreign_string_conversion_before_cp_access(field): + from openpilot.starpilot.flm.controller_replay import SelectedControllerFactory + from openpilot.starpilot.flm.live_tracking_probe import _factory_configuration + from openpilot.starpilot.lateral.controller_selection import ControllerMode + calls = [] + + class Trap: + def __str__(self): + calls.append('str') + return 'standard' + factory = object.__new__(SelectedControllerFactory) + factory.mode, factory.policy = ControllerMode.STANDARD, 'invalid-only' + setattr(factory, field, Trap()) + with pytest.raises(ValueError): + _factory_configuration(factory) + assert calls == [] + + +def test_genuine_ioniq6_native_namedtuple_projection_roundtrip_and_unknown_leaf_rejection(): + from opendbc.car.hyundai.values import CAR as HYUNDAI + from opendbc.car.steering_projection import TorqueProjection, projection_from_dict, projection_to_dict + from openpilot.starpilot.flm.tests.test_state_aligned_encoder import captured_event + from openpilot.starpilot.flm.live_tracking_probe import _projection_literal + # Real registered CI/CAN packer unit fixture; no permission/model verdict is + # mocked or obtained by copying this observation into restricted transport. + witness = captured_event(HYUNDAI.HYUNDAI_IONIQ_6).witness + assert type(witness) is TorqueProjection and witness.can_data + body = _projection_literal(witness) + assert body == projection_to_dict(witness) + assert projection_from_dict(body) == witness + calls = [] + + class Trap: + def __deepcopy__(self, memo): + calls.append('deepcopy') + return self + def __bytes__(self): + calls.append('bytes') + return b'' + with pytest.raises(ValueError): + _projection_literal(witness._replace(can_data=Trap())) + assert calls == [] + + +def test_historical_native_expiry_does_not_become_current_permission(): + from openpilot.starpilot.flm.live_tracking_probe import _native_lease, _evidence_clock + # Historical numeric clock/lease subchecks only, not counterfeit actual CAN. + control = SimpleNamespace(native_allowed=True, native_observed_ns=1_000_000_000, + control_ns=1_050_000_000, native_expires_ns=1_200_000_000) + output = SimpleNamespace(native_allowed=True, apply_ns=1_060_000_000) + binding = SimpleNamespace(native_required=True) + real_queue = 90_000_000_000 + historical, evidence = _evidence_clock(evidence_ns=1_650_000_000, queued_ns=real_queue, + now_ns=real_queue+500_000_000, current_native=False) + assert historical and evidence == 1_650_000_000 + _native_lease(control, output, binding, now_ns=real_queue+500_000_000, historical=True) + with pytest.raises(ValueError, match='native lease'): + _native_lease(control, output, binding, now_ns=real_queue+500_000_000, historical=False) + output.apply_ns = control.native_expires_ns+1 + with pytest.raises(ValueError, match='native lease'): + _native_lease(control, output, binding, now_ns=real_queue, historical=True) + + +@pytest.mark.parametrize('change', [{'current_native': True}, {'now_ns': 31_000_000_002}, + {'queued_ns': 2_000_000_000}, {'evidence_ns': 0}]) +def test_historical_numerical_clock_rejects_current_grant_and_invalid_queue(change): + from openpilot.starpilot.flm.live_tracking_probe import _evidence_clock + args = {'evidence_ns': 123, 'queued_ns': 1_000_000_000, 'now_ns': 1_500_000_000, 'current_native': False} + args.update(change) + with pytest.raises(ValueError): + _evidence_clock(**args) + + +def test_future_live_evidence_requires_explicit_archival_diagnostic_clock_domain(): + from openpilot.starpilot.flm.live_tracking_probe import _evidence_clock + args = {'evidence_ns': 9_000_000_000, 'queued_ns': 1_000_000_000, + 'now_ns': 1_500_000_000, 'current_native': False} + with pytest.raises(ValueError): + _evidence_clock(**args) + assert _evidence_clock(**args, archived_clock=True) == (True, 9_000_000_000) + with pytest.raises(ValueError): + _evidence_clock(**dict(args, current_native=True), archived_clock=True) + + +def test_normal_process_rejects_archival_diagnostic_jobs_before_spawn(monkeypatch): + from openpilot.starpilot.flm import live_tracking_probe_process as module + payload = {'queued_ns': 100, 'evidence_ns': 9_000_000_000, 'archived_clock': True} + monkeypatch.setattr(module, 'decode_probe_job', lambda job: payload) + process = ProbeProcess() + assert process.submit_probe({}, token=(), now_ns=100) is False + assert process.child is None + + +def test_stale_actual_queue_is_rejected_before_child_start(monkeypatch): + from openpilot.starpilot.flm import live_tracking_probe_process as module + monkeypatch.setattr(module, 'decode_probe_job', lambda job: {'queued_ns': 100}) + process = ProbeProcess() + assert process.submit_probe({}, token=(), now_ns=100+100_000_001) is False + assert process.child is None + + +def _transport_envelope(index): + """Deliberately INVALID replay/model bytes: wire-only, no authority fixture.""" + import hashlib + from openpilot.starpilot.flm.controller_replay import pack_primitive_snapshot + payload = {'version': VERSION, 'replay': f'invalid-transport-only-{index}'.encode(), + 'queued_ns': 100+index*50, 'evidence_ns': 10+index*50, 'archived_clock': False, + 'binding': {'invalid_transport_only': True}, 'accepted_full30': (), 'anchor': (), 'group': 'ff.left', + 'names': ('ff_gain_left',), 'prefix_controls': ({'control_ns': 1, 'replay_snapshot': (('invalid', 0.),)}, + {'control_ns': 10+index*10, 'replay_snapshot': ()}), + 'control': {'control_ns': 10+index*10, 'replay_snapshot': ()}, 'output': {'apply_ns': 11+index*10}} + packed = pack_primitive_snapshot(payload) + return {'version': VERSION, 'payload': packed.hex(), 'sha256': hashlib.sha256(packed).hexdigest()} + + +def test_batch_preserves_separately_observed_clocks_and_literal_snapshots(): + from openpilot.starpilot.flm.live_tracking_probe_process import make_probe_batch + jobs = tuple(_transport_envelope(index) for index in range(4)) + batch = decode_probe_job(make_probe_batch(jobs, queued_ns=300)) + assert batch['queued_ns'] == 300 + assert len(batch['batch']) == 4 + for index, row in enumerate(batch['batch']): + assert row == decode_probe_job(jobs[index]) + assert row['evidence_ns'] == 10+index*50 + assert row['queued_ns'] == 100+index*50 + # These invalid bytes MUST NOT be mistaken for a real child/controller proof. + from openpilot.starpilot.flm.live_tracking_probe_worker import evaluate_job + with pytest.raises((TypeError, ValueError)): + evaluate_job(make_probe_batch(jobs, queued_ns=300), now_ns=400) + + +def test_batch_rejects_duplicate_endpoints_and_unbounded_inputs(): + from openpilot.starpilot.flm.live_tracking_probe_process import make_probe_batch + first = _transport_envelope(0) + with pytest.raises(ValueError): + make_probe_batch((first, first), queued_ns=300) + with pytest.raises(ValueError): + make_probe_batch((first,)*5, queued_ns=300) + with pytest.raises(ValueError): + make_probe_batch((), queued_ns=300) + + +@pytest.mark.parametrize('reason', ['context', 'wall']) +def test_actual_optional_child_cancellation_reaps_without_blocking_publication(reason): + import os + import time + process = ProbeProcess() + process.worker_module = 'openpilot.starpilot.flm.tests._probe_lifecycle_fixture' + process.wall_ns = 200_000_000 + now = time.monotonic_ns() + try: + assert process.submit({'stall': True}, token=('original',), now_ns=now) + pid = process.child.pid + deadline = time.monotonic()+5. + while process.child is not None and time.monotonic() < deadline: + before = time.monotonic() + assert process.poll(token=('changed',) if reason == 'context' else ('original',), now_ns=time.monotonic_ns()) is None + assert time.monotonic()-before < .5 + time.sleep(.01) + assert process.child is None + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + finally: + process.close() + + +def test_actual_optional_child_applies_resource_and_scheduler_bounds(): + import os + import time + process = ProbeProcess() + process.worker_module = 'openpilot.starpilot.flm.tests._probe_lifecycle_fixture' + result = None + try: + assert process.submit({}, token=('resources',), now_ns=time.monotonic_ns()) + deadline = time.monotonic()+10. + while process.child is not None and time.monotonic() < deadline: + value = process.poll(token=('resources',), now_ns=time.monotonic_ns()) + if value is not None: + result = value + time.sleep(.01) + assert process.child is None and result is not None + assert 0 < result['cpu'] <= 20 and 0 < result['as'] <= 2*1024*1024*1024 + assert result['nice'] >= 10 and result['parent'] == os.getpid() + if hasattr(os, 'sched_getscheduler'): + assert result['scheduler'] == os.SCHED_OTHER + assert result['real_now'] <= time.monotonic_ns() + finally: + process.close() + + +@pytest.mark.parametrize('job', [None, {}, {'version': VERSION, 'payload': 'zz', 'sha256': 'x'}, + {'version': VERSION, 'payload': '00', 'sha256': '0'*64}]) +def test_malformed_or_changed_numerical_envelopes_rejected(job): + with pytest.raises(ValueError): + decode_probe_job(job) + + +def test_second_pending_job_is_rejected_without_inspecting_foreign_payload(): + process = ProbeProcess() + from subprocess import Popen + process.child = object.__new__(Popen) + assert process.submit_probe(object(), token=(), now_ns=1) is False + process.child = None + + +def test_child_context_change_cannot_return_owned_numerical_result(monkeypatch): + from openpilot.starpilot.flm.fit_process import FitProcess + process = ProbeProcess() + process.expected = ('a'*64, {}) + monkeypatch.setattr(FitProcess, 'poll', lambda *args, **kwargs: {'version': VERSION, 'sha256': 'b'*64, 'rows': []}) + with pytest.raises(ValueError, match='owned child'): + process.poll_probe(token=(), binding=None, now_ns=1) + + +@pytest.mark.parametrize('mode', ['standard', 'starpilot']) +@pytest.mark.parametrize('speed,curvature', [(30., 0.), (5., .005)]) +def test_genuine_ioniq6_configured_assist_snapshot_replays_exact_actual_controller(mode, speed, curvature): + from opendbc.car import structs + from openpilot.starpilot.flm.tests.test_controller_replay import _params + from openpilot.starpilot.flm.controller_replay import ( + SelectedControllerFactory, capture_snapshot, restore_snapshot, frame_input, _run, same_final_command, + ) + from openpilot.starpilot.flm.torque_adapter import Objective + from openpilot.starpilot.flm.runtime_adapter_registry import selected_descriptor + factory = SelectedControllerFactory(_params().as_reader(), mode, turn_assist=True) + actual, vm = factory.create() + clone, clone_vm = factory.create() + assert factory.turn_assist is True + descriptor = selected_descriptor(factory.cp, factory.interface_class(factory.cp), actual) + assert descriptor.turn_assist is True + cs = structs.CarState.new_message(vEgo=speed, gearShifter='drive', steeringAngleDeg=0.) + params = SimpleNamespace(angleOffsetDeg=0., roll=0.) + before = capture_snapshot(actual, vm) + assert any(name.startswith('turn_assist.') for name, _ in before.values) + if mode == 'starpilot': + assert dict(before.values)['policy.turn_assist'] == 1. + inputs = frame_input(cs, vm, params, active=True, safety_limited=False, + desired_curvature=curvature, curvature_limited=False, lat_delay=.2) + expected, _ = _run(actual, vm, inputs, Objective('feedforward', 'low', 'steady'), (0., 0.), + actual_stage_residual=(0., 0.)) + restore_snapshot(clone, clone_vm, before) + reproduced, _ = _run(clone, clone_vm, inputs, Objective('feedforward', 'low', 'steady'), (0., 0.), + actual_stage_residual=(0., 0.)) + assert same_final_command(expected, reproduced) + assert capture_snapshot(clone, clone_vm) == capture_snapshot(actual, vm) + assert factory.turn_assist is True + # Configuration differs in exact source receipts; it is never rewritten False. + assert replace(descriptor, turn_assist=False) != descriptor + + +def test_genuine_ioniq6_assist_can_be_noop_or_active_without_changing_configuration(): + from opendbc.car import structs + from openpilot.starpilot.flm.tests.test_controller_replay import _params + from openpilot.starpilot.flm.controller_replay import SelectedControllerFactory + factory = SelectedControllerFactory(_params().as_reader(), 'standard', turn_assist=True) + controller, vm = factory.create() + cs = structs.CarState.new_message(vEgo=30., gearShifter='drive', steeringAngleDeg=0.) + params = SimpleNamespace(angleOffsetDeg=0., roll=0.) + assert controller.turn_assist.apply(cs, vm, params, 0., .1, 1.) == .1 + cs.vEgo = 5. + assert controller.turn_assist.apply(cs, vm, params, .005, 0., 1.) != 0. + assert factory.turn_assist is True diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_recovery_null_adverse.py b/openpilot/starpilot/flm/tests/test_live_tracking_recovery_null_adverse.py new file mode 100644 index 0000000000..61593ca0d6 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_recovery_null_adverse.py @@ -0,0 +1,89 @@ +"""Interrupted numerical null/adverse stress; no native/source authority claim. + +The seed ranges, interruption cadence and acceptance gates are declared here +before execution. This suite is an empirical regression, not a probability +bound under arbitrary correlated road observations. +""" +from dataclasses import asdict, dataclass, field +import json + +import pytest + +from openpilot.starpilot.flm.live_tracking import LiveTrackingOptimizer, TrackingEpisode +from openpilot.starpilot.flm.tests.test_live_tracking_rollout import rollout + +SEEDS = range(64) # Same fixed size/range as the original stationary null sweep. +INTERRUPT_EVERY = 11 # Same cadence as the original interrupted progress test. + + +@dataclass +class RecoveryCoverage: + allocation_calls: int = 0 + allocations: list[tuple[int, int]] = field(default_factory=list) + recovery_launches: int = 0 + + +def interrupted_case(monkeypatch: pytest.MonkeyPatch, *, seed: int, gain: float): + coverage = RecoveryCoverage() + original_allocation = LiveTrackingOptimizer._comparison_allocation + original_observe = LiveTrackingOptimizer.observe_episode + + def allocation(owner: LiveTrackingOptimizer, episode: TrackingEpisode, expected: float, + target: dict[str, float]) -> tuple[int, int] | None: + result = original_allocation(owner, episode, expected, target) + coverage.allocation_calls += 1 + if result is not None: + coverage.allocations.append(result) + return result + + def observe(owner: LiveTrackingOptimizer, episode: TrackingEpisode, *, now_ns: int) -> bool: + selection = owner.discovery + prior = owner.offer_state + result = original_observe(owner, episode, now_ns=now_ns) + state = owner.offer_state + if (selection is not None and selection['recovery'] and state is not None and state['phase'] == 'trial' and + (prior is None or prior['trial_id'] != state['trial_id'])): + coverage.recovery_launches += 1 + return result + + # The observer returns the original decisions unchanged. The context always + # restores both methods, including when the original helper raises. + with monkeypatch.context() as patch: + patch.setattr(LiveTrackingOptimizer, '_comparison_allocation', allocation) + patch.setattr(LiveTrackingOptimizer, 'observe_episode', observe) + result = rollout(seed=seed, gain=gain, interrupt_every=INTERRUPT_EVERY) + return result, coverage + + +def sweep(monkeypatch: pytest.MonkeyPatch, *, gain: float): + cases = [interrupted_case(monkeypatch, seed=seed, gain=gain) for seed in SEEDS] + results = [result for result, _ in cases] + coverage = [item for _, item in cases] + failures = [row for row in results if row['confirmations'] or row['accepted_coordinate'] != 0.] + summary = {'gain': gain, 'seeds': [SEEDS.start, SEEDS.stop], 'routes': len(results), + 'interrupt_every': INTERRUPT_EVERY, 'false_acceptance_routes': len(failures), + 'attempts': sum(len(row['attempts']) for row in results), + 'interruptions': sum(row['interruptions'] for row in results), + 'allocation_calls': sum(item.allocation_calls for item in coverage), + 'successful_allocations': sum(len(item.allocations) for item in coverage), + 'recovery_launches': sum(item.recovery_launches for item in coverage), + 'route_coverage': [dict(seed=seed, **asdict(item)) for seed, item in zip(SEEDS, coverage, strict=True)], + 'failures': failures} + print('NUMERICAL_INTERRUPTED_RECOVERY_SWEEP '+json.dumps(summary, sort_keys=True)) + # False-acceptance checks must actually exercise the newly allocated path, + # rather than passing through a learner that never creates a trial. + assert all(row['attempts'] for row in results), summary + assert any(row['interruptions'] for row in results) and any(item.allocation_calls for item in coverage), summary + assert any(item.allocations for item in coverage) and any(item.recovery_launches for item in coverage), summary + return results, summary + + +def test_interrupted_correlated_null_preserves_original_family_false_acceptance_gate(monkeypatch: pytest.MonkeyPatch): + results, summary = sweep(monkeypatch, gain=0.) + false_acceptances = [row for row in results if row['confirmations']] + assert len(false_acceptances) <= 3, summary # Original stationary null criterion. + + +def test_interrupted_correlated_adverse_trials_never_accumulate_wrong_correction(monkeypatch: pytest.MonkeyPatch): + results, summary = sweep(monkeypatch, gain=-5.) + assert all(row['confirmations'] == 0 and row['accepted_coordinate'] == 0. for row in results), summary diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_report.py b/openpilot/starpilot/flm/tests/test_live_tracking_report.py new file mode 100644 index 0000000000..d295e52a7d --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_report.py @@ -0,0 +1,77 @@ +"""Actual report writer/reader contract; all source points below are synthetic.""" +import json +from types import SimpleNamespace + +import pytest + +from openpilot.starpilot.flm.training_report import TrainingReporter, read_report +from openpilot.starpilot.flm.tests.test_live_tracking_rollout import _optimizer, _refresh, NS + + +def test_actual_report_writer_reader_preserve_tracking_values_and_original_points(tmp_path): + now = 100*NS + owner = _optimizer() + _refresh(owner, now) + points = [{'controlMonoTime': now-NS+index*50_000_000, + 'applyMonoTime': now-NS+index*50_000_000, + 'poseMonoTime': now-NS+index*50_000_000+170_000_000, + 'desiredForceAccel': .2, 'measuredForceAccel': .1, 'errorForceAccel': .2-.1} for index in range(4)] + plot = {'version': 'live-tracking-paired-plot-v1', 'episodeId': '1'*64, + 'profileSha256': owner._hash(owner.accepted), 'lagNs': 170_000_000, 'units': 'm/s²', 'sampled': False, + 'totalEndpoints': 4, 'points': points, 'bindingCurrent': True, + 'ageSeconds': (now-points[-1]['poseMonoTime'])/NS} + live = SimpleNamespace(latest_offer=None, report=lambda stamp: {'tracking': owner.report(stamp), + 'trackingValues': owner.tracking_values(), 'trackingPlot': plot}) + collector = SimpleNamespace(adapter=SimpleNamespace(raw_collection=False, event_collection=False, objective=None), + blocks=(), points=(), session='synthetic-test-only', sealed=False, reason='collecting', + training_interruptions={}, dropped=0, live_tracking=live) + source = SimpleNamespace(enabled=True, calibration_generation='synthetic', observed_ns=now) + message = SimpleNamespace(autoSteeringLearning=SimpleNamespace(learning=SimpleNamespace(status='collecting', reason='synthetic'))) + path = tmp_path/'producer.json' + TrainingReporter('synthetic test source', path=path).publish(now_ns=now, message=message, + collector=collector, source=source, objective='synthetic', pending=False, native_current=False) + actual = read_report(path, now_ns=now) + assert actual['current'] is not None + current = actual['current'] + assert current['tracking'] == owner.report(now) + assert current['trackingValues'] == owner.tracking_values() + assert current['trackingPlot'] == plot + assert current['application']['applied'] is False + assert len(path.read_bytes()) <= 32768 + # Root may retain this exact read_report result for the canonical JS consumer. + (tmp_path/'producer-read-report.json').write_text(json.dumps(actual, allow_nan=False)) + stale = read_report(path, now_ns=now+3*NS)['current'] + assert stale['tracking']['bindingCurrent'] is False + assert stale['tracking']['applicationAcknowledged'] is False + assert stale['trackingPlot']['bindingCurrent'] is False + assert stale['trackingValues']['accepted'] == current['trackingValues']['accepted'] + + +@pytest.mark.parametrize('field,value', [('poseMonoTime', 0), ('applyMonoTime', 10**20)]) +def test_tracking_plot_timestamp_malformed_data_is_not_renderable(field, value): + from openpilot.starpilot.flm.live_tracking_report import tracking_objects + point = {'controlMonoTime': 10, 'applyMonoTime': 11, 'poseMonoTime': 12, + 'desiredForceAccel': .2, 'measuredForceAccel': .1, 'errorForceAccel': .2-.1} + point[field] = value + plot = {'version': 'live-tracking-paired-plot-v1', 'episodeId': '1'*64, 'profileSha256': '2'*64, + 'lagNs': 0, 'units': 'm/s²', 'sampled': False, 'totalEndpoints': 4, 'points': [point]*4, + 'bindingCurrent': True, 'ageSeconds': 0.} + with pytest.raises(ValueError, match='timestamps'): + tracking_objects({'trackingPlot': plot}, fresh=True) + + +@pytest.mark.parametrize('extra_ns,accepted', [(0, True), (30_000_000, True), (30_000_001, False), (-1, False)]) +def test_report_plot_preserves_exact_lag_bracket_boundary(extra_ns, accepted): + from openpilot.starpilot.flm.live_tracking_report import tracking_objects + points = [{'controlMonoTime': 1_000_000_000+index*50_000_000, + 'applyMonoTime': 1_000_000_000+index*50_000_000, + 'poseMonoTime': 1_170_000_000+index*50_000_000+extra_ns, + 'desiredForceAccel': .2, 'measuredForceAccel': .1, 'errorForceAccel': .2-.1} for index in range(4)] + plot = {'version': 'live-tracking-paired-plot-v1', 'episodeId': '1'*64, 'profileSha256': '2'*64, + 'lagNs': 170_000_000, 'units': 'm/s²', 'sampled': False, 'totalEndpoints': 4, 'points': points, + 'bindingCurrent': True, 'ageSeconds': 0.} + if accepted: + assert tracking_objects({'trackingPlot': plot}, fresh=True)['trackingPlot'] == plot + else: + with pytest.raises(ValueError, match='timestamps'): + tracking_objects({'trackingPlot': plot}, fresh=True) diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_requalification.py b/openpilot/starpilot/flm/tests/test_live_tracking_requalification.py new file mode 100644 index 0000000000..75c0f0a6a2 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_requalification.py @@ -0,0 +1,620 @@ +"""Synthetic numerical/protocol tests; never actual native or road proof.""" + +from dataclasses import replace +from openpilot.starpilot.flm.runtime_adapter_registry import digest +import pytest +import json + +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding +from openpilot.starpilot.flm.live_tracking_response import ( + ResponseCertificate, + VERSION, + _revision, + validate_certificate, + physical_calibration_generation, +) +from openpilot.starpilot.flm.live_tracking_requalification import ( + StableResponseBasis, + RetainedResponseRequalifier, + _primitive_key, + _rebound, + compatible_basis, + retained_to_dict, + retained_from_dict, +) + + +def certificate(): + binding = LiveTrialBinding( + 'a' * 64, + 'b' * 64, + 'actual-controller-registry-v2:' + 'c' * 64, + 'd' * 64, + 'e' * 64, + 'drive1-reset', + 'f' * 32, + digest((15.0, 100000.0, 110000.0)), + '', + '0' * 64, + 'b' * 64, + False, + ) + raw = ResponseCertificate( + VERSION, + binding, + '0' * 64, + 170_000_000, + 3.0, + 0.0, + -0.3, + 0.3, + 0.005, + ('train0', 'train1', 'train2', 'train3'), + ('hold0', 'hold1', 'hold2', 'hold3'), + 'c' * 64, + 'd' * 64, + 1, + ) + revision = _revision(raw) + return validate_certificate(replace(raw, model_revision=revision, binding=replace(binding, physical_model_revision=revision))) + + +def basis(cert): + b = cert.binding + return StableResponseBasis( + b.identity, + b.cp_sha256, + b.controller_revision, + b.baseline_revision, + (0.0, 0.01, -0.01), + (15.0, 100000.0, 110000.0, 15.0, 2.8, 1.1, 1700.0), + ('ordinary-hardware-v1', ('measured-hardware-type',), 0, (('hyundai', 0),), ((0, 'hyundai', 0, 0),)), + False, + ) + + +def new_drive(cert): + return replace( + cert.binding, identity='6' * 64, source_epoch='2' * 64, reset_generation='new-UUID', calibration_generation='3' * 32, persistent_source_epoch='5' * 64 + ) + + +def test_session_and_generation_rotation_does_not_require_eight_new_identification_slots(): + old = certificate() + owner = RetainedResponseRequalifier(old, basis(old), new_drive(old)) + assert owner.original is old + assert owner._candidate.binding.source_epoch == '2' * 64 + assert owner.qualified(10_000_000_000) is None + assert owner.reason == 'fresh_independent_requalification_required' + assert not hasattr(owner, 'current_permission') + assert not hasattr(owner, 'profile') + + +@pytest.mark.parametrize( + 'name,value', + [ + ('cp_sha256', '1' * 64), + ('controller_revision', 'other-controller'), + ('baseline_revision', '1' * 64), + ('native_required', True), + ], +) +def test_actual_car_tune_controller_or_native_domain_change_cannot_rebind(name, value): + old = certificate() + changed = replace(new_drive(old), **{name: value}) + if name == 'native_required': + changed = replace(changed, native_identity='1' * 64) + with pytest.raises(ValueError, match='car/controller/tune/native domain'): + RetainedResponseRequalifier(old, basis(old), changed) + + +@pytest.mark.parametrize( + 'name,value', + [ + ('identity', '1' * 64), + ('cp_sha256', '1' * 64), + ('controller_revision', 'other-controller'), + ('baseline_revision', '1' * 64), + ('native_required', True), + ], +) +def test_historical_basis_cannot_be_attached_to_a_different_model(name, value): + old = certificate() + with pytest.raises(ValueError, match='retained certificate/basis'): + RetainedResponseRequalifier(old, replace(basis(old), **{name: value}), new_drive(old)) + + +@pytest.mark.parametrize( + 'name,value', + [ + ('calibration_rpy', (0.0, 0.02, -0.01)), + ('vehicle_model', (15.0, 100000.0, 110000.0, 16.0, 2.8, 1.1, 1700.0)), + ('hardware', ('different-measured-inventory',)), + ], +) +def test_measured_calibration_geometry_or_hardware_is_not_an_ephemeral_generation(name, value): + old = certificate() + original = basis(old) + assert original.key() != replace(original, **{name: value}).key() + + +@pytest.mark.parametrize('bad', [float('nan'), float('inf'), True, object(), [1]]) +def test_basis_rejects_nonfinite_custom_or_mutable_physical_leaves(bad): + old = certificate() + with pytest.raises(ValueError): + replace(basis(old), calibration_rpy=(bad, 0.0, 0.0)).key() + + +def test_typed_context_never_coerces_signed_zero_or_bool_int(): + assert _primitive_key((True,)) != _primitive_key((1,)) + assert _primitive_key((0.0,)) != _primitive_key((-0.0,)) + + +def test_rebound_keeps_historical_identification_and_marks_only_numerical_provenance(): + old = certificate() + rebound = _rebound(old, new_drive(old), '6' * 64) + assert rebound.provenance == 'retained-current-residual-v1' + assert rebound.parent_model_revision == old.model_revision + assert rebound.requalification_sha256 == '6' * 64 + assert rebound.model_revision != old.model_revision + assert rebound.identified_ns == old.identified_ns + assert rebound.train_block_ids == old.train_block_ids + assert rebound.heldout_block_ids == old.heldout_block_ids + assert rebound.identification_sha256 == old.identification_sha256 + assert rebound.source_inventory_sha256 == old.source_inventory_sha256 + assert (rebound.slope, rebound.intercept, rebound.lag_ns) == (old.slope, old.intercept, old.lag_ns) + + +def test_missing_actual_pose_native_or_hardware_cannot_qualify_retained_model(): + old = certificate() + owner = RetainedResponseRequalifier(old, basis(old), new_drive(old)) + assert not owner.observe(object(), object(), object(), now_ns=1, current_native=False) + assert owner.qualified(1) is None + assert not owner._rows + assert not owner._monitor.rows + + +def test_vm_uses_existing_measured_two_percent_regime_not_a_session_generation(): + original = basis(certificate()) + nearby = replace(original, vehicle_model=(15.1, 100500.0, 110100.0, *original.vehicle_model[3:])) + outside = replace(original, vehicle_model=(16.0, *original.vehicle_model[1:])) + assert compatible_basis(original, nearby) + assert not compatible_basis(original, outside) + assert not compatible_basis(original, replace(nearby, calibration_rpy=(0.0, 0.02, -0.01))) + + +def test_historical_model_json_roundtrip_never_contains_a_pending_trial_or_permission(): + original = certificate() + raw = json.loads(json.dumps(retained_to_dict(original, basis(original)))) + restored, measured = retained_from_dict(raw) + assert restored == original + assert measured.key() == basis(original).key() + assert raw["current_permission"] is False + assert raw["pending_trial"] is None + assert RetainedResponseRequalifier(restored, measured, new_drive(restored)).qualified(1) is None + + +@pytest.mark.parametrize("changes", [{"current_permission": True}, {"pending_trial": {"target": 1}}, {"unknown": 1}]) +def test_history_never_restores_pending_target_or_permission(changes): + original = certificate() + raw = retained_to_dict(original, basis(original)) + raw.update(changes) + with pytest.raises(ValueError, match="historical numerical model"): + retained_from_dict(raw) + + +def test_frozen_bypass_basis_mutation_cannot_renew_cached_compatibility(): + original = certificate() + measured = basis(original) + owner = RetainedResponseRequalifier(original, measured, new_drive(original)) + object.__setattr__(measured, "hardware", ("swapped-hardware",)) + assert not owner.observe(object(), object(), object(), now_ns=1, current_native=True) + assert owner.reason == "retained stable basis mutated" + with pytest.raises(ValueError, match="retained stable basis mutated"): + owner.qualified(1) + + +def synthetic_pair(owner, index, *, command=None): + # Genuine evidence classes/validators, but explicitly synthetic clock/native + # fixtures. This checks the protocol path, never road or native acceptance. + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence + from openpilot.starpilot.flm.original_source_capture import OriginalSourcePose + from opendbc.car.steering_projection import TorqueProjection + + b = owner._binding + mono = 10_000_000_000 + index * 50_000_000 + apply = mono - owner.original.lag_ns + value = (0.1 if index % 2 else -0.1) if command is None else command + raw = int(-value * 1000) + limits = ( + ("STEER_DELTA_UP", 10.0), + ("STEER_DELTA_DOWN", 10.0), + ("STEER_DRIVER_ALLOWANCE", 50.0), + ("STEER_DRIVER_MULTIPLIER", 2.0), + ("STEER_DRIVER_FACTOR", 1.0), + ) + projection = TorqueProjection( + 1, + "hyundai", + "synthetic", + 0, + "hyundai.LKAS11", + "round", + index, + -value, + -value, + raw, + raw, + 1000.0, + (), + (), + "driver", + -value, + 0.0, + limits, + (), + True, + 832, + 0, + b"12345678", + -value, + ) + control = ControlEvidence( + b.identity, + 1, + apply, + apply - 1_000_000, + "normalized_torque", + "stage", + "response", + -value, + (), + 0.0, + True, + False, + True, + True, + False, + False, + False, + False, + False, + False, + source_epoch=b.source_epoch, + baseline_revision=b.baseline_revision, + controller_revision=b.controller_revision, + native_required=False, + native_identity="", + vehicle_model_anchor=(15.0, 100000.0, 110000.0), + replay_inputs=(("vm.sR", 15.0), ("vm.cF", 100000.0), ("vm.cR", 110000.0)), + ) + output = AppliedEvidence(apply, apply - 1_000_000, apply, "normalized_torque", -value, True, True, False, False, False, True, projection=projection) + config = (0, (("hyundai", 0),), ((0, "hyundai", 0, 0, True, False, False, ()),)) + context = ( + "new-session", + b.identity, + 1, + b.source_epoch, + b.baseline_revision, + b.controller_revision, + physical_calibration_generation(b), + ("clock-generation", 2), + ("synthetic",), + "", + (15.0, 100000.0, 110000.0), + 15.0, + 2.8, + 1.1, + 1700.0, + ) + pose = OriginalSourcePose( + mono, + mono, + 0, + 0, + 0, + physical_calibration_generation(b), + "7" * 64, + context, + 3 * value / 20.0, + 0.0, + "", + True, + True, + b.source_epoch, + b.identity, + b.baseline_revision, + b.controller_revision, + mono - 1_000_000, + mono - 1_000_000, + observed_ns=mono, + completed_ns=mono, + native_required=False, + ordinary_safety=("ordinary-panda-current-v1", b.cp_sha256, mono, mono, config), + source_lower_ns=mono, + source_upper_ns=mono, + source_timestamp_ns=mono, + source_clock_domain="monotonic", + source_clock_contract=("monotonic", "8" * 64, "9" * 64, "synthetic-fixture"), + calibration_rpy=(0.0, 0.01, -0.01), + angular_velocity_device=(0.0, 0.0, 3 * value / 20.0), + orientation_ned=(0.0, 0.0, 0.0), + speed_mps=20.0, + left_state_ns=mono, + right_state_ns=mono, + left_state=(20.0, 0.0), + right_state=(20.0, 0.0), + ) + return control, output, pose + + +def feed(owner, index, *, command=None, current_native=True): + control, output, pose = synthetic_pair(owner, index, command=command) + return owner.observe(control, output, pose, now_ns=pose.mono_ns, current_native=current_native, hardware_inventory=("measured-hardware-type",)) + + +def test_actual_evidence_validators_accept_fresh_synthetic_pairs_without_relabelling_eight_blocks(): + old = certificate() + owner = RetainedResponseRequalifier(old, basis(old), new_drive(old)) + for index in range(7): + assert feed(owner, index) + assert owner.qualified(10_300_000_000) is None + assert feed(owner, 7) + cert, witness = owner.qualified(10_350_000_000) + assert cert.provenance == "retained-current-residual-v1" + assert cert.train_block_ids == old.train_block_ids + assert cert.heldout_block_ids == old.heldout_block_ids + assert cert.identified_ns == old.identified_ns + assert witness.independent_poses == 8 + assert witness.binding.source_epoch == "2" * 64 + assert owner.qualified(10_450_000_001) is None + + +@pytest.mark.parametrize("command", [0.0, 0.001, 0.1]) +def test_constant_encoded_command_cannot_define_a_local_range(command): + old = certificate() + owner = RetainedResponseRequalifier(old, basis(old), new_drive(old)) + for index in range(8): + assert feed(owner, index, command=command) + assert owner.qualified(10_350_000_000) is None + assert owner.reason == 'fresh_local_range_required' + + +def test_small_one_sided_highway_tracking_requalifies_only_its_fresh_encoded_local_scope(): + old = certificate() + owner = RetainedResponseRequalifier(old, basis(old), new_drive(old)) + for index in range(8): + assert feed(owner, index, command=0.010 if index % 2 else 0.011) + cert, witness = owner.qualified(10_350_000_000) + assert (cert.command_low, cert.command_high) == (0.010, 0.011) + assert cert.command_low > old.command_low + assert cert.command_high < old.command_high + assert cert.slope == old.slope + assert cert.parent_model_revision == old.model_revision + assert witness.independent_poses == 8 + # Publication and a new passing window cannot silently expand this scope. + assert feed(owner, 8, command=0.010) + same, _ = owner.qualified(10_400_000_000) + assert same is cert + assert not feed(owner, 9, command=0.012) + assert owner.qualified(10_450_000_000) is None + assert owner._frozen_certificate is cert + + +def test_local_scope_cannot_expand_old_independently_identified_support(): + old = certificate() + for proposed in ((old.command_low - 0.001, 0.1), (-0.1, old.command_high + 0.001), (0.1, 0.1)): + with pytest.raises(ValueError, match='cannot extrapolate'): + _rebound(old, new_drive(old), '6' * 64, command_range=proposed) + + +def test_current_native_loss_erases_fresh_rows_without_erasing_historical_model(): + old = certificate() + owner = RetainedResponseRequalifier(old, basis(old), new_drive(old)) + for index in range(8): + assert feed(owner, index) + assert owner.qualified(10_350_000_000) is not None + assert not feed(owner, 8, current_native=False) + assert owner.qualified(10_400_000_000) is None + assert owner.original is old + + +def test_capture_epoch_rotation_alone_can_requalify_with_fresh_pairs(): + old = certificate() + rotated = replace(old.binding, source_epoch="2" * 64) + owner = RetainedResponseRequalifier(old, basis(old), rotated) + for index in range(8): + assert feed(owner, index) + cert, witness = owner.qualified(10_350_000_000) + assert cert.binding.reset_generation == old.binding.reset_generation + assert witness.binding.source_epoch != old.binding.source_epoch + + +@pytest.mark.parametrize("bad", ["residual", "manual", "limited", "calibration", "controller", "lag"]) +def test_fresh_requalification_rejects_actual_evidence_guard_failures(bad): + old = certificate() + owner = RetainedResponseRequalifier(old, basis(old), new_drive(old)) + control, output, pose = synthetic_pair(owner, 0) + if bad == "residual": + pose = replace(pose, yaw_rate=pose.yaw_rate + 0.1) + # A single outlier may be collected; it is never a physical witness. + # Current policy tests the complete independent rolling RMS window. + assert owner.observe(control, output, pose, now_ns=pose.mono_ns, current_native=True, hardware_inventory=("measured-hardware-type",)) + assert owner.qualified(pose.mono_ns) is None + for index in range(1, 8): + assert feed(owner, index) + assert owner.qualified(10_000_000_000 + index * 50_000_000) is None + assert owner._monitor.witness(10_350_000_000) is None + assert owner._frozen_certificate is None + return + elif bad == "manual": + control = replace(control, manual=True) + elif bad == "limited": + output = replace(output, limited=True) + elif bad == "calibration": + pose = replace(pose, calibration_rpy=(0.0, 0.02, -0.01)) + elif bad == "controller": + control = replace(control, controller_revision="wrong-controller") + else: + pose = replace(pose, source_lower_ns=pose.mono_ns - 40_000_000) + assert not owner.observe(control, output, pose, now_ns=pose.mono_ns, current_native=True, hardware_inventory=("measured-hardware-type",)) + assert owner.qualified(pose.mono_ns) is None + + +def test_axis_session_can_change_only_with_fresh_native_inventory_and_residuals(): + from openpilot.starpilot.flm.runtime_adapter_registry import native_identity_digest + + key = ("old-axis-session", "actual-serial", 1, 0, ((0, "actual-serial", 1, 0, 0),)) + old = certificate() + raw = replace( + old, + model_revision="0" * 64, + binding=replace(old.binding, native_required=True, native_identity=native_identity_digest(key), physical_model_revision="0" * 64), + ) + revision = _revision(raw) + old = validate_certificate(replace(raw, model_revision=revision, binding=replace(raw.binding, physical_model_revision=revision))) + measured = replace(basis(old), native_required=True, hardware=("axis-hardware-v1", *key[1:])) + new_key = ("new-axis-session", *key[1:]) + binding = replace(new_drive(old), native_identity=native_identity_digest(new_key)) + owner = RetainedResponseRequalifier(old, measured, binding) + assert owner.qualified(1) is None + for index in range(8): + control, output, pose = synthetic_pair(owner, index) + control = replace( + control, + native_required=True, + native_identity=binding.native_identity, + native_allowed=True, + native_observed_ns=control.control_ns - 1_000_000, + native_expires_ns=control.control_ns + 100_000_000, + ) + pose = replace( + pose, + native_required=True, + native_identity=binding.native_identity, + native_control_ns=pose.mono_ns - 1_000_000, + native_observed_ns=pose.mono_ns - 2_000_000, + native_expires_ns=pose.mono_ns + 100_000_000, + ordinary_safety=(), + ) + assert owner.observe(control, output, pose, now_ns=pose.mono_ns, current_native=True, native_key=new_key) + rebound, witness = owner.qualified(10_350_000_000) + assert rebound.binding.native_identity == native_identity_digest(new_key) + assert rebound.parent_model_revision == old.model_revision + assert witness.independent_poses == 8 + changed_hardware = (new_key[0], "different-serial", *new_key[2:]) + assert not owner.observe(control, output, pose, now_ns=pose.mono_ns, current_native=True, native_key=changed_hardware) + assert owner.qualified(pose.mono_ns) is None + + +def test_raw_live_binding_calibration_and_physical_pose_hash_are_distinct_actual_representations(): + original = certificate() + owner = RetainedResponseRequalifier(original, basis(original), new_drive(original)) + _, _, pose = synthetic_pair(owner, 0) + assert len(owner._binding.calibration_generation) == 32 + assert len(pose.calibration) == 64 + assert pose.calibration != owner._binding.calibration_generation + assert pose.calibration == physical_calibration_generation(owner._binding) + + +def test_selected_identity_is_ephemeral_but_historical_identity_provenance_remains_exact(): + original = certificate() + current = new_drive(original) + assert current.identity != original.binding.identity + owner = RetainedResponseRequalifier(original, basis(original), current) + for index in range(8): + assert feed(owner, index) + cert, witness = owner.qualified(10_350_000_000) + assert cert.binding.identity == current.identity + assert original.binding.identity == basis(original).identity + assert witness.binding.identity == current.identity + + +def test_monotone_startup_does_not_freeze_a_range_before_original_input_revisits_it(): + old = certificate() + owner = RetainedResponseRequalifier(old, basis(old), new_drive(old)) + for index in range(8): + assert feed(owner, index, command=0.010 + 0.001 * index) + assert owner.qualified(10_350_000_000) is None + assert owner.reason == 'fresh_local_range_revisit_required' + assert owner._frozen_certificate is None + assert feed(owner, 8, command=0.016) + assert owner.qualified(10_400_000_000) is None + assert owner._frozen_certificate is None + assert feed(owner, 9, command=0.017) + cert, witness = owner.qualified(10_450_000_000) + assert (cert.command_low, cert.command_high) == (0.010, 0.017) + assert cert.command_low >= old.command_low and cert.command_high <= old.command_high + assert witness.independent_poses == 10 + assert cert.slope == old.slope and cert.lag_ns == old.lag_ns + assert not feed(owner, 10, command=0.018) + assert owner.qualified(10_500_000_000) is None + assert owner._frozen_certificate is cert + + +def test_unfinished_restore_retries_only_input_scope_with_no_monitor_or_permission_reuse(): + from types import SimpleNamespace + from openpilot.starpilot.flm.steerlearnerd import _LiveHistoryRestore + + old = certificate() + current = new_drive(old) + owner = RetainedResponseRequalifier(old, basis(old), current) + for index in range(8): + assert feed(owner, index, command=0.010 if index % 2 else 0.011) + local, _ = owner.qualified(10_350_000_000) + assert not feed(owner, 8, command=0.020) + assert owner.reason == 'response_command_outside_identified_range' + h = _LiveHistoryRestore(()) + h.prior = SimpleNamespace(certificate=old, response_basis=basis(old)) + h.requalifier = owner + h.offer = object() + calls = [] + ingress = SimpleNamespace(optimizer=SimpleNamespace(offer_state=None), monitor=owner._monitor) + actor = SimpleNamespace( + seed=current, + restore_pending=True, + ingress=ingress, + certificate=local, + application=object(), + initial_basis=object(), + interrupt=lambda *args, **kw: calls.append((args, kw)), + hold_restore=lambda value: calls.append(value), + ) + collector = SimpleNamespace(live_tracking=ingress, original_source_capture=SimpleNamespace(rows=[SimpleNamespace(mono_ns=10_400_000_000)])) + assert h._retry_local_scope(actor, collector, current) + assert actor.ingress is actor.certificate is actor.application is actor.initial_basis is None + assert collector.live_tracking is h.offer is None + assert h.last_pose_ns == 10_400_000_000 + assert h.requalifier is not owner and h.requalifier.original is old + assert not h.requalifier._rows and h.requalifier.qualified(10_400_000_000) is None + assert h.blocked_seed is None and not h.restored + assert calls[0] == (('retained_local_input_scope_left',), {'context_changed': True}) + assert calls[-1] is True + # The old issued certificate NEVER grows, including after the new owner + # measures a distinct local range under fresh physical/source validation. + for index in range(9, 17): + assert feed(h.requalifier, index, command=0.020 if index % 2 else 0.021) + fresh, _ = h.requalifier.qualified(10_800_000_000) + assert (fresh.command_low, fresh.command_high) == (0.020, 0.021) + assert (local.command_low, local.command_high) == (0.010, 0.011) + assert fresh.model_revision != local.model_revision + + +@pytest.mark.parametrize( + 'reason', ('physical_response_drift', 'response_current_source_or_native_unavailable', 'response_original_source_pair_unqualified', 'response_source_gap') +) +def test_restore_scope_retry_never_reclassifies_faults_as_input_departure(reason): + from types import SimpleNamespace + from openpilot.starpilot.flm.steerlearnerd import _LiveHistoryRestore + + h = _LiveHistoryRestore(()) + h.prior = object() + h.requalifier = object() + actor = SimpleNamespace( + restore_pending=True, + ingress=SimpleNamespace(optimizer=SimpleNamespace(offer_state=None), monitor=SimpleNamespace(reason=reason)), + certificate=object(), + seed='unchanged', + ) + assert not h._retry_local_scope(actor, object(), 'unchanged') + assert h.prior is not None and h.requalifier is not None diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_residual_daemon.py b/openpilot/starpilot/flm/tests/test_live_tracking_residual_daemon.py new file mode 100644 index 0000000000..5f8c4ff655 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_residual_daemon.py @@ -0,0 +1,88 @@ +"""Restore-attempt quarantine; numerical lifecycle only, no native permission.""" +from dataclasses import replace +from types import SimpleNamespace + +from openpilot.starpilot.flm.steerlearnerd import _LiveHistoryRestore +from openpilot.starpilot.flm.tests.test_live_trial_owner import offer + + +def actor_for(binding): + holds = [] + actor = SimpleNamespace(seed=binding, response_recovery_generation=0, + binding_for=lambda collector, source: binding, hold_restore=holds.append) + return actor, holds + + +def test_new_response_recovery_clears_only_attempt_and_survives_collector_reset(monkeypatch): + binding = offer().binding + historical_rows = (object(),) + history = _LiveHistoryRestore(historical_rows) + actor, holds = actor_for(binding) + history.seed = binding + placeholder = object() + monkeypatch.setattr(history, 'prior', placeholder) + monkeypatch.setattr(history, 'requalifier', placeholder) + monkeypatch.setattr(history, 'offer', placeholder) + actor.response_recovery_generation = 1 + assert history.observe_response_recovery(actor) + assert history.priors is historical_rows + assert history.prior is history.requalifier is history.offer is None + assert history.blocked_seed == binding + assert holds == [False] + history.reset() # An unrelated collector reset cannot reselect this prior. + collector = SimpleNamespace(ready=[None]) + assert history.advance(actor, collector, object(), now_ns=1, current_native=True, + native_key=(), hardware_inventory=()) is None + assert history.seed == history.blocked_seed == binding + assert history.requalifier is None + assert not history.observe_response_recovery(actor) + assert history.response_recovery_generation == 1 + + +def test_only_actual_binding_change_clears_current_history_quarantine(): + binding = offer().binding + history = _LiveHistoryRestore(()) + actor, _ = actor_for(binding) + actor.response_recovery_generation = 1 + assert history.observe_response_recovery(actor) + collector = SimpleNamespace(ready=[None]) + history.reset() + assert history.blocked_seed == binding + assert history.advance(actor, collector, object(), now_ns=100_000_000_001, + current_native=False, native_key=(), hardware_inventory=()) is None + assert history.blocked_seed == binding + changed = replace(binding, source_epoch='9'*64) + actor.binding_for = lambda collector, source: changed + assert history.advance(actor, collector, object(), now_ns=100_000_000_002, + current_native=True, native_key=(), hardware_inventory=()) is None + assert history.blocked_seed is None and history.seed == changed + assert history.priors == () + + +def test_fresh_incoming_control_rechecks_native_instead_of_inheriting_expired_previous(monkeypatch): + from openpilot.starpilot.flm.steerlearnerd import _collect_source_control + from openpilot.starpilot.flm import runtime_adapter_registry as registry + from openpilot.starpilot.flm.native_health import current_native_health + identity = '8'*64 + monkeypatch.setattr(registry, 'current_authority', lambda sm, cp, **kwargs: (True, object(), identity)) + now = 300_000_000 + old = SimpleNamespace(native_required=True, native_allowed=True, native_identity=identity, + native_observed_ns=1, control_ns=100_000_000, native_expires_ns=200_000_001) + fresh = SimpleNamespace(native_required=True, native_allowed=True, native_identity=identity, + native_observed_ns=200_000_000, control_ns=290_000_000, native_expires_ns=400_000_000) + observed=[] + collector=SimpleNamespace(_learning_current_native_allowed=current_native_health(None,None,old,now)) + assert collector._learning_current_native_allowed is False + collector.control=lambda control, stamp: observed.append((control,stamp,collector._learning_current_native_allowed)) + _collect_source_control(collector,fresh,None,None,now) + assert observed==[(fresh,now,True)] + # Reusing this same current grant cannot renew the old record's lease. + _collect_source_control(collector,old,None,None,now) + assert observed[-1]==(old,now,False) + fresh.native_identity='9'*64 + _collect_source_control(collector,fresh,None,None,now) + assert observed[-1]==(fresh,now,False) + fresh.native_identity=identity + monkeypatch.setattr(registry,'current_authority',lambda sm,cp,**kwargs:(False,object(),identity)) + _collect_source_control(collector,fresh,None,None,now) + assert observed[-1]==(fresh,now,False) diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_residual_recovery.py b/openpilot/starpilot/flm/tests/test_live_tracking_residual_recovery.py new file mode 100644 index 0000000000..dc2455b4d8 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_residual_recovery.py @@ -0,0 +1,81 @@ +"""Fresh physical drift recovery, without new numerical admission or permission.""" +from dataclasses import replace +from types import SimpleNamespace + +from openpilot.starpilot.flm.live_tracking_response import ResponseMonitor +from openpilot.starpilot.flm.live_tracking_actor import LiveTrackingActor +from openpilot.starpilot.flm.tests.test_live_tracking_requalification import certificate, synthetic_pair + + +def feed(monitor, index, *, bad=False, current_native=True): + c, o, p = synthetic_pair(SimpleNamespace(_binding=monitor.certificate.binding, original=monitor.certificate), index) + if bad: + p = replace(p, yaw_rate=p.yaw_rate+.12/p.speed_mps) + return monitor.observe(c, o, p, now_ns=p.mono_ns, binding=monitor.certificate.binding, current_native=current_native), p.mono_ns + + +def test_one_bad_row_and_missing_permission_do_not_request_reidentification(): + monitor = ResponseMonitor(certificate()) + for i in range(10): + assert feed(monitor, i)[0] + assert feed(monitor, 10, bad=True)[0] + for i in range(11, 100): + accepted, now = feed(monitor, i) + assert accepted + assert not monitor.persistent_residual_failure(now) + assert not feed(monitor, 100, bad=True, current_native=False)[0] + assert not monitor.persistent_residual_failure(now+50_000_000) + + +def test_distinct_fresh_physical_residual_failure_needs_full_persistent_window(): + monitor = ResponseMonitor(certificate()) + for i in range(60): + accepted, now = feed(monitor, i, bad=True) + assert accepted + assert not monitor.persistent_residual_failure(now) + for i in range(60, 69): + accepted, now = feed(monitor, i, bad=True) + assert accepted + assert monitor.persistent_residual_failure(now) + assert monitor.witness(now) is None + assert not monitor.persistent_residual_failure(now+100_000_001) + assert not feed(monitor, 69, bad=True, current_native=False)[0] + assert not monitor.persistent_residual_failure(now+50_000_000) + + +def test_withdraw_once_cancel_probe_no_same_epoch_retry_then_new_epoch(monkeypatch): + from openpilot.starpilot.flm.tests.test_live_tracking_actor import PollProcess + from openpilot.starpilot.flm import live_tracking_bootstrap_producer + b = certificate().binding + process, probe = PollProcess(), PollProcess(reap_after=2) + process.cancel = lambda reason: None + cancellations = [] + probe.cancel = cancellations.append + actor = LiveTrackingActor(process, probe_process=probe) + actor.seed = b + monkeypatch.setattr(actor, 'binding_for', lambda collector, source: b) + monitor = ResponseMonitor(certificate()) + for i in range(69): + _, now = feed(monitor, i, bad=True) + interrupted = [] + monkeypatch.setattr(actor, 'ingress', SimpleNamespace(monitor=monitor, interrupt=lambda reason, **kwargs: interrupted.append(reason))) + actor.certificate = monitor.certificate + collector = SimpleNamespace(ready=[None], original_source_capture=SimpleNamespace(epoch=1), live_tracking=actor.ingress) + snapshots = [] + def snapshot(*args): + snapshots.append(1) + return {'lifecycle_only': True} + monkeypatch.setattr(live_tracking_bootstrap_producer, 'response_snapshot', snapshot) + actor.tick(collector, object(), now_ns=now, current_native=True) + assert actor.ingress is actor.certificate is collector.live_tracking is None + assert actor.attempted_epoch == actor.response_recovery_generation == 1 + assert interrupted == cancellations == ['persistent_physical_response_drift'] + actor.tick(collector, object(), now_ns=now+1, current_native=True) + assert not snapshots and not process.submissions + assert actor.response_recovery_generation == 1 + actor.tick(collector, object(), now_ns=now+2, current_native=False) + assert not snapshots + collector.original_source_capture.epoch = 2 + actor.tick(collector, object(), now_ns=now+3, current_native=True) + assert len(snapshots) == len(process.submissions) == 1 + assert actor.certificate is None # Recalibration result + fresh residual are still required. diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_response_policy.py b/openpilot/starpilot/flm/tests/test_live_tracking_response_policy.py new file mode 100644 index 0000000000..891db2d14f --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_response_policy.py @@ -0,0 +1,163 @@ +"""Purpose/admission boundaries, not a fabricated physical calibration result.""" +import ast +from pathlib import Path + +import pytest + +from openpilot.starpilot.flm.live_tracking_response_policy import ( + MAX_AS, MAX_CPU, MAX_WALL_NS, PURPOSE, WORKER_MODULE, production_result, validate_production_result, +) + + +def measured_shape(): + from openpilot.starpilot.flm.live_tracking_response import certificate_to_dict + from openpilot.starpilot.flm.tests.test_live_tracking import fixture_certificate + return {'live_job_version': PURPOSE, 'status': 'response_identified', + 'reason': 'independent_train_heldout_response_passed', 'certificate': certificate_to_dict(fixture_certificate()), + 'profile': None, 'current_permission': False} + + +def test_exact_production_worker_only_emits_numerical_admission(): + raw = measured_shape() + report = production_result(raw) + assert validate_production_result(report) == report + assert report['worker_purpose'] == PURPOSE and report['production_admissible'] is True + assert report['current_permission'] is False and report['profile'] is None + assert 'worker_purpose' not in raw and 'production_admissible' not in raw + + +@pytest.mark.parametrize('change', [ + {'diagnostic_purpose': 'live-response-bootstrap-diagnostic-v1'}, + {'worker_purpose': PURPOSE}, {'production_admissible': True}, + {'current_permission': True}, {'profile': {}}, {'certificate': {}}, + {'live_job_version': 'live-response-bootstrap-diagnostic-v1'}, +]) +def test_math_result_cannot_supply_admission_or_permission(change): + with pytest.raises((KeyError, TypeError, ValueError)): + production_result(dict(measured_shape(), **change)) + + +def test_collecting_never_becomes_admissible(): + report = production_result({'live_job_version': PURPOSE, 'status': 'collecting', + 'reason': 'insufficient_original_support', 'certificate': None, 'profile': None, 'current_permission': False}) + assert report['production_admissible'] is False + with pytest.raises(ValueError, match='production response'): + validate_production_result(report) + + +@pytest.mark.parametrize('change', [ + {'worker_purpose': 'live-response-bootstrap-diagnostic-v1'}, {'production_admissible': False}, + {'diagnostic_purpose': 'live-response-bootstrap-diagnostic-v1'}, {'current_permission': True}, {'profile': {}}, +]) +def test_process_report_rejects_diagnostics_and_permission(change): + report = production_result(measured_shape()) + with pytest.raises((KeyError, TypeError, ValueError)): + validate_production_result(dict(report, **change)) + + +def test_legacy_and_diagnostic_limits_are_unchanged(): + from openpilot.starpilot.flm.fit_process import FitProcess + from openpilot.starpilot.flm.fit_worker import MAX_CPU as LEGACY_CPU + from openpilot.starpilot.flm.live_tracking_bootstrap_process import LiveBootstrapProcess + from openpilot.starpilot.flm.live_tracking_bootstrap_worker import PURPOSE as DIAGNOSTIC_PURPOSE + from openpilot.starpilot.flm.live_tracking_response_process import ProductionBootstrapProcess + assert LEGACY_CPU == 20 and FitProcess.wall_ns == 30_000_000_000 + assert FitProcess.interval_ns == 60_000_000_000 + assert LiveBootstrapProcess.wall_ns == 300_000_000_000 + assert DIAGNOSTIC_PURPOSE == 'live-response-bootstrap-diagnostic-v1' + assert MAX_CPU == 180 and MAX_WALL_NS == 240_000_000_000 and MAX_AS == 2*1024**3 + assert ProductionBootstrapProcess.worker_module == WORKER_MODULE + assert ProductionBootstrapProcess.wall_ns == MAX_WALL_NS + assert ProductionBootstrapProcess.cancel is FitProcess.cancel + assert ProductionBootstrapProcess.close is FitProcess.close + + +def test_production_selection_is_constructor_not_environment(monkeypatch): + from openpilot.starpilot.flm.fit_process import FitProcess + from openpilot.starpilot.flm.live_tracking_response_process import ProductionBootstrapProcess + observed = [] + def submit(process, snapshot, *, token, now_ns): + observed.append((process.worker_module, snapshot, token, now_ns)) + return True + monkeypatch.setattr(FitProcess, 'submit', submit) + for purpose in (None, 'anything', 'live-response-bootstrap-diagnostic-v1'): + if purpose is None: + monkeypatch.delenv('STARPILOT_LIVE_BOOTSTRAP_PURPOSE', raising=False) + else: + monkeypatch.setenv('STARPILOT_LIVE_BOOTSTRAP_PURPOSE', purpose) + assert ProductionBootstrapProcess().submit({'source': 'test-only'}, token=('test',), now_ns=1) + assert observed == [(WORKER_MODULE, {'source': 'test-only'}, ('test',), 1)]*3 + + +def test_parent_discards_unqualified_successful_child_report(monkeypatch): + from openpilot.starpilot.flm.fit_process import FitProcess + from openpilot.starpilot.flm.live_tracking_response_process import ProductionBootstrapProcess + report = dict(measured_shape(), production_admissible=True, + diagnostic_purpose='live-response-bootstrap-diagnostic-v1') + monkeypatch.setattr(FitProcess, 'poll', lambda self, **kwargs: report) + process = ProductionBootstrapProcess() + assert process.poll(token=('test',), now_ns=1) is None + assert process.reason == 'production_response_report_unqualified' + + +def test_daemon_explicitly_constructs_production_response_process(): + import openpilot.starpilot.flm.steerlearnerd as daemon + tree = ast.parse(Path(daemon.__file__).read_text()) + calls = [node for node in ast.walk(tree) if isinstance(node, ast.Call) and + isinstance(node.func, ast.Name) and node.func.id == 'LiveTrackingActor'] + assert len(calls) == 1 + keywords = {item.arg: item.value for item in calls[0].keywords} + assert isinstance(keywords['bootstrap_process'], ast.Call) + assert isinstance(keywords['bootstrap_process'].func, ast.Name) + assert keywords['bootstrap_process'].func.id == 'ProductionBootstrapProcess' + assert isinstance(keywords['probe_process'], ast.Call) + assert isinstance(keywords['probe_process'].func, ast.Name) + assert keywords['probe_process'].func.id == 'ProbeProcess' + + +def test_real_pending_child_survives_capture_rollover_but_not_changed_physical_basis(): + # Real optional-child IPC/lifecycle and actual canonical binding builder; the + # numerical evidence fixture remains explicitly synthetic, not road proof. + import os + import subprocess + import sys + from dataclasses import replace + from types import SimpleNamespace + from openpilot.starpilot.flm.live_tracking_actor import LiveTrackingActor + from openpilot.starpilot.flm.live_tracking_response_process import ProductionBootstrapProcess + from openpilot.starpilot.flm.live_tracking_requalification import RetainedResponseRequalifier + from openpilot.starpilot.flm.tests.test_live_tracking_requalification import certificate, basis, new_drive, synthetic_pair + old = certificate() + owner = RetainedResponseRequalifier(old, basis(old), new_drive(old)) + control, output, _pose = synthetic_pair(owner, 0) + binding = owner._binding + source = SimpleNamespace(cp_sha256=binding.cp_sha256, source_epoch=binding.source_epoch, + reset=binding.reset_generation, calibration_generation=binding.calibration_generation, + persistent_source_epoch=binding.persistent_source_epoch) + collector = SimpleNamespace(ready=[(control, output)], original_source_capture=SimpleNamespace(epoch=0), live_tracking=None) + process = ProductionBootstrapProcess() + actor = LiveTrackingActor(bootstrap_process=process) + seed = actor.binding_for(collector, source) + token = tuple(vars(seed).values()) + actor.seed, actor.token, actor.attempted_epoch = seed, token, 0 + child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, start_new_session=True) + process.child, process.token, process.started_ns = child, token, 39_000_000_000 + assert child.stdin is not None and child.stdout is not None + os.set_blocking(child.stdin.fileno(), False) + os.set_blocking(child.stdout.fileno(), False) + try: + assert actor.tick(collector, source, now_ns=99_000_000_000, current_native=True) is None + assert process.child is child and child.poll() is None and not process.reaping + collector.original_source_capture.epoch = 1 + assert actor.tick(collector, source, now_ns=101_000_000_000, current_native=True) is None + assert actor.seed == seed and process.token == token + assert process.child is child and child.poll() is None and not process.reaping + assert actor.attempted_epoch == 0 # No duplicate child at capture rotation. + collector.ready = [(replace(control, vehicle_model_anchor=(16., 100000., 110000.)), output)] + assert actor.tick(collector, source, now_ns=102_000_000_000, current_native=True) is None + assert process.reaping or process.child is None + assert actor.reason == 'actual_response_source_unavailable' + finally: + assert process.close('test_actual_child_cleanup') + assert child.returncode is not None and process.child is None diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_retained_drift.py b/openpilot/starpilot/flm/tests/test_live_tracking_retained_drift.py new file mode 100644 index 0000000000..1b9016a018 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_retained_drift.py @@ -0,0 +1,131 @@ +"""Actual typed retainer/history path on synthetic qualified physical evidence.""" +from dataclasses import replace +from types import SimpleNamespace +import pytest + +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, full_profile_sha256 +from openpilot.starpilot.flm.live_trial_store import LiveConfirmedPrior +from openpilot.starpilot.flm.live_tracking_actor import LiveTrackingActor +from openpilot.starpilot.flm.live_tracking_requalification import RetainedResponseRequalifier +from openpilot.starpilot.flm.steerlearnerd import _LiveHistoryRestore +from openpilot.starpilot.flm.tests.test_live_tracking_actor import PollProcess +from openpilot.starpilot.flm.tests.test_live_tracking_requalification import certificate, basis, synthetic_pair + + +def setup(monkeypatch): + old = certificate() + current = replace(old.binding, identity='6'*64, source_epoch='2'*64, calibration_generation='3'*32) + zero = tuple((name, 0.) for name in NAMES) + prior = LiveConfirmedPrior(old.binding, zero, full_profile_sha256(old.binding.baseline_revision, dict(zero)), + '7'*64, 1, old, response_basis=basis(old)) + history = _LiveHistoryRestore((prior,)) + process = PollProcess() + actor = LiveTrackingActor(process) + class SnapshotPreparation: + pid=epoch=token=None + reason='snapshot_submitted' + pending=None + def poll(self,*,token,now_ns): + value,self.pending=self.pending,None + self.pid=None + return value + def submit(self,collector,seed,*,token,now_ns): + from openpilot.starpilot.flm.live_tracking_bootstrap_producer import response_snapshot + self.epoch,self.token,self.pid=collector.original_source_capture.epoch,token,1 + self.pending={'snapshot':response_snapshot(collector,seed,now_ns)} + return True + actor.snapshot_process=SnapshotPreparation() + + actor.seed = current + monkeypatch.setattr(actor, 'binding_for', lambda collector, source: current) + # Exact provisional numerical binding, never returned as current admission. + template = RetainedResponseRequalifier(old, basis(old), current) + collector = SimpleNamespace(ready=[], original_source_capture=SimpleNamespace(rows=[], epoch=1), live_tracking=None, + adapter=SimpleNamespace(replay=SimpleNamespace(emissions=SimpleNamespace(events={})))) + return history, actor, collector, template, process + + +def feed(history, actor, collector, template, index, *, bad=True, native=True, stale=False): + c, o, p = synthetic_pair(template, index, command=.1) + if bad: + yaw = p.yaw_rate+.12/p.speed_mps + p = replace(p, yaw_rate=yaw, angular_velocity_device=(0., 0., yaw)) + now = p.mono_ns+(100_000_001 if stale else 0) + collector.ready = [(c, o)] + collector.original_source_capture.rows = [p] + history.advance(actor, collector, object(), now_ns=now, current_native=native, + native_key=(), hardware_inventory=('measured-hardware-type',)) + return now + + +@pytest.mark.parametrize('already_attempted', (False, True)) +def test_new_drive_bad_historical_model_quarantines_then_allows_only_unused_epoch_fit(monkeypatch, already_attempted): + from openpilot.starpilot.flm import live_tracking_bootstrap_producer + history, actor, collector, template, process = setup(monkeypatch) + if already_attempted: + actor.attempted_epoch = 1 + original_rows = history.priors + for i in range(60): + now = feed(history, actor, collector, template, i) + assert history.blocked_seed is None + assert actor.certificate is actor.ingress is None + for i in range(60, 69): + now = feed(history, actor, collector, template, i) + assert history.blocked_seed == actor.seed + assert history.prior is history.requalifier is history.offer is None + assert history.priors is original_rows + assert not actor.restore_pending + snapshots = [] + def snapshot(*args): + snapshots.append(1) + return {'scheduling_test_only': True} + monkeypatch.setattr(live_tracking_bootstrap_producer, 'response_snapshot', snapshot) + actor.tick(collector, object(), now_ns=now, current_native=True, + allow_bootstrap=history.requalifier is None) + actor.tick(collector,object(),now_ns=now+1,current_native=True,allow_bootstrap=history.requalifier is None) + assert len(process.submissions) == (0 if already_attempted else 1) + assert actor.certificate is actor.ingress is None + # A reset cannot bypass source quarantine or promote the old target. + history.reset() + feed(history, actor, collector, template, 70) + assert history.blocked_seed == actor.seed and history.prior is None + if already_attempted: + collector.original_source_capture.epoch = 2 + actor.tick(collector, object(), now_ns=now+2, current_native=True) + actor.tick(collector, object(), now_ns=now+3, current_native=True) + assert len(process.submissions) == 1 + + +@pytest.mark.parametrize('failure', ('transient', 'stale', 'native')) +def test_nonphysical_or_transient_failure_never_quarantines_history(failure, monkeypatch): + history, actor, collector, template, _ = setup(monkeypatch) + for i in range(80): + feed(history, actor, collector, template, i, bad=(failure != 'transient' or i == 0), + native=(failure != 'native'), stale=(failure == 'stale')) + assert history.blocked_seed is None + assert actor.certificate is actor.ingress is None + + +def test_permission_loss_clears_bad_observation_count_before_fresh_requalification(monkeypatch): + history, actor, collector, template, _ = setup(monkeypatch) + for i in range(55): + feed(history, actor, collector, template, i) + assert history.requalifier is not None + feed(history, actor, collector, template, 55, native=False) + assert not history.requalifier.persistent_residual_failure(12_750_000_000) + for i in range(56, 80): + feed(history, actor, collector, template, i) + assert history.blocked_seed is None + + +def test_source_pose_gap_requires_new_full_persistent_failure_window(monkeypatch): + history, actor, collector, template, _ = setup(monkeypatch) + for i in range(55): + feed(history, actor, collector, template, i) + assert history.blocked_seed is None + # Genuine next source sample is200ms later; no intermediate samples invented. + for i in range(58, 83): + feed(history, actor, collector, template, i) + assert history.blocked_seed is None + assert history.requalifier is not None + assert not history.requalifier.persistent_residual_failure(14_100_000_000) diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_rollout.py b/openpilot/starpilot/flm/tests/test_live_tracking_rollout.py new file mode 100644 index 0000000000..e68f5f5b9e --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_rollout.py @@ -0,0 +1,194 @@ +"""Known-plant numerical trials; NO source/controller/native authority evidence. + +These tests synthesize certificates, physical residual witnesses and target +acknowledgments to exercise the optional learner's decisions. Coordinate +quantization below is a declared model assumption, not a real CAN encoder. +Root must run these inside the separately qualified candidate closure. +""" +from dataclasses import replace +import hashlib +import json +import math +import random +import statistics + +import pytest + +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES +from openpilot.starpilot.flm.live_tracking import HookSensitivity, LiveTrackingOptimizer, TrackingEpisode, _critical +from openpilot.starpilot.flm.live_tracking_response import ResponseCertificate, ResponseWitness, VERSION, _revision, validate_certificate +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding + +NS = 1_000_000_000 +HOOK = 'ff_gain_left' + + +def _optimizer(): + binding = LiveTrialBinding('a'*64, 'b'*64, 'actual-controller-registry-v2:'+'c'*64, 'd'*64, 'e'*64, + 'numerical-reset', 'f'*64, 'a'*64, '', '0'*64, 'b'*64, False) + raw = ResponseCertificate(VERSION, binding, '0'*64, 170_000_000, 3., 0., -.5, .5, .005, + ('train0', 'train1', 'train2', 'train3'), ('hold0', 'hold1', 'hold2', 'hold3'), 'c'*64, 'd'*64, 1) + revision = _revision(raw) + certificate = validate_certificate(replace(raw, model_revision=revision, + binding=replace(binding, physical_model_revision=revision))) + return LiveTrackingOptimizer(certificate.binding, tuple((name, 0.) for name in NAMES), + controller_domain='torque', response_certificate=certificate) + + +def _refresh(owner, now): + witness = ResponseWitness(owner.certificate, owner.binding, now-180_000_000, now-180_000_000, + now, now-350_000_000, 8, .005, now+100_000_000) + assert owner.response_current(witness, now_ns=now) + + +def _ack(owner, offer, now): + _refresh(owner, now) + assert owner.acknowledge_application(trial_id=offer.trial_id, target_sequence=offer.target_sequence, + phase=offer.phase, target_sha256=offer.target_sha256, applied_full30=offer.target, + control_ns=now, binding=offer.binding, now_ns=now, status='applied', + confirmed_target_sha256=offer.target_sha256 if offer.phase == 'confirmed' else '') + + +def _episode(owner, index, now, profile, goal, score, speed): + # A physically visible algorithmic probe can still have the wrong causal + # direction. Future observations, rather than this probe, must decide keep. + sensitivity = HookSensitivity(HOOK, .06, 1., 20, 80, .015, native_gradient_complete=True) + return TrackingEpisode(owner.binding, owner._hash(profile), hashlib.sha256(f'rollout-{index}'.encode()).hexdigest(), + now-4*NS, now, 'large', 1, speed, speed*1.02, goal, score*goal, .1, + 80, 10_000_000, (sensitivity,), (1.,)*8, native_gradient_complete=True) + + +def rollout(*, seed, gain=5., noise_sd=.008, correlation=.65, demand_change=False, interrupt_every=0): + """Twenty minutes, one eligible 4s maneuver every8s, AR1 road/noise. + + gain=0 is the causal null; gain<0 is an adverse controller change. Probes + remain optimistic in both cases to force the independent future test to + reject false starts. The desired magnitude and speed vary independently. + """ + owner, rng = _optimizer(), random.Random(seed) + first_trial = first_confirmation = None + confirmations = rollbacks = interruptions = 0 + noise = 0. + attempts = [] + decisions = [] + rolled_back = set() + scores = [] + for index in range(1, 151): + now = index*8*NS + _refresh(owner, now) + state = owner.offer_state + active = state is not None and state['phase'] == 'trial' and not state.get('accepted', False) + profile = state['target'] if active else owner.accepted + # Explicit model-domain quantization: 0.0001 coordinate resolution. + encoded_coordinate = round(profile[HOOK]/.0001)*.0001 + noise = correlation*noise+math.sqrt(1-correlation**2)*rng.gauss(0., noise_sd) + goal = (.48 if active else .60) if demand_change else .60*(1+rng.uniform(-.06, .06)) + speed = 22.*(1+rng.uniform(-.025, .025)) + score = max(.025, .35-gain*encoded_coordinate+noise+rng.gauss(0., noise_sd*.35)) + scores.append(score) + row = _episode(owner, index, now, profile, goal, score, speed) + if active and interrupt_every and index % interrupt_every == 0: + owner.interrupt('numerical_driver_override') + interruptions += 1 + else: + owner.observe_episode(row, now_ns=now) + offer = owner.offer(now) + if offer is None: + continue + if offer.phase == 'trial' and owner.settled_ns is None: + first_trial = first_trial or now/NS + attempts.append({'seconds': now/NS, 'quota': owner.offer_state['required_episodes'], + 'alpha': owner.offer_state['alpha'], 'baseline_count': len(owner.before), + 'critical': owner.offer_state['critical'], + 'baseline_score_sd': statistics.stdev(row.error_rms/row.goal_rms for row in owner.before)}) + # The bounded .005 step needs1s at the specified .005/s slew. This is + # simulated receipt time, never a claim of actual owner application. + _ack(owner, offer, now+NS) + elif offer.phase == 'confirmed' and not owner.offer_state.get('accepted', False): + decisions.append({'seconds': now/NS, 'phase': offer.phase, 'reason': owner.reason, + 'future_count': len(owner.future), 'elapsed_seconds': (now-offer.started_ns)/NS}) + _ack(owner, offer, now+10_000_000) + confirmations += 1 + first_confirmation = first_confirmation or (now+.01*NS)/NS + elif offer.phase == 'rollback' and (offer.trial_id, offer.target_sequence) not in rolled_back: + decisions.append({'seconds': now/NS, 'phase': offer.phase, 'reason': owner.reason, + 'future_count': len(owner.future), 'elapsed_seconds': (now-offer.started_ns)/NS}) + _ack(owner, offer, now+NS) + rolled_back.add((offer.trial_id, offer.target_sequence)) + rollbacks += 1 + result = {'seed': seed, 'seconds': 1200, 'first_trial_seconds': first_trial, + 'first_confirmation_seconds': first_confirmation, 'confirmations': confirmations, + 'rollbacks': rollbacks, 'interruptions': interruptions, 'attempts': attempts, 'decisions': decisions, + 'accepted_coordinate': owner.accepted[HOOK], 'initial_expected_normalized_error': .35, + 'final_expected_normalized_error': max(.025, .35-gain*round(owner.accepted[HOOK]/.0001)*.0001), + 'last_10_episode_mean_normalized_error': sum(scores[-10:])/10} + return result + + +def test_student_critical_matches_independent_published_reference_values(): + # Standard Student t quantiles: no SciPy dependency and no self-derived oracle. + for df, alpha, expected in ((1, .025, 12.7062047364), (2, .025, 4.3026527297), + (8, .025, 2.3060041352), (23, .01, 2.4998667395)): + assert _critical(df, alpha) == pytest.approx(expected, rel=2e-9) + + +def test_twenty_minute_correlated_noisy_known_plant_makes_multiple_useful_corrections(): + result = rollout(seed=20261019) + print('NUMERICAL_TWENTY_MINUTE_ROLLOUT '+json.dumps(result, sort_keys=True)) + assert result['first_trial_seconds'] is not None and result['first_trial_seconds'] <= 180, result + assert result['first_confirmation_seconds'] is not None and result['first_confirmation_seconds'] <= 300, result + assert result['confirmations'] >= 3, result + assert result['accepted_coordinate'] > 0, result + assert result['final_expected_normalized_error'] <= .8*result['initial_expected_normalized_error'], result + + +def test_proportional_demand_change_without_controller_benefit_never_confirms(): + result = rollout(seed=1, gain=0., noise_sd=0., demand_change=True) + assert len(result['attempts']) >= 3 and result['rollbacks'] >= 3, result + assert result['confirmations'] == 0 and result['accepted_coordinate'] == 0., result + + +def test_subtle_proportional_demand_change_below_score_floor_never_confirms(): + owner = _optimizer() + for index in range(1, 25): + now = index*8*NS + _refresh(owner, now) + state = owner.offer_state + active = state is not None and state['phase'] == 'trial' + profile = state['target'] if active else owner.accepted + # Both errors exceed the subtle eligibility floor. Tracking quality is + # exactly unchanged: error/desired=1 on both sets of maneuvers. + goal = .016 if active else .020 + row = replace(_episode(owner, index, now, profile, goal, 1., 22.), kind='subtle', + sensitivity=(HookSensitivity('center_taper_max', .06, 1., 20, 80, .015),)) + owner.observe_episode(row, now_ns=now) + offer = owner.offer(now) + if offer is None: + continue + assert offer.phase != 'confirmed', owner.report(now) + if offer.phase == 'trial' and owner.settled_ns is None or offer.phase == 'rollback': + _ack(owner, offer, now+NS) + + +def test_repeated_noisy_adverse_trials_revert_without_accumulating_wrong_correction(): + result = rollout(seed=20261020, gain=-5.) + assert len(result['attempts']) >= 3 and result['rollbacks'] >= 3, result + assert result['confirmations'] == 0 and result['accepted_coordinate'] == 0., result + + +def test_driver_interruptions_do_not_prevent_later_independent_useful_trials(): + result = rollout(seed=20261021, interrupt_every=11) + assert result['interruptions'] >= 1 and result['rollbacks'] >= 1, result + assert result['confirmations'] >= 2 and result['accepted_coordinate'] > 0., result + + +def test_correlated_null_seed_sweep_reports_family_false_acceptances(): + # This finite deterministic stress suite is an empirical regression gate, + # not a proof of a5% probability guarantee under arbitrary road correlation. + results = [rollout(seed=seed, gain=0.) for seed in range(64)] + false_acceptances = [row for row in results if row['confirmations']] + summary = {'routes': len(results), 'false_acceptance_routes': len(false_acceptances), + 'attempts': sum(len(row['attempts']) for row in results), 'failures': false_acceptances} + print('NUMERICAL_CORRELATED_NULL_SWEEP '+json.dumps(summary, sort_keys=True)) + assert all(row['attempts'] for row in results), summary + assert len(false_acceptances) <= 3, summary diff --git a/openpilot/starpilot/flm/tests/test_live_tracking_selector_recovery.py b/openpilot/starpilot/flm/tests/test_live_tracking_selector_recovery.py new file mode 100644 index 0000000000..b3ecc843bb --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_tracking_selector_recovery.py @@ -0,0 +1,279 @@ +"""Informational recovery only; synthetic numerical/protocol evidence.""" +from dataclasses import replace + +import pytest + +from openpilot.starpilot.flm.live_trial_owner import TrackingTrialOwner +from openpilot.starpilot.flm.tests.test_live_tracking import optimizer as _protocol_optimizer +from openpilot.starpilot.flm.tests.test_live_tracking_rollout import NS, HOOK, _episode, _refresh, _ack, _optimizer + + +def interrupted_owner(): + # Existing clean protocol certificate isolates lifecycle assertions. + # Original noisy rollout certificates, seeds and floors are unchanged. + owner = _protocol_optimizer() + for index in range(1, 61): + now = index*8*NS + _refresh(owner, now) + owner.observe_episode(_episode(owner, index, now, owner.accepted, .6, .35, 22.), now_ns=now) + if owner.offer_state is not None: + break + offer = owner.offer(now) + assert offer is not None and offer.phase == 'trial' + _ack(owner, offer, now+NS) + before = tuple(owner.before) + _refresh(owner, now+8*NS) + owner.observe_episode(_episode(owner, 90, now+8*NS, dict(offer.target), .6, .325, 22.), now_ns=now+8*NS) + assert len(owner.future) == 1 + future = tuple(owner.future) + assert owner.selector_hint is not None + hint = owner.selector_hint.copy() + owner.interrupt('temporary_current_source_or_native_unavailable') + assert owner.before == () and owner.future == [] and owner.comparison == [] + _refresh(owner, now+10*NS) + rollback = owner.offer(now+10*NS) + assert rollback is not None and rollback.phase == 'rollback' and rollback.target == rollback.baseline + # Even a fresh numerical rollback lease cannot bypass current permission. + actual_owner = TrackingTrialOwner(owner.binding) + assert not actual_owner.prepare(rollback, now_ns=now+10*NS, permission=False, binding=owner.binding) + _ack(owner, rollback, now+11*NS) + return owner, offer, hint, before, future, now+16*NS + + +def test_recovery_uses_only_information_and_new_probe_revalidation_then_new_uncensored_rows(): + owner, old, hint, before, future, now = interrupted_owner() + assert set(hint) == {'group', 'name', 'delta', 'demand', 'binding', 'profile_sha256', 'model_revision', + 'controller_domain', 'selected_ns', 'expires_ns', 'source_episode_ids', 'quota'} + assert owner.preferred_probe_group(now) == hint['group'] + _refresh(owner, now) + revalidation = _episode(owner, 100, now, owner.accepted, .6, .35, 22.) + revalidation = replace(revalidation, sensitivity=(replace(revalidation.sensitivity[0], error_feature_sum=-999.),)) + owner.observe_episode(revalidation, now_ns=now) + assert owner.discovery is not None and not owner.comparison + assert owner.discovery['expires_ns'] == hint['expires_ns'] + quota = owner.discovery['quota'] + compared = [] + for index in range(quota): + now += 8*NS + _refresh(owner, now) + row = _episode(owner, 101+index, now, owner.accepted, .6, .35, 22.) + row = replace(row, sensitivity=(replace(row.sensitivity[0], error_feature_sum=-999.),)) + compared.append(row) + owner.observe_episode(row, now_ns=now) + new = owner.offer(now) + assert new is not None and new.phase == 'trial' and new.trial_id != old.trial_id + assert new.target_sequence == old.target_sequence+1 and new.started_ns == now + assert new.deadline_ns == now+120*NS and new.deadline_ns != old.deadline_ns + assert dict(new.target)[HOOK] > dict(new.baseline)[HOOK] # No direction selection from the new negative correlations. + assert owner.before == tuple(compared) and all(row.error_rms == .35*row.goal_rms for row in owner.before) + spent = {row.episode_id for row in before+future+(revalidation,)} + assert spent.isdisjoint(row.episode_id for row in owner.before) + assert owner.selector_hint is not None and owner.offer_state is not None + assert owner.selector_hint['expires_ns'] == hint['expires_ns'] + assert owner.offer_state['alpha'] == .05/(2*3) # No aborted-attempt alpha refund. + assert owner.preferred_probe_group(now) is None # Round-robin resumes during the actual trial. + + +def test_zero_error_recovery_rows_are_uncensored_but_do_not_force_an_underpowered_trial(): + owner, old, hint, _before, _future, now = interrupted_owner() + _refresh(owner, now) + revalidation = _episode(owner, 100, now, owner.accepted, .6, .35, 22.) + owner.observe_episode(revalidation, now_ns=now) + assert owner.discovery is not None and owner.comparison == [] + quota = owner.discovery['quota'] + compared = [] + for index in range(quota): + now += 8*NS + _refresh(owner, now) + row = _episode(owner, 101+index, now, owner.accepted, .6, 0., 22.) + compared.append(row) + owner.observe_episode(row, now_ns=now) + if index+1 < quota: + assert owner.comparison == compared # Every zero row counts; none is replaced. + assert all(row.episode_id in owner.seen for row in compared) + assert all(row in owner.planning_history for row in compared) + assert owner.discovery is None and owner.comparison == [] and owner.selector_hint is None + assert owner.reason == 'independent_baseline_trial_power_insufficient' + assert owner.sequence == old.target_sequence and owner.before == () + offer = owner.offer(now) + assert offer is not None and offer.phase == 'rollback' and offer.trial_id == old.trial_id + + +@pytest.mark.parametrize('bad', ('missing', 'invisible', 'energy', 'physical')) +def test_retained_information_cannot_replace_fresh_native_visibility(bad): + owner, old, _hint, _before, _future, now = interrupted_owner() + _refresh(owner, now) + row = _episode(owner, 100, now, owner.accepted, .6, .35, 22.) + if bad == 'missing': + row = replace(row, sensitivity=()) + else: + feature = row.sensitivity[0] + if bad == 'invisible': + feature = replace(feature, native_separated=0) + elif bad == 'energy': + feature = replace(feature, feature_energy=0.) + else: + feature = replace(feature, max_step_physical_rms=0.) + row = replace(row, sensitivity=(feature,)) + owner.observe_episode(row, now_ns=now) + assert owner.discovery is None and owner.comparison == [] + assert owner.offer_state is not None + assert owner.offer_state['trial_id'] == old.trial_id and owner.offer_state['phase'] == 'rollback' + assert owner.reason == 'fresh_selected_native_probe_required' + + +def test_recovery_requires_current_response_and_current_source_binding(): + owner, old, _hint, _before, _future, now = interrupted_owner() + row = _episode(owner, 100, now, owner.accepted, .6, .35, 22.) + owner.observe_episode(row, now_ns=now) # Previous pose is stale; no refresh. + assert owner.discovery is None and owner.reason == 'fresh_selector_response_required' + _refresh(owner, now+8*NS) + wrong = _episode(owner, 101, now+8*NS, owner.accepted, .6, .35, 22.) + wrong = replace(wrong, binding=replace(wrong.binding, source_epoch='9'*64)) + assert not owner.observe_episode(wrong, now_ns=now+8*NS) + assert owner.offer_state is not None + assert owner.discovery is None and owner.offer_state['trial_id'] == old.trial_id + + +@pytest.mark.parametrize('change', ('binding', 'profile', 'model', 'domain', 'context', 'expiry')) +def test_informational_selector_invalidates_without_stale_group_priority(change): + owner, _old, hint, _before, _future, now = interrupted_owner() + if change == 'binding': + owner.binding = replace(owner.binding, source_epoch='9'*64) + elif change == 'profile': + owner.accepted[HOOK] = .001 + elif change == 'model': + owner.certificate = replace(owner.certificate, model_revision='9'*64) + elif change == 'domain': + owner.domain = 'torque_pid' + elif change == 'context': + owner.interrupt('changed-physical-context', context_changed=True) + else: + now = hint['expires_ns']+1 + assert owner.preferred_probe_group(now) is None + assert owner.selector_hint is None and not owner.selector_pending and owner.discovery is None + + +def test_numerical_failed_future_cannot_become_a_recovery_hint(): + owner, _old, _hint, _before, _future, _now = interrupted_owner() + owner._rollback('independent_future_tracking_not_improved') + assert owner.selector_hint is None and not owner.selector_pending + + +def test_joint_allocation_uses_more_fresh_baseline_to_shorten_active_exposure(): + owner = _optimizer() # Original noisy .005 model floor, unchanged. + history = tuple(_episode(owner, index, index*8*NS, owner.accepted, .6, .35, 22.) + for index in range(1, 10)) + owner.planning_history.extend(history) + target = dict(owner.accepted) + target[HOOK] = .005 + allocation = owner._comparison_allocation(history[-1], .023, target) + assert allocation is not None + before_count, future_count = allocation + assert 3 < before_count <= 24 and future_count == 3 + # With the SAME whole-episode noise floor, a three-row baseline needs more + # target exposure. The independently collected larger baseline needs three. + short = owner._trial_plan(history[:3], .023, target) + assert short is not None and short[0] > future_count + fresh = tuple(_episode(owner, 100+index, (10+index)*8*NS, owner.accepted, .6, .35, 22.) + for index in range(before_count)) + actual = owner._trial_plan(fresh, .023, target) + assert actual is not None and actual[0] == future_count + assert owner.sequence == 0 and owner.before == () and owner.future == [] + + +def test_cold_selection_predeclares_more_clean_baseline_for_shorter_target_exposure(): + owner = _optimizer() # Same .005 whole-episode model floor as the noisy route. + selected = [] + for index in range(1, 4): + now = index*8*NS + _refresh(owner, now) + row = _episode(owner, index, now, owner.accepted, .6, .35, 22.) + row = replace(row, sensitivity=(replace(row.sensitivity[0], max_step_physical_rms=.0138),)) + selected.append(row) + owner.observe_episode(row, now_ns=now) + assert owner.offer_state is None and owner.discovery is not None + quota = owner.discovery['quota'] + assert 3 < quota <= 24 and owner.comparison == [] + assert not owner.discovery['recovery'] + compared = [] + for index in range(quota): + now += 8*NS + _refresh(owner, now) + row = _episode(owner, 100+index, now, owner.accepted, .6, .35, 22.) + row = replace(row, sensitivity=(replace(row.sensitivity[0], error_feature_sum=-999.),)) + compared.append(row) + owner.observe_episode(row, now_ns=now) + if index+1 < quota: + assert owner.offer_state is None and owner.discovery is not None + assert owner.discovery['quota'] == quota and owner.comparison == compared + offer = owner.offer(now) + assert offer is not None and offer.phase == 'trial' + assert owner.offer_state is not None and owner.offer_state['required_episodes'] == 3 + assert owner.before == tuple(compared) and dict(offer.target)[HOOK] > dict(offer.baseline)[HOOK] + assert {row.episode_id for row in selected}.isdisjoint(row.episode_id for row in owner.before) + assert owner.sequence == 1 and owner.offer_state['alpha'] == .025 + + +def _planned_unissued(owner, now): + _refresh(owner,now) + rows=tuple(_episode(owner,100+i,now-(2-i)*8*NS,owner.accepted,.6,.35,22.)for i in range(3)) + target=dict(owner.accepted) + target[HOOK]+=.005 + assert owner._launch('ff.left',HOOK,target,rows,(3,2.0,.025),now) + preview=owner.offer(now,publication=False) + assert preview is not None and owner.offer_state['published'] is False + return preview + + +def test_unissued_proposal_withdrawal_has_no_phantom_consumer_rollback_or_budget_refund(): + owner=_protocol_optimizer() + preview=_planned_unissued(owner,30*NS) + owner.interrupt('fresh_response_witness_unavailable') + assert owner.offer_state is None and owner.before==() and owner.future==[] + assert owner.sequence==preview.target_sequence==1 and owner.last_attempt==30*NS + _refresh(owner,31*NS) + assert owner.offer(31*NS,publication=False) is None + + +def test_before_send_publication_marker_preserves_real_rollback_and_original_deadline(): + owner=_protocol_optimizer() + preview=_planned_unissued(owner,30*NS) + owner.mark_published(preview) + owner.interrupt('fresh_response_witness_unavailable') + _refresh(owner,31*NS) + rollback=owner.offer(31*NS,publication=False) + assert rollback.phase=='rollback' and rollback.trial_id==preview.trial_id + assert rollback.target_sequence==preview.target_sequence and rollback.target==preview.baseline + assert rollback.started_ns==preview.started_ns and rollback.deadline_ns==preview.deadline_ns + + +def test_abandoned_unissued_attempt_preserves_prior_accepted_sequence_and_target(): + owner=_protocol_optimizer() + prior=_planned_unissued(owner,30*NS) + # Synthetic preexisting accepted fixture; this is a sequence/lifecycle unit, + # never evidence that this test fitted or physically accepted a correction. + state=owner.offer_state.copy() + state.update(phase='confirmed',accepted=True,published=True) + state.pop('prior_accepted',None) + owner.offer_state=state + owner.accepted=dict(prior.target) + proposal=_planned_unissued(owner,60*NS) + assert proposal.target_sequence==2 + owner.interrupt('fresh_response_witness_unavailable') + _refresh(owner,61*NS) + retained=owner.offer(61*NS,publication=False) + assert retained.trial_id==prior.trial_id and retained.target==prior.target + assert retained.target_sequence==1 and owner.sequence==2 + assert owner.acknowledge_application(trial_id=retained.trial_id,target_sequence=1,phase='confirmed', + target_sha256=retained.target_sha256,applied_full30=retained.target,control_ns=61*NS, + binding=retained.binding,now_ns=61*NS,status='applied',confirmed_target_sha256=retained.target_sha256) + assert owner.report(61*NS)['applicationAcknowledged'] is True + + +def test_unissued_proposal_timeout_does_not_start_a_later_deadline(): + owner=_protocol_optimizer() + proposal=_planned_unissued(owner,30*NS) + _refresh(owner,proposal.deadline_ns+1) + assert owner.offer(proposal.deadline_ns+1,publication=False) is None + assert owner.offer_state is None and owner.sequence==1 diff --git a/openpilot/starpilot/flm/tests/test_live_trial_actual_controller.py b/openpilot/starpilot/flm/tests/test_live_trial_actual_controller.py new file mode 100644 index 0000000000..a455150b0f --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_trial_actual_controller.py @@ -0,0 +1,99 @@ +"""Real selected LaC/native calls; unit-issued live witness is not physical proof.""" + +from dataclasses import replace +from types import SimpleNamespace +import math +import pytest +from openpilot.starpilot.flm.live_trial_owner import TrackingTrialOwner +from openpilot.starpilot.flm.tests.test_live_trial_owner import offer + + +@pytest.mark.parametrize('domain', ['torque', 'starpilot', 'pid']) +@pytest.mark.parametrize('curvature', [-0.0003, 0.0003]) +def test_actual_controller_hooks_and_native_packet_are_preserved(domain, curvature): + from openpilot.starpilot.flm.controller_replay import SelectedControllerFactory + from openpilot.starpilot.flm.pid_controller_replay import PidControllerFactory + from openpilot.starpilot.flm.tests.test_event_lazy_native import TOYOTA, HYUNDAI, HONDA, make_interface, install_observer, settle, control, apply, BASE_NS + + platform = {'torque': TOYOTA.TOYOTA_COROLLA_TSS2, 'starpilot': HYUNDAI.HYUNDAI_IONIQ_6, 'pid': HONDA.HONDA_CIVIC}[domain] + ci = make_interface(platform) + assert install_observer(ci) + settle(ci, control(0.0)) + factory = PidControllerFactory(ci.CP, 'native') if domain == 'pid' else SelectedControllerFactory(ci.CP, 'starpilot' if domain == 'starpilot' else 'standard') + controller, vm = factory.create() + from openpilot.starpilot.flm.global_runtime import GlobalRuntime + runtime = GlobalRuntime(ci.CP, ci, controller) + assert runtime.descriptor is not None + assert runtime.descriptor.domain == ('torque_pid' if domain == 'pid' else 'torque') + assert controller.automatic_flm_observation is runtime.observation and runtime.observation is not None + if domain != 'pid': + assert controller.automatic_flm_stages is runtime and controller.automatic_flm_centre is runtime + start = BASE_NS + ci.CC.frame * 10_000_000 + o = offer(start) + owner = TrackingTrialOwner(o.binding, domain='torque_pid' if domain == 'pid' else 'torque') + seen = 0 + from openpilot.starpilot.flm.live_tracking_response import ResponseWitness, prepare_response_witness, validate_response_witness + + raw = validate_response_witness(o.response_witness, o.binding, start) + for tick in range(120): + now = start + tick * 10_000_000 + if tick % 5 == 0: + w = prepare_response_witness( + ResponseWitness( + raw.certificate, o.binding, now - raw.certificate.lag_ns, now - raw.certificate.lag_ns, now, start - 500_000_000, 8, 0.001, now + 100_000_000 + ), + o.binding, + now, + ) + current = replace(o, sequence=tick // 5 + 1, observed_ns=now, expires_ns=now + 100_000_000, response_witness=w) + assert owner.prepare(current, now_ns=now, permission=True, binding=o.binding) + runtime.observation.begin_frame() + controller.automatic_flm_effects = owner + result, _, _ = controller.update(True, ci.CS.out, vm, SimpleNamespace(angleOffsetDeg=0.0, roll=0.0), False, curvature, False, 0.2) + assert math.isfinite(float(result)) and abs(float(result)) <= 1.0 + assert owner.used and owner.source.stage_calls == 1 + assert owner.frame_complete(owner.domain) + ack = owner.acknowledge_frame(True) + assert ack is not None and ack.status == 'applied' + if domain == 'pid': + assert owner.source.pid_error_calls == 1 + else: + assert owner.source.friction_calls == 1 + _, packets, witness = apply(ci, control(float(result))) + if witness is not None: + seen += 1 + assert (witness.can_address, witness.can_data, witness.can_bus) in packets + assert seen > 0 + assert owner.applied_full['ff_gain_left'] == pytest.approx(0.005) + # A confirmed acknowledgement requires the actual domain hooks again. + now = start + 1_200_000_000 + w = prepare_response_witness( + ResponseWitness( + raw.certificate, o.binding, now - raw.certificate.lag_ns, now - raw.certificate.lag_ns, now, start - 500_000_000, 8, 0.001, now + 100_000_000 + ), + o.binding, + now, + ) + confirmed = replace(o, phase='confirmed', sequence=25, observed_ns=now, expires_ns=now + 100_000_000, response_witness=w) + assert owner.prepare(confirmed, now_ns=now, permission=True, binding=o.binding) + runtime.observation.begin_frame() + controller.automatic_flm_effects = owner + controller.update(True, ci.CS.out, vm, SimpleNamespace(angleOffsetDeg=0.0, roll=0.0), False, curvature, False, 0.2) + ack = owner.acknowledge_frame(True) + assert ack is not None and ack.status == 'applied' and ack.confirmed_target_sha256 == o.target_sha256 + assert owner.confirmed_full == o.target + owner.neutral('driver_override') + controller.automatic_flm_effects = owner + assert owner.source is None and not any(owner.applied_full.values()) + assert owner.acknowledge_frame(True) is None + + +def test_pid_registry_keeps26_and_exact4_unsupported(): + from openpilot.starpilot.flm.pid_stages import SUPPORTED, UNSUPPORTED + from openpilot.starpilot.flm.canonical_surface_adapter import NAMES + + assert len(NAMES) == 30 and len(SUPPORTED) == 26 and len(UNSUPPORTED) == 4 + assert set(SUPPORTED) | set(UNSUPPORTED) == set(NAMES) + assert UNSUPPORTED == frozenset( + ('turn_in_threshold_reduction_left', 'turn_in_threshold_reduction_right', 'unwind_threshold_increase_left', 'unwind_threshold_increase_right') + ) diff --git a/openpilot/starpilot/flm/tests/test_live_trial_basis_store.py b/openpilot/starpilot/flm/tests/test_live_trial_basis_store.py new file mode 100644 index 0000000000..569b31902b --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_trial_basis_store.py @@ -0,0 +1,73 @@ +"""Native Params persistence; explicit unit basis is not physical identification.""" + +from dataclasses import replace +from pathlib import Path +import json +import pytest +from openpilot.common.params import Params +from openpilot.starpilot.flm.live_trial_store import LiveTrialPersistence, KEY, decode, load_confirmed_history +from openpilot.starpilot.flm.live_tracking_requalification import StableResponseBasis +from openpilot.starpilot.flm.tests.test_live_trial_store import confirmed + + +def unit_basis(binding): + return StableResponseBasis( + binding.identity, + binding.cp_sha256, + binding.controller_revision, + binding.baseline_revision, + (0.0, 0.0, 0.0), + (1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0), + ('unit-hardware-not-device-proof',), + binding.native_required, + ) + + +def test_confirmed_response_basis_roundtrips_without_current_permission(tmp_path): + params = Params(str(tmp_path)) + store = LiveTrialPersistence(params) + offer, app, cert = confirmed() + basis = unit_basis(offer.binding) + assert store.save_confirmed(offer, app, cert, now_ns=offer.observed_ns, current_binding=lambda: offer.binding, authorized=lambda: True, response_basis=basis) + raw = params.get(KEY) + row = next(iter(decode(raw)['confirmed'].values())) + assert row['retained_response']['current_permission'] is False + assert row['retained_response']['pending_trial'] is None + prior = load_confirmed_history(params)[0] + assert prior.historical_only is True and prior.response_basis == basis + assert prior.certificate == cert and prior.binding == offer.binding + assert store.save_confirmed(offer, app, cert, now_ns=offer.observed_ns, current_binding=lambda: offer.binding, authorized=lambda: True, response_basis=basis) + assert params.get(KEY) == raw + + +def test_legacy_confirmed_document_without_basis_keeps_target_for_fresh_bootstrap(tmp_path): + params = Params(str(tmp_path)) + store = LiveTrialPersistence(params) + offer, app, cert = confirmed() + assert store.save_confirmed(offer, app, cert, now_ns=offer.observed_ns, current_binding=lambda: offer.binding, authorized=lambda: True) + doc = decode(params.get(KEY)) + for row in doc['confirmed'].values(): + del row['retained_response'] + raw = json.dumps(doc).encode() + params.put(KEY, raw, block=True) + assert params.get(KEY) == raw + assert Path(params.get_param_path(KEY)).read_bytes() == raw + prior = load_confirmed_history(params)[0] + assert prior.target == offer.target and prior.response_basis is None and prior.historical_only is True + assert params.get(KEY) == raw + assert Path(params.get_param_path(KEY)).read_bytes() == raw + + +@pytest.mark.parametrize('kind', ['pending', 'denied', 'wrong_basis']) +def test_unconfirmed_or_wrong_context_basis_cannot_be_saved(tmp_path, kind): + params = Params(str(tmp_path)) + offer, app, cert = confirmed() + basis = unit_basis(offer.binding) + if kind == 'pending': + offer = replace(offer, phase='trial') + if kind == 'wrong_basis': + basis = replace(basis, cp_sha256='f' * 64) + assert not LiveTrialPersistence(params).save_confirmed( + offer, app, cert, now_ns=offer.observed_ns, current_binding=lambda: offer.binding, authorized=lambda: kind != 'denied', response_basis=basis + ) + assert params.get(KEY) is None diff --git a/openpilot/starpilot/flm/tests/test_live_trial_global_runtime.py b/openpilot/starpilot/flm/tests/test_live_trial_global_runtime.py new file mode 100644 index 0000000000..7570070fdf --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_trial_global_runtime.py @@ -0,0 +1,417 @@ +"""Actual Controls -> GlobalRuntime -> LaC at100Hz with real cereal deliveries. + +Response certificates below are explicit unit fixtures, not physical model +qualification. Admission, native permission, source leases, controller hooks, +application receipts and steering encoding execute their production paths. +""" + +from contextlib import contextmanager +from dataclasses import replace +import math +from unittest import mock + +import pytest + +from opendbc.car.car_helpers import interfaces +from opendbc.car.toyota.values import CAR +from openpilot.cereal import messaging +from openpilot.cereal.services import SERVICE_LIST +from openpilot.starpilot.flm.canonical_surface_adapter import full_profile_sha256 +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding +from openpilot.starpilot.flm.live_trial_owner import LiveTrialApplication +from openpilot.starpilot.flm.live_trial_wire import write_trial, write_tracking_source +from openpilot.starpilot.flm.runtime_adapter_registry import read_source +from openpilot.starpilot.flm.tests.test_live_trial_owner import offer, response +from openpilot.starpilot.flm.tests.test_registered_controls_global import controls_for, feed, source_packet + + +LIVE = 'starpilotSteeringTracking' +LEGACY = 'autoSteeringLearning' +BASE = 800_000_000_000 +DT = 10_000_000 +GENERATION = 'a' * 32 +FED = frozenset(('carState', 'selfdriveState', 'deviceState', 'vehicleParameters', 'longitudinalPlan', + 'lateralDelay', 'onroadEvents', 'modelV2', 'driverMonitoringState', 'extrinsicsCalibration', + LEGACY, LIVE, 'pandaStates', 'aolAxisState', 'aolSafetyWire')) + + +class LiveControls: + def __init__(self, controls): + self.controls = controls + self.tick = -1 + self.publications = 0 + self.latest = None + self.last_command = None + self.extra_last = {} + + @property + def runtime(self): + return self.controls.global_learning + + def source(self, now): + return read_source(self.controls.CP, self.controls.params, now_ns=now, + calibration=(0.0, 0.0, 0.0), calibration_generation=GENERATION, + vehicle_session='live-consumer-integration-drive') + + def initial_offer(self): + now = BASE + (self.tick + 1) * DT + runtime = self.runtime + source = self.source(now) + assert runtime.identity and runtime.live_vehicle_model_generation + assert runtime.identity_source_epoch == source.source_epoch + binding = LiveTrialBinding(runtime.identity, runtime.descriptor.cp_sha256, + runtime.descriptor.controller_revision, runtime.baseline_revision, + source.source_epoch, source.reset, source.calibration_generation, + runtime.live_vehicle_model_generation, runtime.native_identity, '0' * 64, source.persistent_source_epoch, runtime.native_required) + binding, witness = response(binding, now) + original = offer(now) + return replace(original, binding=binding, response_witness=witness, + baseline_sha256=full_profile_sha256(binding.baseline_revision, original.baseline), + target_sha256=full_profile_sha256(binding.baseline_revision, original.target), + deadline_ns=now + 120_000_000_000) + + def renew(self, proposal, now): + from openpilot.starpilot.flm.live_tracking_response import ResponseWitness, prepare_response_witness, validate_response_witness + + raw = validate_response_witness(proposal.response_witness, proposal.binding, proposal.observed_ns) + witness = prepare_response_witness(ResponseWitness(raw.certificate, proposal.binding, + now, now, now, proposal.started_ns - 500_000_000, 8, 0.001, now + 100_000_000), proposal.binding, now) + return replace(proposal, sequence=self.publications + 1, observed_ns=now, + expires_ns=now + 100_000_000, response_witness=witness) + + def frame(self, proposal=None, *, publish=True, source_valid=True, native_allowed=True, + driver=False, active=True, model_age=0, lease_ns=100_000_000): + self.tick += 1 + now = BASE + self.tick * DT + controls = self.controls + original_message = messaging.new_message + + def packet(service, *args, **kwargs): + event = original_message(service, *args, **kwargs) + if service == 'carState': + event.carState.steeringAngleDeg = math.degrees(controls.VM.get_steer_from_curvature(-0.0002, 30.0, 0.0)) + elif service == 'extrinsicsCalibration': + event.extrinsicsCalibration.calStatus = 'calibrated' + event.extrinsicsCalibration.validBlocks = 50 + event.extrinsicsCalibration.rpyCalibSpread = [0.0, 0.001, 0.001] + return event + + from openpilot.starpilot.aol.wire import encode_safety + + def native_packet(value): + # Change only the typed Panda input on denial. The positive path uses + # the existing real wire encoder and Controls' current_native parser. + if not native_allowed: + value = replace(value, lateralAllowed=False, longitudinalAllowed=False, + pandaInventory=tuple(replace(slot, controlsAllowed=False) for slot in value.pandaInventory)) + return encode_safety(value) + + with (mock.patch.object(messaging, 'new_message', side_effect=packet), + mock.patch('openpilot.starpilot.aol.wire.encode_safety', side_effect=native_packet)): + feed(controls, now, self.tick, speed=30.0, calibration=(0.0, 0.0, 0.0), + override=driver, active=active, model_age=model_age) + events = [] + # The existing feed fixture supplies the actual GlobalRuntime inputs. + # Populate other ordinary Controls subscriptions at their declared rates + # too, so all_checks() is a real aggregation assertion, not a flag patch. + for service in controls.sm.services: + frequency = SERVICE_LIST[service].frequency + if service in FED or service in controls.sm.ignore_alive or frequency <= 0: + continue + period = round(1e9 / frequency) + if service not in self.extra_last: + for stamp in (now - 2 * period, now - period): + event = messaging.new_message(service, valid=True, logMonoTime=stamp) + controls.sm.update_msgs(stamp / 1e9, [messaging.log_from_bytes(event.to_bytes())]) + self.extra_last[service] = now - period + if now - self.extra_last[service] >= period: + event = messaging.new_message(service, valid=True, logMonoTime=now) + events.append(messaging.log_from_bytes(event.to_bytes())) + self.extra_last[service] = now + if self.tick % 25 == 0: + events.append(source_packet(controls, now, self.tick, (0.0, 0.0, 0.0))) + panda_period = round(100 / SERVICE_LIST['pandaStates'].frequency) + if self.tick % panda_period == 0: + event = messaging.new_message('pandaStates', len(controls.CP.safetyConfigs), valid=True, logMonoTime=now) + for panda, config in zip(event.pandaStates, controls.CP.safetyConfigs, strict=True): + panda.safetyModel, panda.safetyParam = config.safetyModel, config.safetyParam + panda.alternativeExperience = controls.CP.alternativeExperience + panda.controlsAllowed = native_allowed and str(config.safetyModel) not in ('silent', 'noOutput') + events.append(messaging.log_from_bytes(event.to_bytes())) + if publish and self.tick % 5 == 0: + event = messaging.new_message(LIVE, valid=source_valid, logMonoTime=now) + write_tracking_source(event, self.source(now)) + if proposal is not None: + self.latest = replace(self.renew(proposal, now), expires_ns=now + lease_ns) + write_trial(event, self.latest) + self.publications += 1 + events.append(messaging.log_from_bytes(event.to_bytes())) + controls.sm.update_msgs(now / 1e9, events) + with mock.patch('openpilot.selfdrive.controls.controlsd.time.monotonic_ns', return_value=now): + command, _ = controls.state_control() + assert controls.global_learning is not None # Optional exceptions must not silently satisfy denials. + assert not controls.sm.simulation + assert math.isfinite(command.actuators.torque) and abs(command.actuators.torque) <= 1.0 + self.last_command = command + return now, command + + def assert_applied(self): + runtime = self.runtime + owner = runtime.live_trial_owner + assert runtime.reason == 'qualified', (self.tick, runtime.reason) + assert owner is not None and self.controls.LaC.automatic_flm_effects is owner + assert owner.used and owner.frame_complete(runtime.descriptor.domain) + assert type(runtime.outcome) is LiveTrialApplication + assert runtime.outcome.status == 'applied' + assert runtime.outcome.control_ns == BASE + self.tick * DT + assert runtime.outcome.sequence == self.latest.sequence + assert runtime.profile is None and runtime.validation is None + assert owner.confirmed_full == self.latest.baseline or owner.offer.phase == 'confirmed' + return owner.applied_full['ff_gain_left'] + + def assert_neutral(self): + assert self.controls.LaC.automatic_flm_effects is None + owner = self.runtime.live_trial_owner + if owner is not None: + assert owner.source is None and not any(owner.applied_full.values()) + assert owner.application is None + outcome = self.runtime.outcome + if outcome is not None: + from openpilot.starpilot.flm.adaptive_profile import ApplicationOutcome + # Active original control still emits fresh legacy neutral bookkeeping. + # This must never be a stale live-trial acknowledgement or correction. + assert type(outcome) is ApplicationOutcome + assert outcome.observed_ns == BASE + self.tick * DT + assert outcome.identity == self.runtime.identity and outcome.baseline_revision == self.runtime.baseline_revision + assert outcome.status in ('collecting', 'paused') and outcome.reason == self.runtime.reason + assert outcome.profile_sequence == 0 and outcome.actual_coordinates == (0.0, 0.0) + assert outcome.physical_model_revision == '' + assert outcome.effect_manifest_sha256 == outcome.effect_original_anchor == outcome.effect_applied_profile_sha256 == '' + assert self.runtime.profile is None and self.runtime.validation is None + + +@contextmanager +def live_controls(): + cp = interfaces[CAR.TOYOTA_RAV4_TSS2].get_non_essential_params(CAR.TOYOTA_RAV4_TSS2) + with controls_for(cp, 'standard', True) as controls: + fixture = LiveControls(controls) + assert LIVE in controls.sm.services and 'pandaStates' in controls.sm.services + from openpilot.starpilot.aol.vehicle import policy_for + assert controls.aol_replay and controls.ordinary_axis_ack_required + assert policy_for(controls.CP).ordinary_axis_ack_required + assert {'aolAxisState', 'aolSafetyWire'} <= set(controls.sm.services) + for _ in range(60): + fixture.frame() + assert controls.sm.freq_ok[LIVE] and controls.sm.freq_ok[LEGACY] + assert controls.sm.freq_ok['pandaStates'] + assert controls.sm.freq_ok['aolAxisState'] and controls.sm.freq_ok['aolSafetyWire'] + assert fixture.runtime.native_required and fixture.runtime.native_identity + assert controls.flm_native_witness is not None + yield fixture + + +def test_production100hz_consumer_across4hz_metadata_and20hz_trials_reaches_actual_can(): + from openpilot.starpilot.flm.tests.test_steering_projection import make_interface, install_observer, settle, control, apply + + with live_controls() as fixture: + proposal = fixture.initial_offer() + original_tune = fixture.controls.LaC.torque_params.to_dict() + native = make_interface(CAR.TOYOTA_RAV4_TSS2) + assert install_observer(native) + settle(native, control(0.0)) + previous = 0.0 + actual_packets = 0 + for _ in range(120): + _, command = fixture.frame(proposal) + value = fixture.assert_applied() + assert 0 < value <= 0.005 + assert 0 <= value - previous <= 0.00005 + 1e-12 + previous = value + assert fixture.controls.sm.freq_ok[LIVE] and fixture.controls.sm.freq_ok[LEGACY] + assert fixture.controls.sm.all_checks() # Ordinary aggregation sees no artificial rate failure. + # Card receives the serialized SubMaster reader, not the Controls builder. + reader = command.as_reader() + assert reader.actuators.torque == command.actuators.torque and reader.latActive == command.latActive + _, packets, witness = apply(native, reader) + if witness is not None: + actual_packets += 1 + assert (witness.can_address, witness.can_data, witness.can_bus) in packets + assert actual_packets > 0 + assert previous == pytest.approx(0.005) + assert fixture.controls.LaC.torque_params.to_dict() == original_tune + assert fixture.controls.params.get('AutomaticSteeringLiveTrials') is None + + +def test_missed20hz_publications_expire_at100ms_and_old_pending_cannot_resume_or_confirm(): + with live_controls() as fixture: + proposal = fixture.initial_offer() + for _ in range(21): + fixture.frame(proposal) + fixture.assert_applied() + last = fixture.controls.sm.recv_time[LIVE] + for _ in range(10): + now, _ = fixture.frame(proposal, publish=False) + assert now <= round(last * 1e9) + 100_000_000 + fixture.assert_applied() + fixture.frame(proposal, publish=False) + fixture.assert_neutral() + assert fixture.controls.sm.alive[LIVE] # Generic20Hz liveness is500ms, not the100ms permission lease. + for _ in range(9): + fixture.frame(proposal) + fixture.assert_neutral() + confirmed = replace(proposal, phase='confirmed') + for _ in range(5): + fixture.frame(confirmed) + fixture.assert_neutral() + assert fixture.runtime.live_trial_owner.confirmed_full == proposal.baseline + assert fixture.controls.params.get('AutomaticSteeringLiveTrials') is None + replacement = replace(proposal, trial_id='c' * 64, target_sequence=2) + for _ in range(10): + fixture.frame(replacement) + if fixture.tick % 5 == 0 or fixture.runtime.reason == 'qualified': + fixture.assert_applied() + + +@pytest.mark.parametrize('fault', ['driver', 'native', 'source', 'model', 'inactive']) +def test_current_fault_neutralizes_then_rollback_requires_fresh_permission(fault): + with live_controls() as fixture: + proposal = fixture.initial_offer() + for _ in range(20): + fixture.frame(proposal) + fixture.assert_applied() + assert fixture.tick == 79 + before = fixture.runtime.live_trial_owner.applied_full['ff_gain_left'] + assert before > 0 + kwargs = {'driver': fault == 'driver', 'native_allowed': fault != 'native', + 'source_valid': fault != 'source', 'model_age': 1_000_000_000 if fault == 'model' else 0, + 'active': fault != 'inactive'} + fixture.frame(proposal, **kwargs) + fixture.assert_neutral() + rollback = replace(proposal, phase='rollback', target=proposal.baseline, target_sha256=proposal.baseline_sha256) + # The rollback offer is configuration; fresh native/current source admission + # remains necessary before it can submit a controller command. + for _ in range(10): + fixture.frame(rollback, **{**kwargs, 'native_allowed': False}) + fixture.assert_neutral() + assert fixture.runtime.live_trial_owner.confirmed_full == proposal.baseline + for _ in range(15): + fixture.frame(rollback) + if fixture.tick >= 100: + fixture.assert_applied() + else: + fixture.assert_neutral() + assert fixture.runtime.live_trial_owner.confirmed_full == proposal.baseline + assert not any(fixture.runtime.live_trial_owner.applied_full.values()) + assert fixture.runtime.outcome.confirmed_target_sha256 == '' + from openpilot.starpilot.flm.learning_wire import write_control, read_control + + now = BASE + fixture.tick * DT + evidence = fixture.runtime.capture_evidence(control_ns=now, + state_ns=int(fixture.controls.sm.logMonoTime['carState']), active=bool(fixture.last_command.latActive), limited=False) + assert evidence is not None and evidence.live_applied_coordinates == (0.0,) * 30 + event = messaging.new_message('starpilotLateralState', valid=True, logMonoTime=now) + write_control(event.starpilotLateralState, evidence) + assert read_control(messaging.log_from_bytes(event.to_bytes())).live_applied_coordinates == (0.0,) * 30 + + +def test_confirmed_requires_actual_controller_ack_and_retains_exact_older_publication_sequence(): + from openpilot.starpilot.flm.live_trial_wire import write_application, read_application + + with live_controls() as fixture: + proposal = fixture.initial_offer() + for _ in range(110): + fixture.frame(proposal) + fixture.assert_applied() + assert fixture.runtime.live_trial_owner.confirmed_full == proposal.baseline + confirmed = replace(proposal, phase='confirmed') + for _ in range(5): + now, _ = fixture.frame(confirmed) + fixture.assert_applied() + application = fixture.runtime.outcome + assert application.confirmed_target_sha256 == proposal.target_sha256 + assert fixture.runtime.live_trial_owner.confirmed_full == proposal.target + event = messaging.new_message('starpilotLateralState', valid=True, logMonoTime=now) + write_application(event.starpilotLateralState, application) + later = replace(fixture.latest, sequence=fixture.latest.sequence + 1) + parsed = read_application(messaging.log_from_bytes(event.to_bytes()).starpilotLateralState, now_ns=now, offer=later) + assert parsed is not None and parsed.sequence == application.sequence + assert parsed.sequence < later.sequence + assert parsed.applied == proposal.target + assert parsed.confirmed_target_sha256 == proposal.target_sha256 + assert read_application(event.starpilotLateralState, now_ns=now + 100_000_001, offer=later) is None + + +def test_actual_control_evidence_keeps_owner_vm_anchor_and_typed_wire_bounds(): + from openpilot.starpilot.flm.learning_wire import write_control, read_control + + with live_controls() as fixture: + proposal = fixture.initial_offer() + now, command = fixture.frame(proposal) + fixture.assert_applied() + evidence = fixture.runtime.capture_evidence(control_ns=now, + state_ns=int(fixture.controls.sm.logMonoTime['carState']), active=bool(command.latActive), limited=False) + assert evidence is not None + assert evidence.vehicle_model_anchor == tuple(fixture.runtime.vm_anchor) + assert len(evidence.vehicle_model_anchor) == 3 + actual = tuple(value for _, value in fixture.runtime.outcome.applied) + assert len(evidence.live_applied_coordinates) == 30 + assert evidence.live_applied_coordinates == actual + assert evidence.live_applied_coordinates != tuple(value for _, value in proposal.target) + assert evidence.effect_trace is None # A live stage acknowledgement is not a replay trace. + assert evidence.applied_coordinates == (0.0, 0.0) + event = messaging.new_message('starpilotLateralState', valid=True, logMonoTime=now) + write_control(event.starpilotLateralState, evidence) + decoded = read_control(messaging.log_from_bytes(event.to_bytes())) + assert decoded.vehicle_model_anchor == evidence.vehicle_model_anchor + assert decoded.live_applied_coordinates == actual + assert decoded.effect_trace is None + # Historical recordings did not carry an anchor; no reconstruction mints one. + write_control(event.starpilotLateralState, replace(evidence, vehicle_model_anchor=(), live_applied_coordinates=())) + historical = read_control(messaging.log_from_bytes(event.to_bytes())) + assert historical.vehicle_model_anchor == () + assert historical.live_applied_coordinates == () + for invalid in ((1.0,), (1.0, 2.0), (1.0, 2.0, 3.0, 4.0), + (1.0, 0.0, 3.0), (1.0, math.nan, 3.0), (1.0, math.inf, 3.0)): + with pytest.raises(ValueError, match='control evidence'): + write_control(event.starpilotLateralState, replace(evidence, vehicle_model_anchor=invalid)) + write_control(event.starpilotLateralState, evidence) + event.starpilotLateralState.learningEvidence.vehicleModelAnchor = invalid + with pytest.raises(ValueError, match='control version or bound'): + read_control(messaging.log_from_bytes(event.to_bytes())) + for invalid in ((0.0,), (0.0,) * 29, (0.0,) * 31, (math.nan,) * 30, (math.inf,) * 30): + with pytest.raises(ValueError, match='control evidence'): + write_control(event.starpilotLateralState, replace(evidence, live_applied_coordinates=invalid)) + write_control(event.starpilotLateralState, evidence) + event.starpilotLateralState.learningEvidence.liveAppliedCoordinates = invalid + with pytest.raises(ValueError, match='control version or bound'): + read_control(messaging.log_from_bytes(event.to_bytes())) + now, command = fixture.frame(proposal, driver=True) + fixture.assert_neutral() + interrupted = fixture.runtime.capture_evidence(control_ns=now, + state_ns=int(fixture.controls.sm.logMonoTime['carState']), active=bool(command.latActive), limited=False) + assert interrupted is None # No previous live map or acknowledgement survives as current evidence. + assert fixture.runtime.outcome is None + assert not any(fixture.runtime.live_trial_owner.applied_full.values()) + + +@pytest.mark.parametrize('lease_ns,publication_delay_ns', [(10_000_000, 20_000_000), (100_000_000, 30_000_001)]) +def test_consumed_live_map_without_current_publication_proof_is_absent_not_neutral(lease_ns, publication_delay_ns): + with live_controls() as fixture: + proposal = fixture.initial_offer() + now, command = fixture.frame(proposal, lease_ns=lease_ns) + fixture.assert_applied() + actual = fixture.runtime.outcome.applied + assert any(value != 0.0 for _, value in actual) + assert fixture.runtime.live_trial_owner.used + assert fixture.runtime.live_trial_owner.frame_complete(fixture.runtime.descriptor.domain) + # The first case expires its real offer before publication within30ms; + # the second still has an offer lease but misses same-frame publication. + publication_ns = now + publication_delay_ns + if lease_ns < publication_delay_ns: + assert publication_ns > fixture.latest.expires_ns + else: + assert publication_ns <= fixture.latest.expires_ns + evidence = fixture.runtime.capture_evidence(control_ns=publication_ns, + state_ns=int(fixture.controls.sm.logMonoTime['carState']), active=bool(command.latActive), limited=False) + assert evidence is None + assert fixture.runtime.outcome.applied == actual # Historical consumption is not relabelled as neutral. diff --git a/openpilot/starpilot/flm/tests/test_live_trial_messaging.py b/openpilot/starpilot/flm/tests/test_live_trial_messaging.py new file mode 100644 index 0000000000..2f4935605f --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_trial_messaging.py @@ -0,0 +1,94 @@ +"""Real SubMaster/cereal rate and health contracts; no frequency flag injection. + +update_msgs supplies deterministic transport receipt times to the production +SubMaster implementation. These tests do not claim physical source evidence. +""" + +import pytest + +from openpilot.cereal import messaging +from openpilot.cereal.services import SERVICE_LIST +from openpilot.common.prefix import OpenpilotPrefix + + +LEGACY = 'autoSteeringLearning' +LIVE = 'starpilotSteeringTracking' +BASE = 700_000_000_000 +DT = 10_000_000 + + +def packet(service, now, *, valid=True): + event = messaging.new_message(service, valid=valid, logMonoTime=now) + return messaging.log_from_bytes(event.to_bytes()) + + +def deliver(sm, tick, *, legacy=True, live=True, live_period=5, legacy_period=25, live_valid=True): + now = BASE + tick * DT + events = [packet('carState', now)] + if legacy and tick % legacy_period == 0: + events.append(packet(LEGACY, now)) + if live and tick % live_period == 0: + events.append(packet(LIVE, now, valid=live_valid)) + sm.update_msgs(now / 1e9, events) + return now + + +def test_separate_topics_keep_declared4_and20hz_and_ordinary_health(monkeypatch): + monkeypatch.setenv('SIMULATION', '0') + assert SERVICE_LIST[LEGACY].frequency == 4.0 + assert SERVICE_LIST[LIVE].frequency == 20.0 + with OpenpilotPrefix(): + sm = messaging.SubMaster(['carState', LEGACY, LIVE], frequency=100) + assert not sm.simulation + for tick in range(201): + deliver(sm, tick) + if tick >= 25: + assert sm.all_checks(), (tick, sm.alive, sm.valid, sm.freq_ok) + # The optional publisher can stop without changing the legacy publisher's + # rate, validity or health. The live raw health fails at its real threshold. + last_live = sm.recv_time[LIVE] + for tick in range(201, 261): + deliver(sm, tick, live=False) + assert sm.recv_time[LIVE] == last_live + assert not sm.alive[LIVE] + assert not sm.all_checks() + assert sm.all_checks(['carState', LEGACY]) + for tick in range(261, 301): + deliver(sm, tick) + assert sm.all_checks() + + +@pytest.mark.parametrize('wrong_service', [LEGACY, LIVE]) +def test_standard_submaster_rejects_swapped_rates_without_forged_freq_flags(monkeypatch, wrong_service): + monkeypatch.setenv('SIMULATION', '0') + with OpenpilotPrefix(): + sm = messaging.SubMaster(['carState', LEGACY, LIVE], frequency=100) + for tick in range(301): + deliver(sm, tick, legacy_period=5 if wrong_service == LEGACY else 25, + live_period=25 if wrong_service == LIVE else 5) + assert sm.seen[wrong_service] and sm.valid[wrong_service] and sm.alive[wrong_service] + assert not sm.freq_ok[wrong_service] + assert not sm.all_freq_ok() and not sm.all_checks() + other = LIVE if wrong_service == LEGACY else LEGACY + assert sm.freq_ok[other] + + +def test_optional_live_failure_does_not_create_ordinary_comm_issue(monkeypatch): + monkeypatch.setenv('SIMULATION', '0') + with OpenpilotPrefix(): + sm = messaging.SubMaster(['carState', LEGACY, LIVE], frequency=100, + ignore_alive=[LIVE], ignore_valid=[LIVE], ignore_avg_freq=[LIVE]) + for tick in range(101): + deliver(sm, tick) + assert sm.freq_ok[LEGACY] and sm.freq_ok[LIVE] + deliver(sm, 101, live_period=1, live_valid=False) + assert not sm.valid[LIVE] + assert sm.all_checks() + for tick in range(102, 162): + deliver(sm, tick, live=False) + assert not sm.alive[LIVE] + assert sm.all_checks() + # A genuine ordinary input failure must still reach ordinary aggregation. + now = BASE + 162 * DT + sm.update_msgs(now / 1e9, [packet('carState', now, valid=False)]) + assert not sm.all_valid() and not sm.all_checks() diff --git a/openpilot/starpilot/flm/tests/test_live_trial_owner.py b/openpilot/starpilot/flm/tests/test_live_trial_owner.py new file mode 100644 index 0000000000..5463f88f62 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_trial_owner.py @@ -0,0 +1,171 @@ +"""Live owner unit contracts; unit model fixtures are NOT physical qualification.""" + +from dataclasses import replace +import pytest +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, full_profile_sha256 +from openpilot.starpilot.flm.live_trial_contract import LiveTrialBinding, LiveTrialOffer, VERSION +from openpilot.starpilot.flm.live_trial_owner import TrackingTrialOwner + + +def binding(): + return LiveTrialBinding( + '1' * 64, '2' * 64, 'actual-controller-registry-v2:' + '3' * 64, '4' * 64, '5' * 64, 'reset', 'generation', '6' * 64, '', '7' * 64, '8' * 64, False + ) + + +def response(b, now): + from openpilot.starpilot.flm.live_tracking_response import ResponseCertificate, ResponseWitness, _revision, validate_certificate, prepare_response_witness + + c = ResponseCertificate( + 'live-tracking-response-v1', b, '0' * 64, 0, 1.0, 0.0, -1.0, 1.0, 0.001, ('a', 'b', 'c', 'd'), ('e', 'f', 'g', 'h'), '9' * 64, 'a' * 64, now - 1_000_000_000 + ) + revision = _revision(c) + b = replace(b, physical_model_revision=revision) + c = validate_certificate(replace(c, binding=b, model_revision=revision)) + witness = prepare_response_witness(ResponseWitness(c, b, now, now, now, now - 500_000_000, 8, 0.001, now + 100_000_000), b, now) + return b, witness + + +def offer(now=2_000_000_000): + b, w = response(binding(), now) + baseline = tuple((n, 0.0) for n in NAMES) + target = tuple((n, 0.005 if n == 'ff_gain_left' else 0.0) for n in NAMES) + return LiveTrialOffer( + VERSION, + b, + 'b' * 64, + 1, + 'trial', + 'ff.left', + ('ff_gain_left',), + baseline, + target, + full_profile_sha256(b.baseline_revision, baseline), + full_profile_sha256(b.baseline_revision, target), + now, + now, + now + 100_000_000, + w, + 1, + now + 5_000_000_000, + ) + + +def test_pending_trial_is_distinct_from_replay_and_small_slewed(): + o = offer() + owner = TrackingTrialOwner(o.binding) + assert owner.prepare(o, now_ns=o.observed_ns, permission=True, binding=o.binding) + assert 0 < owner.applied_full['ff_gain_left'] <= 0.00005 + assert owner.confirmed_full == o.baseline + assert owner.application.confirmed_target_sha256 == '' + assert owner.trace is None + + +@pytest.mark.parametrize('field,value', [('phase', 'restore'), ('sequence', 0), ('trial_id', 'bad'), ('version', 'replay-v1')]) +def test_unknown_or_reordered_offer_denies(field, value): + o = replace(offer(), **{field: value}) + owner = TrackingTrialOwner(o.binding) + with pytest.raises(ValueError): + owner.prepare(o, now_ns=o.observed_ns, permission=True, binding=o.binding) + + +def test_no_permission_immediately_neutral_no_rollback_command(): + o = offer() + owner = TrackingTrialOwner(o.binding) + owner.prepare(o, now_ns=o.observed_ns, permission=True, binding=o.binding) + assert not owner.prepare(o, now_ns=o.observed_ns + 10_000_000, permission=False, binding=o.binding) + assert owner.source is None and not any(owner.applied_full.values()) + assert owner.confirmed_full == o.baseline + + +def test_interrupted_pending_cannot_resume_and_new_trial_keeps_confirmed_cas(): + o = offer() + owner = TrackingTrialOwner(o.binding) + owner.prepare(o, now_ns=o.observed_ns, permission=True, binding=o.binding) + owner.neutral('driver') + with pytest.raises(ValueError): + owner.prepare(replace(o, sequence=2), now_ns=o.observed_ns + 10_000_000, permission=True, binding=o.binding) + new = replace(o, trial_id='c' * 64, sequence=2, target_sequence=2, observed_ns=o.observed_ns + 10_000_000) + assert owner.prepare(new, now_ns=new.observed_ns, permission=True, binding=new.binding) + assert owner.confirmed_full == o.baseline + + +def test_pending_never_becomes_confirmed_without_actual_stage_ack(): + o = offer() + owner = TrackingTrialOwner(o.binding) + owner.prepare(o, now_ns=o.observed_ns, permission=True, binding=o.binding) + c = replace(o, phase='confirmed', sequence=2, observed_ns=o.observed_ns + 10_000_000) + owner.prepare(c, now_ns=o.observed_ns + 10_000_000, permission=True, binding=o.binding) + assert owner.confirmed_full == o.baseline + assert owner.acknowledge_frame(False).confirmed_target_sha256 == '' + assert owner.confirmed_full == o.baseline + + +def test_original_baseline25percent_bound_and_trial_delta_checked(): + o = offer() + bad = tuple((n, 0.251 if n == 'ff_gain_left' else v) for n, v in o.target) + with pytest.raises(ValueError): + replace(o, target=bad).validate() + bad = tuple((n, 0.006 if n == 'ff_gain_left' else v) for n, v in o.target) + with pytest.raises(ValueError): + replace(o, target=bad, target_sha256=full_profile_sha256(o.binding.baseline_revision, bad)).validate() + + +def test_missed_pose_publication_does_not_restamp_witness(): + o = offer() + owner = TrackingTrialOwner(o.binding) + with pytest.raises(ValueError): + owner.prepare(o, now_ns=o.observed_ns + 110_000_000, permission=True, binding=o.binding) + + +def test_100hz_current_checks_with_genuine20hz_publications(): + base = offer() + owner = TrackingTrialOwner(base.binding) + for tick in range(50): + now = base.observed_ns + tick * 10_000_000 + if tick % 5 == 0: + # This helper constructs a NEW immutable raw source observation per20Hz tick. + # Certificate revision must remain the same model; only witness times renew. + from openpilot.starpilot.flm.live_tracking_response import ResponseWitness, prepare_response_witness, validate_response_witness + + raw = validate_response_witness(base.response_witness, base.binding, base.observed_ns) + w = prepare_response_witness( + ResponseWitness(raw.certificate, base.binding, now, now, now, base.observed_ns - 500_000_000, 8, 0.001, now + 100_000_000), base.binding, now + ) + current = replace(base, sequence=tick // 5 + 1, observed_ns=now, expires_ns=now + 100_000_000, response_witness=w) + assert owner.prepare(current, now_ns=now, permission=True, binding=base.binding) + assert owner.source is not None + + +def test_interrupted_pending_cannot_be_confirmed_even_with_new_publication(): + o = offer() + owner = TrackingTrialOwner(o.binding) + owner.prepare(o, now_ns=o.observed_ns, permission=True, binding=o.binding) + owner.neutral('native_or_driver_interruption') + rejected = replace(o, phase='confirmed', sequence=2, observed_ns=o.observed_ns + 10_000_000) + with pytest.raises(ValueError, match='interrupted pending'): + owner.prepare(rejected, now_ns=rejected.observed_ns, permission=True, binding=o.binding) + assert owner.confirmed_full == o.baseline + assert owner.source is None + + +@pytest.mark.parametrize('name', ['baseline_sha256', 'target_sha256']) +def test_offer_hash_requires_exact_digest_string(name): + from openpilot.starpilot.flm.live_trial_contract import validate_offer + + class EqualToAnything: + def __eq__(self, other): + return True + + with pytest.raises(ValueError, match='full30'): + validate_offer(replace(offer(), **{name: EqualToAnything()})) + + +def test_owned_offer_content_rejects_class_accessor_mutation(monkeypatch): + from openpilot.starpilot.flm.live_trial_contract import validate_offer, LiveTrialOffer + + o = offer() + validate_offer(o) + monkeypatch.setattr(LiveTrialOffer, 'phase', property(lambda self: 'confirmed'), raising=False) + with pytest.raises(ValueError, match='plain live offer shape'): + validate_offer(o) diff --git a/openpilot/starpilot/flm/tests/test_live_trial_store.py b/openpilot/starpilot/flm/tests/test_live_trial_store.py new file mode 100644 index 0000000000..5faef73d78 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_live_trial_store.py @@ -0,0 +1,90 @@ +"""Actual native Params document behavior; unit ack is NOT actuation proof.""" + +from dataclasses import replace +from pathlib import Path +import pytest +from openpilot.common.params import Params +from openpilot.starpilot.flm.live_trial_owner import LiveTrialApplication +from openpilot.starpilot.flm.live_trial_store import KEY, LiveTrialPersistence, decode, load_confirmed_history +from openpilot.starpilot.flm.tests.test_live_trial_owner import offer + + +def confirmed(): + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + + pending = offer() + o = replace(pending, phase='confirmed', sequence=2, observed_ns=pending.observed_ns + 10_000_000) + app = LiveTrialApplication( + 1, + o.binding, + o.trial_id, + o.sequence, + o.observed_ns, + 'applied', + o.target, + o.target_sha256, + o.target_sha256, + 'unit_actual_application_fixture', + o.target_sequence, + ) + return o, app, validate_response_witness(o.response_witness, o.binding, o.observed_ns).certificate + + +def test_actual_native_params_confirmed_only_and_new_drive_is_historical(tmp_path): + params = Params(str(tmp_path)) + params.put('AutomaticSteeringProfiles', b'legacy-replay-document-untouched', block=True) + store = LiveTrialPersistence(params) + o, app, c = confirmed() + assert store.save_confirmed(o, app, c, now_ns=o.observed_ns, current_binding=lambda: o.binding, authorized=lambda: True) + raw = params.get(KEY) + assert raw is not None + assert params.get('AutomaticSteeringProfiles') == b'legacy-replay-document-untouched' + assert len(decode(raw)['confirmed']) == 1 + new = replace(o.binding, source_epoch='c' * 64, calibration_generation='new-drive-generation', physical_model_revision='d' * 64) + prior = LiveTrialPersistence(Params(str(tmp_path))).load_prior(new) + assert prior is not None and prior.historical_only is True and prior.target == o.target + assert prior.binding == o.binding and prior.binding != new + assert not hasattr(prior, 'expires_ns') and not hasattr(prior, 'permission') + assert load_confirmed_history(params) == (prior,) + before = params.get(KEY) + assert store.save_confirmed(o, app, c, now_ns=o.observed_ns, current_binding=lambda: o.binding, authorized=lambda: True) + assert params.get(KEY) == before + + +@pytest.mark.parametrize('kind', ['pending', 'neutral_ack', 'wrong_ack_target', 'stale_ack', 'denied_permission', 'changed_reset', 'unknown_vehicle']) +def test_no_pending_or_invalid_application_can_persist(tmp_path, kind): + params = Params(str(tmp_path)) + store = LiveTrialPersistence(params) + o, app, c = confirmed() + current = o.binding + allowed = True + now = o.observed_ns + if kind == 'pending': + o = replace(o, phase='trial') + elif kind == 'neutral_ack': + app = replace(app, status='neutral', confirmed_target_sha256='') + elif kind == 'wrong_ack_target': + app = replace(app, applied=o.baseline) + elif kind == 'stale_ack': + now += 110_000_000 + elif kind == 'denied_permission': + allowed = False + elif kind == 'changed_reset': + current = replace(current, reset_generation='newreset') + elif kind == 'unknown_vehicle': + o = replace(o, binding=replace(o.binding, persistent_source_epoch='')) + assert not store.save_confirmed(o, app, c, now_ns=now, current_binding=lambda: current, authorized=lambda: allowed) + assert params.get(KEY) is None + + +def test_pending_document_is_never_loaded_or_repaired(tmp_path): + params = Params(str(tmp_path)) + raw = b'{"version":"live-confirmed-history-v1","confirmed":{},"pending":{"target":1}}' + params.put(KEY, raw, block=True) + assert params.get(KEY) == raw + assert Path(params.get_param_path(KEY)).read_bytes() == raw + assert LiveTrialPersistence(params).load_prior(offer().binding) is None + assert params.get(KEY) == raw + assert Path(params.get_param_path(KEY)).read_bytes() == raw + with pytest.raises(ValueError): + decode(raw) diff --git a/openpilot/starpilot/flm/tests/test_native_cbuffer.py b/openpilot/starpilot/flm/tests/test_native_cbuffer.py new file mode 100644 index 0000000000..7d666f9d9c --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_native_cbuffer.py @@ -0,0 +1,183 @@ +"""ROOT ONLY: actual rebuilt source-bound extension; author did not import/run. +No fixture substitutes the native serializer or grants learning authority. +""" +import hashlib +import importlib +import json +import math +import sys +from typing import cast + +import pytest + + +def typed(value): + kind = type(value) + if value is None or kind in (bool, int, str): + return [kind.__name__, value] + if kind is float: + return ['float64', value.hex()] + if kind is bytes: + return ['bytes', value.hex()] + if kind in (tuple, list): + return [kind.__name__, [typed(child) for child in value]] + return ['dict', [[key, typed(value[key])] for key in sorted(value)]] + + +def oracle(value): + return hashlib.sha256(json.dumps(typed(value), separators=(',', ':'), allow_nan=False).encode()).hexdigest() + + +@pytest.fixture +def native(): + return importlib.import_module('openpilot.starpilot.flm._primitive_walk') + + +def large(child): + return {'rows': [child for _ in range(257)], 'version': 'test-only-byte-oracle'} + + +def test_complete_typed_alphabet_and_exact_ordering(native): + value = large({'z': [None, True, False, 0, 1, -1, 2**511, -(2**511), 0.0, -0.0, + math.ldexp(1.0, -1074), sys.float_info.max, b'\x00\xff', (), [], {}], + 'a': ('quote"', 'slash\\', '\n\r\t\b\f', '\x00\x1f', 'é', '\u2028', + '\U0001f680', '\ud800', '\udc00', '\ud800\udc00')}) + assert native.whole_body_digest(value) == oracle(value) + assert native.whole_body_digest(large([True, 1])) != native.whole_body_digest(large([1, True])) + assert native.whole_body_digest(large([0.0])) != native.whole_body_digest(large([-0.0])) + assert native.whole_body_digest(large((1, 2))) != native.whole_body_digest(large([1, 2])) + assert native.whole_body_digest(large({'b': 2, 'a': 1})) == oracle(large({'a': 1, 'b': 2})) + + +def test_fresh_call_mutable_alias_and_changed_body(native): + shared = {'cp': bytes(range(256)) * 8, 'revision': ['a', 0.0], 'config': {'enabled': True}} + value = large(shared) + first = native.whole_body_digest(value) + assert first == oracle(value) + shared['revision'][1] = -0.0 + shared['config']['enabled'] = False + shared['cp'] += b'changed' + second = native.whole_body_digest(value) + assert second == oracle(value) and second != first + value['rows'][128] = {'new': ('distinct', 4)} + assert native.whole_body_digest(value) == oracle(value) + + +@pytest.mark.parametrize('length', [5460, 5461, 5462, 32761, 32762, 32763]) +def test_scalar_entry_boundary_unicode_bytes(native, length): + # Crosses conservative Unicode and exact byte-token64KiB entry thresholds. + child = {'s': '\U0001f680' * length, 'b': b'\xff' * length} + value = {'rows': [child, child] + [None] * 255} + assert native.whole_body_digest(value) == oracle(value) + + +def test_split_unicode_escape_boundaries(native): + value = large({'s': 'x' * 2047 + '\U0001f680' + '\ud800' + '\udc00' + '\\"\n' + 'z' * 8200}) + assert native.whole_body_digest(value) == oracle(value) + + +def test_large_int_encoder_and_digit_limit(native): + value = large({'n': 10**600, 'm': -(10**600)}) + assert native.whole_body_digest(value) == oracle(value) + before = sys.get_int_max_str_digits() + try: + sys.set_int_max_str_digits(640) + value = large({'n': 10**700}) + with pytest.raises(ValueError): + oracle(value) + with pytest.raises(ValueError): + native.whole_body_digest(value) + finally: + sys.set_int_max_str_digits(before) + + +def test_whole_graph_prevalidation_and_no_callbacks(native): + class Trap: + def __repr__(self): + raise AssertionError('callback ran') + class DictSubclass(dict): + def items(self): + raise AssertionError('subclass callback ran') + for bad in [Trap(), DictSubclass(), {1: 'nonstring key'}, float('nan'), float('inf')]: + assert native.whole_body_digest(large(bad)) is native.FALLBACK + cycle = [] + cycle.append(cycle) + assert native.whole_body_digest(large(cycle)) is native.FALLBACK + deep = 0 + for _ in range(34): + deep = [deep] + assert native.whole_body_digest(large(deep)) is native.FALLBACK + + +def test_no_cache_state_between_calls_after_failed_prevalidation(native): + shared = {'x': b'original' * 128} + value = large(shared) + expected = native.whole_body_digest(value) + shared['bad'] = object() + assert native.whole_body_digest(value) is native.FALLBACK + del shared['bad'] + assert native.whole_body_digest(value) == expected == oracle(value) + + + +def test_every_ascii_and_unicode_escape_class(native): + value = large(''.join(chr(i) for i in range(256)) + '\U00010000\U0010ffff\ud800\udfff') + assert native.whole_body_digest(value) == oracle(value) + + +def test_native_buffer_boundaries_and_maximum_update(native, monkeypatch): + value = {'rows': [b'\x00\x7f\xff' * 21900, 'x' * 65535, 'q' * 65536, + '\u007f' * 10923, '\U0001f680' * 5462] + [None] * 257} + expected = oracle(value) + real_sha = hashlib.sha256 + updates = [] + class RecordingDigest: + def __init__(self): + self.inner = real_sha() + def update(self, chunk): + updates.append(chunk) + self.inner.update(chunk) + def hexdigest(self): + return self.inner.hexdigest() + monkeypatch.setattr(hashlib, 'sha256', RecordingDigest) + assert native.whole_body_digest(value) == expected + assert max(map(len, updates)) <= 65536 + assert b''.join(updates) == json.dumps(typed(value), separators=(',', ':'), allow_nan=False).encode() + + +def test_small_path_still_original_encoder(native, monkeypatch): + real_dumps = json.dumps + calls = [] + def recording(value, *args, **kwargs): + calls.append(value) + return real_dumps(value, *args, **kwargs) + monkeypatch.setattr(json, 'dumps', recording) + assert native.whole_body_digest({'x': -0.0}) == oracle({'x': -0.0}) + assert len(calls) == 2 + + +@pytest.mark.parametrize('key', [1, None, True, 1.5, ('tuple', 1)]) +def test_post_prepass_nonstring_dict_key_matches_safe_original_json(native, monkeypatch, key): + # Authoritative old algorithm also accepts this post-prepass mutation: the + # dict key occupies a raw JSON pair slot, rather than a typed leaf slot. + # Root must run against original and successor SOs, proving equal digests. + value = [b'x' * 40000] + [None] * 255 + [{'marker': 'unchanged'}] + real_sha = hashlib.sha256 + changed = False + class ReentrantDigest: + def __init__(self): + self.inner = real_sha() + def update(self, chunk): + nonlocal changed + if not changed: + cast(dict[object, str], value[-1]).clear() + cast(dict[object, str], value[-1])[key] = 'injected' + changed = True + self.inner.update(chunk) + def hexdigest(self): + return self.inner.hexdigest() + monkeypatch.setattr(hashlib, 'sha256', ReentrantDigest) + result = native.whole_body_digest(value) + assert changed + # Use the real digest because expected bytes are formed after mutation. + assert result == real_sha(json.dumps(typed(value), separators=(',', ':'), allow_nan=False).encode()).hexdigest() diff --git a/openpilot/starpilot/flm/tests/test_native_primitives.py b/openpilot/starpilot/flm/tests/test_native_primitives.py new file mode 100644 index 0000000000..aadcaf202f --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_native_primitives.py @@ -0,0 +1,141 @@ +from collections.abc import Callable +from typing import cast, NoReturn +import hashlib +import json +from types import SimpleNamespace + +import pytest +from openpilot.starpilot.flm import native_primitives as native + + +@pytest.fixture +def fixture_binding(tmp_path, monkeypatch): + here = tmp_path / 'repo/openpilot/starpilot/flm' + here.mkdir(parents=True) + monkeypatch.setattr(native, 'HERE', here) + monkeypatch.setattr(native, '_enabled', False) + clear_identity = native.native_identity.cache_clear + clear_extension = native._extension.cache_clear + clear_identity() + clear_extension() + for name, path in native.source_paths().items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('identity fixture ' + name) + def write_binding(): + artifact = here / '_primitive_walk.so' + artifact.write_bytes(b'fixture only; not executable native qualification') + value = {'schema': 1, 'revision': native.REVISION, 'abi': native.runtime_abi(), + 'sources': {name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in native.source_paths().items()}, + 'artifact_sha256': hashlib.sha256(artifact.read_bytes()).hexdigest()} + (here / '_primitive_walk.binding.json').write_text(json.dumps(value)) + native.native_identity.cache_clear() + native._extension.cache_clear() + return value + yield here, write_binding + clear_identity() + clear_extension() + + +def test_producer_never_imports_or_loads_native_and_disabled_call_does_not_read_identity(fixture_binding, monkeypatch): + _, write_binding = fixture_binding + write_binding() + monkeypatch.setattr(native.importlib.util, 'spec_from_file_location', lambda *args: cast(Callable[[str], NoReturn], pytest.fail)('producer native import')) + assert json.loads(native.native_identity())['artifact_sha256'] + monkeypatch.setattr(native, 'native_identity', lambda: cast(Callable[[str], NoReturn], pytest.fail)('per-row source or ABI lookup')) + assert native.try_native('whole_body_digest', {'value': -0.0}) is native.FALLBACK + + +def test_identity_binds_sources_executed_artifact_receipt_abi_and_absence(fixture_binding, monkeypatch): + here, write_binding = fixture_binding + missing = native.native_identity() + assert json.loads(missing)['status'] == 'python-missing' + write_binding() + present = native.native_identity() + assert present != missing + for name in ('_primitive_walk.pyx', 'native_primitives.py', 'SConscript', 'SConstruct'): + write_binding() + path = native.source_paths()[name] + path.write_text(path.read_text() + ' changed') + native.native_identity.cache_clear() + assert json.loads(native.native_identity())['status'] == 'python-invalid-binding' + write_binding() + (here / '_primitive_walk.so').write_bytes(b'different fixture artifact') + native.native_identity.cache_clear() + assert json.loads(native.native_identity())['status'] == 'python-invalid-binding' + write_binding() + monkeypatch.setattr(native, 'runtime_abi', lambda: {'different': 'ABI'}) + native.native_identity.cache_clear() + assert json.loads(native.native_identity())['status'] == 'python-unsupported-abi' + + +def test_cached_binding_refuses_artifact_replacement_and_wrong_module_origin(fixture_binding, monkeypatch): + here, write_binding = fixture_binding + monkeypatch.setattr(native.sys, 'platform', 'linux') + write_binding() + # Fixture binds metadata only; it is never accepted as an executed extension. + assert json.loads(native.native_identity())['status'] == 'native-bound' + (here / '_primitive_walk.so').write_bytes(b'changed after identity capture') + with pytest.raises(ValueError, match='artifact changed'): + native._extension() + write_binding() + fake = SimpleNamespace(__file__=str(here.parent / 'unexpected.so'), compiled_abi=lambda: (3, 12, 64, native.REVISION)) + monkeypatch.setitem(native.sys.modules, 'openpilot.starpilot.flm._primitive_walk', fake) + with pytest.raises(ValueError, match='unowned preloaded module'): + native._extension() + + +def test_import_failure_is_fit_failure_not_native_marked_python_fallback(fixture_binding, monkeypatch): + _, write_binding = fixture_binding + monkeypatch.setattr(native.sys, 'platform', 'linux') + write_binding() + assert json.loads(native.native_identity())['status'] == 'native-bound' + monkeypatch.setattr(native.importlib.util, 'spec_from_file_location', lambda *args: None) + with pytest.raises(ValueError, match='module missing'): + native._extension() + + +def test_worker_enable_requires_existing_caps_niceness_and_normal_scheduler(monkeypatch): + import os + import resource + monkeypatch.setattr(native.sys, 'platform', 'linux') + monkeypatch.setattr(resource, 'getrlimit', lambda kind: (20, 20) if kind == resource.RLIMIT_CPU else (2 * 1024**3, 2 * 1024**3)) + monkeypatch.setattr(os, 'getpriority', lambda *args: 10) + monkeypatch.setattr(os, 'SCHED_OTHER', 0, raising=False) + monkeypatch.setattr(os, 'sched_getscheduler', lambda pid: os.SCHED_OTHER, raising=False) + monkeypatch.setattr(native, '_enabled', False) + native.enable_fit_worker() + assert native._enabled + monkeypatch.setattr(os, 'getpriority', lambda *args: 0) + monkeypatch.setattr(native, '_enabled', False) + with pytest.raises(ValueError, match='worker context'): + native.enable_fit_worker() + assert not native._enabled + + +def test_whole_python_fallbacks_preserve_bits_and_alias_independence(monkeypatch): + from openpilot.starpilot.flm import primitive_source_memo as memo, physical_input_observation as physical + monkeypatch.setattr(native, '_enabled', False) + shared = [True, 1, 1., -0.0, 'é\ud800', b'\x00\xff'] + value = {'first': shared, 'second': shared} + result = memo.independent_primitive_graph(value) + assert result['first'] is not result['second'] + assert physical.body_digest(result) == physical._body_digest_python(value) + result['first'].append('mutation') + assert result['second'] == shared + cyclic = [] + cyclic.append(cyclic) + with pytest.raises(ValueError, match='cyclic primitive memo input'): + memo.independent_primitive_graph(cyclic) + + +def test_matching_origin_metadata_cannot_substitute_a_preloaded_python_object(fixture_binding, monkeypatch): + here, write_binding = fixture_binding + monkeypatch.setattr(native.sys, 'platform', 'linux') + write_binding() + name = 'openpilot.starpilot.flm._primitive_walk' + path = here / '_primitive_walk.so' + fake = SimpleNamespace(__spec__=native.importlib.util.spec_from_file_location(name, path), __file__=str(path), + compiled_abi=lambda: (3, 12, 64, native.REVISION), whole_body_digest=lambda value: 'forged') + monkeypatch.setitem(native.sys.modules, name, fake) + with pytest.raises(ValueError, match='unowned preloaded module'): + native._extension() diff --git a/openpilot/starpilot/flm/tests/test_native_response_worker.py b/openpilot/starpilot/flm/tests/test_native_response_worker.py new file mode 100644 index 0000000000..2e2a4195e5 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_native_response_worker.py @@ -0,0 +1,122 @@ +"""Guard-unit fixtures only; actual source/native binding and CPU remain root gates.""" +import ctypes +from importlib.machinery import ModuleSpec, SourceFileLoader +import os +import resource +import signal +from types import ModuleType +from typing import cast + +import pytest + +from openpilot.starpilot.flm import native_primitives as native + +PURPOSE = 'live-response-bootstrap-v1' +MODULE = 'openpilot.starpilot.flm.live_tracking_response_worker' + + +@pytest.fixture +def context(monkeypatch, tmp_path): + # These controlled guard values are NOT an actual executed worker receipt. + worker = tmp_path/'live_tracking_response_worker.py' + worker.write_text('# guard fixture, never executable production authority\n') + main = ModuleType('__main__') + main.__dict__.update(__package__='openpilot.starpilot.flm', __file__=str(worker), + __spec__=ModuleSpec(MODULE, SourceFileLoader(MODULE, str(worker)), origin=str(worker)), + PURPOSE=PURPOSE, MAX_CPU=180, MAX_AS=2*1024**3, MAX_WALL_NS=240_000_000_000) + monkeypatch.setattr(native, 'HERE', tmp_path) + monkeypatch.setattr(native, '_enabled', False) + monkeypatch.setattr(native.sys, 'platform', 'linux') + monkeypatch.setattr(native.sys, 'orig_argv', [native.sys.executable, '-B', '-m', MODULE]) + monkeypatch.setitem(native.sys.modules, '__main__', main) + monkeypatch.setattr(resource, 'getrlimit', lambda kind: (180, 180) if kind == resource.RLIMIT_CPU else (2*1024**3, 2*1024**3)) + monkeypatch.setattr(os, 'getpriority', lambda *args: 10) + monkeypatch.setattr(os, 'SCHED_OTHER', 0, raising=False) + monkeypatch.setattr(os, 'sched_getscheduler', lambda pid: 0, raising=False) + monkeypatch.setattr(os, 'getppid', lambda: 1234) + monkeypatch.setenv('STARPILOT_FLM_PARENT_PID', '1234') + for name in ('OPENBLAS_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS', 'NUMEXPR_NUM_THREADS'): + monkeypatch.setenv(name, '1') + + class Library: + def prctl(self, operation, pointer, *args): + assert operation == 2 + pointer._obj.value = signal.SIGKILL + return 0 + monkeypatch.setattr(ctypes, 'CDLL', lambda name: Library()) + return main + + +def test_context_enable_does_not_import_or_claim_a_native_binding(context, monkeypatch): + def forbidden(): + raise AssertionError('enable must not load extension or manufacture receipt') + monkeypatch.setattr(native, '_extension', forbidden) + monkeypatch.setattr(native, 'native_identity', forbidden) + assert native.enable_response_worker(PURPOSE) is True + assert native._enabled is True + + +@pytest.mark.parametrize('purpose', ['live-response-bootstrap-diagnostic-v1', '', 180, True]) +def test_wrong_explicit_purpose_never_enables(context, purpose): + with pytest.raises(ValueError): + native.enable_response_worker(purpose) + assert native._enabled is False + + +@pytest.mark.parametrize('change', ['argv', 'package', 'spec', 'file', 'policy', 'cpu', 'memory', + 'priority', 'scheduler', 'threads', 'parent', 'death_signal', 'platform']) +def test_each_actual_execution_guard_is_required(context, monkeypatch, tmp_path, change): + if change == 'argv': + monkeypatch.setattr(native.sys, 'orig_argv', [native.sys.executable, '-B', '-m', 'openpilot.starpilot.flm.fit_worker']) + elif change == 'package': + context.__package__ = 'other' + elif change == 'spec': + context.__spec__ = ModuleSpec('other', SourceFileLoader(MODULE, context.__file__)) + elif change == 'file': + context.__file__ = str(tmp_path/'other.py') + elif change == 'policy': + context.__dict__['MAX_CPU'] = 181 + elif change == 'cpu': + monkeypatch.setattr(resource, 'getrlimit', lambda kind: (181, 181) if kind == resource.RLIMIT_CPU else (2*1024**3, 2*1024**3)) + elif change == 'memory': + monkeypatch.setattr(resource, 'getrlimit', lambda kind: (180, 180) if kind == resource.RLIMIT_CPU else (-1, -1)) + elif change == 'priority': + monkeypatch.setattr(os, 'getpriority', lambda *args: 9) + elif change == 'scheduler': + monkeypatch.setattr(os, 'sched_getscheduler', lambda pid: 1) + elif change == 'threads': + monkeypatch.setenv('OMP_NUM_THREADS', '2') + elif change == 'parent': + monkeypatch.setenv('STARPILOT_FLM_PARENT_PID', '2345') + elif change == 'death_signal': + class Library: + def prctl(self, *args): + return 0 + monkeypatch.setattr(ctypes, 'CDLL', lambda name: Library()) + else: + monkeypatch.setattr(native.sys, 'platform', 'darwin') + with pytest.raises(ValueError): + native.enable_response_worker(PURPOSE) + assert native._enabled is False + + +def test_response_allowance_does_not_widen_existing_legacy20_guard(context, monkeypatch): + monkeypatch.setattr(resource, 'getrlimit', lambda kind: (21, 180) if kind == resource.RLIMIT_CPU else (2*1024**3, 2*1024**3)) + with pytest.raises(ValueError, match='worker context'): + native.enable_fit_worker() + assert native._enabled is False + assert native.enable_response_worker(PURPOSE) is True + + +def test_unknown_purpose_and_main_metadata_cannot_invoke_custom_equality(context): + calls = [] + class Trap: + def __eq__(self, other): + calls.append('eq') + return True + with pytest.raises(ValueError): + native.enable_response_worker(cast(str, Trap())) + context.__dict__['__package__'] = Trap() + with pytest.raises(ValueError): + native.enable_response_worker(PURPOSE) + assert calls == [] diff --git a/openpilot/starpilot/flm/tests/test_normalized_trace_archive.py b/openpilot/starpilot/flm/tests/test_normalized_trace_archive.py new file mode 100644 index 0000000000..a394fad191 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_normalized_trace_archive.py @@ -0,0 +1,308 @@ +"""Representation vectors only; no fitted profile, native or physical proof.""" +from copy import deepcopy +from dataclasses import asdict, dataclass, replace +import gc +import hashlib +import pickle +from types import SimpleNamespace +from collections.abc import Callable +from typing import cast, TypedDict +import weakref + +import pytest + +from openpilot.starpilot.flm import normalized_trace_archive as n +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, full_profile_sha256 +from openpilot.starpilot.flm.effect_trace import FullEffectTrace, ordered, step_full, trace_dict +from openpilot.starpilot.flm.full_surface_profile import FullSurfaceEnvelope, neutral_coordinates +from openpilot.starpilot.flm.primitive_source_memo import independent_primitive_graph, MAX_LOGICAL_BYTES +from openpilot.starpilot.flm.tests.test_fullrate_pipeline import snapshots +from openpilot.starpilot.flm.torque_raw_fit import control_from_dict + + +def structural_trace(stamp=10_000_000_000): + envelope = FullSurfaceEnvelope.initial(cp_sha256='1' * 64, controller_revision='source-only-structural-vector', + source_epoch='2' * 64, reset_generation='0', selected_baseline_revision='3' * 64, + trial_names=('ff_gain_left',), persistent_source_epoch='4' * 64) + before = tuple((name, -0.0 if index == 0 else value) for index, (name, value) in enumerate(neutral_coordinates())) + applied = ordered(step_full(dict(before), dict(envelope.coordinates), now_ns=stamp, previous_ns=0, qualified=True)) + return FullEffectTrace(1,envelope,stamp,0,True,before,envelope.coordinates,applied, + full_profile_sha256(envelope.selected_baseline_revision,applied)).validate() + + +def fixture(): + trace = structural_trace() + control, _, _ = snapshots(trace.control_ns) + control = replace(control,effect_trace=trace,effect_prepare_ns=trace.control_ns) + builder = n.ArchiveBuilder() + wire = builder.record(control) + archive = n.TraceArchive(builder.literal()) + owner = SimpleNamespace(trace_archive=archive) + decoded = control_from_dict(wire,trace_archive=archive) + return trace,control,wire,archive,owner,decoded + + +def _shape_frame(evidence): + # Representation-only batch/dispatch vector. No selected-stage/native proof. + from openpilot.starpilot.flm.controller_replay import ReplayFrame + return ReplayFrame(evidence.control_ns,evidence.replay_inputs,evidence.requested, + evidence.applied_coordinates,effect_trace=evidence.effect_trace, + effect_prepare_ns=evidence.effect_prepare_ns) + + +def test_exact_all_fields_types_bits_and_scalar_refs(): + trace,control,wire,archive,owner,decoded = fixture() + assert type(decoded.effect_trace) is n.TraceRef + assert set(vars(decoded.effect_trace)) == {'index','prepare_ns'} + assert not any(type(value) is n.TraceArchive for value in vars(decoded.effect_trace).values()) + restored = n.trace_for(decoded,owner) + assert pickle.dumps(trace_dict(restored),protocol=5) == pickle.dumps(trace_dict(trace),protocol=5) + assert restored.before[0][1].hex() == '-0x0.0p+0' + assert n.stored_record(decoded) == wire + assert n.current_record(control,archive.literal) == wire + assert n.record_body(decoded,owner)['effect_trace']['declaration'] == archive.literal['envelopes'][0] + assert n.record_body(decoded,owner)['effect_trace']['row'] == archive.literal['rows'][0] + assert len(restored.before) == len(restored.target) == len(restored.applied) == len(NAMES) == 30 + + +def test_value_deduplication_not_object_identity_or_hash_only(): + trace = structural_trace() + builder = n.ArchiveBuilder() + first = builder.encode(trace) + same = builder.encode(deepcopy(trace)) + assert first == same and len(builder.rows) == len(builder.envelopes) == 1 + second_trace = structural_trace(trace.control_ns + 10_000_000) + second = builder.encode(second_trace) + assert second['index'] != first['index'] + assert len(builder.rows) == 2 and len(builder.envelopes) == 1 + assert set(builder.literal()) == {'version','manifest','envelopes','rows'} + + +@pytest.mark.parametrize('mutation', ['index','clock','manifest','missing_metadata','applied','hash','nonfinite','row_type']) +def test_changed_or_missing_literal_fields_fail(mutation): + _,_,wire,archive,_,_ = fixture() + literal = deepcopy(archive.literal) + ref = deepcopy(wire['effect_trace']) + row = list(literal['rows'][0]) + if mutation == 'index': + ref['index'] = 10 + elif mutation == 'clock': + ref['prepare_ns'] += 1 + elif mutation == 'manifest': + literal['manifest'] = '0' * 64 + elif mutation == 'missing_metadata': + literal['envelopes'][0].pop('source_epoch') + elif mutation == 'applied': + row[7] = (0.1, *row[7][1:]) + elif mutation == 'hash': + row[8] = '0' * 64 + elif mutation == 'nonfinite': + row[5] = (float('nan'), *row[5][1:]) + elif mutation == 'row_type': + row[4] = 1 + literal['rows'] = (tuple(row),) + with pytest.raises((ValueError,TypeError)): + changed = n.TraceArchive(literal) + changed.resolve(n.read_ref(ref,changed)).validate() + + +def test_legacy_expanded_control_path_remains_exact_and_missing_owner_declines(): + trace,control,_,_,_,decoded = fixture() + assert control_from_dict(asdict(control)) == control + with pytest.raises(ValueError,match='literal source archive'): + n.trace_for(decoded,SimpleNamespace()) + with pytest.raises(ValueError): + control_from_dict(n.stored_record(decoded)) + + +def test_entire_literal_owner_key_changes_for_unconsumed_rows(): + _,_,_,archive,owner,_ = fixture() + before = hashlib.sha256(pickle.dumps(n.archive_key(owner),protocol=5)).digest() + declaration = archive.literal['envelopes'][0] + declaration['controller_revision'] += '-changed' + after = hashlib.sha256(pickle.dumps(n.archive_key(owner),protocol=5)).digest() + assert before != after + + +def test_archive_parse_and_reference_parse_do_not_expand_all_traces(monkeypatch): + trace = structural_trace() + builder = n.ArchiveBuilder() + refs = [builder.encode(replace(trace,control_ns=trace.control_ns+index*10_000_000)) for index in range(64)] + # Same neutral .01s first-step trace remains valid for these distinct clocks. + import openpilot.starpilot.flm.effect_trace as module + original = module.FullEffectTrace + reserved_bound = n.reservation_upper_bound() + monkeypatch.setattr(n,'reservation_upper_bound',lambda: reserved_bound) + live, peaks = [], [] + def construct(*args, **kwargs): + value = original(*args, **kwargs) + live.append(weakref.ref(value)) + peaks.append(sum(ref() is not None for ref in live)) + return value + monkeypatch.setattr(module,'FullEffectTrace',construct) + archive = n.TraceArchive(builder.literal()) + parsed = [n.read_ref(ref,archive) for ref in refs] + assert not live + for ref in parsed: + expanded = archive.resolve(ref) + assert len(expanded.applied) == 30 + del expanded + gc.collect() + assert max(peaks) == 1 and not any(ref() is not None for ref in live) + + +def test_reserved_allocation_uses_same_combined_logical_cap_before_clone(monkeypatch): + import openpilot.starpilot.flm.primitive_source_memo as module + from openpilot.starpilot.flm.native_primitives import FALLBACK + monkeypatch.setattr(module,'_try_native',lambda *args: FALLBACK) + # Bytes cost9, tuple9; outer reservation wrapper adds18 plus reserved bytes. + amount = MAX_LOGICAL_BYTES - n.RESERVED_BYTES - 27 + exact = b'x' * amount + copied = independent_primitive_graph(exact,reserved_bytes=n.RESERVED_BYTES) + assert copied == exact + with pytest.raises(ValueError,match='logical expansion bound'): + independent_primitive_graph(exact+b'x',reserved_bytes=n.RESERVED_BYTES) + with pytest.raises(ValueError,match='reservation'): + independent_primitive_graph((),reserved_bytes=n.RESERVED_BYTES+1) + + +def test_complete_packet_membership_and_unused_rows(): + _,_,wire,archive,_,_ = fixture() + packet = {'version':n.SOURCE_VERSION,'trace_archive':archive.literal, + 'context_receipts':{'controls':(wire,)},'actual_frames':(deepcopy(wire),)} + assert n.reserve_for_packet(packet) == n.RESERVED_BYTES + changed = deepcopy(packet) + class PreparedFrame(TypedDict): + effect_prepare_ns: int + cast(tuple[PreparedFrame, ...], changed['actual_frames'])[0]['effect_prepare_ns'] += 1 + with pytest.raises(ValueError,match='membership'): + n.validate_packet(changed) + changed = deepcopy(packet) + class ArchiveRows(TypedDict): + rows: tuple[object, ...] + changed_archive = cast(ArchiveRows, changed['trace_archive']) + changed_archive['rows'] += (changed_archive['rows'][0],) + with pytest.raises(ValueError,match='unused or omitted'): + n.validate_packet(changed) + with pytest.raises(ValueError,match='legacy'): + n.reserve_for_packet({**packet,'version':'unpaired-original-source-v8'}) + + +def test_fresh_complete_owner_key_rejects_unknown_objects_without_reduce_callbacks(): + _,_,_,archive,owner,_ = fixture() + calls = [] + class Executable: + def __reduce__(self): + calls.append('executed') + raise AssertionError('must reject before pickle') + archive.literal['envelopes'] += ({**archive.literal['envelopes'][0],'controller_revision':Executable()},) + archive.literal['rows'] += (archive.literal['rows'][0],) # Header remains valid; ALL declarations must be checked. + with pytest.raises(ValueError): + n.archive_key(owner) + assert not calls + + +def test_mutable_work_guard_rechecks_whole_table_without_permission_cache(): + _,_,_,archive,owner,_ = fixture() + @n.guarded_archive() + def work(engine): + declaration = engine.trace_archive.literal['envelopes'][0] + declaration['controller_revision'] += '-changed' + return 'not-admission' + with pytest.raises(ValueError,match='changed during mutable work'): + work(owner) + @n.guarded_archive() + def ordinary(engine): + return 'legacy' + assert ordinary(SimpleNamespace()) == 'legacy' + + +def test_transient_and_retained_cache_reservation_proven_below_combined_limit(): + assert n.reservation_upper_bound() <= n.RESERVED_BYTES == 512 * 1024 + assert n.RESERVED_BYTES < MAX_LOGICAL_BYTES + + +def test_unknown_physical_body_keeps_legacy_asdict_callback_and_error_order(): + from openpilot.starpilot.flm.physical_input_observation import _physical_asdict + calls = [] + @dataclass + class Unknown: + value: int = 1 + def __getattr__(self,name): + calls.append(name) + raise AttributeError(name) + body = Unknown() + expected = asdict(body) + oracle_calls = tuple(calls) + calls.clear() + assert _physical_asdict(body) == expected + assert tuple(calls) == oracle_calls + class NonDataclass: + def __getattr__(self,name): + calls.append(name) + raise AttributeError(name) + body = NonDataclass() + calls.clear() + with pytest.raises(TypeError) as original_error: + cast(Callable[[object], object], asdict)(body) + oracle_calls = tuple(calls) + calls.clear() + with pytest.raises(TypeError) as new_error: + _physical_asdict(body) + assert str(new_error.value) == str(original_error.value) + assert tuple(calls) == oracle_calls + + +def test_snapshot_frame_batch_has_one_full_guard_with_fresh_row_resolution(monkeypatch): + # Guard-placement unit only; fake normalize never confers native admission. + from openpilot.starpilot.flm.controller_replay import ControllerReplay + _,_,_,archive,_,decoded = fixture() + frame = _shape_frame(decoded) + wire = n.stored_record(frame) + keys, parsed = [], [] + original_key = n.archive_key + def counted(owner): + keys.append('fresh-whole-table') + return original_key(owner) + monkeypatch.setattr(n,'archive_key',counted) + owner = ControllerReplay.__new__(ControllerReplay) + owner.trace_archive, owner.failed = archive, False + monkeypatch.setattr(owner, 'checkpoint', lambda *args: None, raising=False) + def normalize(frame,**kwargs): + parsed.append(n.trace_for(frame,owner).control_ns) + return 'shape-only' + monkeypatch.setattr(owner, '_normalize_frame', normalize) + ControllerReplay._load_snapshot_frames(owner,(wire,),{frame.control_ns:None},normalized=True) + assert keys == ['fresh-whole-table','fresh-whole-table'] + assert parsed == [frame.control_ns] + def mutate(frame,**kwargs): + owner.trace_archive.literal['envelopes'][0]['controller_revision'] += '-changed' + return 'not-admitted' + monkeypatch.setattr(owner, '_normalize_frame', mutate) + with pytest.raises(ValueError,match='changed during mutable work'): + ControllerReplay._load_snapshot_frames(owner,(wire,),{frame.control_ns:None},normalized=True) + + +@pytest.mark.parametrize('normalized',[False,True]) +def test_snapshot_subclass_retains_public_normalize_dispatch(normalized, monkeypatch): + from openpilot.starpilot.flm.controller_replay import ControllerReplay + _,control,_,archive,_,decoded = fixture() + calls = [] + class Override(ControllerReplay): + def normalize(self,frame,**kwargs): + calls.append(frame.control_ns) + return 'subclass-shape-only' + def _normalize_frame(self,*args,**kwargs): + raise AssertionError('unknown subclass must retain original public dispatch') + owner = Override.__new__(Override) + owner.failed = False + monkeypatch.setattr(owner, 'checkpoint', lambda *args: None, raising=False) + if normalized: + owner.trace_archive = archive + frame = _shape_frame(decoded) + body = n.stored_record(frame) + else: + frame = _shape_frame(replace(control,effect_trace=None)) + body = asdict(frame) + owner._load_snapshot_frames((body,),{frame.control_ns:None},normalized=normalized) + assert calls == [frame.control_ns] diff --git a/openpilot/starpilot/flm/tests/test_parent_complete_geometry.py b/openpilot/starpilot/flm/tests/test_parent_complete_geometry.py new file mode 100644 index 0000000000..03978ab16e --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_parent_complete_geometry.py @@ -0,0 +1,63 @@ +"""Actual parent snapshot availability, not fabricated fit/native qualification.""" + +import pytest + +from openpilot.starpilot.flm import source_plant_runtime as runtime +from openpilot.starpilot.flm.tests.test_compact_parent_timing import SECOND, START, parent_fixture + + +def subtle_parent(age_seconds): + collector, now, exported = parent_fixture(age_seconds * SECOND) + # A full 4.5-second small-amplitude cycle has one geometry bundle at20Hz. + # Use actual whole_event_windows via runtime.snapshot; no geometry mock. + for frame in collector.adapter.replay.records: + elapsed = frame.control_ns - START + frame.desired_physical_accel = ( + (0.15 if ((elapsed - 39 * SECOND) // 2_250_000_000) % 2 == 0 else -0.15) + if elapsed >= 39 * SECOND else 0. + ) + return collector, now, exported + + +@pytest.mark.parametrize('age_seconds,attempt,windows', [(80, False, 8), (85, True, 9)]) +def test_actual_parent_waits_for_ninth_complete_subtle_cycle(age_seconds, attempt, windows): + collector, now, exported = subtle_parent(age_seconds) + assert runtime.snapshot(collector, now) is None + assert bool(exported) is attempt + assert collector.snapshot_attempts == int(attempt) + assert collector.last_fit == (now if attempt else 0) + if not attempt: + readiness = collector.adapter.snapshot_readiness + assert readiness['reason'] == 'plant_later_original_goal_geometry' + assert readiness['candidateBundles'] == 8 + assert readiness['requiredBundles'] == 8 + assert readiness['completeGoalWindows'] == windows + assert readiness['requiredCompleteGoalWindows'] == 9 + assert readiness['childQualification'] == 'not_yet_evaluated' + else: + assert exported == [(((START, now),), now)] + # The parent-only fixture stops at export rather than inventing a packet. + assert collector.adapter.snapshot_readiness['reason'] == 'plant_snapshot_unavailable' + + +def test_eighth_cycle_does_not_consume_epoch_or_cooldown(): + collector, now, exported = subtle_parent(80) + assert runtime.snapshot(collector, now) is None + assert not exported + assert collector.snapshot_attempts == collector.last_fit == 0 + assert collector.adapter.source_plant_submitted_epoch == -1 + later, later_ns, _ = subtle_parent(85) + collector.adapter.replay = later.adapter.replay + collector.original_source_capture = later.original_source_capture + collector.ready = later.ready + assert runtime.snapshot(collector, later_ns) is None + assert exported == [(((START, later_ns),), later_ns)] + assert collector.snapshot_attempts == 1 + assert collector.last_fit == later_ns + + +def test_geometry_wait_does_not_enable_retry_of_submitted_epoch(): + collector, now, exported = subtle_parent(85) + collector.adapter.source_plant_submitted_epoch = collector.original_source_capture.epoch + assert runtime.snapshot(collector, now) is None + assert not exported and collector.snapshot_attempts == collector.last_fit == 0 diff --git a/openpilot/starpilot/flm/tests/test_physical_formatting.py b/openpilot/starpilot/flm/tests/test_physical_formatting.py new file mode 100644 index 0000000000..d18b8e1900 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_physical_formatting.py @@ -0,0 +1,169 @@ +from collections.abc import Callable +from typing import cast, NoReturn +"""Exact formatting vectors only; no vehicle/native/profile acceptance.""" +from dataclasses import asdict, fields + +import pytest + +from openpilot.starpilot.flm import feedback_observation as feedback +from openpilot.starpilot.flm import physical_input_observation as physical + + +def test_hash_ascii_fullmatch_matches_original_all_byte_positions_and_unicode(): + def original(value): + return type(value) is str and len(value) == 64 and all(c in '0123456789abcdef' for c in value) + for position in range(64): + for char in (*map(chr, range(256)), '\u0100', '\uff10', '\U0001f642', '\ud800'): + value = 'a' * position + char + 'f' * (63 - position) + assert physical._hash(value) is original(value) + for value in ('', 'a' * 63, 'a' * 65, None, b'a' * 64, 1, True, [], {}): + assert physical._hash(value) is original(value) + + +def test_hash_exact_str_guard_rejects_subclass_without_callbacks(): + class Text(str): + def __len__(self): + raise AssertionError('unsupported subclass callback') + assert physical._hash(Text('a' * 64)) is False + + +def feedback_record(kind, **changes): + defaults: dict[object, object] = {str: 'a' * 64, int: 1, float: -0., bool: True} + values = {field.name: defaults[field.type] for field in fields(kind)} + values.update(changes) + return kind(**values) + + +@pytest.mark.parametrize('value', [None, True, 1, 2**120, 1., -0., float('inf'), float('nan'), + 'plain', '\ud800', b'\x00\xff']) +def test_feedback_exact_record_all_literal_types_and_hashes_match_asdict(value): + consumption = feedback_record(feedback.CoConsumption, reported_torque=value) + publication = feedback_record(feedback.CoPublication, reported_torque=value) + for record in (consumption, publication, feedback.FeedbackJoin(consumption, publication)): + expected = asdict(record) + actual = feedback._feedback_asdict(record) + # Whole typed digest compares NaN/error and signed-zero semantics honestly. + outcomes = [] + for body in (expected, actual): + try: + outcomes.append(('result', physical.body_digest(body))) + except Exception as error: + outcomes.append(('error', type(error), str(error))) + assert outcomes[0] == outcomes[1] + + +def test_feedback_unknown_leaf_whole_fallback_keeps_deepcopy_mutation_order(): + outcomes = [] + for converter in (asdict, feedback._feedback_asdict): + consumption = feedback_record(feedback.CoConsumption) + publication = feedback_record(feedback.CoPublication) + log = [] + class Callback: + def __deepcopy__(self, memo, log=log, publication=publication): + log.append('consumption callback') + object.__setattr__(publication, 'sequence', 99) + return 'copied' + object.__setattr__(consumption, 'requested_torque', Callback()) + result = converter(feedback.FeedbackJoin(consumption, publication)) + outcomes.append((result, tuple(log))) + assert outcomes[0] == outcomes[1] + assert outcomes[0][0]['publication']['sequence'] == 99 + + +def test_feedback_subclass_falls_back_to_original_asdict(monkeypatch): + class Publication(feedback.CoPublication): + pass + value = Publication(**asdict(feedback_record(feedback.CoPublication))) + observed = [] + original = feedback.asdict + monkeypatch.setattr(feedback, 'asdict', lambda item: (observed.append(item), original(item))[1]) + assert feedback._feedback_asdict(value) == original(value) + assert observed == [value] + + +def test_feedback_exact_record_avoids_deepcopy_and_still_reads_every_field(monkeypatch): + record = feedback_record(feedback.CoConsumption) + expected = asdict(record) + monkeypatch.setattr(feedback, 'asdict', lambda *args: cast(Callable[[str], NoReturn], pytest.fail)('closed immutable primitive deepcopy')) + assert feedback._feedback_asdict(record) == expected + object.__setattr__(record, 'requested_torque', .5) + assert feedback._feedback_asdict(record)['requested_torque'] == .5 + + +def test_feedback_early_unknown_deepcopy_restores_missing_later_field(): + outcomes = [] + for converter in (asdict, feedback._feedback_asdict): + record = feedback_record(feedback.CoConsumption) + log = [] + class Restore: + def __deepcopy__(self, memo, log=log, record=record): + log.append('early deepcopy') + object.__setattr__(record, 'cp_sha256', 'b' * 64) + return 'copied' + object.__setattr__(record, 'session' if hasattr(record, 'session') else 'version', Restore()) + object.__delattr__(record, 'cp_sha256') + outcomes.append((converter(record), tuple(log))) + assert outcomes[0] == outcomes[1] + assert outcomes[0][0]['cp_sha256'] == 'b' * 64 + + +def test_feedback_unknown_consumption_can_restore_publication_before_original_read(): + outcomes = [] + for converter in (asdict, feedback._feedback_asdict): + consumption = feedback_record(feedback.CoConsumption) + publication = feedback_record(feedback.CoPublication) + class Restore: + def __deepcopy__(self, memo, publication=publication): + object.__setattr__(publication, 'cp_sha256', 'b' * 64) + return 'copied' + object.__setattr__(consumption, 'version', Restore()) + object.__delattr__(publication, 'cp_sha256') + outcomes.append(converter(feedback.FeedbackJoin(consumption, publication))) + assert outcomes[0] == outcomes[1] + + +def test_feedback_descriptor_unknown_leaf_runs_once_in_original_order(monkeypatch): + outcomes = [] + for converter in (asdict, feedback._feedback_asdict): + record = feedback_record(feedback.CoConsumption) + log = [] + class Copy: + def __deepcopy__(self, memo, log=log): + log.append('deepcopy') + return 'copied' + def getter(self, log=log, Copy=Copy): + log.append('getter') + return Copy() + with monkeypatch.context() as scope: + scope.setattr(feedback.CoConsumption, 'version', property(getter), raising=False) + outcomes.append((converter(record), tuple(log))) + assert outcomes[0] == outcomes[1] + assert outcomes[0][1] == ('getter', 'deepcopy') + + +def test_feedback_colliding_extra_key_falls_back_before_any_key_callback(): + outcomes = [] + for converter in (asdict, feedback._feedback_asdict): + record = feedback_record(feedback.CoConsumption) + owned = object.__getattribute__(record, '__dict__') + log = [] + class Collision: + def __hash__(self): + return hash('cp_sha256') + def __eq__(self, other, log=log): + log.append('unexpected key comparison') + return False + key = Collision() + class Restore: + def __deepcopy__(self, memo, log=log, owned=owned, key=key, record=record): + log.append('early deepcopy') + owned.pop(key) + object.__setattr__(record, 'cp_sha256', 'b' * 64) + return 'copied' + object.__setattr__(record, 'version', Restore()) + object.__delattr__(record, 'cp_sha256') + owned[key] = 'extra ignored by original asdict' + log.clear() + outcomes.append((converter(record), tuple(log))) + assert outcomes[0] == outcomes[1] + assert outcomes[0][1] == ('early deepcopy',) diff --git a/openpilot/starpilot/flm/tests/test_physical_input_observation.py b/openpilot/starpilot/flm/tests/test_physical_input_observation.py new file mode 100644 index 0000000000..558e57fc46 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_physical_input_observation.py @@ -0,0 +1,302 @@ +"""Declared synthetic arithmetic/type contracts, NOT native or route evidence. + +Root executes in the matching composed project. Author did not import/run. +Actual registered producer/native/CAN positive remains a separate root fixture. +""" +from dataclasses import asdict, replace +import math +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from openpilot.starpilot.flm import plant_lag as p +from openpilot.starpilot.flm import physical_input_observation as o + + +def h(value): + return p.digest(value) + + +def reseal(row, **changed): + row = replace(row, **changed) + return replace(row, observation_digest=o.observation_digest(row)) + + +def retape(binding, tape, commands): + commands = tuple(commands) + return replace(tape, commands=commands, verified_tape_digest=o.physical_tape_digest(binding, commands)) + + +def physical_fixture(*, tags=None): + """All51 complete varying OBSERVED inputs, all performance tags originally True. + + Evidence hashes are explicitly synthetic placeholders; this does not test the + real from_original constructor or grant native permission/profile authority. + """ + binding = p.SourceBinding(*(h(name) for name in ('identity', 'cp', 'source', 'controller', 'epoch', + 'baseline', 'native', 'calibration', 'vm')), session='SYNTHETIC_ARITHMETIC_NOT_NATIVE_PROOF', + source_clock_contract_sha256=h('synthetic-clock-proof'), native_required=True) + bound = p.binding_digest(binding) + tags = tags if tags is not None else o.OriginalPerformanceTags(True, True, True, True, True) + commands, poses, windows = [], [], [] + for block in range(8): + start = 20_000_000_000 + block * 6_000_000_000 + sign = 1 if block % 2 == 0 else -1 + + def value(stamp, *, start=start, sign=sign): + t = (stamp - start) * 1e-9 + return sign * (.25 + .13 * math.sin(1.9 * t) + .02 * math.sin(7.3 * t)) + + for index in range(501): + apply = start + index * 10_000_000 + control = apply - 1_000_000 + row = o.PhysicalDeliveredInput(o.PROTOCOL, control, control, control, apply, value(apply), + control - 10_000_000, control - 1_000_000, apply + 100_000_000, len(commands) + 1, + apply + 1_000_000, h(('encoded', block, index)), h(('original-control', block, index)), + h(('original-applied', block, index)), h(('effective-applied', block, index)), + h(('declared-synthetic-emission', block, index)), binding.identity, 1, binding.source_epoch, + binding.baseline_revision, binding.controller_revision, binding.native_identity, True, + 'measured', tags, 'observed_input_performance_tagged' if any(asdict(tags).values()) else + 'observed_input_clean', bound, '') + commands.append(reseal(row)) + ids = [] + for index in range(80): + source = start + 605_000_000 + index * 50_000_000 + pid = h(('pose', block, index)) + ids.append(pid) + selected = start + ((source - 170_000_000 - start) // 10_000_000) * 10_000_000 + y = 1.98 * value(selected) + .01 + poses.append(p.SourcePose(pid, source, source + 40_000_000, source, + 'mono_boot_equivalent_interval_v1', 0, 1_000, source - 1_000, source + 1_000, + 0, 25., y / 25., 0., bound)) + windows.append(p.CalibrationWindow(h(('time-block', block)), start, start + 5_000_000_000, + tuple(ids), h(('declared-original-slot', block)))) + commands = tuple(commands) + tape = o.VerifiedPhysicalTape(o.PROTOCOL, commands, bound, + o.physical_tape_digest(binding, commands), h('synthetic-not-native-verifier')) + return binding, tape, tuple(poses), tuple(windows) + + +def test_known_limited_physical_inputs_preserve_all51_original_rows_and_affine_gates(): + evidence = physical_fixture() + result = p.identify_physical_plant_lag(*evidence) + assert result.status == 'physical_affine_test_passed' + assert result.reason == 'NO_PROFILE_NO_CURRENT_AUTHORITY' + assert result.decision.protocol == p.PHYSICAL_PROTOCOL != p.PROTOCOL + assert result.decision.lag_ns == 170_000_000 + assert result.decision.slope == pytest.approx(1.98) + assert result.decision.intercept == pytest.approx(.01) + assert len(result.decision.training_scores) == 51 + assert all(score is not None for _, score in result.decision.training_scores) + assert len(result.rows) == 640 + assert tuple(row.pose_id for row in result.rows) == tuple(pose.pose_id for pose in evidence[2]) + assert all(error < 1e-10 and outside == 0 for _, error, outside in result.heldout_blocks) + assert all(all(asdict(row.tags).values()) for row in evidence[1].commands) + assert not isinstance(evidence[1].commands[0], p.DeliveredCommand) + assert not hasattr(result, 'profile') and not hasattr(result, 'admission') + + +def test_performance_tag_disposition_is_bound_but_not_a_numeric_physics_parameter(): + tagged = p.identify_physical_plant_lag(*physical_fixture()) + clean = p.identify_physical_plant_lag(*physical_fixture(tags=o.OriginalPerformanceTags(False, False, False, False, False))) + assert (tagged.decision.lag_ns, tagged.decision.slope, tagged.decision.intercept) == ( + clean.decision.lag_ns, clean.decision.slope, clean.decision.intercept) + assert tagged.rows == clean.rows + assert tagged.experiment_digest != clean.experiment_digest + assert tagged.decision.train_source_digest != clean.decision.train_source_digest + + +def test_legacy_strict_types_cannot_receive_or_coerce_physical_observations(): + binding, tape, poses, windows = physical_fixture() + with pytest.raises(ValueError, match='independently verified command tape'): + p.identify_plant_lag(binding, tape, poses, windows) + disguised = p.VerifiedCommandTape(tape.commands, p.binding_digest(binding), + p.command_tape_digest(binding, tape.commands), h('fake-strict-wrapper'), True) + with pytest.raises(ValueError, match='actual delivered command type'): + p.identify_plant_lag(binding, disguised, poses, windows) + + +def test_tag_or_original_body_tamper_cannot_keep_an_old_observation_digest(): + binding, tape, poses, windows = physical_fixture() + for changed in ( + replace(tape.commands[0], tags=replace(tape.commands[0].tags, output_limited=False)), + replace(tape.commands[0], original_applied_sha256=h('forged-original-body')), + replace(tape.commands[0], actual_emission_sha256=h('different-native-emission')), + ): + attacked = retape(binding, tape, (changed, *tape.commands[1:])) + with pytest.raises(ValueError, match='custody'): + p.identify_physical_plant_lag(binding, attacked, poses, windows) + + +@pytest.mark.parametrize('changed', [ + {'state_ns': 1}, {'output_state_ns': 1}, {'source_receipt_ns': 1}, + {'native_expires_ns': 1}, {'native_observed_ns': 0}, {'native_required': False}, + {'native_identity': h('different-native')}, {'identity': h('different-car')}, + {'source_epoch': h('different-epoch')}, {'baseline_revision': h('different-baseline')}, + {'controller_revision_sha256': h('different-controller')}, {'binding_digest': h('different-binding')}, + {'limiter_kind': 'unknown'}, {'disposition': 'live_permission'}, {'command': 1.01}, {'generation': -1}, +]) +def test_hard_clock_native_identity_support_guards_survive_resealed_attack(changed): + binding, tape, poses, windows = physical_fixture() + changed_row = reseal(tape.commands[0], **changed) + attacked = retape(binding, tape, (changed_row, *tape.commands[1:])) + with pytest.raises(ValueError): + p.identify_physical_plant_lag(binding, attacked, poses, windows) + + +def test_boolean_flags_are_not_truthy_integer_permissions(): + binding, tape, poses, windows = physical_fixture() + changed = reseal(tape.commands[0], tags=replace(tape.commands[0].tags, control_limited=1)) + with pytest.raises(ValueError, match='boolean tags'): + p.identify_physical_plant_lag(binding, retape(binding, tape, (changed, *tape.commands[1:])), poses, windows) + + +def test_fifo_previous_carrier_and_stale_tape_hash_reject(): + binding, tape, poses, windows = physical_fixture() + for commands in ( + (tape.commands[0], reseal(tape.commands[1], carrier_frame=tape.commands[1].carrier_frame + 4), *tape.commands[2:]), + (tape.commands[1], tape.commands[0], *tape.commands[2:]), + ): + with pytest.raises(ValueError, match='FIFO/carrier'): + p.identify_physical_plant_lag(binding, retape(binding, tape, commands), poses, windows) + with pytest.raises(ValueError, match='tape custody'): + p.identify_physical_plant_lag(binding, replace(tape, commands=tape.commands[1:]), poses, windows) + + +def test_missing_one_maximum_lookback_input_rejects_entire_common51_experiment(): + binding, tape, poses, windows = physical_fixture() + commands = tuple(row for row in tape.commands if row.apply_ns >= poses[0].mono_ns - 100_000_000) + result = p.identify_physical_plant_lag(binding, retape(binding, tape, commands), poses, windows) + assert result.status == 'collecting' and result.reason.startswith('common_command_') + assert result.decision is None and not result.rows + + +def test_heldout_failure_cannot_choose_another_lag_or_mutate_training_receipt(): + binding, tape, poses, windows = physical_fixture() + before = p.identify_physical_plant_lag(binding, tape, poses, windows) + attacked = tuple(replace(pose, calibrated_yaw_rate=pose.calibrated_yaw_rate + .5) + if index >= 320 else pose for index, pose in enumerate(poses)) + result = p.identify_physical_plant_lag(binding, tape, attacked, windows) + assert result.status == 'rejected' and result.reason == 'heldout_rmse' + assert result.decision == before.decision + assert result.model_receipt_digest is None and len(result.heldout_blocks) == 4 + + +@pytest.mark.parametrize('changed', [{'healthy': False}, {'native_allowed_at_receipt': False}, + {'overridden': True}, {'fault': True}, {'clock_generation': 7}]) +def test_performance_tags_cannot_cover_invalid_original_pose(changed): + binding, tape, poses, windows = physical_fixture() + attacked = (replace(poses[0], **changed), *poses[1:]) + with pytest.raises(ValueError, match='source pose'): + p.identify_physical_plant_lag(binding, tape, attacked, windows) + + +def test_heldout_range_and_training_excitation_remain_original_gates(): + binding, tape, poses, windows = physical_fixture() + original = p.identify_physical_plant_lag(binding, tape, poses, windows) + edge = windows[4].original_start_ns + changed = tuple(reseal(row, command=row.command * 1.5) if row.apply_ns >= edge else row + for row in tape.commands) + result = p.identify_physical_plant_lag(binding, retape(binding, tape, changed), poses, windows) + assert result.reason == 'heldout_command_range' and result.decision == original.decision + constant = tuple(reseal(row, command=.1) for row in tape.commands) + quiet = p.identify_physical_plant_lag(binding, retape(binding, tape, constant), poses, windows) + assert quiet.status == 'collecting' and quiet.reason == 'training_excitation_slope_direction' + + +def test_observation_packet_does_not_expand_original_resource_or_row_bounds(): + binding, tape, poses, windows = physical_fixture() + enlarged = retape(binding, tape, tape.commands * 4) + with pytest.raises(ValueError, match='tape custody/bounds'): + p.identify_physical_plant_lag(binding, enlarged, poses, windows) + with pytest.raises(ValueError, match='40|declaration'): + p.identify_physical_plant_lag(binding, tape, poses, (replace(windows[0], pose_ids=windows[0].pose_ids[:39]), *windows[1:])) + + +def _boundary_objects(): + """Only negative trust-boundary tests use an uninitialized real engine type.""" + from openpilot.starpilot.flm.controller_replay import ControllerReplay + from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedEvidence + binding = physical_fixture()[0] + stamp = 30_000_000_000 + control = ControlEvidence(binding.identity, 1, stamp, stamp, 'normalized_torque', 'stage', 'response', + .1, (.1,), .1, True, False, True, True, False, False, False, False, False, False, + source_epoch=binding.source_epoch, source_receipt_ns=stamp - 10_000_000, + baseline_revision=binding.baseline_revision, controller_revision='declared-boundary-negative', + native_required=True, native_allowed=True, native_identity=binding.native_identity, + native_observed_ns=stamp - 1_000_000, native_expires_ns=stamp + 100_000_000) + output = AppliedEvidence(stamp, stamp, stamp + 1_000_000, 'normalized_torque', .1, + True, True, False, False, True, True, carrier_frame=1, carrier_observed_ns=stamp + 1_000_000) + engine = object.__new__(ControllerReplay) + engine.failed, engine.raw_mode = False, True + engine.applied_matches = Mock(side_effect=AssertionError('hard rejection must precede any proof call')) + return control, output, engine, binding + + +@pytest.mark.parametrize('field,value', [ + ('enabled', False), ('manual', True), ('active', False), ('valid', False), + ('override', True), ('fault', True), ('applied_coordinates', (.01,)), +]) +def test_constructor_keeps_original_hard_control_guard_even_when_performance_tagged(field, value): + control, output, engine, binding = _boundary_objects() + control = replace(control, saturated=True, limited=True, controller_chatter=True, model_wobble=True, + **{field: value}) + with pytest.raises(ValueError, match='hard validity'): + o.from_original(control, output, engine, binding, original_output=output) + engine.applied_matches.assert_not_called() + + +@pytest.mark.parametrize('field,value', [('valid', False), ('active', False), ('override', True), ('fault', True), ('native_allowed', False)]) +def test_constructor_keeps_original_hard_output_guard(field, value): + control, output, engine, binding = _boundary_objects() + output = replace(output, **{field: value}) + with pytest.raises(ValueError, match='hard validity'): + o.from_original(control, output, engine, binding, original_output=output) + engine.applied_matches.assert_not_called() + + +def test_caller_callback_or_claimed_qualified_engine_cannot_create_an_observation(): + control, output, _, binding = _boundary_objects() + forged_engine = SimpleNamespace(applied_matches=lambda *_: True, verified_source_native_can=True) + with pytest.raises(ValueError, match='actual replay types'): + o.from_original(control, output, forged_engine, binding, original_output=output) + + +def test_real_class_without_actual_original_context_cannot_claim_a_receipt(): + control, output, engine, binding = _boundary_objects() + with pytest.raises(ValueError, match='original context owner body'): + o.from_original(control, output, engine, binding, original_output=output) + engine.applied_matches.assert_not_called() + + +def test_changed_original_control_body_is_not_blessed_by_a_matching_epoch(): + control, output, engine, binding = _boundary_objects() + engine.original_context_controls = {control.control_ns: replace(control, controller_chatter=True)} + with pytest.raises(ValueError, match='original context owner body'): + o.from_original(control, output, engine, binding, original_output=output) + engine.applied_matches.assert_not_called() + + +@pytest.mark.parametrize('changed', [{'native_allowed': False}, {'native_observed_ns': 0}, + {'native_expires_ns': 1}, {'source_receipt_ns': 1}, {'generation': -1}]) +def test_constructor_rejects_original_expired_native_source_or_generation_before_proof(changed): + control, output, engine, binding = _boundary_objects() + with pytest.raises(ValueError, match='lease/carrier'): + o.from_original(replace(control, **changed), output, engine, binding, original_output=output) + engine.applied_matches.assert_not_called() + + +@pytest.mark.parametrize('control_field,output_field', [ + ('saturated', None), ('model_wobble', None), ('controller_chatter', None), + ('limited', None), (None, 'limited'), +]) +def test_old_stage_command_verifier_retains_performance_veto(control_field, output_field): + from openpilot.starpilot.flm.source_plant_fit import _command + control, output, _, binding = _boundary_objects() + control = replace(control, **({control_field: True} if control_field else {})) + output = replace(output, limited=False, **({output_field: True} if output_field else {})) if output_field != 'limited' else replace(output, limited=True) + proof = Mock(return_value=True) + with pytest.raises(ValueError, match='plant actual endpoint'): + _command(control, output, SimpleNamespace(applied_matches=proof), binding) + proof.assert_not_called() diff --git a/openpilot/starpilot/flm/tests/test_registered_accepted_profile.py b/openpilot/starpilot/flm/tests/test_registered_accepted_profile.py index 82dacf574b..d9e2a8eec3 100644 --- a/openpilot/starpilot/flm/tests/test_registered_accepted_profile.py +++ b/openpilot/starpilot/flm/tests/test_registered_accepted_profile.py @@ -9,6 +9,7 @@ registered representatives are source coverage, not firmware/fleet equivalence. from dataclasses import replace from types import SimpleNamespace import math +from pathlib import Path import sys import pytest @@ -293,7 +294,12 @@ def test_registered_controls_computed_profile_is_used_and_revoked(vehicle, mode, assert candidate.params.get_type('AutomaticSteeringProfiles') == ParamKeyType.BYTES assert candidate.params.check_key('UseVOACC') assert candidate.params.check_key('DeveloperSidebar') - assert len(candidate.params.all_keys()) == 347 + from tools.laptop_device_build.validate_params_registry import expected_registry + root = Path(__file__).resolve().parents[4] + registry = expected_registry((root / 'openpilot/common/params_keys.h').read_text(), + (root / 'openpilot/common/params.h').read_text(), + (root / 'openpilot/cereal/log.capnp').read_text()) + assert set(candidate.params.all_keys()) == {key.encode() for key in registry} runtime = candidate.global_learning assert runtime.identity and runtime.eligible assert candidate.sm['selfdriveState'].active and candidate.sm['selfdriveState'].enabled diff --git a/openpilot/starpilot/flm/tests/test_regular_publication_cadence.py b/openpilot/starpilot/flm/tests/test_regular_publication_cadence.py new file mode 100644 index 0000000000..f5cd724554 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_regular_publication_cadence.py @@ -0,0 +1,242 @@ +"""Actual daemon publication + ordinary receiver flags; no controller authority.""" + +from dataclasses import replace +from types import SimpleNamespace +import json + +import pytest + +from openpilot.cereal import messaging +from openpilot.starpilot.flm import steerlearnerd +from openpilot.starpilot.flm.tests.test_live_tracking_daemon import ( + BASE, + LoopComplete, + NoLegacyFitter, + RecordingActor, + source, +) + +TOPIC = 'autoSteeringLearning' + + +def _run_regular(monkeypatch, offsets, *, mode, delay_first_publication=0): + now = [BASE] + reads = [] + + class SM: + def __init__(self): + self.index = 0 + self.updated = {'deviceMotion': False} + self.valid = {'deviceMotion': True} + self.logMonoTime = {'deviceMotion': 0} + + def update(self, timeout=None): + if timeout == 0: + self.updated['deviceMotion'] = False + return + if self.index == len(offsets): + raise LoopComplete + now[0] = BASE + offsets[self.index] + self.index += 1 + self.updated['deviceMotion'] = True + self.logMonoTime['deviceMotion'] = now[0] + + sent = [] + pm = SimpleNamespace(send=lambda topic, message: sent.append((now[0], topic, message.as_reader()))) + actor = RecordingActor() if mode == 'live' else None + fitter = ( + NoLegacyFitter() if actor is not None else SimpleNamespace(child=None, poll=lambda **kwargs: None, cancel=lambda reason: None, close=lambda reason: True) + ) + + def read(stamp): + reads.append(stamp) + value = source(stamp) + return replace(value, enabled=False) if mode == 'off' else value + + publication_reads = 0 + + def publication_read(stamp): + nonlocal publication_reads + publication_reads += 1 + if publication_reads == 1: + now[0] += delay_first_publication + return read(now[0]) + + monkeypatch.setattr(steerlearnerd, 'drain', lambda *args: None) + with pytest.raises(LoopComplete): + steerlearnerd.run( + SimpleNamespace(), + SM(), + object(), + object(), + pm, + None, + None, + read, + None, + None, + None, + None, + fitter, + None, + clock=lambda: now[0], + publication_source_reader=publication_read, + live_actor=actor, + ) + if actor is not None: + assert actor.closed and fitter.closed + return [(stamp, message) for stamp, topic, message in sent if topic == TOPIC], reads + + +def _ordinary_receiver(monkeypatch, publications): + # Actual Controls topology: selfdriveState100Hz poll, learning service4Hz. + # No ignore_alive/valid/frequency options, simulation bypass or padded msgs. + monkeypatch.delenv('SIMULATION', raising=False) + receiver = messaging.SubMaster(['selfdriveState', TOPIC], poll='selfdriveState') + assert receiver.ignore_alive == receiver.ignore_valid == receiver.ignore_average_freq == [] + assert not receiver.simulation + for stamp, message in publications: + receiver.update_msgs(stamp / 1e9, [message]) + return receiver + + +@pytest.mark.parametrize('mode', ('live', 'nonlive', 'off')) +def test_actual_regular_run_preserves_four_hz_phase_under_49_51_ms_poll_jitter(monkeypatch, mode): + offsets = [0] + for index in range(200): + offsets.append(offsets[-1] + (49_000_000 if index % 2 == 0 else 51_000_000)) + publications, reads = _run_regular(monkeypatch, offsets, mode=mode) + receiver = _ordinary_receiver(monkeypatch, publications) + tracker = receiver.freq_tracker[TOPIC] + average = 1 / tracker.avg_dt.get_average() + summary = { + 'mode': mode, + 'count': len(publications), + 'average_hz': average, + 'alive': receiver.alive[TOPIC], + 'valid': receiver.valid[TOPIC], + 'freq_ok': receiver.freq_ok[TOPIC], + 'stamps': [stamp for stamp, _message in publications], + } + print('ACTUAL_REGULAR_PUBLICATION_CADENCE ' + json.dumps(summary, sort_keys=True)) + assert len(publications) == 41, summary + assert average == pytest.approx(4.0, abs=1e-10), summary + assert receiver.all_checks([TOPIC]), summary + assert len({stamp for stamp, _message in publications}) == len(publications) + assert all(stamp in {BASE + offset for offset in offsets} for stamp, _message in publications) + for stamp, message in publications: + row = message.autoSteeringLearning.learning + assert stamp in reads and message.logMonoTime == stamp + assert row.observedMonoTime == stamp and row.validUntilMonoTime == stamp + 500_000_000 + assert row.sourceReceipt.observedMonoTime == stamp + assert row.sourceReceipt.validUntilMonoTime == stamp + 500_000_000 + assert row.status == ('collecting' if mode == 'live' else 'unavailable') + + +def test_actual_regular_run_skips_missed_periods_without_burst_or_fake_receiver_health(monkeypatch): + offsets = (0, 49_000_000, 100_000_000, 600_000_000, 649_000_000, 700_000_000, 749_000_000, 800_000_000, 1_000_000_000) + publications, _reads = _run_regular(monkeypatch, offsets, mode='live') + assert [stamp - BASE for stamp, _message in publications] == [0, 600_000_000, 800_000_000, 1_000_000_000] + assert len({stamp for stamp, _message in publications}) == len(publications) + receiver = _ordinary_receiver(monkeypatch, publications) + assert not receiver.freq_ok[TOPIC] and not receiver.all_checks([TOPIC]) + for stamp, message in publications: + row = message.autoSteeringLearning.learning + assert message.logMonoTime == row.observedMonoTime == stamp + assert row.validUntilMonoTime == stamp + 500_000_000 + + +def test_actual_regular_run_skips_periods_spent_inside_current_source_read(monkeypatch): + offsets = (0, 649_000_000, 700_000_000, 749_000_000, 800_000_000, 1_000_000_000) + publications, reads = _run_regular(monkeypatch, offsets, mode='live', delay_first_publication=600_000_000) + assert [stamp - BASE for stamp, _message in publications] == [600_000_000, 800_000_000, 1_000_000_000] + for stamp, message in publications: + row = message.autoSteeringLearning.learning + assert stamp in reads and message.logMonoTime == row.observedMonoTime == stamp + assert row.validUntilMonoTime == stamp + 500_000_000 + assert row.sourceReceipt.observedMonoTime == stamp + assert row.sourceReceipt.validUntilMonoTime == stamp + 500_000_000 + + +def test_final_native_refresh_is_processed_before_another_blocking_motion_poll(): + from openpilot.starpilot.flm.steerlearnerd import _SourceDelivery + + class SM: + def __init__(self): + self.updated = {'deviceMotion': False, 'pandaStates': False} + self.logMonoTime = {'deviceMotion': 0, 'pandaStates': 0} + self.bodies = {} + self.calls = [] + + def update(self, timeout=None): + self.calls.append(timeout) + self.updated = dict.fromkeys(self.updated, False) + stamp = 100 + len(self.calls) * 50 + self.updated['deviceMotion'] = True + self.logMonoTime['deviceMotion'] = stamp + self.bodies['deviceMotion'] = ('original', stamp) + + old = SM() + old.update() + first = old.logMonoTime['deviceMotion'] + old.update(0) + skipped = old.logMonoTime['deviceMotion'] + old.update() + assert old.logMonoTime['deviceMotion'] != skipped + sm = SM() + delivery = _SourceDelivery() + assert delivery.next(sm) == first + delivery.refresh(sm) + body = sm.bodies['deviceMotion'] + flags = sm.updated.copy() + header = sm.logMonoTime.copy() + assert delivery.next(sm) == skipped + assert sm.calls == [None, 0] + assert sm.bodies['deviceMotion'] is body and sm.updated == flags and sm.logMonoTime == header + assert delivery.next(sm) == 250 + assert sm.calls == [None, 0, None] + + +def test_empty_incidental_refresh_cannot_erase_a_real_pending_pose_or_create_new_health(): + from types import SimpleNamespace + from openpilot.starpilot.flm.steerlearnerd import _SourceDelivery + + sm = SimpleNamespace(updated={'deviceMotion': False, 'pandaStates': False}, logMonoTime={'deviceMotion': 100}) + calls = [] + + def update(timeout=None): + calls.append(timeout) + sm.updated = {'deviceMotion': len(calls) == 1, 'pandaStates': False} + if len(calls) == 1: + sm.logMonoTime['deviceMotion'] = 150 + + sm.update = update + delivery = _SourceDelivery() + delivery.refresh(sm) + delivery.refresh(sm) + assert not sm.updated['deviceMotion'] + assert delivery.next(sm) == 150 + assert calls == [0, 0] and not any(sm.updated.values()) + # The saved ORIGINAL header was consumed once, not renewed on empty input. + assert delivery.next(sm) == 0 + assert calls == [0, 0, None] + + +def test_native_only_refresh_is_consumed_once_without_inventing_a_motion_delivery(): + from types import SimpleNamespace + from openpilot.starpilot.flm.steerlearnerd import _SourceDelivery + + sm = SimpleNamespace(updated={'deviceMotion': False, 'pandaStates': False}, logMonoTime={'deviceMotion': 100}) + calls = [] + + def update(timeout=None): + calls.append(timeout) + sm.updated = {'deviceMotion': False, 'pandaStates': len(calls) == 1} + + sm.update = update + delivery = _SourceDelivery() + delivery.refresh(sm) + flags = sm.updated.copy() + assert delivery.next(sm) == 0 and sm.updated == flags + assert calls == [0] + assert delivery.next(sm) == 0 and calls == [0, None] diff --git a/openpilot/starpilot/flm/tests/test_subset_baseline_view.py b/openpilot/starpilot/flm/tests/test_subset_baseline_view.py new file mode 100644 index 0000000000..2d16fffa1c --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_subset_baseline_view.py @@ -0,0 +1,175 @@ +"""Actual controller/native numeric equality; original source seals mocked. + +No manufactured fitting/profile/current permission. Whole source keys are real; +only the existing numeric fixture's original feedback seal is unit-isolated. +""" +from dataclasses import replace +from unittest import mock +import pickle + +import pytest + +from openpilot.starpilot.flm.event_controller_replay import EventControllerReplay +from openpilot.starpilot.flm.event_objective import EffectObjective +from openpilot.starpilot.flm.tests.test_closed_fd_reference_reuse import closed_engine + + +def full_subset(monkeypatch,domain='torque'): + full,evidence=closed_engine(monkeypatch,domain,neutral_full=True) + # Remove the older fixture's deliberately synthetic key: exercise actual full + # source serialization/key freshness in this numerical equality test. + del full.__dict__['_causal_input_key'] + assert type(full._causal_input_key(.01)) is bytes + full.rebuild_causal_features() + from openpilot.starpilot.flm.tests.test_full_surface_profile import base + from openpilot.starpilot.flm.canonical_surface_adapter import GROUPS + envelope=base(names=GROUPS['ff.left']).for_trial(('ff_gain_left',)) + objective=EffectObjective('ff.left',envelope.trial_names,envelope.trial_anchor,envelope.context_key) + subset=EventControllerReplay(full.engine,objective,envelope.replay_anchor(),activation_plan=full.activation_plan) + subset.verified_feedback_tape=evidence + assert subset.reuse_subset_causal_features(full) + assert subset._EventControllerReplay__subset_baseline_view is not None + return full,subset,evidence + + +@pytest.mark.parametrize('domain',['torque','starpilot','pid']) +def test_numeric_view_matches_independent_actual_selected_controller_native_baseline(monkeypatch,domain): + full,subset,_=full_subset(monkeypatch,domain) + expected=subset._rollout_uncached(subset.reference) + original_proof=full._EventControllerReplay__strict_rollouts[1]['baseline'][2][0] + subset._EventControllerReplay__candidate_feedback_proof=None + with mock.patch.object(subset,'_rollout_uncached',side_effect=AssertionError('duplicate baseline traversal')): + actual=subset._rollout(subset.reference) + assert pickle.dumps(actual,protocol=5)==pickle.dumps(expected,protocol=5) + assert subset._EventControllerReplay__strict_rollouts is None + assert subset._EventControllerReplay__candidate_feedback_proof is None + assert full._EventControllerReplay__strict_rollouts[1]['baseline'][2][0] is original_proof + first=next(iter(actual[0])) + actual[0][first]+=.1 + assert subset._rollout(subset.reference)==expected + + +@pytest.mark.parametrize('change',['frame','native','coordinates','anchor','proof','missing_cache','replaced_cache', + 'replaced_entry','restore','changed_feedback','signed_zero','factory_callback']) +def test_changed_source_native_coordinates_anchor_proof_and_cache_decline_view(monkeypatch,change): + full,subset,evidence=full_subset(monkeypatch) + coordinates=subset.reference + if change=='frame': + full.records[0]=replace(full.records[0],selected_command=-0.) + elif change=='native': + stamp,event=next(iter(full.emissions.events.items())) + full.emissions.events[stamp]=replace(event,witness=event.witness._replace(final_raw=event.witness.final_raw+1)) + elif change=='coordinates': + coordinates=(.001,) + elif change=='signed_zero': + coordinates=(-0.,) + elif change=='anchor': + subset.original_anchor['accumulated']=list(subset.original_anchor['accumulated']) + elif change=='proof': + proof=full._EventControllerReplay__strict_rollouts[1]['baseline'][2][0] + object.__setattr__(proof,'chronology_digest','0'*64) + elif change=='missing_cache': + full._EventControllerReplay__strict_rollouts=None + elif change=='replaced_cache': + cache=full._EventControllerReplay__strict_rollouts + full._EventControllerReplay__strict_rollouts=(cache[0],dict(cache[1])) + elif change=='replaced_entry': + slots=full._EventControllerReplay__strict_rollouts[1] + entry=slots['baseline'] + slots['baseline']=tuple(value for value in entry) + elif change=='restore': + subset.restore_envelope=object() + elif change=='changed_feedback': + object.__setattr__(evidence,'original',object()) + else: + class Callback: + def __str__(self): + raise AssertionError('unknown mode must not become reusable numeric proof') + subset.factory.mode=Callback() + # This direct helper is a structural negative, not a native-fallback claim. + source_key=subset._EventControllerReplay__subset_baseline_view.selected_key + assert subset._equivalent_subset_baseline(coordinates,source_key) is None + + +def test_full_source_keys_are_rechecked_after_effect_basis_callbacks(monkeypatch): + full,subset,_=full_subset(monkeypatch) + from openpilot.starpilot.flm.selected_effects import EffectTrial + original=EffectTrial.source + calls=[] + def mutate(trial): + source=original(trial) + calls.append(trial) + full.factory.manual_source_epoch='changed-during-proof' + return source + monkeypatch.setattr(EffectTrial,'source',mutate) + view=subset._EventControllerReplay__subset_baseline_view + assert subset._equivalent_subset_baseline(subset.reference,view.selected_key) is None + assert calls + + +def test_candidate_shadow_restore_and_fd_do_not_borrow_baseline(monkeypatch): + full,subset,_=full_subset(monkeypatch) + calls=[] + original=subset._rollout_uncached + def run(*args,**kwargs): + calls.append((args,kwargs)) + return original(*args,**kwargs) + monkeypatch.setattr(subset,'_rollout_uncached',run) + subset._rollout((.00001,)) + assert len(calls)==1 + assert subset._EventControllerReplay__strict_rollouts[1].get('baseline') is None + subset._rollout(subset.reference,_identification_token=subset._EventControllerReplay__identification_token) + assert len(calls)==2 + assert subset._strict_fd_reference(subset.reference,None) is None + with pytest.raises(ValueError,match='restore'): + subset._rollout(subset.reference,restore_target=True) + assert len(calls)==3 + + +def test_unknown_private_view_and_argument_shapes_never_run_callbacks(monkeypatch): + _,subset,_=full_subset(monkeypatch) + view=subset._EventControllerReplay__subset_baseline_view + class Unknown: + def __reduce__(self): + raise AssertionError('unknown coordinates must not be pickled') + def __eq__(self,other): + raise AssertionError('unknown source key must not compare') + assert subset._equivalent_subset_baseline((Unknown(),),view.selected_key) is None + assert subset._equivalent_subset_baseline(subset.reference,Unknown()) is None + class Slots(dict): + def get(self,*args): + raise AssertionError('unknown cache dictionary must not be read') + subset._EventControllerReplay__subset_baseline_view=view._replace(cache=(view.original_key,Slots())) + assert subset._equivalent_subset_baseline(subset.reference,view.selected_key) is None + + +@pytest.mark.parametrize('custom_metaclass',[False,True]) +def test_spoofed_interface_name_is_not_registered_constructor(monkeypatch,custom_metaclass): + _,subset,_=full_subset(monkeypatch) + actual=subset.factory.interface_class + if custom_metaclass: + class SpoofMeta(type(actual)): + def __getattribute__(cls,name): + if name in ('__module__','__qualname__'): + raise AssertionError('unregistered metaclass metadata callback') + return super().__getattribute__(name) + class Interface(metaclass=SpoofMeta): + pass + else: + Interface=type('CarInterface',(),{}) + Interface.__module__=actual.__module__ + Interface.__qualname__=actual.__qualname__ + subset.factory.interface_class=Interface + view=subset._EventControllerReplay__subset_baseline_view + assert subset._equivalent_subset_baseline(subset.reference,view.selected_key) is None + + +def test_unknown_factory_property_is_not_read_before_fallback(monkeypatch): + full,subset,_=full_subset(monkeypatch) + view=subset._EventControllerReplay__subset_baseline_view + class UnknownFactory: + @property + def interface_class(self): + raise AssertionError('unregistered factory property callback') + full.engine.factory=UnknownFactory() + assert subset._equivalent_subset_baseline(subset.reference,view.selected_key) is None diff --git a/openpilot/starpilot/flm/tests/test_tracking_source_context.py b/openpilot/starpilot/flm/tests/test_tracking_source_context.py new file mode 100644 index 0000000000..22a3596aab --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_tracking_source_context.py @@ -0,0 +1,389 @@ +"""Source-context and numerical optimization contracts; NOT native/road proof.""" + +from dataclasses import replace +import json +import pytest +from openpilot.starpilot.flm.live_tracking_response import ( + TrackingSourceContext, + SOURCE_VERSION, + ResponseWitness, + _revision, + validate_certificate, + prepare_response_witness, + certificate_to_dict, + certificate_from_dict, + source_context, +) +from openpilot.starpilot.flm.canonical_surface_adapter import NAMES +from openpilot.starpilot.flm.tests.test_live_tracking import fixture_certificate, episode +from openpilot.starpilot.flm.live_tracking import LiveTrackingOptimizer + + +def context(binding=None): + b = binding or fixture_certificate().binding + c = TrackingSourceContext(SOURCE_VERSION, b, '0' * 64, 170_000_000, 'a' * 64, 1) + revision = _revision(c) + return validate_certificate(replace(c, model_revision=revision, binding=replace(b, physical_model_revision=revision))) + + +def owner(): + c = context() + return LiveTrackingOptimizer(c.binding, tuple((n, 0.0) for n in NAMES), controller_domain='torque', response_certificate=c) + + +def refresh(o, now): + w = ResponseWitness(o.certificate, o.binding, now - o.lag_ns, now - o.lag_ns, now, max(1, now - 350_000_000), 8, None, now + 100_000_000) + assert o.response_current(w, now_ns=now) + + +def test_source_context_has_no_fabricated_model_precision_or_training_results(): + c = context() + assert not any(hasattr(c, n) for n in ('slope', 'intercept', 'heldout_rmse', 'train_block_ids', 'heldout_block_ids')) + raw = certificate_to_dict(c) + assert certificate_from_dict(json.loads(json.dumps(raw))) == c + assert raw['version'] == SOURCE_VERSION + assert not {'profile', 'permission', 'current_native', 'heldout_rmse'} & raw.keys() + + +def test_legacy_point_precision_is_unchanged(): + old = fixture_certificate() + bad = replace(old, heldout_rmse=0.1) + revision = _revision(bad) + bad = replace(bad, model_revision=revision, binding=replace(bad.binding, physical_model_revision=revision)) + with pytest.raises(ValueError): + validate_certificate(bad) + + +@pytest.mark.parametrize('name', NAMES) +def test_global_thirty_coordinates_require_new_unselected_baseline(name): + o = owner() + kind = 'subtle' if name.startswith('center_') or name == 'highway_center_taper_max' else 'large' + count = 9 if kind == 'subtle' else 3 + discovery = set() + for i in range(2 * count): + e = episode(o, i, name, kind=kind, error=0.2) + e = replace(e, normalization_accel=max(0.1, e.goal_rms)) + refresh(o, e.end_ns) + o.observe_episode(e, now_ns=e.end_ns) + if i < count: + discovery.add(e.episode_id) + if i == count - 1: + assert o.offer_state is None + selected = o.offer(e.end_ns) + assert selected is not None and selected.names == (name,) and len(o.before) == count + assert discovery.isdisjoint(e.episode_id for e in o.before) + assert o.offer_state['alpha'] == 0.0 and o.offer_state['critical'] == 0.0 + assert 'noise_prior' not in o.offer_state and len(selected.target) == 30 and not any(o.accepted.values()) + changes = [n for n, v in selected.target if v != dict(selected.baseline)[n]] + assert changes == [name] and abs(dict(selected.target)[name]) <= 0.005 + + +def test_source_wire_roundtrip_and_model_field_smuggling_rejection(): + from openpilot.cereal import messaging + from openpilot.starpilot.flm.tests.test_live_trial_owner import offer + from openpilot.starpilot.flm.live_trial_wire import write_trial, read_trial + + old = offer(10_000_000_000) + c = context(old.binding) + w = ResponseWitness( + c, c.binding, old.observed_ns - c.lag_ns, old.observed_ns - c.lag_ns, old.observed_ns, old.observed_ns - 350_000_000, 8, None, old.observed_ns + 100_000_000 + ) + new = replace(old, binding=c.binding, response_witness=prepare_response_witness(w, c.binding, old.observed_ns)) + event = messaging.new_message('starpilotSteeringTracking', valid=True, logMonoTime=old.observed_ns) + write_trial(event, new) + read = read_trial(messaging.log_from_bytes(event.to_bytes()).starpilotSteeringTracking) + assert read.response_witness.certificate == c and read.response_witness.residual_rmse is None + event = event.as_reader().as_builder() + event.starpilotSteeringTracking.trial.responseWitness.certificate.slope = 1.0 + with pytest.raises(ValueError, match='cannot claim fitted'): + read_trial(event.starpilotSteeringTracking) + + +def source_retainer(): + from openpilot.starpilot.flm.tests.test_live_tracking_requalification import certificate, basis, new_drive + from openpilot.starpilot.flm.live_tracking_requalification import RetainedResponseRequalifier + + old = context(certificate().binding) + return RetainedResponseRequalifier(old, basis(old), new_drive(old)) + + +def test_new_drive_reuses_no_old_permission_and_needs_eight_original_sources(): + from openpilot.starpilot.flm.tests.test_live_tracking_requalification import feed + + retainer = source_retainer() + assert retainer.qualified(1) is None + for i in range(7): + assert feed(retainer, i, command=0.4) + assert retainer.qualified(10_300_000_000) is None + assert feed(retainer, 7, command=0.4) + c, w = retainer.qualified(10_350_000_000) + assert c.provenance == 'retained-current-source-v3' and w.independent_poses == 8 + assert w.residual_rmse is None and not hasattr(c, 'slope') + assert retainer.qualified(10_450_000_001) is None + + +@pytest.mark.parametrize('fault', ['native', 'source', 'override', 'fault', 'stale', 'hardware', 'controller']) +def test_source_context_never_bypasses_current_source_native_or_context_guards(fault): + from openpilot.starpilot.flm.tests.test_live_tracking_requalification import synthetic_pair + + retainer = source_retainer() + for i in range(8): + c, o, p = synthetic_pair(retainer, i, command=0.4) + if fault == 'source': + c = replace(c, source_epoch='0' * 64) + if fault == 'override': + c = replace(c, override=True) + if fault == 'fault': + c = replace(c, fault=True) + if fault == 'controller': + c = replace(c, controller_revision='changed') + now = p.mono_ns + (100_000_001 if fault == 'stale' else 0) + assert not retainer.observe( + c, o, p, now_ns=now, current_native=fault != 'native', hardware_inventory=('different',) if fault == 'hardware' else ('measured-hardware-type',) + ) + assert retainer.qualified(now) is None + + +def test_construct_context_uses_actual_consumed_delay_not_a_fitted_or_default_value(): + from openpilot.starpilot.flm.tests.test_live_tracking_requalification import synthetic_pair + + r = source_retainer() + c, o, p = synthetic_pair(r, 0) + with pytest.raises(KeyError): + source_context(r._binding, c, p, now_ns=p.mono_ns) + c = replace(c, replay_inputs=(*c.replay_inputs, ('lat_delay', 0.21))) + result = source_context(r._binding, c, p, now_ns=p.mono_ns) + assert result.lag_ns == 210_000_000 and result.observed_ns == p.mono_ns + assert not hasattr(result, 'heldout_rmse') + + +def test_direct_kinematic_target_does_not_mix_two_roll_estimates(): + from openpilot.starpilot.flm.live_tracking_episodes import tracking_reference_acceleration, tracking_reference_force + from openpilot.starpilot.flm.tests.test_live_tracking_requalification import synthetic_pair + + r = source_retainer() + c, _, _ = synthetic_pair(r, 0) + c = replace(c, stage_values=(('setpoint', 0.2),), replay_inputs=(*c.replay_inputs, ('roll', 0.1))) + assert tracking_reference_acceleration(c) == 0.2 + assert tracking_reference_force(c) < 0 + + +def test_unbroken_long_curve_yields_input_scheduled_windows_without_turn_ending(): + from openpilot.starpilot.flm.live_tracking_episodes import TrackingEpisodeBuilder + + c = context() + b = TrackingEpisodeBuilder(c.binding, c) + for tick in range(601): + stamp = 10_000_000_000 + tick * 10_000_000 + b._geometry(stamp, 0.3) + b.last = stamp + assert len(b.pending) == 3 + assert all(kind == 'large' and direction == 1 and end - start == 1_990_000_000 for start, end, kind, direction, _ in b.pending) + # No independent response, native sample, probe or fit is invented. + assert not b.endpoints and not b.ready and not b.probes + + +def test_straight_raw_goal_stays_zero_with_an_explicit_numerical_score_scale(): + from openpilot.starpilot.flm.live_tracking import _score, _matches + + o = owner() + e = episode(o, 0, 'center_taper_max', kind='subtle', error=0.02) + straight = replace(e, direction=0, goal_rms=0.0, goal_shape=(0.0,) * 8, normalization_accel=0.1) + straight.validate() + assert straight.goal_rms == 0.0 and _score(straight) == pytest.approx(0.2) + assert _matches(straight, replace(straight, episode_id='f' * 64)) + with pytest.raises(ValueError): + replace(straight, normalization_accel=0.001).validate() + + +def test_source_gap_drops_partial_window_without_changing_input_clock_lattice(): + from openpilot.starpilot.flm.live_tracking_episodes import TrackingEpisodeBuilder + + c = context() + b = TrackingEpisodeBuilder(c.binding, c) + origin = 10_000_000_000 + for tick in range(101): + stamp = origin + tick * 10_000_000 + b._geometry(stamp, 0.3) + b.last = stamp + b.interrupt('genuine-source-gap') + assert b.input_window_origin == origin + for tick in range(110, 601): + stamp = origin + tick * 10_000_000 + b._geometry(stamp, 0.3) + b.last = stamp + assert len(b.pending) == 2 + assert [row[0] for row in b.pending] == [origin + 2_000_000_000, origin + 4_000_000_000] + assert not b.endpoints and not b.ready + + +def test_source_proposal_does_not_claim_instantaneous_gradient_is_plant_direction(): + o = owner() + for i in range(100): + e = episode(o, i, 'ff_gain_left', kind='large', error=0.2) + factor = (1.0, -1.0, 2.0)[i % 3] + e = replace( + e, normalization_accel=max(0.1, e.goal_rms), sensitivity=tuple(replace(f, error_feature_sum=f.error_feature_sum * factor) for f in e.sensitivity) + ) + refresh(o, e.end_ns) + o.observe_episode(e, now_ns=e.end_ns) + if o.offer_state is not None: + break + assert o.offer_state is not None and o.offer_state['phase'] == 'trial' + assert not any(o.accepted.values()) and len(o.before) == 3 + assert o.offer_state['alpha'] == 0.0 and abs(o.offer_state['target']['ff_gain_left']) <= 0.005 + assert not o.offer_state.get('accepted', False) + + +def test_direct_input_matching_uses_demand_not_exact_waveform_phase(): + from openpilot.starpilot.flm.live_tracking import _matches + + o = owner() + e = replace(episode(o, 0, 'ff_gain_left', kind='subtle'), normalization_accel=1.0, direction=0, goal_shape=(0.0, 0.2, 0.0, -0.2, 0.0, 0.2, 0.0, -0.2)) + shifted = replace(e, goal_shape=(0.2, 0.0, -0.2, 0.0, 0.2, 0.0, -0.2, 0.0)) + assert _matches(e, shifted) + assert not _matches(e, replace(shifted, speed_low=2.0, speed_high=2.0)) + assert not _matches(e, replace(shifted, normalization_accel=2.0)) + assert not _matches(e, replace(shifted, goal_shape=(0.9,) * 8)) + assert not _matches(e, replace(shifted, goal_shape=(1.0, -1.0) * 4)) + # The old physical whole-maneuver protocol retains exact waveform matching. + assert not _matches(replace(e, normalization_accel=None), replace(shifted, normalization_accel=None)) + + +def test_selected_input_class_does_not_silently_move_to_first_baseline(): + from openpilot.starpilot.flm.live_tracking import _matches, _demand, _matches_demand + + o = owner() + + def row(i, scale): + return replace(episode(o, i, 'ff_gain_left', kind='large'), goal_rms=scale, normalization_accel=scale, goal_shape=(1.0,) * 8) + + anchor = row(0, 1.0) + different = row(1, 0.81) + before = tuple(row(i, 1.24) for i in range(2, 5)) + assert _matches(anchor, different) and _matches(anchor, before[0]) and not _matches(before[0], different) + o.selector_hint = { + 'binding': o.binding, + 'model_revision': o.certificate.model_revision, + 'profile_sha256': anchor.profile_sha256, + 'selected_ns': anchor.end_ns, + 'expires_ns': before[-1].end_ns + 1, + 'demand': _demand(anchor), + } + target = dict(o.accepted, ff_gain_left=0.005) + plan = o._trial_plan(before, 0.001, target, minimum_episodes=3) + assert plan is not None and o._launch('ff.left', 'ff_gain_left', target, before, plan, before[-1].end_ns) + assert o.offer_state['demand'] == _demand(anchor) and _matches_demand(o.offer_state['demand'], different) + assert not any(o.accepted.values()) + + +def test_source_probe_window_keeps_reference_membership_and_original_native_clocks(): + from types import SimpleNamespace as S + from openpilot.starpilot.flm.live_tracking_actor import _original_probe_windows + + points = tuple(S(control_ns=i, reference_ns=i + 170) for i in range(100, 801, 100)) + e = S(source_tracking=True, completed={}, pending=((200, 800, 'subtle', 0, 'profile'),), endpoints=points, probe_requests={100: ('all30', NAMES, 900)}) + assert _original_probe_windows(e, 'profile', 'all30', NAMES) == ((100, 600, frozenset({100})),) + assert tuple((p.control_ns, p.reference_ns) for p in points) == tuple((i, i + 170) for i in range(100, 801, 100)) + e.source_tracking = False + assert _original_probe_windows(e, 'profile', 'all30', NAMES) == ((200, 800, frozenset()),) + + +def test_old_history_eviction_cannot_block_the_new_fresh_baseline(): + o = owner() + i = 0 + while o.discovery is None and i < 80: + e = episode(o, i, 'ff_gain_left', kind='large', error=0.2) + e = replace(e, normalization_accel=max(0.1, e.goal_rms)) + refresh(o, e.end_ns) + o.observe_episode(e, now_ns=e.end_ns) + i += 1 + assert o.discovery is not None + o.planning_history.clear() + for _ in range(3): + e = episode(o, i, 'ff_gain_left', kind='large', error=0.2) + e = replace(e, normalization_accel=max(0.1, e.goal_rms)) + refresh(o, e.end_ns) + o.observe_episode(e, now_ns=e.end_ns) + i += 1 + assert o.offer_state is not None and o.offer_state['phase'] == 'trial' and len(o.before) == 3 + assert 'noise_prior' not in o.offer_state and not any(o.accepted.values()) + + +@pytest.mark.parametrize( + 'before,future,expected', + [ + ([1.0] * 9, [0.99] * 8 + [1.01], 1), + ([1.0] * 9, [1.0] * 9, 0), + ([1.0] * 9, [0.9] * 7 + [1.1] * 2, 0), + ([1.0] * 9, [0.1] + [1.01] * 8, 0), + ([1.0] * 9, [1.02] * 9, -1), + ([1.0] * 9, [0.9999] * 9, 0), + ([1.0] * 3, [0.8] * 3, 1), + ([1.0] * 3, [0.8, 0.8, 1.1], 0), + ], +) +def test_empirical_feedback_needs_mean_median_and_repeated_later_improvement(before, future, expected): + from openpilot.starpilot.flm.live_tracking import _feedback_result + + assert _feedback_result(before, future) == expected + + +@pytest.mark.parametrize( + 'before,future', [([1.0] * 2, [0.5] * 2), ([1.0] * 3, [0.5] * 9), ([1.0] * 3, [float('nan')] * 3), ([True] * 3, [0.5] * 3), ([-1.0] * 3, [0.5] * 3)] +) +def test_empirical_feedback_never_accepts_malformed_measurements(before, future): + from openpilot.starpilot.flm.live_tracking import _feedback_result + + with pytest.raises(ValueError): + _feedback_result(before, future) + + +def source_ack(o, offer, stamp): + refresh(o, stamp) + return o.acknowledge_application( + trial_id=offer.trial_id, + target_sequence=offer.target_sequence, + phase=offer.phase, + target_sha256=offer.target_sha256, + applied_full30=offer.target, + control_ns=stamp, + binding=offer.binding, + now_ns=stamp, + status='applied', + confirmed_target_sha256=offer.target_sha256 if offer.phase == 'confirmed' else '', + ) + + +def test_feedback_confirmation_still_needs_actual_target_ack_and_new_current_witness(): + o = owner() + for i in range(6): + e = replace(episode(o, i, 'ff_gain_left', error=0.2), normalization_accel=0.5) + refresh(o, e.end_ns) + o.observe_episode(e, now_ns=e.end_ns) + offer = o.offer(e.end_ns) + assert offer is not None and offer.phase == 'trial' + assert not any(o.accepted.values()) and source_ack(o, offer, e.end_ns + 10_000_000) + now = e.end_ns + for i in range(3): + e = replace( + episode(o, 100 + i, 'ff_gain_left', profile=offer.target_sha256, error=0.15, start=now + 1_000_000_000 + i * 1_000_000_000), normalization_accel=0.5 + ) + refresh(o, e.end_ns) + assert o.observe_episode(e, now_ns=e.end_ns) + confirmed = o.offer(e.end_ns) + assert confirmed is not None and confirmed.phase == 'confirmed' and not any(o.accepted.values()) + assert source_ack(o, confirmed, e.end_ns + 10_000_000) + assert o.accepted['ff_gain_left'] == dict(offer.target)['ff_gain_left'] + + +def test_small_step_can_improve_mean_and_median_with_normal_baseline_variation(): + from openpilot.starpilot.flm.live_tracking import _feedback_result + + before = [0.1246, 0.1275, 0.1264, 0.1251, 0.1256, 0.1239, 0.1234, 0.1297, 0.1278] + future = [0.1243, 0.1209, 0.1240, 0.1207, 0.1221, 0.1256, 0.1208, 0.1275, 0.1251] + assert _feedback_result(before, future) == 1 + # A lower average from one lucky point is still insufficient. + assert _feedback_result([1.0] * 9, [0.1] + [1.01] * 8) == 0 + # Repeated new errors outside the original envelope are not kept. + assert _feedback_result([1.0] * 9, [0.9] * 7 + [1.1] * 2) == 0 diff --git a/openpilot/starpilot/flm/tests/test_upstream_message_factories.py b/openpilot/starpilot/flm/tests/test_upstream_message_factories.py new file mode 100644 index 0000000000..d371b22e56 --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_upstream_message_factories.py @@ -0,0 +1,18 @@ +"""Actual local IPC timestamp/body transport, never physical clock proof.""" +from openpilot.cereal import messaging + + +def test_current_message_factories_preserve_event_body_and_explicit_header(): + publisher = messaging.PubMaster(['carState']) + subscriber = messaging.SubMaster(['carState'], poll='carState') + for index in range(2): + # This is a literal transport sentinel, not a fresh source/native lease. + stamp = 1_234_567_890 + index + event = messaging.new_message('carState', valid=True, logMonoTime=stamp) + event.carState.vEgo = 12.5 + index + publisher.send('carState', event) + subscriber.update(1000) + assert subscriber.updated['carState'] and subscriber.seen['carState'] and subscriber.valid['carState'] + assert subscriber.logMonoTime['carState'] == stamp + assert subscriber['carState'].vEgo == 12.5 + index + messaging.reset_context() # Compatibility no-op must not break owned sockets. diff --git a/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py b/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py new file mode 100644 index 0000000000..a70b8ae6dd --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py @@ -0,0 +1,49 @@ +"""Installed source closure only; no kernel or model-output equivalence proof.""" +import hashlib +from pathlib import Path + +import pytest + +from openpilot.starpilot.flm import producer_clock_equivalence as clock + + +APPROVED = { + 'selfdrive/modeld/modeld.py': '43ef68eaef76540b562a64b1f4dda4f1c155de1476f3caf3500279db2202e588', + 'visionipc/visionipc_server.cc': 'a4325a5171eacb931eb271d06a992d2f27216040df113d92099f62cdfa67d81a', + 'visionipc/visionipc_client.cc': 'd142cd60eedabfc581e9770fcd6f0ca81b762ba9dbc608b54dd20a27a317a52d', +} + + +def test_exact_installed_upstream_closure_is_checked_before_contract(): + contract = clock.pinned_default_contract() + assert len(clock.PRODUCER_SOURCE_HASHES) == 19 + assert {relative: expected for _, relative, expected in clock.PRODUCER_SOURCE_HASHES if relative in APPROVED} == APPROVED + assert contract.producer_revision == clock.CLOSURE_SHA256 + assert contract.proof_sha256 == clock.CLOSURE_SHA256 + + +@pytest.mark.parametrize('relative', tuple(APPROVED)) +def test_unapproved_change_to_each_new_producer_fails_closed(monkeypatch, relative): + original = Path.read_bytes + reached = [] + def changed(path): + body = original(path) + if path.as_posix().endswith(relative): + reached.append(path) + assert hashlib.sha256(body).hexdigest() == APPROVED[relative] + return body + b'\n' + return body + monkeypatch.setattr(Path, 'read_bytes', changed) + with pytest.raises(ValueError, match='unsupported original mixed-producer source closure'): + clock.pinned_default_contract() + assert len(reached) == 1 + + +def test_unapproved_second_producer_cannot_be_blessed_by_new_modeld_pin(monkeypatch): + rows = list(clock.PRODUCER_SOURCE_HASHES) + index = next(index for index, (_, relative, _) in enumerate(rows) if relative == 'system/sensord/sensord.py') + package, relative, _ = rows[index] + rows[index] = (package, relative, '0' * 64) + monkeypatch.setattr(clock, 'PRODUCER_SOURCE_HASHES', tuple(rows)) + with pytest.raises(ValueError, match='unsupported original mixed-producer source closure'): + clock.pinned_default_contract() diff --git a/openpilot/starpilot/flm/torque_emission_replay.py b/openpilot/starpilot/flm/torque_emission_replay.py index a6daf1d8e9..2ed779b632 100644 --- a/openpilot/starpilot/flm/torque_emission_replay.py +++ b/openpilot/starpilot/flm/torque_emission_replay.py @@ -6,7 +6,7 @@ steps and finite differences belong to the child or existing bounded4Hz probe. from typing import cast -from collections import OrderedDict +from collections import OrderedDict, deque from dataclasses import dataclass, asdict from types import SimpleNamespace import math @@ -262,6 +262,29 @@ class EncoderBranch: return value + def candidate_reported_torque(self, event): + """Exact registered CC actuator report after this SAME native branch step. + + Honda reports the shaped/rate-limited normalized INPUT; inverse CAN lookup + is a distinct physical coordinate. Other registered round codecs report + final raw torque divided by the same effective maximum, all Float32 on CO. + """ + witness=event.witness + if (self.last_frame!=witness.frame or self.last_apply_ns!=event.apply_ns or + self.family!=(witness.family,witness.fingerprint,witness.flags,witness.codec_id) or + witness.codec_id not in CODECS or self.previous is None): + raise ValueError('candidate reported CO missing same native step') + if witness.family=='honda' and witness.mapping_kind=='honda_lookup': + value=float32(self.previous) + elif witness.family!='honda' and witness.mapping_kind=='round' and not witness.shape: + value=float32(float(self.previous)/witness.effective_max) + else: + raise ValueError('unproven candidate reported CO codec') + if not math.isfinite(value): + raise ValueError('nonfinite candidate reported CO') + return value + + class EmissionHistory: def __init__(self, cp, *, identity='', baseline_revision=''): self.cp = cp @@ -269,6 +292,9 @@ class EmissionHistory: self.events = OrderedDict() self.by_control = {} self.decoder = None + self.decode_queue = deque() + self.decode_errors = OrderedDict() + self.decode_history_truncated = False self.carriers = OrderedDict() def record_carrier(self, output): @@ -355,9 +381,14 @@ class EmissionHistory: if getattr(output, 'carrier_frame', witness.frame + 1) != witness.frame + 1: return False self.events[event.apply_ns] = event + self.decode_queue.append(event.apply_ns) self.by_control[event.control_ns] = event while len(self.events) > MAX_EMISSIONS: _, removed = self.events.popitem(last=False) + self.decode_errors.pop(removed.apply_ns, None) + if self.decode_queue and self.decode_queue[0] <= removed.apply_ns: + self.decode_queue.popleft() + self.decode_history_truncated = True if self.decoder is not None: self.decoder.verified.pop(removed.apply_ns, None) if self.by_control.get(removed.control_ns) == removed: @@ -387,9 +418,26 @@ class EmissionHistory: return event def decode(self, event): + # CANParser has stateful counter checks. A query for an occasional clipped + # input must consume the intervening original emissions too, rather than + # presenting that sparse query stream as the actual CAN sequence. + if self.events.get(event.apply_ns) != event or self.decode_history_truncated: + raise ValueError('actual CAN decoder original history unavailable') if self.decoder is None: self.decoder = EmissionDecoder(self.cp) - self.decoder.check(event) + for _ in range(64): + if not self.decode_queue or self.decode_queue[0] > event.apply_ns: + break + stamp = self.decode_queue.popleft() + original = self.events[stamp] + try: + self.decoder.check(original) + except (AttributeError, KeyError, TypeError, ValueError, OverflowError) as error: + self.decode_errors[stamp] = str(error) + if event.apply_ns in self.decode_errors: + raise ValueError(self.decode_errors[event.apply_ns]) + if event.apply_ns not in self.decoder.verified: + raise ValueError('actual CAN decoder original sequence pending') def one_apply(self, apply_ns, request): event = self.events.get(apply_ns) @@ -406,9 +454,11 @@ class EmissionHistory: return base, features def rollout(self, requests, start_ns, end_ns, windows=(), *, quality_contract=RECORDED_DEBT, - identification_policy=STRICT_IDENTIFICATION): + identification_policy=STRICT_IDENTIFICATION, feedback_reported=None): """Advance on EVERY actual emission, including unscored warmup emissions.""" branch = EncoderBranch(quality_contract=quality_contract, identification_policy=identification_policy) + if feedback_reported is not None and (type(feedback_reported) is not dict or feedback_reported): + raise ValueError('empty private candidate reported CO sink required') values = {} previous = None last_window = None @@ -423,6 +473,8 @@ class EmissionHistory: previous = None self.decode(event) value = branch.step(event, requests[event.control_ns]) + if feedback_reported is not None: + feedback_reported[event.apply_ns]=branch.candidate_reported_torque(event) values[event.apply_ns] = value last_window = window if previous is not None and event.apply_ns - previous > 100_000_000: diff --git a/openpilot/starpilot/flm/torque_raw_fit.py b/openpilot/starpilot/flm/torque_raw_fit.py index 64384fa96a..171cee69ce 100644 --- a/openpilot/starpilot/flm/torque_raw_fit.py +++ b/openpilot/starpilot/flm/torque_raw_fit.py @@ -9,9 +9,9 @@ from openpilot.starpilot.flm.fullrate_collector import ControlEvidence, AppliedE from openpilot.starpilot.flm.runtime_adapter_registry import canonical_physical_context, physical_model_revision -def control_from_dict(value): +def control_from_dict(value, *, trace_archive=None): value = dict(value) - for key in ('features', 'applied_coordinates', 'applied_parameter_names', 'baseline_values'): + for key in ('features', 'applied_coordinates', 'applied_parameter_names', 'baseline_values', 'vehicle_model_anchor', 'live_applied_coordinates'): if key in value: value[key] = tuple(value[key]) for key in ('stage_values', 'replay_inputs', 'replay_snapshot'): @@ -19,8 +19,26 @@ def control_from_dict(value): value[key] = tuple(tuple(row) for row in value[key]) if value.get('effect_trace') is not None: from openpilot.starpilot.flm.effect_trace import trace_from_dict - value['effect_trace'] = trace_from_dict(value['effect_trace']) - return ControlEvidence(**value) + if trace_archive is None: + value['effect_trace'] = trace_from_dict(value['effect_trace']) + else: + from openpilot.starpilot.flm.normalized_trace_archive import read_ref + value['effect_trace'] = read_ref(value['effect_trace'],trace_archive) + control = ControlEvidence(**value) + import math + from openpilot.starpilot.flm.canonical_surface_adapter import NAMES, coordinate_bounds + if (type(control.vehicle_model_anchor) is not tuple or len(control.vehicle_model_anchor) not in (0, 3) or + any(type(v) is not float or not math.isfinite(v) or v == 0. for v in control.vehicle_model_anchor) or + type(control.live_applied_coordinates) is not tuple or len(control.live_applied_coordinates) not in (0, 30)): + raise ValueError('explicit actual live source metadata shape') + if control.live_applied_coordinates: + if control.effect_trace is not None: + raise ValueError('distinct live and replay effect sources') + for name, coordinate in zip(NAMES, control.live_applied_coordinates, strict=True): + low, high = coordinate_bounds(name) + if type(coordinate) not in (int, float) or not math.isfinite(coordinate) or not low <= coordinate <= high: + raise ValueError('actual live coordinate domain') + return control def output_from_dict(value): diff --git a/openpilot/starpilot/flm/torque_stages.py b/openpilot/starpilot/flm/torque_stages.py index cfeb6929e8..49f0b42d07 100644 --- a/openpilot/starpilot/flm/torque_stages.py +++ b/openpilot/starpilot/flm/torque_stages.py @@ -10,6 +10,7 @@ from dataclasses import dataclass import math from openpilot.starpilot.flm.selected_effects import ( apply_selected_stage, + bounded_profile_on_limited_frame, apply_selected_friction as apply_selected_friction, apply_selected_deadzone as apply_selected_deadzone, apply_selected_output as apply_selected_output, @@ -28,6 +29,7 @@ class StageFrame: active: bool driver_override: bool limited: bool + curvature_limited: bool = False def valid(self): return ( @@ -37,21 +39,32 @@ class StageFrame: ) -def apply_stages(parent, cs, setpoint, jerk, ff, threshold, *, active, limited): +def apply_stages(parent, cs, setpoint, jerk, ff, threshold, *, active, limited, curvature_limited=False): """One selected-policy call after baseline FF/threshold, before friction. Missing source and inactive/manual/off source emit exact unchanged baseline. Source methods are owned by core and may only return admitted residuals. """ source = getattr(parent, 'automatic_flm_stages', None) - frame = StageFrame(float(cs.vEgo), float(setpoint), float(jerk), float(ff), float(threshold), bool(active), bool(cs.steeringPressed), bool(limited)) + frame = StageFrame( + float(cs.vEgo), float(setpoint), float(jerk), float(ff), float(threshold), bool(active), bool(cs.steeringPressed), bool(limited), bool(curvature_limited) + ) observer = getattr(parent, 'automatic_flm_observation', None) if observer is not None and frame.valid(): observer.observe_stage(frame, ff, (0.0, 0.0)) if source is None: return ff, threshold - if not frame.valid() or not frame.active or frame.driver_override or frame.limited: + if not frame.valid() or not frame.active or frame.driver_override: return ff, threshold + if frame.limited: + # Preserve the limiter flag and PID freeze. Only an admitted bounded full + # profile may shape its request before the unchanged native actuator limit. + if curvature_limited or not bounded_profile_on_limited_frame(parent): + return ff, threshold + candidate_ff, candidate_threshold = apply_selected_stage(parent, frame) + if observer is not None: + observer.observe_stage(frame, candidate_ff, tuple(parent.automatic_flm_effects.applied_coordinates), candidate_threshold) + return candidate_ff, candidate_threshold try: # Core owns monotonic tick, identity/manual/TTL/policy validation and slew. # delta is relative to this selected controller's fully shaped baseline. @@ -67,8 +80,7 @@ def apply_stages(parent, cs, setpoint, jerk, ff, threshold, *, active, limited): # Only one admitted profile exists; legacy stage residual is neutral for V2. candidate_ff, candidate_threshold = apply_selected_stage(parent, frame) if observer is not None: - observer.observe_stage(frame, candidate_ff, - tuple(effect.applied_coordinates if effect is not None else source.applied_coordinates), candidate_threshold) + observer.observe_stage(frame, candidate_ff, tuple(effect.applied_coordinates if effect is not None else source.applied_coordinates), candidate_threshold) return candidate_ff, candidate_threshold except (AttributeError, TypeError, ValueError, OverflowError): return ff, threshold diff --git a/openpilot/starpilot/flm/training_report.py b/openpilot/starpilot/flm/training_report.py index 0507414cc2..f9ed8c2ad1 100644 --- a/openpilot/starpilot/flm/training_report.py +++ b/openpilot/starpilot/flm/training_report.py @@ -10,6 +10,7 @@ from pathlib import Path from collections import Counter, deque from openpilot.starpilot.flm.application_wire import read_application +from openpilot.starpilot.flm.live_tracking_report import tracking_objects READ_MAX_BYTES = 32_768 READ_MAX_AGE_NS = 2_000_000_000 @@ -77,7 +78,8 @@ def _report_current(value, age_ns): 'status': value['status'], 'reason': value['reason'], 'progress': {key: progress[key] for key in ('samples', 'blocks', 'requiredBlocks')}, 'fit': None if fit is None else {key: fit[key] for key in ('status', 'reason', 'trainWindows', 'heldoutWindows')}, - 'application': application, 'details': details} + 'application': application, 'details': details, + **tracking_objects(value, fresh=fresh)} def read_report(path, *, now_ns): @@ -210,6 +212,46 @@ class TrainingReporter: coordinates = list(outcome.actual_coordinates) if matched else [] applied_sequence = outcome.profile_sequence if matched else 0 actual_full = trace if matched and trace is not None else None + actual_live = None + live = getattr(collector, 'live_tracking', None) if collector is not None else None + live_offer = None if live is None else live.latest_offer + restoring = live_offer is None and live is not None and getattr(live, 'latest_restore_offer', None) is not None + if restoring: + live_offer = live.latest_restore_offer + if self.control is not None and live_offer is not None and native_current and source is not None: + event, evidence = self.control + try: + from openpilot.starpilot.flm.live_trial_wire import read_application as read_tracking_application, read_restore_application + from openpilot.starpilot.flm.live_tracking_response import validate_response_witness + validate_response_witness(live_offer.response_witness, live_offer.binding, now_ns) + reader = read_restore_application if restoring else read_tracking_application + ack = reader(event.starpilotLateralState, now_ns=now_ns, offer=live_offer) + matched = bool(ack is not None and ack.status == ('restored' if restoring else 'applied') and + event.which() == 'starpilotLateralState' and evidence.live_applied_coordinates and + tuple(value for _, value in ack.applied) == evidence.live_applied_coordinates and + ack.control_ns == evidence.effect_prepare_ns and + 0 <= evidence.control_ns-ack.control_ns <= 30_000_000 and + 0 < evidence.control_ns <= now_ns <= evidence.control_ns+100_000_000 and + evidence.identity == live_offer.binding.identity and evidence.source_epoch == source.source_epoch and + source.source_epoch == live_offer.binding.source_epoch and source.enabled and not source.manual and + (source.cp_sha256, source.reset, source.calibration_generation) == + (live_offer.binding.cp_sha256, live_offer.binding.reset_generation, live_offer.binding.calibration_generation) and + (evidence.controller_revision, evidence.baseline_revision) == + (live_offer.binding.controller_revision, live_offer.binding.baseline_revision) and + source.observed_ns <= now_ns <= source.expires_ns and + live_offer.observed_ns <= now_ns <= live_offer.expires_ns and + evidence.active and evidence.enabled and evidence.valid and not evidence.manual and not evidence.fault and + 0 < evidence.source_receipt_ns <= evidence.control_ns <= now_ns <= evidence.source_receipt_ns+500_000_000 and + (not evidence.native_required or evidence.native_allowed and + evidence.native_observed_ns <= evidence.control_ns <= now_ns <= evidence.native_expires_ns)) + if matched: + actual_live = ack + application = {'applied': any(value for _, value in ack.applied), 'reason': ack.reason} + applied_sequence = ack.sequence + coordinates = list(evidence.live_applied_coordinates) + except (AttributeError, TypeError, ValueError, OverflowError): + # Telemetry never grants application or substitutes an offered target. + pass details = { 'Vehicle': self.vehicle, 'Collection session': '' if collector is None else collector.session, 'Current objective': str(objective)[:128], @@ -230,6 +272,25 @@ class TrainingReporter: 'Collection duration seconds': (now_ns - self.started) / 1e9, } if collector is not None and getattr(collector.adapter, 'event_collection', False): + if getattr(collector.adapter, 'source_plant_protocol', None) is not None: + capture = getattr(collector, 'original_source_capture', None) + replay = collector.adapter.replay + receipts = getattr(collector.adapter, 'original_context_receipts', None) + details['GLOBAL original source capture'] = { + 'poses': 0 if capture is None else len(capture.rows), + 'epoch': None if capture is None else capture.epoch, + 'epochStartNs': None if capture is None else capture.epoch_start_ns, + 'epochCutoffNs': None if capture is None else capture.epoch_cutoff_ns, + 'sealed': False if capture is None else capture.sealed, + 'reason': None if capture is None else capture.reason, + 'rejected': 0 if capture is None else capture.rejected, + 'frames': len(replay.records), 'checkpoints': len(replay.checkpoints), + 'emissions': len(replay.emissions.events), + 'controls': 0 if receipts is None else len(receipts.controls), + 'readyPairs': len(collector.ready), + } + details['GLOBAL response bootstrap readiness'] = dict(getattr(collector, '_live_bootstrap_readiness', + {'reason': 'response_snapshot_not_attempted'})) readiness_reader = getattr(collector, 'snapshot_readiness_report', None) readiness: dict[str, object] = readiness_reader() if callable(readiness_reader) else {'reason': 'snapshot_not_attempted'} stamp = cast(int | None, readiness.pop('observedMonoTime', None)) @@ -241,6 +302,12 @@ class TrainingReporter: 'Last unsuccessful fit': list(self.last_fit_rejection) if self.last_fit_rejection else [], 'Parent fit profile retained': bool(collector.fit_result is not None and collector.fit_result.profile is not None), 'Collector admission result retained': bool(collector.result is not None and collector.result.profile is not None)}) + if actual_live is not None: + details.update({'Actual full surface coordinates': [[name, value] for name, value in actual_live.applied], + 'Actual full profile hash': actual_live.applied_sha256, + 'Actual live trial ID': actual_live.trial_id, + 'Actual live target sequence': actual_live.target_sequence, + 'Actual live control timestamp': actual_live.control_ns}) if actual_full is not None: details.update({'Actual full surface coordinates': [[n,v] for n,v in actual_full.applied], 'Actual full profile hash': actual_full.applied_full_profile_sha256, @@ -272,6 +339,9 @@ class TrainingReporter: reason = 'capturing_steering_evidence' current = {'enabled': None if source is None else bool(source.enabled), 'status': status, 'reason': reason, 'progress': progress, 'fit': fit, 'application': application, 'details': details} + live = getattr(collector, 'live_tracking', None) if collector is not None else None + if live is not None: + current.update(tracking_objects(live.report(now_ns), fresh=True)) state = (status, reason, application['applied'], None if fit is None else (fit['status'], fit['reason'])) if state != self.last_state: self.history.append({'time': f'{(now_ns - self.started) / 1e9:.1f} s', 'status': status, diff --git a/openpilot/starpilot/galaxy/tests/test_flm_tracking_results.mjs b/openpilot/starpilot/galaxy/tests/test_flm_tracking_results.mjs new file mode 100644 index 0000000000..309c820b34 --- /dev/null +++ b/openpilot/starpilot/galaxy/tests/test_flm_tracking_results.mjs @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict' +import { compile } from '../web/vendor/vue/vue.esm-browser.js' +import { FlmPage } from '../web/js/flm.js' +import { SURFACE_NAMES, PID_UNSUPPORTED, validTrainingResults, validTrackingResults, trackingView, surfaceRows, measurementText, validTrackingValues, trackingCoordinates, validTrackingPlot, trackingPlotView, sourceSeconds } from '../web/js/flm-training-results.js' + +const coordinates = amount => SURFACE_NAMES.map((name, i) => [name, i === 0 ? amount : 0]) +const tracking = () => ({ version: 1, status: 'trial_active', reason: 'scoring_future_tracking', group: 'feedforward', + names: ['ff_gain_left'], trialId: 'a'.repeat(64), phase: 'trial', requiredEpisodes: 3, + beforeEpisodes: 3, futureEpisodes: 1, modelStatus: 'valid', lagMs: 120, + baselineErrorRms: .2, futureErrorRms: .18, baselineReversalHz: 1.2, futureReversalHz: 1.1, + applicationAcknowledged: true, applicationSettledMonoTime: 1_000_000, lastCorrectionMonoTime: null, + bindingCurrent: true }) +const report = () => ({ version: 1, history: [], current: { fresh: true, ageSeconds: .1, enabled: true, + status: 'collecting', reason: 'independent_evidence', progress: { samples: 120, blocks: 3, requiredBlocks: 16 }, + fit: null, application: { applied: true, reason: 'current_controller_application' }, + details: { 'Full surface coordinates': coordinates(.004), 'Actual full surface coordinates': coordinates(.001) }, + tracking: tracking() } }) +assert(validTrackingResults(tracking())) +for (const requiredEpisodes of [3, 8, 9, 12, 24]) { + const frozenQuota = report(); frozenQuota.current.tracking.requiredEpisodes = requiredEpisodes + assert(validTrainingResults(frozenQuota)) +} +const excessiveQuota = report(); excessiveQuota.current.tracking.requiredEpisodes = 25 +assert(!validTrainingResults(excessiveQuota)) +assert(validTrainingResults(report())) +assert.equal(trackingView(report(), true).label, 'Evaluating correction') +assert.equal(trackingView(report(), true).changedParameters, 'ff gain left') +assert.equal(trackingView(report(), true).measurements[0].future, .18) +assert.equal(surfaceRows(report(), true)[0].learned, .004, 'Do not relabel or overwrite replay learned proposals') +assert.equal(surfaceRows(report(), true)[0].applied, .001, 'Current values come from actual controller map') +assert.equal(surfaceRows(report(), true).filter(row => row.pidSupported).length, 26) +assert.equal(PID_UNSUPPORTED.length, 4) +const old = report(); delete old.current.tracking +assert(validTrainingResults(old)); assert.equal(trackingView(old, true), null) +old.current.tracking = null +assert(validTrainingResults(old)); assert.equal(trackingView(old, true), null) +for (const mutate of [value => value.current.tracking.applicationAcknowledged = false, + value => value.current.tracking.applicationSettledMonoTime = null, + value => value.current.tracking.trialId = null, + value => value.current.tracking.bindingCurrent = false, + value => value.current.application.applied = false, + value => delete value.current.details['Actual full surface coordinates']]) { + const value = report(); mutate(value) + assert(validTrainingResults(value)) + assert.equal(trackingView(value, true).acknowledged, false) + assert.notEqual(trackingView(value, true).label, 'Evaluating correction') +} +const pending = report(); pending.current.tracking.status = 'trial_pending'; pending.current.tracking.applicationAcknowledged = false +assert.match(trackingView(pending, true).label, /awaiting controller/) +const confirmed = report(); confirmed.current.tracking.status = 'confirmed'; confirmed.current.tracking.phase = 'confirmed' +confirmed.current.tracking.applicationAcknowledged = false +assert.equal(trackingView(confirmed, true).label, 'Future checks passed') +assert(!/applied|saved|persist/i.test(trackingView(confirmed, true).label)) +const rollback = report(); rollback.current.tracking.status = 'rollback'; rollback.current.tracking.phase = 'rollback' +rollback.current.tracking.applicationAcknowledged = false +assert.match(trackingView(rollback, true).label, /awaiting controller/) +rollback.current.tracking.applicationAcknowledged = true +assert.equal(trackingView(rollback, true).label, 'Correction rollback acknowledged') +const neutralRollback = report(); neutralRollback.current.tracking.status = 'rollback'; neutralRollback.current.tracking.phase = 'rollback' +neutralRollback.current.application.applied = false +neutralRollback.current.details['Actual full surface coordinates'] = coordinates(0) +assert.equal(trackingView(neutralRollback, true).label, 'Correction rollback acknowledged') +for (const mutate of [value => value.current.tracking.applicationAcknowledged = false, + value => value.current.tracking.bindingCurrent = false, + value => value.current.tracking.applicationSettledMonoTime = null, + value => value.current.details['Actual full surface coordinates'] = coordinates(.001), + value => delete value.current.details['Actual full surface coordinates']]) { + const noProof = structuredClone(neutralRollback); mutate(noProof) + assert.equal(trackingView(noProof, true).acknowledged, false) +} +const neutralTrial = structuredClone(neutralRollback); neutralTrial.current.tracking.status = 'trial_active'; neutralTrial.current.tracking.phase = 'trial' +assert.equal(trackingView(neutralTrial, true).acknowledged, false) +for (const value of [report(), pending, confirmed, rollback]) { + const stale = trackingView(value, false) + assert.equal(stale.acknowledged, false) + assert.equal(stale.label, 'Earlier learning report; current application unknown') +} +for (const [status, phase] of [['trial_active', 'rollback'], ['trial_active', 'confirmed'], ['confirmed', 'trial'], ['confirmed', 'rollback'], ['rollback', 'confirmed'], ['trial_active', null]]) { + const mismatch = report(); mismatch.current.tracking.status = status; mismatch.current.tracking.phase = phase + assert(validTrainingResults(mismatch)) + assert.equal(trackingView(mismatch, true).acknowledged, false) + assert.equal(trackingView(mismatch, true).label, 'Learning state reported; trial application unknown') +} +const pausedRollback = report(); pausedRollback.current.tracking.status = 'paused'; pausedRollback.current.tracking.phase = 'rollback' +assert.equal(trackingView(pausedRollback, true).label, 'Learning paused') +assert.equal(trackingView(pausedRollback, true).acknowledged, false) +const unknownPhase = report(); unknownPhase.current.tracking.phase = 'restore' +assert(validTrainingResults(unknownPhase)) +assert.equal(trackingView(unknownPhase, true).acknowledged, false) +assert.equal(trackingView(unknownPhase, true).label, 'Learning state reported; trial application unknown') +const staleProducer = report(); staleProducer.current.fresh = false +assert.equal(trackingView(staleProducer, true).acknowledged, false) +assert.equal(trackingView(staleProducer, true).label, 'Earlier learning report; current application unknown') +const unmeasured = report() +for (const key of ['lagMs', 'baselineErrorRms', 'futureErrorRms', 'baselineReversalHz', 'futureReversalHz']) unmeasured.current.tracking[key] = null +assert(validTrainingResults(unmeasured)) +assert.equal(measurementText(trackingView(unmeasured, true).measurements[0].future), 'Not measured') +assert.equal(measurementText(0), '0') +assert.equal(measurementText(.1234), '0.123') +assert.equal(measurementText(.000001), '1.00e-6') +for (const mutate of [value => value.version = 2, value => value.status = 'done', + value => value.names = ['unknown'], value => value.names = ['ff_gain_left', 'ff_gain_left'], + value => value.names = SURFACE_NAMES.slice(0, 5), value => value.trialId = 'bad', value => value.phase = 123, + value => value.requiredEpisodes = 2, value => value.beforeEpisodes = -1, value => value.futureEpisodes = 25, + value => value.modelStatus = 'assumed', value => value.lagMs = 501, value => value.lagMs = true, + value => value.baselineErrorRms = -1, value => value.futureErrorRms = Infinity, value => value.futureReversalHz = NaN, + value => value.applicationAcknowledged = 1, value => value.bindingCurrent = 'yes', + value => value.applicationSettledMonoTime = 0, value => value.lastCorrectionMonoTime = 1.5]) { + const value = report(); mutate(value.current.tracking) + assert(!validTrackingResults(value.current.tracking)) + assert(!validTrainingResults(value)) + assert.equal(trackingView(value, true), null) +} +const namedCoordinates = amount => SURFACE_NAMES.map((name, i) => ({ name, value: i === 0 ? amount : 0 })) +const valuesReport = report() +valuesReport.current.trackingValues = { version: 1, accepted: namedCoordinates(.003), pending: namedCoordinates(.004), + acceptedSha256: 'b'.repeat(64), pendingSha256: 'c'.repeat(64) } +assert(validTrainingResults(valuesReport)) +assert.equal(trackingCoordinates(valuesReport.current.trackingValues.accepted).ff_gain_left, .003) +assert.equal(surfaceRows(valuesReport, true)[0].accepted, .003) +assert.equal(surfaceRows(valuesReport, true)[0].pending, .004) +assert.equal(surfaceRows(valuesReport, true)[0].learned, .004, 'Old replay proposal is preserved separately') +assert.equal(surfaceRows(valuesReport, false)[0].accepted, .003, 'Accepted baseline is historical learning, not current authority') +assert.equal(surfaceRows(valuesReport, false)[0].pending, null, 'An old offer must not be presented as pending now') +const noPending = structuredClone(valuesReport); noPending.current.trackingValues.pending = null; noPending.current.trackingValues.pendingSha256 = null +assert(validTrainingResults(noPending)); assert.equal(surfaceRows(noPending, true)[0].pending, null) +for (const mutate of [value => value.version = 2, value => value.accepted[0].value = .251, + value => value.accepted[0].name = 'unknown', value => value.accepted = value.accepted.slice(1), + value => value.accepted = value.accepted.reverse(), value => value.pending[0].value = NaN, + value => value.pendingSha256 = null, value => value.acceptedSha256 = 'not-a-hash']) { + const invalid = structuredClone(valuesReport); mutate(invalid.current.trackingValues) + assert(!validTrackingValues(invalid.current.trackingValues)); assert(!validTrainingResults(invalid)) +} +const plot = () => ({ version: 'live-tracking-paired-plot-v1', bindingCurrent: true, ageSeconds: 1, + episodeId: 'd'.repeat(64), profileSha256: 'e'.repeat(64), lagNs: 100_000_000, units: 'm/s²', sampled: false, totalEndpoints: 4, + points: Array.from({ length: 4 }, (_, i) => ({ controlMonoTime: 1_000_000_000 + i * 50_000_000, + applyMonoTime: 1_002_000_000 + i * 50_000_000, poseMonoTime: 1_102_000_000 + i * 50_000_000, + desiredForceAccel: [.2, .3, .4, .5][i], measuredForceAccel: [.18, .28, .38, .48][i], errorForceAccel: .02 })) }) +const plotted = report(); plotted.current.trackingPlot = plot() +assert(validTrainingResults(plotted)); assert(validTrackingPlot(plot())) +const displayed = trackingPlotView(plotted, 500) +assert.equal(displayed.age, 1.6, 'Episode age adds actual report age and browser elapsed time without comparing clock domains') +assert.deepEqual(displayed.points.map(point => point.at), [0, 50, 100, 150]) +assert.equal(displayed.points.length, plotted.current.trackingPlot.points.length, 'No extra interpolated or upsampled measurements') +for (const [i, point] of displayed.points.entries()) { + for (const key of ['controlMonoTime', 'applyMonoTime', 'poseMonoTime', 'desiredForceAccel', 'measuredForceAccel', 'errorForceAccel']) + assert.equal(point[key], plotted.current.trackingPlot.points[i][key]) +} +assert.equal(sourceSeconds(1_002_000_000), '1.002000') +for (const boundary of [0, 30_000_000]) { + const edge = plot() + for (const point of edge.points) point.poseMonoTime = point.applyMonoTime + edge.lagNs + boundary + assert(validTrackingPlot(edge), 'Preserve outer bounds of the genuine original-source lag-pair interval') +} +const sampled = plot(); sampled.sampled = true; sampled.totalEndpoints = 100 +assert(validTrackingPlot(sampled)) +const earlier = report(); earlier.current.fresh = false; earlier.current.trackingPlot = plot(); earlier.current.trackingPlot.bindingCurrent = false +assert(validTrainingResults(earlier)); assert(trackingPlotView(earlier), 'An earlier episode remains explicitly historical data') +for (const mutate of [value => value.version = 'unknown', value => value.bindingCurrent = 1, + value => value.ageSeconds = -1, value => value.ageSeconds = Infinity, value => value.lagNs = 500_000_001, + value => value.units = 'degrees', value => value.sampled = true, value => value.totalEndpoints = 3, + value => value.points = value.points.slice(0, 1), value => value.points = Array(65).fill(value.points[0]), + value => value.points[1].poseMonoTime = value.points[0].poseMonoTime, + value => value.points[1].controlMonoTime = value.points[0].controlMonoTime, + value => value.points[0].applyMonoTime = value.points[0].controlMonoTime - 1, + value => value.points[0].poseMonoTime = '1102000000', + value => value.points[0].poseMonoTime = value.points[0].applyMonoTime - 1, + value => value.points[0].poseMonoTime = value.points[0].applyMonoTime + value.lagNs + 30_000_001, + value => value.points[0].poseMonoTime = value.points[0].applyMonoTime + value.lagNs - 1, + value => value.lagNs = 0, + value => value.totalEndpoints = 241, + value => value.points[0].measuredForceAccel = NaN, value => value.points[0].errorForceAccel = 1]) { + const invalid = report(); invalid.current.trackingPlot = plot(); mutate(invalid.current.trackingPlot) + assert(!validTrackingPlot(invalid.current.trackingPlot)); assert(!validTrainingResults(invalid)); assert.equal(trackingPlotView(invalid), null) +} +assert.match(FlmPage.template, /Latest completed delay-paired steering episode/) +assert.match(FlmPage.template, /not actuator-delay aligned/) +assert.match(FlmPage.template, /Pending trial target/) +assert.match(FlmPage.template, /Accepted baseline/) +assert.match(FlmPage.template, /No delay-paired steering episode plot/) +compile(FlmPage.template, { onError: error => { throw error }, decodeEntities: value => value.replaceAll('&', '&') }) +assert.match(FlmPage.template, /tracking\?\.label \|\| summary/) +assert.match(FlmPage.template, /Future trial episodes/) +assert.match(FlmPage.template, /Passing future checks does not itself confirm/) +assert.doesNotMatch(FlmPage.template, /v-html/) +console.log('Optional versioned FLM tracking report, paired observed metrics, honest trial acknowledgements and old report compatibility passed') diff --git a/openpilot/starpilot/galaxy/web/js/flm-training-results.js b/openpilot/starpilot/galaxy/web/js/flm-training-results.js index 12c342cc71..876efac941 100644 --- a/openpilot/starpilot/galaxy/web/js/flm-training-results.js +++ b/openpilot/starpilot/galaxy/web/js/flm-training-results.js @@ -16,6 +16,9 @@ export function validTrainingResults(value) { if (row.fit !== null && (!row.fit || !text(row.fit.status, 64) || !text(row.fit.reason) || ![row.fit.trainWindows, row.fit.heldoutWindows].every(count))) return false if (row.application !== null && (!row.application || typeof row.application.applied !== "boolean" || !text(row.application.reason))) return false + if (row.tracking !== undefined && row.tracking !== null && !validTrackingResults(row.tracking)) return false + if (row.trackingValues !== undefined && row.trackingValues !== null && !validTrackingValues(row.trackingValues)) return false + if (row.trackingPlot !== undefined && row.trackingPlot !== null && !validTrackingPlot(row.trackingPlot)) return false return row.details === undefined || row.details && typeof row.details === "object" && !Array.isArray(row.details) && Object.keys(row.details).length <= 64 && Object.entries(row.details).every(([key, item]) => text(key, 80) && (["Full surface coordinates", "Actual full surface coordinates"].includes(key) ? @@ -98,8 +101,105 @@ export function surfaceRows(report, live = false) { const learned = fullCoordinates(details["Full surface coordinates"]) const applied = live && report.current?.application?.applied === true ? fullCoordinates(details["Actual full surface coordinates"]) : null - return SURFACE_NAMES.map(name => ({ name, baseline: 0, learned: learned?.[name] ?? null, + const values = validTrackingValues(report?.current?.trackingValues) ? report.current.trackingValues : null + const accepted = values ? trackingCoordinates(values.accepted) : null + const pending = values && live && report.current?.tracking?.bindingCurrent === true ? trackingCoordinates(values.pending) : null + return SURFACE_NAMES.map(name => ({ name, baseline: 0, accepted: accepted?.[name] ?? null, pending: pending?.[name] ?? null, learned: learned?.[name] ?? null, applied: applied?.[name] ?? null, pidSupported: !PID_UNSUPPORTED.includes(name) })) } export const coordinateText = value => value === null ? "Not reported" : value === 0 ? "0" : Math.abs(value) < 1e-6 ? value.toExponential(3) : (value > 0 ? "+" : "") + value.toFixed(6).replace(/0+$/, "").replace(/\.$/, "") + + +const TRACKING_STATES = ['calibrating', 'collecting', 'not_identifiable', 'ready', 'trial_pending', 'trial_active', 'confirmed', 'rollback', 'paused'] +const optionalMeasurement = value => value === null || typeof value === 'number' && Number.isFinite(value) && value >= 0 +const episodeCount = value => Number.isSafeInteger(value) && value >= 0 && value <= 24 +const optionalStamp = value => value === null || Number.isSafeInteger(value) && value > 0 +export function validTrackingResults(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value) && value.version === 1 && + TRACKING_STATES.includes(value.status) && text(value.reason) && + (value.group === null || text(value.group, 80)) && Array.isArray(value.names) && value.names.length <= 4 && + value.names.every(name => SURFACE_NAMES.includes(name)) && new Set(value.names).size === value.names.length && + (value.trialId === null || typeof value.trialId === 'string' && /^[0-9a-f]{64}$/.test(value.trialId)) && + (value.phase === null || text(value.phase, 64)) && + (value.requiredEpisodes === null || Number.isSafeInteger(value.requiredEpisodes) && value.requiredEpisodes >= 3 && value.requiredEpisodes <= 24) && episodeCount(value.beforeEpisodes) && episodeCount(value.futureEpisodes) && + ['valid', 'stale', 'drift', 'unavailable'].includes(value.modelStatus) && + optionalMeasurement(value.lagMs) && (value.lagMs === null || value.lagMs <= 500) && + [value.baselineErrorRms, value.futureErrorRms, value.baselineReversalHz, value.futureReversalHz].every(optionalMeasurement) && + typeof value.applicationAcknowledged === 'boolean' && typeof value.bindingCurrent === 'boolean' && + optionalStamp(value.applicationSettledMonoTime) && optionalStamp(value.lastCorrectionMonoTime) +} +const trackingLabels = { + calibrating: 'Measuring steering response', collecting: 'Collecting comparable driving examples', + not_identifiable: 'Waiting for identifiable steering response', ready: 'Ready to evaluate a small correction', + trial_pending: 'Correction offered; awaiting controller acknowledgement', + trial_active: 'Trial awaiting current application evidence', confirmed: 'Future checks passed', + rollback: 'Rollback awaiting controller acknowledgement', paused: 'Learning paused', +} +export function trackingView(report, live = false) { + const value = report?.current?.tracking + if (!validTrackingResults(value)) return null + live = live && report.current.fresh && report.current.ageSeconds <= 2 + const expectedPhase = { trial_pending: 'trial', trial_active: 'trial', confirmed: 'confirmed', rollback: 'rollback' }[value.status] + const matchingPhase = expectedPhase !== undefined && value.phase === expectedPhase + const actual = fullCoordinates(report.current.details?.['Actual full surface coordinates']) + // The producer acknowledgement binds the actual owner target; a zero offer alone is never evidence. + const neutralRollback = value.status === 'rollback' && matchingPhase && actual !== null && + Object.values(actual).every(coordinate => coordinate === 0) + const acknowledged = live && matchingPhase && value.status !== 'trial_pending' && value.bindingCurrent && value.applicationAcknowledged && + value.applicationSettledMonoTime !== null && value.trialId !== null && actual !== null && + (report.current.application?.applied === true || neutralRollback) + let label = trackingLabels[value.status] + if (value.status === 'trial_active' && acknowledged) label = 'Evaluating correction' + if (value.status === 'confirmed' && acknowledged) label = 'Confirmed correction acknowledged' + if (value.status === 'rollback' && acknowledged) label = 'Correction rollback acknowledged' + if (value.status !== 'paused' && (value.phase !== null && !matchingPhase || expectedPhase !== undefined && !matchingPhase)) label = 'Learning state reported; trial application unknown' + if (live && !value.bindingCurrent) label = 'Learning paused; steering context changed' + if (!live) label = 'Earlier learning report; current application unknown' + return { ...value, label, acknowledged, changedParameters: value.names.map(readable).join(', '), + measurements: [ + { label: 'Tracking error RMS · m/s²', before: value.baselineErrorRms, future: value.futureErrorRms }, + { label: 'Steering reversals · Hz', before: value.baselineReversalHz, future: value.futureReversalHz }, + ] } +} +export const measurementText = value => value === null ? 'Not measured' : + value !== 0 && value < .001 ? value.toExponential(2) : value.toFixed(3).replace(/0+$/, '').replace(/\.$/, '') + + +const sha256 = value => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value) +export function trackingCoordinates(rows) { + if (!Array.isArray(rows) || rows.length !== SURFACE_NAMES.length || rows.some((row, i) => + !row || typeof row !== 'object' || Array.isArray(row) || row.name !== SURFACE_NAMES[i] || + typeof row.value !== 'number' || !Number.isFinite(row.value) || Math.abs(row.value) > .25)) return null + return Object.fromEntries(rows.map(row => [row.name, row.value])) +} +export function validTrackingValues(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value) && value.version === 1 && + trackingCoordinates(value.accepted) !== null && sha256(value.acceptedSha256) && + (value.pending === null && value.pendingSha256 === null || trackingCoordinates(value.pending) !== null && sha256(value.pendingSha256)) +} +export function validTrackingPlot(value) { + if (!value || value.version !== 'live-tracking-paired-plot-v1' || typeof value.bindingCurrent !== 'boolean' || + typeof value.ageSeconds !== 'number' || !Number.isFinite(value.ageSeconds) || value.ageSeconds < 0 || + !sha256(value.episodeId) || !sha256(value.profileSha256) || value.units !== 'm/s²' || + !Number.isSafeInteger(value.lagNs) || value.lagNs < 0 || value.lagNs > 500_000_000 || + !Array.isArray(value.points) || value.points.length < 2 || value.points.length > 64 || + !Number.isSafeInteger(value.totalEndpoints) || value.totalEndpoints < value.points.length || value.totalEndpoints < 4 || value.totalEndpoints > 240 || + value.sampled !== (value.totalEndpoints > value.points.length)) return false + return value.points.every((point, i) => point && + ['controlMonoTime', 'applyMonoTime', 'poseMonoTime'].every(key => Number.isSafeInteger(point[key]) && point[key] > 0) && + point.applyMonoTime >= point.controlMonoTime && point.poseMonoTime >= point.applyMonoTime + value.lagNs && + point.poseMonoTime <= point.applyMonoTime + value.lagNs + 30_000_000 && + ['desiredForceAccel', 'measuredForceAccel', 'errorForceAccel'].every(key => typeof point[key] === 'number' && Number.isFinite(point[key])) && + Math.abs(point.desiredForceAccel - point.measuredForceAccel - point.errorForceAccel) <= + 1e-9 * Math.max(1, Math.abs(point.desiredForceAccel), Math.abs(point.measuredForceAccel)) && + (i === 0 || point.controlMonoTime > value.points[i - 1].controlMonoTime && point.poseMonoTime > value.points[i - 1].poseMonoTime)) +} +export function trackingPlotView(report, elapsedMs = 0) { + const value = report?.current?.trackingPlot + if (!validTrackingPlot(value)) return null + const first = value.points[0].poseMonoTime + return { ...value, age: value.ageSeconds + report.current.ageSeconds + Math.max(0, elapsedMs) / 1000, + points: value.points.map(point => ({ ...point, at: (point.poseMonoTime - first) / 1e6 })) } +} +export const sourceSeconds = value => (value / 1e9).toFixed(6) diff --git a/openpilot/starpilot/galaxy/web/js/flm.js b/openpilot/starpilot/galaxy/web/js/flm.js index 4c5a9f1b5f..3d2628f25d 100644 --- a/openpilot/starpilot/galaxy/web/js/flm.js +++ b/openpilot/starpilot/galaxy/web/js/flm.js @@ -3,7 +3,7 @@ import { GxNotice } from "./notice.js" import { PollTimer, connectionError } from "./polling.js" import { requestJson } from "./startup.js" import { PlotGraph, PlotsFeed, chartGeometry } from "./plots.js" -import { validTrainingResults, trainingSummary, readable, detailText, surfaceRows, coordinateText } from "./flm-training-results.js" +import { validTrainingResults, trainingSummary, readable, detailText, surfaceRows, coordinateText, trackingView, measurementText, validTrackingValues, trackingPlotView, sourceSeconds } from "./flm-training-results.js" export const FlmPage = { name: "FlmPage", @@ -39,6 +39,11 @@ export const FlmPage = { summary() { return trainingSummary(this.report, this.error || (this.current && !this.live ? "stale" : "")) }, currentApplied() { return this.live && this.current?.application?.applied === true }, rows() { return surfaceRows(this.report, this.live) }, + tracking() { return trackingView(this.report, this.live) }, + trackingValues() { return validTrackingValues(this.current?.trackingValues) ? this.current.trackingValues : null }, + trackingPlot() { return trackingPlotView(this.report, this.now - this.receivedAt) }, + pairedTrackingChart() { return chartGeometry(this.trackingPlot?.points || [], ['desiredForceAccel', 'measuredForceAccel']) }, + pairedErrorChart() { return chartGeometry(this.trackingPlot?.points || [], ['errorForceAccel']) }, readiness() { return this.current?.details?.["Snapshot readiness"] }, details() { return Object.entries(this.current?.details || {}) }, losses() { @@ -50,7 +55,7 @@ export const FlmPage = { termsChart() { return chartGeometry(this.history, ["lateralP", "lateralI", "lateralD", "lateralF"]) }, }, methods: { - readable, detailText, coordinateText, + readable, detailText, coordinateText, measurementText, sourceSeconds, amount(value) { return value === null || value === undefined ? "Not reported" : value.toLocaleString() }, togglePlots() { this.showPlots = !this.showPlots @@ -85,7 +90,7 @@ export const FlmPage = { Auto Learn results require authenticated local Galaxy access.