Qualify split HKG angle EPS publications

.github/workflows/hkg-angle-recovery-check.yaml

@@ -0,0 +1,59 @@
+name: HKG angle recovery qualification
+on:
+  push:
+    branches: [codex/hkg-angle-retention-20261106]
+permissions:
+  contents: read
+jobs:
+  recovery:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    env:
+      PYTHONPATH: ${{ github.workspace }}
+      SCALE: '1'
+      RAYLIB_BACKEND: headless
+    steps:
+      - uses: actions/checkout@v7
+      - name: Fetch exact baseline source
+        run: git -c gc.auto=0 -c maintenance.auto=false fetch --no-tags --depth=1 --filter=blob:none origin 68434cf39c501604fe962e9488a03bde0fed0d73
+      - uses: actions/setup-go@v7
+        with:
+          go-version: '1.25.1'
+          cache-dependency-path: mapd_repo/go.sum
+      - name: Fetch pinned map provider dependencies
+        working-directory: mapd_repo
+        env:
+          GOTOOLCHAIN: local
+        run: go mod download
+      - run: ./tools/op.sh setup
+      - name: Build matching Linux dependencies
+        run: >-
+          .venv/bin/scons -j2
+          openpilot/starpilot/ui/native/_raylib_extra.so
+          openpilot/common/libparams_c.so
+          msgq_repo/msgq/_ipc.so
+          msgq_repo/msgq/visionipc/_visionipc.so
+          opendbc_repo/opendbc/can/_raw_value.so
+          opendbc_repo/opendbc/dbc
+          openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so
+          openpilot/selfdrive/locationd/models/generated/libcar.so
+      - name: Run steering retention and independent safety veto regressions
+        run: |
+          .venv/bin/python -m pytest -q --junitxml=qualification.xml             openpilot/starpilot/aol/tests/test_hkg_angle_eps_retention.py             openpilot/starpilot/aol/tests/test_pe_combined_profile_contract.py             openpilot/starpilot/aol/tests/test_stock_eps_recovery.py             openpilot/starpilot/aol/tests/test_runtime.py             openpilot/starpilot/aol/tests/test_source_pause_driver_monitoring.py             openpilot/system/hardware/tests             openpilot/starpilot/drive_state/tests/test_hardware_lifecycle.py             openpilot/starpilot/ui/tests/test_weather.py openpilot/starpilot/flm/tests/test_pid_global_capture.py openpilot/starpilot/flm/tests/test_registered_controls_global.py::test_actual_controls_source_freshness_calibration_and_override_rejection openpilot/starpilot/lateral/tests/test_ev6_default.py openpilot/starpilot/lateral/tests/test_ioniq6_policy.py tools/ci/tests/test_replay_references.py
+      - name: Check owned Python files
+        run: |
+          .venv/bin/ruff check openpilot/starpilot/aol/temporary_steering.py openpilot/starpilot/aol/tests/test_hkg_angle_eps_retention.py openpilot/starpilot/aol/transport_pause.py openpilot/system/hardware/hardwared.py openpilot/system/hardware/hardware_timing.py openpilot/system/hardware/tests/test_hardware_timing.py openpilot/starpilot/ui/tests/test_weather.py openpilot/starpilot/lateral/tests/test_ev6_default.py openpilot/starpilot/lateral/tests/test_ioniq6_policy.py
+          .venv/bin/ty check openpilot/starpilot/aol/temporary_steering.py openpilot/starpilot/aol/tests/test_hkg_angle_eps_retention.py openpilot/starpilot/aol/transport_pause.py openpilot/system/hardware/hardwared.py openpilot/system/hardware/hardware_timing.py openpilot/system/hardware/tests/test_hardware_timing.py openpilot/starpilot/ui/tests/test_weather.py openpilot/starpilot/lateral/tests/test_ev6_default.py openpilot/starpilot/lateral/tests/test_ioniq6_policy.py
+      - name: Compare both reviewed default-controller recordings
+        run: .venv/bin/python openpilot/selfdrive/test/process_replay/test_processes.py --whitelist-cars HYUNDAI HYUNDAI2 --whitelist-procs controlsd --output reviewed-defaults -j2
+      - name: Save exact results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: hkg-angle-recovery-results
+          path: |
+            baseline.xml
+            baseline-result.json
+            qualification.xml
+            reviewed-defaults/coverage.json
+            reviewed-defaults/diff.txt

openpilot/starpilot/aol/temporary_steering.py

@@ -49,7 +49,8 @@ def temporary_steering_intent_current(self, sm, cp, cs, axis, stamp, now_ns, *,
     now_ns - previous[8] <= AXIS_MAX_AGE_NS and
     0 < self.lateral_source_ns <= now_ns and
     now_ns - self.lateral_source_ns <= AXIS_MAX_AGE_NS)
-  held_original = bool(axis.desiredLateral and axis.lateralActive and axis.nativeAcknowledged and
+  held_original = bool(axis.desiredLateral and (axis.lateralActive or
+    cp.brand == 'hyundai' and str(cp.steerControlType) == 'angle' and fresh_active_successor) and axis.nativeAcknowledged and
     (signature == previous or fresh_active_successor))
   if not withdrawn and not held_original:
     return False
@@ -150,7 +151,7 @@ def temporary_steering_intent_current(self, sm, cp, cs, axis, stamp, now_ns, *,
   if (native is None or not native.pandaInventory or
       stock_eps_user and (native.requestedLongitudinal or native.longitudinalAllowed) or
       held_original and not (native.requestedLateral and (native.lateralAllowed or
-        restricted_eps and (ford_alpha_eps or not native.requestedLongitudinal and not native.longitudinalAllowed))) or
+        restricted_eps and (ford_alpha_eps or hkg_angle_eps or not native.requestedLongitudinal and not native.longitudinalAllowed))) or
       axis.longitudinalActive and not native.longitudinalAllowed or
       native.observedMonoTime < self.lateral_native_ns or
       (native.pandaSerial, tuple(p.hardwareSerial for p in native.pandaInventory)) != self.lateral_inventory):

openpilot/starpilot/aol/tests/test_hkg_angle_eps_retention.py

@@ -121,3 +121,103 @@ def test_angle_eps_retention_preserves_independent_vetoes(fault):
       sm.update_msgs(now / 1e9, frames)
       rejected = feedback.observe(sm, sd.CP, state, now, latched=True, intent_producer_session='authority-card')
       assert rejected == (tick == 3)
+
+
+@pytest.mark.parametrize('identity', ('pe_long', 'ev9_long'))
+@pytest.mark.parametrize('full', (False, True))
+@pytest.mark.parametrize('steering', (False, True))
+@pytest.mark.parametrize('delivery', ('native_first', 'event_first', 'together', 'native_pause_first'))
+def test_angle_eps_split_publications_preserve_only_armed_intent(identity, full, steering, delivery):
+  case = fixture.IoniqAvailabilityTests(methodName='runTest')
+  with ExitStack() as cleanup:
+    cleanup.callback(case.doCleanups)
+    cleanup.enter_context(OpenpilotPrefix())
+    cp = profile(identity)
+    assert cp is not None
+    sd, _, drive = case._authority_callers(full, cp=cp, genuine_events=True)
+    feedback = TransportPauseFeedback(sd.CP)
+    sm = messaging.SubMaster(['aolAxisState', 'onroadEvents', 'aolSafetyWire', 'pandaStates'])
+    state = fixture.car_state()
+    state.vEgoRaw = state.vEgo
+    state.cruiseState.enabled = full and cp.pcmCruise
+    for tick in range(9):
+      fault = 3 <= tick < 8
+      state.steerFaultTemporary = fault and not (tick == 3 and delivery == 'native_pause_first')
+      axis, _, command = drive(tick, observed_state=state, steering=steering if fault or delivery == 'native_pause_first' else False,
+        gas=delivery == 'native_pause_first',
+        allowed=not fault, native_request_lat=True, native_request_long=full)
+      now = 10_000_000_000 + tick * 10_000_000
+      frames = [sd.pm.messages['aolAxisState'], sd.pm.messages['onroadEvents']]
+      for name in ('aolSafetyWire', 'pandaStates'):
+        message = messaging.new_message(name, len(sd.sm[name]), valid=sd.sm.valid[name],
+          logMonoTime=sd.sm.logMonoTime[name])
+        setattr(message, name, sd.sm[name])
+        frames.append(message.as_reader())
+      assert bool(axis.lateralActive and command.latActive) == (not fault)
+      if tick == 3 and delivery == 'native_pause_first':
+        assert axis.desiredLateral and not axis.lateralActive and axis.nativeAcknowledged
+        state.steerFaultTemporary = True
+      if tick == 3 and delivery not in ('together', 'native_pause_first'):
+        sm.update_msgs(now / 1e9, frames[2:] if delivery == 'native_first' else frames[:2])
+        assert not feedback.observe(sm, sd.CP, state, now, latched=True,
+          intent_producer_session='authority-card'), 'NATIVE_FIRST_EPS_DISARMED'
+        assert not feedback.failed
+      joined = (frames[:2] if delivery == 'native_first' else frames[2:]) if tick == 3 and delivery in ('native_first', 'event_first') else frames
+      sm.update_msgs(now / 1e9, joined)
+      assert not feedback.observe(sm, sd.CP, state, now, latched=True,
+        intent_producer_session='authority-card'), 'NATIVE_PAUSE_EPS_DISARMED' if delivery == 'native_pause_first' else 'EPS_JOIN_DISARMED'
+      assert not feedback.failed
+      if fault:
+        assert not axis.lateralActive and not command.latActive
+
+
+@pytest.mark.parametrize('identity', ('pe_long', 'ev9_long'))
+@pytest.mark.parametrize('native_long_pending', (False, True))
+def test_stopped_full_request_keeps_existing_lateral_intent_without_granting_commands(identity, native_long_pending):
+  from opendbc.car import structs
+  case = fixture.IoniqAvailabilityTests(methodName='runTest')
+  with ExitStack() as cleanup:
+    cleanup.callback(case.doCleanups)
+    cleanup.enter_context(OpenpilotPrefix())
+    cp = profile(identity)
+    assert cp is not None
+    sd, _, drive = case._authority_callers(False, cp=cp, genuine_events=True)
+    feedback = TransportPauseFeedback(sd.CP)
+    sm = messaging.SubMaster(['aolAxisState', 'onroadEvents', 'aolSafetyWire', 'pandaStates'])
+    state = fixture.car_state()
+    state.vEgoRaw = state.vEgo
+    for tick in range(7):
+      state.standstill = tick in (3, 4, 5)
+      state.vEgo = state.vEgoRaw = 0. if state.standstill else 5.
+      state.buttonEnable = tick == 4
+      state.buttonEvents = ([structs.CarState.ButtonEvent(type='accelCruise', pressed=False)] if tick == 4 else [])
+      requested_long = tick >= 5 or tick == 4 and native_long_pending
+      allowed_long = tick >= 5
+      state.cruiseState.enabled = allowed_long
+      state.cruiseState.standstill = allowed_long and state.standstill
+      axis, _, command = drive(tick, observed_state=state, native_request_lat=True,
+        allowed=not state.standstill, native_request_long=requested_long, long_allowed=allowed_long,
+        panda_controls_allowed=allowed_long)
+      now = 10_000_000_000 + tick * 10_000_000
+      frames = [sd.pm.messages['aolAxisState'], sd.pm.messages['onroadEvents']]
+      for name in ('aolSafetyWire', 'pandaStates'):
+        message = messaging.new_message(name, len(sd.sm[name]), valid=sd.sm.valid[name],
+          logMonoTime=sd.sm.logMonoTime[name])
+        setattr(message, name, sd.sm[name])
+        frames.append(message.as_reader())
+      sm.update_msgs(now / 1e9, frames)
+      if tick == 4:
+        assert axis.desiredLongitudinal and not axis.longitudinalActive
+        assert not axis.lateralActive and not command.latActive and not command.longActive
+      if tick == 5:
+        assert axis.longitudinalActive and not axis.lateralActive and not command.latActive
+        assert all(not p.controlsAllowed for p in sm['pandaStates'])
+        veto = messaging.SubMaster(['aolAxisState', 'onroadEvents', 'aolSafetyWire', 'pandaStates'])
+        latest = messaging.new_message('pandaStates', len(sm['pandaStates']), valid=True, logMonoTime=now)
+        latest.pandaStates = sm['pandaStates']
+        veto.update_msgs(now / 1e9, [*frames[:3], latest.as_reader()])
+        assert not feedback._angle_stop_intent_current(veto, sd.CP, state, axis, now, now,
+          latched=True, producer='authority-card')
+      assert not feedback.observe(sm, sd.CP, state, now, latched=True,
+        intent_producer_session='authority-card'), f'STOPPED_FULL_REQUEST_DISARMED tick={tick}'
+      assert not feedback.failed

openpilot/starpilot/aol/transport_pause.py

@@ -1177,6 +1177,7 @@ class TransportPauseFeedback:
     ccnc_stop = bool((cs.standstill or 0 <= cs.vEgoRaw <= .3) and cp.openpilotLongitudinalControl and
       cp.carFingerprint in (CAR.KIA_EV9, CAR.HYUNDAI_IONIQ_5_PE) and not cs.accFaulted and not cs.steeringDisengage)
     combined_stop = bool(ccnc_stop and axis.desiredLongitudinal and axis.longitudinalActive)
+    pending_stop = bool(ccnc_stop and axis.desiredLongitudinal and not axis.longitudinalActive and not axis.lateralActive)
     subaru_stop = cs.standstill and subaru_restriction(cp, cs)
     if (not (qualified(cp, marked_only=True) or mazda_qualified(cp, marked_only=True) or subaru_stop or ccnc_stop) or
       cs.gearShifter != car.CarState.GearShifter.drive or
@@ -1187,7 +1188,7 @@ class TransportPauseFeedback:
       not producer or producer != self.availability_producer or
       axis.intentProducerSessionId != producer or axis.intentIntegrityEpoch != self.availability_epoch or
       not (axis.desiredLateral or subaru_stop and not axis.lateralActive) or not axis.nativeAcknowledged or
-      (axis.desiredLongitudinal or axis.longitudinalActive) and not combined_stop or
+      (axis.desiredLongitudinal or axis.longitudinalActive) and not (combined_stop or pending_stop) or
       not 0 < axis.sourceCarStateMonoTime <= stamp <= now_ns or
       now_ns - axis.sourceCarStateMonoTime > AXIS_MAX_AGE_NS):
       return False
@@ -1198,10 +1199,19 @@ class TransportPauseFeedback:
             event.name not in (log.OnroadEvent.EventName.pcmDisable, log.OnroadEvent.EventName.pedalPressed)
             for event in sm['onroadEvents'])):
       return False
-    native = _configured_transport_native(sm, cp, self.session, now_ns, allow_restricted_lateral=True)
+    native = _configured_transport_native(sm, cp, self.session, now_ns,
+      require_longitudinal=not pending_stop, allow_restricted_lateral=True)
+    if native is None and combined_stop:
+      newer = _configured_transport_native(sm, cp, self.session, now_ns,
+        require_longitudinal=False, allow_restricted_lateral=True)
+      if (newer is not None and newer.pandaInventory and
+          0 < int(sm.logMonoTime['pandaStates']) < newer.sourcePandaStatesMonoTime and
+          all(slot.controlsAllowed for slot, config in zip(newer.pandaInventory, cp.safetyConfigs, strict=True)
+              if str(config.safetyModel) not in ('silent', 'noOutput'))):
+        native = newer
     return bool(native is not None and native.pandaInventory and native.requestedLateral and
       (not native.lateralAllowed or subaru_stop and not axis.desiredLateral and not axis.lateralActive) and
-      (not native.requestedLongitudinal and not combined_stop or
+      (pending_stop or not native.requestedLongitudinal and not combined_stop or
        combined_stop and native.requestedLongitudinal and native.longitudinalAllowed) and
       native.observedMonoTime >= self.lateral_native_ns and
       (native.pandaSerial, tuple(p.hardwareSerial for p in native.pandaInventory)) == self.lateral_inventory)

openpilot/starpilot/flm/tests/test_registered_controls_global.py

@@ -200,6 +200,7 @@ def controls_for(cp, mode=None, enabled=False):
     mock.patch('openpilot.selfdrive.controls.controlsd.messaging.PubMaster'),
   ):
     params = Params()
+    params.put_bool('OpenpilotEnabledToggle', True, block=True)
     selected = published_cp(cp, params)
     params.put('CarParams', selected.to_bytes(), block=True)
     params.put(KEY, replace_preference(None, cp, enabled), block=True)

openpilot/starpilot/galaxy/web/css/galaxy.css

@@ -840,12 +840,10 @@ border-radius: var(--radius-full); color: var(--text-muted);
 .gx-auth form .gx-btn { justify-self: start; }
 /* Focused System Monitor adaptation from frozen Galaxy material.css:2424-2443. */
 .gx-view, .gx-monitor { min-width: 0; max-width: 100%; }
-/* Every tool page shares the Colors & Layout rhythm: one grid, one gap. */
 .gx-view, .gx-driving, .gx-long-curves, .gx-vehicle { display: grid; gap: var(--sp-4); align-content: start; }
 .gx-view > *, .gx-driving > *, .gx-long-curves > *, .gx-vehicle > * { margin-block: 0; }
 .gx-view h2, .gx-monitor h2 { margin-top: 0; }
 .gx-note { color: var(--text-muted); font-size: var(--fs-sm); }
-/* Shared banners: message first, optional actions on their own row. */
 .gx-alert { --notice-accent: var(--primary); align-items: center; backdrop-filter: var(--glass-blur); -webkit-backdrop-filter: var(--glass-blur); border: 1px solid var(--glass-border); border-left: 4px solid var(--notice-accent); border-radius: var(--radius-md); display: flex; gap: var(--sp-3); width: 100%; margin: 0 0 var(--sp-4); padding: var(--sp-4); background: color-mix(in srgb, var(--notice-accent) 12%, transparent); }
 .gx-alert--warn { --notice-accent: var(--warning); }
 .gx-alert--danger { --notice-accent: var(--error); }
@@ -863,7 +861,6 @@ border-radius: var(--radius-full); color: var(--text-muted);
 .gx-alert__actions .gx-btn { max-width: 100%; white-space: normal; overflow-wrap: anywhere; padding-block: var(--sp-2); box-shadow: none; }
 .gx-alert .gx-spinner { border-top-color: var(--notice-accent); }
 .gx-settings > .gx-card, .gx-settings > .gx-alert, .gx-stack > .gx-card, .gx-stack > .gx-alert { margin-bottom: 0; }
-/* Shared submenu cube so Tools, Logs and Plots render identically. */
 .gx-menu-tile {
 align-items: center;
 background: var(--surface-container);
@@ -1272,7 +1269,6 @@ input[type="checkbox"]:not(.gx-switch input):checked { background: var(--primary
 input[type="checkbox"]:not(.gx-switch input):checked::after { content: ""; display: block; width: .3rem; height: .6rem; border: solid var(--on-primary); border-width: 0 2px 2px 0; transform: translate(.3rem, .08rem) rotate(45deg); }
 input[type="checkbox"]:not(.gx-switch input):focus-visible { outline: 2px solid var(--primary); outline-offset: 3px; }
 input[type="checkbox"]:disabled { opacity: .4; cursor: not-allowed; }
-/* Shared action groups wrap while keeping their controls within the page. */
 .gx-btn { white-space: normal; max-width: 100%; overflow-wrap: anywhere; }
 .gx-actions, .gx-tabs { display: flex; align-items: center; gap: var(--sp-2); flex-wrap: wrap; min-width: 0; max-width: 100%; }
 .gx-actions > * { max-width: 100%; }
@@ -1547,7 +1543,6 @@ border-top: 1px solid var(--outline-variant);
 .gx-update-progress__fill--indeterminate { width: 35%; animation: gx-update-progress 1.6s ease-in-out infinite; }
 @keyframes gx-update-progress { from { transform: translateX(-100%); } to { transform: translateX(385%); } }
 @media (prefers-reduced-motion: reduce) { .gx-update-progress__fill--indeterminate { width: 100%; animation: none; opacity: .6; } }
-/* Composition utilities: shared page chrome and controls need no inline styles. */
 .gx-panel { padding: var(--panel-padding); min-width: 0; }
 .gx-card.gx-panel > .gx-section__header { margin: calc(-1 * var(--panel-padding)) calc(-1 * var(--panel-padding)) var(--sp-4); }
 .gx-panel > :first-child { margin-top: 0; }
@@ -1658,7 +1653,6 @@ progress { max-width: 100%; accent-color: var(--primary); }
 .gx-spotify-now__progress { display: grid; gap: 2px; }
 .gx-spotify-now__controls { align-items: center; justify-content: center; flex-wrap: wrap; }
 .gx-spotify-now__play { min-width: 64px; font-size: 1.4rem; }
-/* Auto Learn results use existing cards and plot colors. */
 .gx-flm__facts { display: flex; flex-wrap: wrap; gap: 1rem 2rem; }
 .gx-flm__facts dd { margin: .3rem 0 0; }
 .gx-flm__table-wrap { overflow-x: auto; max-width: 100%; }

openpilot/starpilot/lateral/tests/test_ev6_default.py

@@ -1,4 +1,4 @@
-"""Untuned EV6 lateral selection and steering command boundaries."""
+"""EV6 vehicle-policy defaults and steering command boundaries."""
 
 import unittest
 
@@ -15,6 +15,7 @@ from opendbc.car.hyundai.values import CAR, DBC, HyundaiFlags, CarControllerPara
 from opendbc.car.lateral import apply_driver_steer_torque_limits
 from openpilot.selfdrive.controls.lib.latcontrol_torque import LatControlTorque
 from openpilot.starpilot.lateral.torque_extension import selected_policy
+from openpilot.starpilot.lateral.hkg_vehicle_policy import HKGVehicleTorquePolicy
 from openpilot.starpilot.lateral.torque_runtime import runtime_enabled
 
 
@@ -26,7 +27,7 @@ def ev6(lka_steering: bool):
 
 
 class TestEV6DefaultLateral(unittest.TestCase):
-  def test_hda_topologies_keep_vehicle_tune_and_generic_controller(self):
+  def test_hda_topologies_keep_vehicle_tune_and_vehicle_policy(self):
     for lka_steering in (False, True):
       with self.subTest(lka_steering=lka_steering), OpenpilotPrefix():
         cp = ev6(lka_steering)
@@ -40,7 +41,8 @@ class TestEV6DefaultLateral(unittest.TestCase):
         self.assertFalse(runtime_enabled(cp, Params()))
         self.assertEqual(cp.lateralTuning.to_dict(), original_tune)
         lateral = LatControlTorque(cp.as_reader(), CarInterface(cp), DT_CTRL)
-        self.assertIsNone(selected_policy(lateral))
+        self.assertIsInstance(selected_policy(lateral), HKGVehicleTorquePolicy)
+        self.assertEqual(selected_policy(lateral).profile, "kia_ev6")
         self.assertAlmostEqual(lateral.torque_params.latAccelFactor, cp.lateralTuning.torque.latAccelFactor)
         controller = CarController(DBC[cp.carFingerprint], cp)
         self.assertIsNone(controller.ioniq6_longitudinal)

openpilot/starpilot/lateral/tests/test_ioniq6_policy.py

@@ -28,7 +28,7 @@ def _legacy_friction(controller, error, deadzone, threshold, _cs, _setpoint):
 
 
 def _rows(car, sequence, *, profile='2023', matched_frozen_cp=False, passive=False, direct_original_controller=False, turn_assist=True,
-          roll=None):
+          roll=None, controller_mode=None):
   cp = interfaces[car].get_non_essential_params(car)
   cp.passive = passive
   if matched_frozen_cp:
@@ -44,7 +44,7 @@ def _rows(car, sequence, *, profile='2023', matched_frozen_cp=False, passive=Fal
       versions[0].fwVersion = b'IONIQ6-230915'
       versions[1].fwVersion = b'IONIQ6-240206'
   ci = interfaces[car](cp)
-  controller = LatControlTorque(cp.as_reader(), ci, DT_CTRL, turn_assist=turn_assist)
+  controller = LatControlTorque(cp.as_reader(), ci, DT_CTRL, turn_assist=turn_assist, controller_mode=controller_mode)
   if direct_original_controller:
     # The older frozen vectors called the controller directly, before the
     # original controlsd drive loop replaced the PID gain curve each tick.
@@ -178,7 +178,8 @@ class Ioniq6PolicyTests(unittest.TestCase):
     parent = json.loads((TESTDATA / 'non_ioniq_parent.json').read_text())
     for car in (TOYOTA.TOYOTA_COROLLA_TSS2, HYUNDAI.HYUNDAI_IONIQ_5):
       with self.subTest(car=str(car)):
-        controller, actual = _rows(car, sequence, passive=car == TOYOTA.TOYOTA_COROLLA_TSS2, turn_assist=False)
+        controller, actual = _rows(car, sequence, passive=car == TOYOTA.TOYOTA_COROLLA_TSS2, turn_assist=False,
+                                   controller_mode='standard')
         self.assertIsNone(selected_policy(controller))
         for row, expected in zip(actual, parent[str(car)], strict=True):
           for value, reference in zip(row, expected, strict=True):

openpilot/starpilot/ui/tests/test_weather.py

@@ -56,7 +56,7 @@ def test_weather_is_nonblocking_single_flight_cached_and_expires():
       original(point)
     finally:
       finished.set()
-  client._refresh = refresh
+  client._refresh = Mock(side_effect=refresh)
   try:
     start = time.monotonic()
     assert client.observe((41., -87.)) is None

openpilot/system/hardware/hardware_timing.py

@@ -0,0 +1,20 @@
+import time
+from contextlib import suppress
+
+from openpilot.common.swaglog import cloudlog
+
+
+class HardwareLoopTiming:
+  def __init__(self):
+    self.phase = ''
+    self.wall_ns = 0
+    self.cpu_ns = 0
+
+  def mark(self, phase: str) -> None:
+    wall_ns, cpu_ns = time.monotonic_ns(), time.thread_time_ns()
+    elapsed_ns = wall_ns - self.wall_ns
+    if self.phase and elapsed_ns >= 500_000_000:
+      with suppress(OSError):
+        cloudlog.event('hardwared.slow_phase', phase=self.phase,
+          elapsedMs=elapsed_ns / 1e6, threadCpuMs=(cpu_ns - self.cpu_ns) / 1e6, error=True)
+    self.phase, self.wall_ns, self.cpu_ns = phase, wall_ns, cpu_ns

openpilot/system/hardware/hardwared.py

@@ -30,6 +30,7 @@ from openpilot.common.linux import LinuxSystemStats
 from openpilot.system.loggerd.config import get_available_percent
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.hardware.power_monitoring import PowerMonitoring
+from openpilot.system.hardware.hardware_timing import HardwareLoopTiming
 from openpilot.system.hardware.fan_controller import FanController
 from openpilot.system.hardware.chestnut.monitoring import chestnut_state_thread
 from openpilot.system.hardware.chestnut.status import ChestnutStatus
@@ -224,6 +225,7 @@ def hw_state_thread(end_event, hw_queue):
 
 def hardware_thread(end_event, hw_queue) -> None:
   system_stats = LinuxSystemStats() if sys.platform == "linux" else None
+  timing = HardwareLoopTiming()
   pm = messaging.PubMaster(['deviceState'])
   sm = messaging.SubMaster(["peripheralState", "gpsLocationExternal", "selfdriveState", "pandaStates", "chestnutState"], poll="pandaStates")
 
@@ -283,8 +285,10 @@ def hardware_thread(end_event, hw_queue) -> None:
   branch = get_short_branch()
 
   while not end_event.is_set():
+    timing.mark("panda_poll")
     sm.update(PANDA_STATES_TIMEOUT)
 
+    timing.mark("drive_mode")
     pandaStates = sm['pandaStates']
     peripheralState = sm['peripheralState']
 
@@ -324,6 +328,7 @@ def hardware_thread(end_event, hw_queue) -> None:
     if (sm.frame % round(SERVICE_LIST['pandaStates'].frequency * DT_HW) != 0) and not ign_edge:
       continue
 
+    timing.mark("thermal_read")
     msg = messaging.new_message('deviceState', valid=True)
     msg.deviceState = thermal_config.get_msg()
     msg.deviceState.deviceType = HARDWARE.get_device_type()
@@ -333,6 +338,7 @@ def hardware_thread(end_event, hw_queue) -> None:
     except queue.Empty:
       pass
 
+    timing.mark("system_stats")
     memory_usage = system_stats.memory_usage_percent() if system_stats is not None else 0.
     cpu_usage = system_stats.cpu_usage_percent() if system_stats is not None else []
 
@@ -352,8 +358,10 @@ def hardware_thread(end_event, hw_queue) -> None:
 
     msg.deviceState.modemTempC = last_hw_state.modem_temps
 
+    timing.mark("brightness")
     msg.deviceState.screenBrightnessPercent = HARDWARE.get_screen_brightness()
 
+    timing.mark("usb_chestnut")
     set_usb_state(msg.deviceState, last_hw_state.usb_state)
     chestnut.update(started_ts is None, last_hw_state.usb_state)
     chestnut_state = sm["chestnutState"]
@@ -374,6 +382,7 @@ def hardware_thread(end_event, hw_queue) -> None:
     all_comp_temp = all_temp_filter.update(max(temp_sources))
     msg.deviceState.maxTempC = all_comp_temp
 
+    timing.mark("fan_control")
     msg.deviceState.fanSpeedPercentDesired = fan_controller.update(all_comp_temp, onroad_conditions["ignition"])
 
     is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (time.monotonic() - off_ts > 60 * 5))
@@ -391,6 +400,7 @@ def hardware_thread(end_event, hw_queue) -> None:
 
     # **** starting logic ****
 
+    timing.mark("startup_conditions")
     startup_conditions["up_to_date"] = params.get("Offroad_ConnectivityNeeded") is None or params.get_bool("DisableUpdates") or params.get_bool("SnoozeUpdate")
     startup_conditions["no_excessive_actuation"] = params.get("Offroad_ExcessiveActuation") is None
     startup_conditions["not_uninstalling"] = not params.get_bool("DoUninstall")
@@ -413,6 +423,7 @@ def hardware_thread(end_event, hw_queue) -> None:
     onroad_conditions["device_temp_good"] = thermal_status < ThermalStatus.critical
     extra_text = f"{offroad_comp_temp:.1f}C"
     show_alert = (not onroad_conditions["device_temp_good"] or not startup_conditions["device_temp_engageable"]) and onroad_conditions["ignition"]
+    timing.mark("alerts")
     set_offroad_alert_if_changed("Offroad_TemperatureTooHigh", show_alert, extra_text=extra_text)
     update_jetlink_alert()
 
@@ -426,6 +437,7 @@ def hardware_thread(end_event, hw_queue) -> None:
       startup_conditions["registered_device"] = params.get("DongleId") != UNREGISTERED_DONGLE_ID
 
     # Handle offroad/onroad transition
+    timing.mark("engagement")
     should_start = resolve_start(drive_mode, onroad_conditions, startup_conditions, already_started=started_ts is not None)
 
     if should_start != should_start_prev or (count == 0):
@@ -444,6 +456,7 @@ def hardware_thread(end_event, hw_queue) -> None:
       except Exception:
         pass
 
+    timing.mark("power_save")
     should_pwrsave = not (onroad_conditions["ignition"] or should_start) and msg.deviceState.screenBrightnessPercent < 1e-3
     if should_pwrsave != pwrsave or (count == 0):
       HARDWARE.set_power_save(should_pwrsave)
@@ -479,12 +492,15 @@ def hardware_thread(end_event, hw_queue) -> None:
 
     # Offroad power monitoring
     voltage = None if peripheralState.pandaType == log.PandaState.PandaType.unknown else peripheralState.voltage
+    timing.mark("power_read")
     power_monitor.calculate(voltage, onroad_conditions["ignition"])
     msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used()
     msg.deviceState.carBatteryCapacityUwh = max(0, power_monitor.get_car_battery_capacity())
+    timing.mark("power_draw")
     current_power_draw = HARDWARE.get_current_power_draw()
     msg.deviceState.powerDrawW = current_power_draw
 
+    timing.mark("som_power")
     som_power_draw = HARDWARE.get_som_power_draw()
     msg.deviceState.somPowerDrawW = som_power_draw
 
@@ -494,6 +510,7 @@ def hardware_thread(end_event, hw_queue) -> None:
       cloudlog.warning(f"shutting device down, offroad since {shutdown_off_ts}")
       params.put_bool("DoShutdown", True, block=True)
 
+    timing.mark("publish")
     msg.deviceState.started = started_ts is not None
     msg.deviceState.startedMonoTime = int(1e9*(started_ts or 0))
 
@@ -524,6 +541,7 @@ def hardware_thread(end_event, hw_queue) -> None:
         except Exception:
           cloudlog.exception("failed to save offroad status")
 
+    timing.mark("bookkeeping")
     params.put_bool("NetworkMetered", msg.deviceState.networkMetered)
 
     now_ts = time.monotonic()

openpilot/system/hardware/tests/test_hardware_timing.py

@@ -0,0 +1,31 @@
+from unittest import TestCase, mock
+
+from openpilot.system.hardware import hardware_timing as timing
+
+
+class TestHardwareTiming(TestCase):
+  def test_slow_phase_preserves_original_wall_and_thread_cpu_clocks(self):
+    for elapsed_ns, reported in ((499_999_999, False), (500_000_000, True), (5_100_019_763, True)):
+      with self.subTest(elapsed_ns=elapsed_ns), \
+           mock.patch.object(timing.time, 'monotonic_ns', side_effect=[1_000_000_000, 1_000_000_000 + elapsed_ns]), \
+           mock.patch.object(timing.time, 'thread_time_ns', side_effect=[80_000_000, 90_000_000]), \
+           mock.patch.object(timing.cloudlog, 'event') as report:
+        watch = timing.HardwareLoopTiming()
+        watch.mark('thermal_read')
+        report.assert_not_called()
+        watch.mark('system_stats')
+        self.assertEqual(bool(report.call_count), reported)
+        if reported:
+          report.assert_called_once_with('hardwared.slow_phase', phase='thermal_read', elapsedMs=elapsed_ns / 1e6,
+            threadCpuMs=10., error=True)
+        self.assertEqual(watch.phase, 'system_stats')
+
+  def test_logging_failure_does_not_interrupt_hardware_work(self):
+    with mock.patch.object(timing.time, 'monotonic_ns', side_effect=[1, 5_100_000_001]), \
+         mock.patch.object(timing.time, 'thread_time_ns', side_effect=[1, 10_000_001]), \
+         mock.patch.object(timing.cloudlog, 'event', side_effect=OSError('logging unavailable')):
+      watch = timing.HardwareLoopTiming()
+      watch.mark('power_read')
+      watch.mark('publish')
+      self.assertEqual(watch.phase, 'publish')
+      self.assertEqual(watch.wall_ns, 5_100_000_001)

prebuilt.json

@@ -1 +1 @@
-{"protected_inventory_sha256": "1364c5a835bd115aea10b2847f46703f087444fd3204e07e508188ad5c97115d", "schema_version": 1}
+{"protected_inventory_sha256": "b1b32d57b53f664878ae381bd92018b4d4b6b57d6d7f92e465c5ed15aaa09728", "schema_version": 1}