This commit is contained in:
whoisdomi
2026-09-16 07:35:48 -05:00
parent 2267f2dce2
commit bc33dc0f48
2 changed files with 35 additions and 3 deletions
+11 -3
View File
@@ -707,13 +707,21 @@ class SelfdriveD:
(contains_event_type(self.events, self.starpilot_events, ET.SOFT_DISABLE) or
contains_event_type(self.events, self.starpilot_events, ET.IMMEDIATE_DISABLE))
no_system_errors = (not has_disable_events) or (len(self.events) == num_events)
# modeld publishes on the small model throughout the background load, so only the model
# swap itself can briefly disturb the stream. Suppressing for the whole load would hide
# a genuinely dead modeld for as long as the load takes.
# The model swap briefly disturbs the stream, so forgive everything around it.
big_model_settling = time.monotonic() < self.big_model_swap_t + 2.
all_checks = self.sm.all_checks()
all_alive = self.sm.all_alive() if not all_checks else True
all_freq_ok = self.sm.all_freq_ok() if not all_checks else True
# Loading the big model realizes its graph on the same QCOM GPU the small model drives
# on, which costs modeld frames for part of the load and drags modelV2 below its target
# rate. commIssueAvgFreq is NO_ENTRY, so that would block engaging on a small model that
# is working fine. Forgive only a slow modelV2, and only while a load is in flight --
# anything dying, and any other service, still reports normally.
if self.big_model_loading and not all_checks and all_alive and not all_freq_ok:
slow = {s for s, freq_ok in self.sm.freq_ok.items() if not freq_ok}
if slow and slow <= {'modelV2', 'drivingModelData', 'cameraOdometry'} and self.sm.all_valid():
all_freq_ok = True
all_checks = True
report_comm_issue, self.valid_only_comm_issue_frames = evaluate_comm_issue(
all_checks, all_alive, all_freq_ok, self.valid_only_comm_issue_frames,
)
@@ -68,6 +68,30 @@ def test_big_model_loading_raises_no_alert():
assert EVENTS[EventName.bigModelLoading] == {}
def test_slow_modelv2_during_a_load_does_not_block_engaging():
"""commIssueAvgFreq is NO_ENTRY, so a load-induced modelV2 slowdown blocks engaging.
Measured on drive 00000abd: modelV2 p95 hit 732 ms during the load while roadCameraState
stayed at 51 ms, and the resulting commIssue/commIssueAvgFreq storm ran 38.5 s - 47.9 s.
The forgiveness must be narrow: only a *slow* modelV2 family, only while loading, and only
when everything is still alive and valid.
"""
from openpilot.selfdrive.selfdrived import selfdrived
source = (Path(selfdrived.__file__)).read_text(encoding="utf-8")
guard = next(
node for node in ast.walk(ast.parse(source))
if isinstance(node, ast.If) and "big_model_loading" in ast.dump(node.test)
and "all_freq_ok" in ast.dump(node.test)
)
test = ast.dump(guard.test)
# A dead service must still be reported, so aliveness is required, not forgiven.
assert "all_alive" in test
body = ast.dump(guard)
assert "all_valid" in body, "stale/invalid services must still raise commIssue"
assert "modelV2" in body, "only the modeld output services may be forgiven"
def test_frame_drop_warning_is_suppressed_while_the_big_model_loads():
"""Loading realizes the big model's graph on QCOM, the GPU the small model drives on.