diff --git a/selfdrive/selfdrived/selfdrived.py b/selfdrive/selfdrived/selfdrived.py index 86cc43d1f9..b33cd627ae 100644 --- a/selfdrive/selfdrived/selfdrived.py +++ b/selfdrive/selfdrived/selfdrived.py @@ -707,13 +707,21 @@ class SelfdriveD: (contains_event_type(self.events, self.starpilot_events, ET.SOFT_DISABLE) or contains_event_type(self.events, self.starpilot_events, ET.IMMEDIATE_DISABLE)) no_system_errors = (not has_disable_events) or (len(self.events) == num_events) - # modeld publishes on the small model throughout the background load, so only the model - # swap itself can briefly disturb the stream. Suppressing for the whole load would hide - # a genuinely dead modeld for as long as the load takes. + # The model swap briefly disturbs the stream, so forgive everything around it. big_model_settling = time.monotonic() < self.big_model_swap_t + 2. all_checks = self.sm.all_checks() all_alive = self.sm.all_alive() if not all_checks else True all_freq_ok = self.sm.all_freq_ok() if not all_checks else True + # Loading the big model realizes its graph on the same QCOM GPU the small model drives + # on, which costs modeld frames for part of the load and drags modelV2 below its target + # rate. commIssueAvgFreq is NO_ENTRY, so that would block engaging on a small model that + # is working fine. Forgive only a slow modelV2, and only while a load is in flight -- + # anything dying, and any other service, still reports normally. + if self.big_model_loading and not all_checks and all_alive and not all_freq_ok: + slow = {s for s, freq_ok in self.sm.freq_ok.items() if not freq_ok} + if slow and slow <= {'modelV2', 'drivingModelData', 'cameraOdometry'} and self.sm.all_valid(): + all_freq_ok = True + all_checks = True report_comm_issue, self.valid_only_comm_issue_frames = evaluate_comm_issue( all_checks, all_alive, all_freq_ok, self.valid_only_comm_issue_frames, ) diff --git a/selfdrive/selfdrived/tests/test_big_model_engagement.py b/selfdrive/selfdrived/tests/test_big_model_engagement.py index bde9410e93..45f7fc17ea 100644 --- a/selfdrive/selfdrived/tests/test_big_model_engagement.py +++ b/selfdrive/selfdrived/tests/test_big_model_engagement.py @@ -68,6 +68,30 @@ def test_big_model_loading_raises_no_alert(): assert EVENTS[EventName.bigModelLoading] == {} +def test_slow_modelv2_during_a_load_does_not_block_engaging(): + """commIssueAvgFreq is NO_ENTRY, so a load-induced modelV2 slowdown blocks engaging. + + Measured on drive 00000abd: modelV2 p95 hit 732 ms during the load while roadCameraState + stayed at 51 ms, and the resulting commIssue/commIssueAvgFreq storm ran 38.5 s - 47.9 s. + The forgiveness must be narrow: only a *slow* modelV2 family, only while loading, and only + when everything is still alive and valid. + """ + from openpilot.selfdrive.selfdrived import selfdrived + + source = (Path(selfdrived.__file__)).read_text(encoding="utf-8") + guard = next( + node for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.If) and "big_model_loading" in ast.dump(node.test) + and "all_freq_ok" in ast.dump(node.test) + ) + test = ast.dump(guard.test) + # A dead service must still be reported, so aliveness is required, not forgiven. + assert "all_alive" in test + body = ast.dump(guard) + assert "all_valid" in body, "stale/invalid services must still raise commIssue" + assert "modelV2" in body, "only the modeld output services may be forgiven" + + def test_frame_drop_warning_is_suppressed_while_the_big_model_loads(): """Loading realizes the big model's graph on QCOM, the GPU the small model drives on.