diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 3c660457d7..775782b724 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -111,6 +111,11 @@ LAT_SMOOTH_BP = [2.0, 8.0] # the 20 Hz publish loop on core 7 (shared with dmonitoringmodeld) and barely run at all. BIG_MODEL_LOADER_CORES = {6} +# A healthy load takes ~30 s once vehicle power is stable. If the AMD device never comes up +# the tinygrad calls can block indefinitely, so give up rather than reporting "loading" +# forever while the small model quietly keeps driving. +BIG_MODEL_LOAD_TIMEOUT_SECONDS = 150.0 + class BigModelLoadCancelled(Exception): """Raised inside the background loader when modeld no longer wants the big model.""" @@ -909,6 +914,14 @@ class BigModelLoader: def in_progress(self) -> bool: return self._thread is not None and self._thread.is_alive() + @property + def timed_out(self) -> bool: + """A load stuck inside tinygrad cannot be interrupted, so the caller gives up on it. + + The thread is a daemon and keeps running, but nothing will consume its result. + """ + return self.in_progress and time.monotonic() - self.started_t > BIG_MODEL_LOAD_TIMEOUT_SECONDS + def start(self, model_id: str, model_version: str = "") -> bool: if self.in_progress: return False @@ -1438,7 +1451,15 @@ def main(demo=False): # Collect a finished background big-model load, then promote it once the driver is # disengaged. Swapping rebuilds the temporal input queues, so it must not happen # under actuation. - if big_loader is not None and not big_loader.in_progress: + if big_loader is not None and big_loader.timed_out: + big_loader.cancel() + big_loader = None + params.put_bool("UsbGpuLoading", False) + params.put_bool("UsbGpuPending", False) + params.put_bool("UsbGpuActive", False) + cloudlog.error(f"big model load exceeded {BIG_MODEL_LOAD_TIMEOUT_SECONDS:.0f}s, " + "staying on the small model") + elif big_loader is not None and not big_loader.in_progress: loaded_big_model, big_load_error = big_loader.take() big_loader = None params.put_bool("UsbGpuLoading", False) diff --git a/selfdrive/modeld/tests/test_usbgpu_helpers.py b/selfdrive/modeld/tests/test_usbgpu_helpers.py index 8d6e272ebf..f2a3d79d99 100644 --- a/selfdrive/modeld/tests/test_usbgpu_helpers.py +++ b/selfdrive/modeld/tests/test_usbgpu_helpers.py @@ -321,6 +321,48 @@ def test_background_big_model_load_leaves_the_running_model_untouched(monkeypatc ] +def test_big_model_load_gives_up_instead_of_loading_forever(monkeypatch): + # A load stuck inside tinygrad cannot be interrupted, so the timeout is what stops modeld + # reporting "loading" for the rest of the drive while the small model quietly drives. + release = threading.Event() + calls = [] + + class HangingModelState: + uses_external_gpu = True + + def __init__(self, *_a, **_k): + release.wait(timeout=10) + + def warmup(self): + pass + + monkeypatch.setattr(modeld, "set_core_affinity", lambda cores: None) + monkeypatch.setattr(modeld, "wait_usbgpu_link", lambda: None) + monkeypatch.setattr(modeld, "wait_for_external_gpu_power_ready", lambda CP, cancel=None: None) + monkeypatch.setattr(modeld, "_close_tinygrad_disk_cache_connection", lambda: calls.append("close")) + monkeypatch.setattr(modeld, "ModelState", HangingModelState) + + loader = modeld.BigModelLoader(1928, 1208, "car-params") + loader.start("big-model", "v15") + try: + assert loader.in_progress + assert not loader.timed_out + + # Pretend the load has been running well past its budget. + loader.started_t -= modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS + 1 + assert loader.timed_out + # take() must not hand over a model from a load we already gave up on. + assert loader.take() == (None, "") + finally: + release.set() + loader._thread.join(timeout=10) + + +def test_big_model_load_timeout_leaves_room_for_a_normal_load(): + # A healthy load measured ~26 s on device; the budget must not cut those off. + assert modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS > 60 + + def test_background_big_model_loader_runs_off_modelds_realtime_core(): # modeld is SCHED_FIFO on core 7 and threads inherit its affinity, so a loader left there # would be starved behind the 20 Hz publish loop. diff --git a/selfdrive/selfdrived/selfdrived.py b/selfdrive/selfdrived/selfdrived.py index bcec82270d..30ff87c0c4 100644 --- a/selfdrive/selfdrived/selfdrived.py +++ b/selfdrive/selfdrived/selfdrived.py @@ -64,6 +64,9 @@ StarPilotEventName = custom.StarPilotOnroadEvent.EventName IGNORED_SAFETY_MODES = (SafetyModel.silent, SafetyModel.noOutput) VALID_ONLY_COMM_ISSUE_GRACE_FRAMES = max(1, round(0.5 / DT_CTRL)) +# How long the "driving on the small model" banner shows before the blinking eGPU icon +# takes over as the indication that the big model is still loading. +BIG_MODEL_LOADING_ALERT_SECONDS = 3.0 def evaluate_comm_issue(all_checks: bool, all_alive: bool, all_freq_ok: bool, @@ -256,6 +259,7 @@ class SelfdriveD: self.big_model_active = False self.big_model_failed = False self.big_model_swap_t = 0. + self.big_model_loading_t = 0. self.experimental_mode = False self.ecu_disable_failed = False self.ecu_disable_failed_checked = not ( @@ -394,8 +398,12 @@ class SelfdriveD: loading = self.params.get_bool("UsbGpuLoading") if loading: self.big_model_attempted = True + if not self.big_model_loading: + self.big_model_loading_t = time.monotonic() self.big_model_loading = loading - if loading: + # Announce the small-model handover briefly; the blinking eGPU icon carries the + # "still loading" state from there so the banner does not sit on screen for minutes. + if loading and time.monotonic() < self.big_model_loading_t + BIG_MODEL_LOADING_ALERT_SECONDS: self.events.add(EventName.bigModelLoading) # The big model loads in the background while the small model drives, so it sits diff --git a/selfdrive/selfdrived/tests/test_big_model_engagement.py b/selfdrive/selfdrived/tests/test_big_model_engagement.py index d22039f1b2..f25dd022de 100644 --- a/selfdrive/selfdrived/tests/test_big_model_engagement.py +++ b/selfdrive/selfdrived/tests/test_big_model_engagement.py @@ -25,6 +25,24 @@ def test_big_model_failure_still_disengages(): assert ET.SOFT_DISABLE in EVENTS[EventName.bigModelFailed] +def test_loading_banner_is_brief_and_hands_over_to_the_icon(): + # The load can run for minutes; the banner announces the handover to the small model and + # then gets out of the way, leaving the blinking eGPU icon as the "still loading" cue. + from openpilot.selfdrive.selfdrived import selfdrived + + assert selfdrived.BIG_MODEL_LOADING_ALERT_SECONDS == 3.0 + + source = (Path(selfdrived.__file__)).read_text(encoding="utf-8") + tree = ast.parse(source) + guard = next( + node for node in ast.walk(tree) + if isinstance(node, ast.If) and "bigModelLoading" in ast.dump(node) + and "BIG_MODEL_LOADING_ALERT_SECONDS" in ast.dump(node.test) + ) + # The alert must be gated on elapsed time, not added unconditionally every frame. + assert "big_model_loading_t" in ast.dump(guard.test) + + def _big_failed(*, attempted, loading, pending, big_active, model_unavailable=False): """Mirror of selfdrived's big_failed expression, extracted from the source.""" source = (Path(__file__).parents[1] / "selfdrived.py").read_text(encoding="utf-8") diff --git a/selfdrive/ui/onroad/starpilot/widgets/model_source.py b/selfdrive/ui/onroad/starpilot/widgets/model_source.py index 7b25208e41..eb78312eca 100644 --- a/selfdrive/ui/onroad/starpilot/widgets/model_source.py +++ b/selfdrive/ui/onroad/starpilot/widgets/model_source.py @@ -72,7 +72,9 @@ class ModelSourceWidget(LayoutWidget): @staticmethod def _status_for(loading: bool, small_model_engaged: bool, big_failed: bool, pending: bool = False) -> ModelSourceStatus: - if loading or pending: + # Pending means the big model finished loading and is waiting for the next disengage, + # so the blinking loading icon stops: its absence is what tells the driver it is ready. + if loading and not pending: return ModelSourceStatus.LOADING if small_model_engaged: return ModelSourceStatus.FALLBACK_ENGAGED diff --git a/selfdrive/ui/tests/test_model_source_widget.py b/selfdrive/ui/tests/test_model_source_widget.py index 9aa83c0941..f0733dcad5 100644 --- a/selfdrive/ui/tests/test_model_source_widget.py +++ b/selfdrive/ui/tests/test_model_source_widget.py @@ -32,13 +32,14 @@ def test_model_source_failure_detection_matches_the_backend_state_contract(): assert not failed(True, True, False, True) -def test_model_source_shows_a_pending_big_model_as_still_loading(): - # The big model loads in the background, so "loaded, waiting for a disengage" must read - # as in-progress rather than as a failure. +def test_model_source_blinks_while_loading_then_clears_once_the_big_model_is_ready(): + # The blinking icon is the driver's "still loading" cue, and its disappearance is how they + # know the next disengage/engage will pick up the big model. status = model_source.ModelSourceWidget._status_for failed = model_source.ModelSourceWidget._big_model_failed - assert status(False, False, False, True) is model_source.ModelSourceStatus.LOADING + assert status(True, False, False, False) is model_source.ModelSourceStatus.LOADING + assert status(True, False, False, True) is not model_source.ModelSourceStatus.LOADING assert not failed(False, True, False, True, True) # Chestnut going away while pending is still a genuine failure. assert failed(False, False, False, True, True)