smol models matter

selfdrive/modeld/modeld.py

@@ -111,6 +111,11 @@ LAT_SMOOTH_BP = [2.0, 8.0]
 # the 20 Hz publish loop on core 7 (shared with dmonitoringmodeld) and barely run at all.
 BIG_MODEL_LOADER_CORES = {6}
 
+# A healthy load takes ~30 s once vehicle power is stable. If the AMD device never comes up
+# the tinygrad calls can block indefinitely, so give up rather than reporting "loading"
+# forever while the small model quietly keeps driving.
+BIG_MODEL_LOAD_TIMEOUT_SECONDS = 150.0
+
 
 class BigModelLoadCancelled(Exception):
   """Raised inside the background loader when modeld no longer wants the big model."""
@@ -909,6 +914,14 @@ class BigModelLoader:
   def in_progress(self) -> bool:
     return self._thread is not None and self._thread.is_alive()
 
+  @property
+  def timed_out(self) -> bool:
+    """A load stuck inside tinygrad cannot be interrupted, so the caller gives up on it.
+
+    The thread is a daemon and keeps running, but nothing will consume its result.
+    """
+    return self.in_progress and time.monotonic() - self.started_t > BIG_MODEL_LOAD_TIMEOUT_SECONDS
+
   def start(self, model_id: str, model_version: str = "") -> bool:
     if self.in_progress:
       return False
@@ -1438,7 +1451,15 @@ def main(demo=False):
     # Collect a finished background big-model load, then promote it once the driver is
     # disengaged. Swapping rebuilds the temporal input queues, so it must not happen
     # under actuation.
-    if big_loader is not None and not big_loader.in_progress:
+    if big_loader is not None and big_loader.timed_out:
+      big_loader.cancel()
+      big_loader = None
+      params.put_bool("UsbGpuLoading", False)
+      params.put_bool("UsbGpuPending", False)
+      params.put_bool("UsbGpuActive", False)
+      cloudlog.error(f"big model load exceeded {BIG_MODEL_LOAD_TIMEOUT_SECONDS:.0f}s, "
+                     "staying on the small model")
+    elif big_loader is not None and not big_loader.in_progress:
       loaded_big_model, big_load_error = big_loader.take()
       big_loader = None
       params.put_bool("UsbGpuLoading", False)

selfdrive/modeld/tests/test_usbgpu_helpers.py

@@ -321,6 +321,48 @@ def test_background_big_model_load_leaves_the_running_model_untouched(monkeypatc
   ]
 
 
+def test_big_model_load_gives_up_instead_of_loading_forever(monkeypatch):
+  # A load stuck inside tinygrad cannot be interrupted, so the timeout is what stops modeld
+  # reporting "loading" for the rest of the drive while the small model quietly drives.
+  release = threading.Event()
+  calls = []
+
+  class HangingModelState:
+    uses_external_gpu = True
+
+    def __init__(self, *_a, **_k):
+      release.wait(timeout=10)
+
+    def warmup(self):
+      pass
+
+  monkeypatch.setattr(modeld, "set_core_affinity", lambda cores: None)
+  monkeypatch.setattr(modeld, "wait_usbgpu_link", lambda: None)
+  monkeypatch.setattr(modeld, "wait_for_external_gpu_power_ready", lambda CP, cancel=None: None)
+  monkeypatch.setattr(modeld, "_close_tinygrad_disk_cache_connection", lambda: calls.append("close"))
+  monkeypatch.setattr(modeld, "ModelState", HangingModelState)
+
+  loader = modeld.BigModelLoader(1928, 1208, "car-params")
+  loader.start("big-model", "v15")
+  try:
+    assert loader.in_progress
+    assert not loader.timed_out
+
+    # Pretend the load has been running well past its budget.
+    loader.started_t -= modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS + 1
+    assert loader.timed_out
+    # take() must not hand over a model from a load we already gave up on.
+    assert loader.take() == (None, "")
+  finally:
+    release.set()
+    loader._thread.join(timeout=10)
+
+
+def test_big_model_load_timeout_leaves_room_for_a_normal_load():
+  # A healthy load measured ~26 s on device; the budget must not cut those off.
+  assert modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS > 60
+
+
 def test_background_big_model_loader_runs_off_modelds_realtime_core():
   # modeld is SCHED_FIFO on core 7 and threads inherit its affinity, so a loader left there
   # would be starved behind the 20 Hz publish loop.

selfdrive/selfdrived/selfdrived.py

@@ -64,6 +64,9 @@ StarPilotEventName = custom.StarPilotOnroadEvent.EventName
 
 IGNORED_SAFETY_MODES = (SafetyModel.silent, SafetyModel.noOutput)
 VALID_ONLY_COMM_ISSUE_GRACE_FRAMES = max(1, round(0.5 / DT_CTRL))
+# How long the "driving on the small model" banner shows before the blinking eGPU icon
+# takes over as the indication that the big model is still loading.
+BIG_MODEL_LOADING_ALERT_SECONDS = 3.0
 
 
 def evaluate_comm_issue(all_checks: bool, all_alive: bool, all_freq_ok: bool,
@@ -256,6 +259,7 @@ class SelfdriveD:
     self.big_model_active = False
     self.big_model_failed = False
     self.big_model_swap_t = 0.
+    self.big_model_loading_t = 0.
     self.experimental_mode = False
     self.ecu_disable_failed = False
     self.ecu_disable_failed_checked = not (
@@ -394,8 +398,12 @@ class SelfdriveD:
     loading = self.params.get_bool("UsbGpuLoading")
     if loading:
       self.big_model_attempted = True
+      if not self.big_model_loading:
+        self.big_model_loading_t = time.monotonic()
     self.big_model_loading = loading
-    if loading:
+    # Announce the small-model handover briefly; the blinking eGPU icon carries the
+    # "still loading" state from there so the banner does not sit on screen for minutes.
+    if loading and time.monotonic() < self.big_model_loading_t + BIG_MODEL_LOADING_ALERT_SECONDS:
       self.events.add(EventName.bigModelLoading)
 
     # The big model loads in the background while the small model drives, so it sits

selfdrive/selfdrived/tests/test_big_model_engagement.py

@@ -25,6 +25,24 @@ def test_big_model_failure_still_disengages():
   assert ET.SOFT_DISABLE in EVENTS[EventName.bigModelFailed]
 
 
+def test_loading_banner_is_brief_and_hands_over_to_the_icon():
+  # The load can run for minutes; the banner announces the handover to the small model and
+  # then gets out of the way, leaving the blinking eGPU icon as the "still loading" cue.
+  from openpilot.selfdrive.selfdrived import selfdrived
+
+  assert selfdrived.BIG_MODEL_LOADING_ALERT_SECONDS == 3.0
+
+  source = (Path(selfdrived.__file__)).read_text(encoding="utf-8")
+  tree = ast.parse(source)
+  guard = next(
+    node for node in ast.walk(tree)
+    if isinstance(node, ast.If) and "bigModelLoading" in ast.dump(node)
+    and "BIG_MODEL_LOADING_ALERT_SECONDS" in ast.dump(node.test)
+  )
+  # The alert must be gated on elapsed time, not added unconditionally every frame.
+  assert "big_model_loading_t" in ast.dump(guard.test)
+
+
 def _big_failed(*, attempted, loading, pending, big_active, model_unavailable=False):
   """Mirror of selfdrived's big_failed expression, extracted from the source."""
   source = (Path(__file__).parents[1] / "selfdrived.py").read_text(encoding="utf-8")

selfdrive/ui/onroad/starpilot/widgets/model_source.py

@@ -72,7 +72,9 @@ class ModelSourceWidget(LayoutWidget):
   @staticmethod
   def _status_for(loading: bool, small_model_engaged: bool, big_failed: bool,
                   pending: bool = False) -> ModelSourceStatus:
-    if loading or pending:
+    # Pending means the big model finished loading and is waiting for the next disengage,
+    # so the blinking loading icon stops: its absence is what tells the driver it is ready.
+    if loading and not pending:
       return ModelSourceStatus.LOADING
     if small_model_engaged:
       return ModelSourceStatus.FALLBACK_ENGAGED

selfdrive/ui/tests/test_model_source_widget.py

@@ -32,13 +32,14 @@ def test_model_source_failure_detection_matches_the_backend_state_contract():
   assert not failed(True, True, False, True)
 
 
-def test_model_source_shows_a_pending_big_model_as_still_loading():
-  # The big model loads in the background, so "loaded, waiting for a disengage" must read
-  # as in-progress rather than as a failure.
+def test_model_source_blinks_while_loading_then_clears_once_the_big_model_is_ready():
+  # The blinking icon is the driver's "still loading" cue, and its disappearance is how they
+  # know the next disengage/engage will pick up the big model.
   status = model_source.ModelSourceWidget._status_for
   failed = model_source.ModelSourceWidget._big_model_failed
 
-  assert status(False, False, False, True) is model_source.ModelSourceStatus.LOADING
+  assert status(True, False, False, False) is model_source.ModelSourceStatus.LOADING
+  assert status(True, False, False, True) is not model_source.ModelSourceStatus.LOADING
   assert not failed(False, True, False, True, True)
   # Chestnut going away while pending is still a genuine failure.
   assert failed(False, False, False, True, True)