napolean

selfdrive/modeld/modeld.py

@@ -972,11 +972,13 @@ class BigModelLoader:
       )
       if not candidate.uses_external_gpu:
         raise RuntimeError("external GPU model resolved to the builtin model")
-      # warmup() is deliberately not called here: the big model's camera warp runs on QCOM,
-      # the same GPU the small model is driving on, so warming up in the background stalls
-      # the 20 Hz loop for seconds at a time. It runs at promotion instead, while disengaged.
+      # warmup() runs the camera warp on QCOM, shared with the driving small model, so it
+      # costs some jitter here. It still belongs on this thread: doing it at promotion
+      # instead blocks the publish loop for ~13 s in one stretch, which trips commIssue and
+      # blocks the driver from engaging exactly when the model becomes available.
+      candidate.warmup()
       if self._cancel.is_set():
-        raise BigModelLoadCancelled("cancelled after load")
+        raise BigModelLoadCancelled("cancelled after warmup")
       with self._lock:
         self._result = candidate
       cloudlog.warning(f"background big model load finished in {time.monotonic() - self.started_t:.1f}s")
@@ -1487,33 +1489,26 @@ def main(demo=False):
       vipc_dropped_frames,
       live_calib_seen,
     ):
-      # Warm up here rather than on the loader thread: this runs on QCOM alongside the small
-      # model, so it has to happen while disengaged. warmup() ends with _reset_state().
-      try:
-        big_model.warmup()
-      except Exception:
-        cloudlog.exception("big model warmup failed, staying on the small model")
-        big_model = None
-        params.put_bool("UsbGpuPending", False)
-        params.put_bool("UsbGpuActive", False)
-      else:
-        model = big_model
-        external_gpu_active = True
-        # Nothing from the small model carries over: re-arm the frame-drop warmup and drop
-        # the rolling probability buffers and previous action, which are model specific.
-        run_count = 0
-        frame_dropped_filter.x = 0.
-        publish_state = PublishState()
-        prev_action = log.ModelDataV2.Action()
-        params.put("ModelVersion", model.policy_generation)
-        params.put("DrivingModelVersion", model.policy_generation)
-        set_runtime_model_params(params, model.model_id, model.policy_generation)
-        params.put_bool("UsbGpuActive", True)
-        params.put_bool("UsbGpuPending", False)
-        params.put_bool("UsbGpuLoading", False)
-        if chestnut_state is not None:
-          chestnut_state.big = True
-        cloudlog.warning(f"now driving on the big model {model.model_id}")
+      # The loader already warmed this model, so the swap itself is just a pointer change
+      # plus a queue reset; it must stay cheap because it runs inside the publish loop.
+      big_model._reset_state()
+      model = big_model
+      external_gpu_active = True
+      # Nothing from the small model carries over: re-arm the frame-drop warmup and drop
+      # the rolling probability buffers and previous action, which are model specific.
+      run_count = 0
+      frame_dropped_filter.x = 0.
+      publish_state = PublishState()
+      prev_action = log.ModelDataV2.Action()
+      params.put("ModelVersion", model.policy_generation)
+      params.put("DrivingModelVersion", model.policy_generation)
+      set_runtime_model_params(params, model.model_id, model.policy_generation)
+      params.put_bool("UsbGpuActive", True)
+      params.put_bool("UsbGpuPending", False)
+      params.put_bool("UsbGpuLoading", False)
+      if chestnut_state is not None:
+        chestnut_state.big = True
+      cloudlog.warning(f"now driving on the big model {model.model_id}")
 
     frame_drop_ratio = frames_dropped / (1 + frames_dropped)
     dropped_frame = vipc_dropped_frames > 0

selfdrive/modeld/tests/test_usbgpu_helpers.py

@@ -311,16 +311,16 @@ def test_background_big_model_load_leaves_the_running_model_untouched(monkeypatc
   loaded, error = loader.take()
   assert isinstance(loaded, fake_model_state)
   assert error == ""
-  # warmup() is absent on purpose: it runs on QCOM alongside the driving small model, so it
-  # is deferred to promotion (while disengaged) rather than run on the loader thread.
+  # warmup() belongs here rather than at promotion: see
+  # test_big_model_warmup_stays_on_the_background_loader.
   assert calls == [
     ("affinity", tuple(sorted(modeld.BIG_MODEL_LOADER_CORES))),
     ("power", "car-params"),
     "link",
     ("model", 1928, 1208, True, "big-model", False, "v15"),
+    "warmup",
     "close_cache",
   ]
-  assert "warmup" not in calls
 
 
 def test_big_model_load_gives_up_instead_of_loading_forever(monkeypatch):
@@ -365,12 +365,15 @@ def test_big_model_load_timeout_leaves_room_for_a_normal_load():
   assert modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS > 60
 
 
-def test_big_model_warmup_is_deferred_out_of_the_background_load():
-  """warmup() runs the camera warp on QCOM, the GPU the small model is driving on.
+def test_big_model_warmup_stays_on_the_background_loader():
+  """Warming at promotion blocks the publish loop in one long stretch; warming on the
+  loader spreads the same work out while the small model is still driving.
 
-  Running it on the loader thread blocked modeld for seconds at a time: measured modelV2
-  gaps of 1.3-2.9 s clustered in the second half of four loads, with core 7 only ~30% busy
-  (blocked, not starved). It belongs at promotion, which is already gated on disengaged.
+  Measured with warmup at promotion: modelV2 went silent for 13.0 s and 12.8 s on two
+  drives, starting the instant the model was collected, which tripped commIssue and blocked
+  the driver from engaging. Measured with warmup on the loader: worst in-load gap was
+  1.3-2.9 s, spread across the load. Neither is free, but only the second one leaves
+  modeld publishing when the driver wants to engage.
   """
   import ast
   from pathlib import Path
@@ -380,13 +383,13 @@ def test_big_model_warmup_is_deferred_out_of_the_background_load():
 
   loader = next(n for n in ast.walk(tree)
                 if isinstance(n, ast.ClassDef) and n.name == "BigModelLoader")
-  assert "warmup" not in ast.dump(loader), \
-    "the loader thread must not warm up the big model; it shares QCOM with the small model"
+  assert "warmup" in ast.dump(loader), "the loader thread must warm the big model"
 
   main_fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "main")
   promote = next(n for n in ast.walk(main_fn)
                  if isinstance(n, ast.If) and "_big_model_swap_allowed" in ast.dump(n.test))
-  assert "warmup" in ast.dump(promote), "promotion must warm the big model before it drives"
+  assert "warmup" not in ast.dump(promote), \
+    "promotion must not warm the big model; it runs inside the 20 Hz publish loop"
 
 
 def test_chestnut_telemetry_is_suppressed_while_a_background_load_runs():

selfdrive/selfdrived/events.py

@@ -537,11 +537,10 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
                                        "Driving on the small model"),
   },
 
-  EventName.bigModelPending: {
-    ET.PERMANENT: NormalPermanentAlert("Big Model Ready",
-                                       "Disengage and re-engage to use it",
-                                       duration=10.),
-  },
+  # bigModelPending carries no alert on purpose: the eGPU icon clearing, and then turning
+  # green on the next engage, is the driver's cue. A banner here announced the model before
+  # it was usable.
+  EventName.bigModelPending: {},
 
   EventName.bigModelFailed: {
     ET.SOFT_DISABLE: soft_disable_alert("Big Model Failed"),

selfdrive/selfdrived/selfdrived.py

@@ -407,11 +407,11 @@ class SelfdriveD:
       self.events.add(EventName.bigModelLoading)
 
     # The big model loads in the background while the small model drives, so it sits
-    # loaded-but-unused until the driver disengages. That is a success, not a failure.
+    # loaded-but-unused until the driver disengages. That is a success, not a failure, and
+    # it raises no alert: the driver sees it through the eGPU icon, not a banner.
     pending = self.params.get_bool("UsbGpuPending")
     if pending:
       self.big_model_attempted = True
-      self.events.add(EventName.bigModelPending)
 
     big_active = self.params.get("UsbGpuActive")
     if self.big_model_active != (big_active is True):

selfdrive/selfdrived/tests/test_big_model_engagement.py

@@ -16,9 +16,10 @@ def test_big_model_loading_does_not_block_engagement():
   assert ET.PERMANENT in EVENTS[EventName.bigModelLoading]
 
 
-def test_big_model_pending_is_advisory_only():
-  pending = EVENTS[EventName.bigModelPending]
-  assert set(pending) == {ET.PERMANENT}
+def test_big_model_pending_raises_no_alert():
+  # The banner fired ~13 s before the model was actually usable, so the driver saw "ready"
+  # and then got errors trying to engage. The eGPU icon carries this state instead.
+  assert EVENTS[EventName.bigModelPending] == {}
 
 
 def test_big_model_failure_still_disengages():