Lil timmy

selfdrive/modeld/modeld.py

@@ -972,9 +972,11 @@ class BigModelLoader:
       )
       if not candidate.uses_external_gpu:
         raise RuntimeError("external GPU model resolved to the builtin model")
-      candidate.warmup()
+      # warmup() is deliberately not called here: the big model's camera warp runs on QCOM,
+      # the same GPU the small model is driving on, so warming up in the background stalls
+      # the 20 Hz loop for seconds at a time. It runs at promotion instead, while disengaged.
       if self._cancel.is_set():
-        raise BigModelLoadCancelled("cancelled after warmup")
+        raise BigModelLoadCancelled("cancelled after load")
       with self._lock:
         self._result = candidate
       cloudlog.warning(f"background big model load finished in {time.monotonic() - self.started_t:.1f}s")
@@ -1485,24 +1487,33 @@ def main(demo=False):
       vipc_dropped_frames,
       live_calib_seen,
     ):
-      big_model._reset_state()
-      model = big_model
-      external_gpu_active = True
-      # Nothing from the small model carries over: re-arm the frame-drop warmup and drop
-      # the rolling probability buffers and previous action, which are model specific.
-      run_count = 0
-      frame_dropped_filter.x = 0.
-      publish_state = PublishState()
-      prev_action = log.ModelDataV2.Action()
-      params.put("ModelVersion", model.policy_generation)
-      params.put("DrivingModelVersion", model.policy_generation)
-      set_runtime_model_params(params, model.model_id, model.policy_generation)
-      params.put_bool("UsbGpuActive", True)
-      params.put_bool("UsbGpuPending", False)
-      params.put_bool("UsbGpuLoading", False)
-      if chestnut_state is not None:
-        chestnut_state.big = True
-      cloudlog.warning(f"now driving on the big model {model.model_id}")
+      # Warm up here rather than on the loader thread: this runs on QCOM alongside the small
+      # model, so it has to happen while disengaged. warmup() ends with _reset_state().
+      try:
+        big_model.warmup()
+      except Exception:
+        cloudlog.exception("big model warmup failed, staying on the small model")
+        big_model = None
+        params.put_bool("UsbGpuPending", False)
+        params.put_bool("UsbGpuActive", False)
+      else:
+        model = big_model
+        external_gpu_active = True
+        # Nothing from the small model carries over: re-arm the frame-drop warmup and drop
+        # the rolling probability buffers and previous action, which are model specific.
+        run_count = 0
+        frame_dropped_filter.x = 0.
+        publish_state = PublishState()
+        prev_action = log.ModelDataV2.Action()
+        params.put("ModelVersion", model.policy_generation)
+        params.put("DrivingModelVersion", model.policy_generation)
+        set_runtime_model_params(params, model.model_id, model.policy_generation)
+        params.put_bool("UsbGpuActive", True)
+        params.put_bool("UsbGpuPending", False)
+        params.put_bool("UsbGpuLoading", False)
+        if chestnut_state is not None:
+          chestnut_state.big = True
+        cloudlog.warning(f"now driving on the big model {model.model_id}")
 
     frame_drop_ratio = frames_dropped / (1 + frames_dropped)
     dropped_frame = vipc_dropped_frames > 0

selfdrive/modeld/tests/test_usbgpu_helpers.py

@@ -311,14 +311,16 @@ def test_background_big_model_load_leaves_the_running_model_untouched(monkeypatc
   loaded, error = loader.take()
   assert isinstance(loaded, fake_model_state)
   assert error == ""
+  # warmup() is absent on purpose: it runs on QCOM alongside the driving small model, so it
+  # is deferred to promotion (while disengaged) rather than run on the loader thread.
   assert calls == [
     ("affinity", tuple(sorted(modeld.BIG_MODEL_LOADER_CORES))),
     ("power", "car-params"),
     "link",
     ("model", 1928, 1208, True, "big-model", False, "v15"),
-    "warmup",
     "close_cache",
   ]
+  assert "warmup" not in calls
 
 
 def test_big_model_load_gives_up_instead_of_loading_forever(monkeypatch):
@@ -363,6 +365,30 @@ def test_big_model_load_timeout_leaves_room_for_a_normal_load():
   assert modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS > 60
 
 
+def test_big_model_warmup_is_deferred_out_of_the_background_load():
+  """warmup() runs the camera warp on QCOM, the GPU the small model is driving on.
+
+  Running it on the loader thread blocked modeld for seconds at a time: measured modelV2
+  gaps of 1.3-2.9 s clustered in the second half of four loads, with core 7 only ~30% busy
+  (blocked, not starved). It belongs at promotion, which is already gated on disengaged.
+  """
+  import ast
+  from pathlib import Path
+
+  source = (Path(modeld.__file__).with_name("modeld.py")).read_text(encoding="utf-8")
+  tree = ast.parse(source)
+
+  loader = next(n for n in ast.walk(tree)
+                if isinstance(n, ast.ClassDef) and n.name == "BigModelLoader")
+  assert "warmup" not in ast.dump(loader), \
+    "the loader thread must not warm up the big model; it shares QCOM with the small model"
+
+  main_fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "main")
+  promote = next(n for n in ast.walk(main_fn)
+                 if isinstance(n, ast.If) and "_big_model_swap_allowed" in ast.dump(n.test))
+  assert "warmup" in ast.dump(promote), "promotion must warm the big model before it drives"
+
+
 def test_chestnut_telemetry_is_suppressed_while_a_background_load_runs():
   """Telemetry shares Chestnut's USB device with the model weight transfer.
 
@@ -508,6 +534,10 @@ def test_big_model_promotion_waits_for_the_driver_to_disengage(monkeypatch):
     def __init__(self):
       self.reset = 0
 
+    def warmup(self):
+      # The real warmup() ends with _reset_state(); it runs at promotion, not on the loader.
+      self.reset += 1
+
     def _reset_state(self):
       self.reset += 1
 
@@ -559,7 +589,7 @@ def test_big_model_promotion_waits_for_the_driver_to_disengage(monkeypatch):
   scope["_engaged"] = False
   exec(promote, scope)  # noqa: S102
   assert scope["model"] is big
-  assert big.reset == 1, "temporal queues must be reset before the big model drives"
+  assert big.reset == 1, "the big model must be warmed and reset before it drives"
   assert scope["run_count"] == 0 and scope["frame_dropped_filter"].x == 0.
   assert scope["publish_state"] == "fresh" and scope["prev_action"] != "stale"
   assert written["UsbGpuActive"] is True and written["UsbGpuPending"] is False