From 7947d848fdf8304cbe29a150fd45461581d7b2b2 Mon Sep 17 00:00:00 2001 From: whoisdomi Date: Tue, 15 Sep 2026 08:02:54 -0500 Subject: [PATCH] Lil timmy --- selfdrive/modeld/modeld.py | 51 +++++++++++-------- selfdrive/modeld/tests/test_usbgpu_helpers.py | 34 ++++++++++++- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index ea3ec0834c..4cf6fac85e 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -972,9 +972,11 @@ class BigModelLoader: ) if not candidate.uses_external_gpu: raise RuntimeError("external GPU model resolved to the builtin model") - candidate.warmup() + # warmup() is deliberately not called here: the big model's camera warp runs on QCOM, + # the same GPU the small model is driving on, so warming up in the background stalls + # the 20 Hz loop for seconds at a time. It runs at promotion instead, while disengaged. if self._cancel.is_set(): - raise BigModelLoadCancelled("cancelled after warmup") + raise BigModelLoadCancelled("cancelled after load") with self._lock: self._result = candidate cloudlog.warning(f"background big model load finished in {time.monotonic() - self.started_t:.1f}s") @@ -1485,24 +1487,33 @@ def main(demo=False): vipc_dropped_frames, live_calib_seen, ): - big_model._reset_state() - model = big_model - external_gpu_active = True - # Nothing from the small model carries over: re-arm the frame-drop warmup and drop - # the rolling probability buffers and previous action, which are model specific. - run_count = 0 - frame_dropped_filter.x = 0. - publish_state = PublishState() - prev_action = log.ModelDataV2.Action() - params.put("ModelVersion", model.policy_generation) - params.put("DrivingModelVersion", model.policy_generation) - set_runtime_model_params(params, model.model_id, model.policy_generation) - params.put_bool("UsbGpuActive", True) - params.put_bool("UsbGpuPending", False) - params.put_bool("UsbGpuLoading", False) - if chestnut_state is not None: - chestnut_state.big = True - cloudlog.warning(f"now driving on the big model {model.model_id}") + # Warm up here rather than on the loader thread: this runs on QCOM alongside the small + # model, so it has to happen while disengaged. warmup() ends with _reset_state(). + try: + big_model.warmup() + except Exception: + cloudlog.exception("big model warmup failed, staying on the small model") + big_model = None + params.put_bool("UsbGpuPending", False) + params.put_bool("UsbGpuActive", False) + else: + model = big_model + external_gpu_active = True + # Nothing from the small model carries over: re-arm the frame-drop warmup and drop + # the rolling probability buffers and previous action, which are model specific. + run_count = 0 + frame_dropped_filter.x = 0. + publish_state = PublishState() + prev_action = log.ModelDataV2.Action() + params.put("ModelVersion", model.policy_generation) + params.put("DrivingModelVersion", model.policy_generation) + set_runtime_model_params(params, model.model_id, model.policy_generation) + params.put_bool("UsbGpuActive", True) + params.put_bool("UsbGpuPending", False) + params.put_bool("UsbGpuLoading", False) + if chestnut_state is not None: + chestnut_state.big = True + cloudlog.warning(f"now driving on the big model {model.model_id}") frame_drop_ratio = frames_dropped / (1 + frames_dropped) dropped_frame = vipc_dropped_frames > 0 diff --git a/selfdrive/modeld/tests/test_usbgpu_helpers.py b/selfdrive/modeld/tests/test_usbgpu_helpers.py index d9066e3092..695cff5666 100644 --- a/selfdrive/modeld/tests/test_usbgpu_helpers.py +++ b/selfdrive/modeld/tests/test_usbgpu_helpers.py @@ -311,14 +311,16 @@ def test_background_big_model_load_leaves_the_running_model_untouched(monkeypatc loaded, error = loader.take() assert isinstance(loaded, fake_model_state) assert error == "" + # warmup() is absent on purpose: it runs on QCOM alongside the driving small model, so it + # is deferred to promotion (while disengaged) rather than run on the loader thread. assert calls == [ ("affinity", tuple(sorted(modeld.BIG_MODEL_LOADER_CORES))), ("power", "car-params"), "link", ("model", 1928, 1208, True, "big-model", False, "v15"), - "warmup", "close_cache", ] + assert "warmup" not in calls def test_big_model_load_gives_up_instead_of_loading_forever(monkeypatch): @@ -363,6 +365,30 @@ def test_big_model_load_timeout_leaves_room_for_a_normal_load(): assert modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS > 60 +def test_big_model_warmup_is_deferred_out_of_the_background_load(): + """warmup() runs the camera warp on QCOM, the GPU the small model is driving on. + + Running it on the loader thread blocked modeld for seconds at a time: measured modelV2 + gaps of 1.3-2.9 s clustered in the second half of four loads, with core 7 only ~30% busy + (blocked, not starved). It belongs at promotion, which is already gated on disengaged. + """ + import ast + from pathlib import Path + + source = (Path(modeld.__file__).with_name("modeld.py")).read_text(encoding="utf-8") + tree = ast.parse(source) + + loader = next(n for n in ast.walk(tree) + if isinstance(n, ast.ClassDef) and n.name == "BigModelLoader") + assert "warmup" not in ast.dump(loader), \ + "the loader thread must not warm up the big model; it shares QCOM with the small model" + + main_fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "main") + promote = next(n for n in ast.walk(main_fn) + if isinstance(n, ast.If) and "_big_model_swap_allowed" in ast.dump(n.test)) + assert "warmup" in ast.dump(promote), "promotion must warm the big model before it drives" + + def test_chestnut_telemetry_is_suppressed_while_a_background_load_runs(): """Telemetry shares Chestnut's USB device with the model weight transfer. @@ -508,6 +534,10 @@ def test_big_model_promotion_waits_for_the_driver_to_disengage(monkeypatch): def __init__(self): self.reset = 0 + def warmup(self): + # The real warmup() ends with _reset_state(); it runs at promotion, not on the loader. + self.reset += 1 + def _reset_state(self): self.reset += 1 @@ -559,7 +589,7 @@ def test_big_model_promotion_waits_for_the_driver_to_disengage(monkeypatch): scope["_engaged"] = False exec(promote, scope) # noqa: S102 assert scope["model"] is big - assert big.reset == 1, "temporal queues must be reset before the big model drives" + assert big.reset == 1, "the big model must be warmed and reset before it drives" assert scope["run_count"] == 0 and scope["frame_dropped_filter"].x == 0. assert scope["publish_state"] == "fresh" and scope["prev_action"] != "stale" assert written["UsbGpuActive"] is True and written["UsbGpuPending"] is False