From a1c6920511e6ba9bdfd37876837c6769ffe7060d Mon Sep 17 00:00:00 2001 From: whoisdomi Date: Tue, 15 Sep 2026 10:30:34 -0500 Subject: [PATCH] napolean --- selfdrive/modeld/modeld.py | 57 +++++++++---------- selfdrive/modeld/tests/test_usbgpu_helpers.py | 25 ++++---- selfdrive/selfdrived/events.py | 9 ++- selfdrive/selfdrived/selfdrived.py | 4 +- .../tests/test_big_model_engagement.py | 7 ++- 5 files changed, 50 insertions(+), 52 deletions(-) diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 4cf6fac85e..1be68efb10 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -972,11 +972,13 @@ class BigModelLoader: ) if not candidate.uses_external_gpu: raise RuntimeError("external GPU model resolved to the builtin model") - # warmup() is deliberately not called here: the big model's camera warp runs on QCOM, - # the same GPU the small model is driving on, so warming up in the background stalls - # the 20 Hz loop for seconds at a time. It runs at promotion instead, while disengaged. + # warmup() runs the camera warp on QCOM, shared with the driving small model, so it + # costs some jitter here. It still belongs on this thread: doing it at promotion + # instead blocks the publish loop for ~13 s in one stretch, which trips commIssue and + # blocks the driver from engaging exactly when the model becomes available. + candidate.warmup() if self._cancel.is_set(): - raise BigModelLoadCancelled("cancelled after load") + raise BigModelLoadCancelled("cancelled after warmup") with self._lock: self._result = candidate cloudlog.warning(f"background big model load finished in {time.monotonic() - self.started_t:.1f}s") @@ -1487,33 +1489,26 @@ def main(demo=False): vipc_dropped_frames, live_calib_seen, ): - # Warm up here rather than on the loader thread: this runs on QCOM alongside the small - # model, so it has to happen while disengaged. warmup() ends with _reset_state(). - try: - big_model.warmup() - except Exception: - cloudlog.exception("big model warmup failed, staying on the small model") - big_model = None - params.put_bool("UsbGpuPending", False) - params.put_bool("UsbGpuActive", False) - else: - model = big_model - external_gpu_active = True - # Nothing from the small model carries over: re-arm the frame-drop warmup and drop - # the rolling probability buffers and previous action, which are model specific. - run_count = 0 - frame_dropped_filter.x = 0. - publish_state = PublishState() - prev_action = log.ModelDataV2.Action() - params.put("ModelVersion", model.policy_generation) - params.put("DrivingModelVersion", model.policy_generation) - set_runtime_model_params(params, model.model_id, model.policy_generation) - params.put_bool("UsbGpuActive", True) - params.put_bool("UsbGpuPending", False) - params.put_bool("UsbGpuLoading", False) - if chestnut_state is not None: - chestnut_state.big = True - cloudlog.warning(f"now driving on the big model {model.model_id}") + # The loader already warmed this model, so the swap itself is just a pointer change + # plus a queue reset; it must stay cheap because it runs inside the publish loop. + big_model._reset_state() + model = big_model + external_gpu_active = True + # Nothing from the small model carries over: re-arm the frame-drop warmup and drop + # the rolling probability buffers and previous action, which are model specific. + run_count = 0 + frame_dropped_filter.x = 0. + publish_state = PublishState() + prev_action = log.ModelDataV2.Action() + params.put("ModelVersion", model.policy_generation) + params.put("DrivingModelVersion", model.policy_generation) + set_runtime_model_params(params, model.model_id, model.policy_generation) + params.put_bool("UsbGpuActive", True) + params.put_bool("UsbGpuPending", False) + params.put_bool("UsbGpuLoading", False) + if chestnut_state is not None: + chestnut_state.big = True + cloudlog.warning(f"now driving on the big model {model.model_id}") frame_drop_ratio = frames_dropped / (1 + frames_dropped) dropped_frame = vipc_dropped_frames > 0 diff --git a/selfdrive/modeld/tests/test_usbgpu_helpers.py b/selfdrive/modeld/tests/test_usbgpu_helpers.py index 695cff5666..86046ca2c4 100644 --- a/selfdrive/modeld/tests/test_usbgpu_helpers.py +++ b/selfdrive/modeld/tests/test_usbgpu_helpers.py @@ -311,16 +311,16 @@ def test_background_big_model_load_leaves_the_running_model_untouched(monkeypatc loaded, error = loader.take() assert isinstance(loaded, fake_model_state) assert error == "" - # warmup() is absent on purpose: it runs on QCOM alongside the driving small model, so it - # is deferred to promotion (while disengaged) rather than run on the loader thread. + # warmup() belongs here rather than at promotion: see + # test_big_model_warmup_stays_on_the_background_loader. assert calls == [ ("affinity", tuple(sorted(modeld.BIG_MODEL_LOADER_CORES))), ("power", "car-params"), "link", ("model", 1928, 1208, True, "big-model", False, "v15"), + "warmup", "close_cache", ] - assert "warmup" not in calls def test_big_model_load_gives_up_instead_of_loading_forever(monkeypatch): @@ -365,12 +365,15 @@ def test_big_model_load_timeout_leaves_room_for_a_normal_load(): assert modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS > 60 -def test_big_model_warmup_is_deferred_out_of_the_background_load(): - """warmup() runs the camera warp on QCOM, the GPU the small model is driving on. +def test_big_model_warmup_stays_on_the_background_loader(): + """Warming at promotion blocks the publish loop in one long stretch; warming on the + loader spreads the same work out while the small model is still driving. - Running it on the loader thread blocked modeld for seconds at a time: measured modelV2 - gaps of 1.3-2.9 s clustered in the second half of four loads, with core 7 only ~30% busy - (blocked, not starved). It belongs at promotion, which is already gated on disengaged. + Measured with warmup at promotion: modelV2 went silent for 13.0 s and 12.8 s on two + drives, starting the instant the model was collected, which tripped commIssue and blocked + the driver from engaging. Measured with warmup on the loader: worst in-load gap was + 1.3-2.9 s, spread across the load. Neither is free, but only the second one leaves + modeld publishing when the driver wants to engage. """ import ast from pathlib import Path @@ -380,13 +383,13 @@ def test_big_model_warmup_is_deferred_out_of_the_background_load(): loader = next(n for n in ast.walk(tree) if isinstance(n, ast.ClassDef) and n.name == "BigModelLoader") - assert "warmup" not in ast.dump(loader), \ - "the loader thread must not warm up the big model; it shares QCOM with the small model" + assert "warmup" in ast.dump(loader), "the loader thread must warm the big model" main_fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "main") promote = next(n for n in ast.walk(main_fn) if isinstance(n, ast.If) and "_big_model_swap_allowed" in ast.dump(n.test)) - assert "warmup" in ast.dump(promote), "promotion must warm the big model before it drives" + assert "warmup" not in ast.dump(promote), \ + "promotion must not warm the big model; it runs inside the 20 Hz publish loop" def test_chestnut_telemetry_is_suppressed_while_a_background_load_runs(): diff --git a/selfdrive/selfdrived/events.py b/selfdrive/selfdrived/events.py index ecac0c5e91..be4d0a8d22 100644 --- a/selfdrive/selfdrived/events.py +++ b/selfdrive/selfdrived/events.py @@ -537,11 +537,10 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = { "Driving on the small model"), }, - EventName.bigModelPending: { - ET.PERMANENT: NormalPermanentAlert("Big Model Ready", - "Disengage and re-engage to use it", - duration=10.), - }, + # bigModelPending carries no alert on purpose: the eGPU icon clearing, and then turning + # green on the next engage, is the driver's cue. A banner here announced the model before + # it was usable. + EventName.bigModelPending: {}, EventName.bigModelFailed: { ET.SOFT_DISABLE: soft_disable_alert("Big Model Failed"), diff --git a/selfdrive/selfdrived/selfdrived.py b/selfdrive/selfdrived/selfdrived.py index 30ff87c0c4..a37d2c4e90 100644 --- a/selfdrive/selfdrived/selfdrived.py +++ b/selfdrive/selfdrived/selfdrived.py @@ -407,11 +407,11 @@ class SelfdriveD: self.events.add(EventName.bigModelLoading) # The big model loads in the background while the small model drives, so it sits - # loaded-but-unused until the driver disengages. That is a success, not a failure. + # loaded-but-unused until the driver disengages. That is a success, not a failure, and + # it raises no alert: the driver sees it through the eGPU icon, not a banner. pending = self.params.get_bool("UsbGpuPending") if pending: self.big_model_attempted = True - self.events.add(EventName.bigModelPending) big_active = self.params.get("UsbGpuActive") if self.big_model_active != (big_active is True): diff --git a/selfdrive/selfdrived/tests/test_big_model_engagement.py b/selfdrive/selfdrived/tests/test_big_model_engagement.py index e175481f28..57061fbbfb 100644 --- a/selfdrive/selfdrived/tests/test_big_model_engagement.py +++ b/selfdrive/selfdrived/tests/test_big_model_engagement.py @@ -16,9 +16,10 @@ def test_big_model_loading_does_not_block_engagement(): assert ET.PERMANENT in EVENTS[EventName.bigModelLoading] -def test_big_model_pending_is_advisory_only(): - pending = EVENTS[EventName.bigModelPending] - assert set(pending) == {ET.PERMANENT} +def test_big_model_pending_raises_no_alert(): + # The banner fired ~13 s before the model was actually usable, so the driver saw "ready" + # and then got errors trying to engage. The eGPU icon carries this state instead. + assert EVENTS[EventName.bigModelPending] == {} def test_big_model_failure_still_disengages():