From 54a836583d2943bc264331e6ef937bf07d0cb6d4 Mon Sep 17 00:00:00 2001 From: whoisdomi Date: Mon, 14 Sep 2026 20:50:56 -0500 Subject: [PATCH] its a shmall world after all --- selfdrive/modeld/modeld.py | 91 +------------------ selfdrive/modeld/tests/test_usbgpu_helpers.py | 56 +++++------- 2 files changed, 24 insertions(+), 123 deletions(-) diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index ea3f290405..efb021a6f6 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -116,13 +116,6 @@ BIG_MODEL_LOADER_CORES = {6} # forever while the small model quietly keeps driving. BIG_MODEL_LOAD_TIMEOUT_SECONDS = 150.0 -# Chestnut's own supply must be healthy before tinygrad touches the device. The vehicle rail -# can read healthy while Chestnut browns out and re-enumerates, and initializing across that -# window blocks forever inside tinygrad instead of failing. Measured on two e-GMP startups: -# the supply recovers ~6 s before PCIe reaches L0, so hold it steady well past the recovery. -EXTERNAL_GPU_SUPPLY_STABLE_SECONDS = 10.0 -EXTERNAL_GPU_LINK_WAIT_TIMEOUT_SECONDS = 60.0 - class BigModelLoadCancelled(Exception): """Raised inside the background loader when modeld no longer wants the big model.""" @@ -226,84 +219,6 @@ def wait_for_external_gpu_power_ready(CP=None, cancel=None) -> None: last_log = now -def _external_gpu_supply_ready(voltage_mv: int, current_ma: int, fault: bool, enumerated: bool, - now: float, stable_since: float | None) -> tuple[bool, float | None]: - """Chestnut's own supply and USB enumeration must both be healthy. - - The vehicle rail does not tell us this: on an e-GMP startup it can read 11 V throughout - while Chestnut browns out to 7 V and re-enumerates. Negative current means its capacitors - are discharging, and the supply recovers seconds before the device is back on the bus, so - both signals have to hold steady together. - """ - if fault or current_ma < 0 or voltage_mv < EXTERNAL_GPU_POWER_READY_MV or not enumerated: - return False, None - - stable_since = now if stable_since is None else stable_since - return now - stable_since >= EXTERNAL_GPU_SUPPLY_STABLE_SECONDS, stable_since - - -def _read_external_gpu_supply() -> tuple[int, int, bool]: - """Read Chestnut's supply telemetry over USB, before tinygrad has opened the device.""" - context = usb1.USBContext() - try: - for vendor_id, product_id in CHESTNUT_USB_IDS: - handle = context.openByVendorIDAndProductID(vendor_id, product_id, skip_on_error=True) - if handle is None: - continue - try: - raw = handle.controlRead(0xC0, 0xC0, 0, 0, 5, timeout=100) - return struct.unpack(" None: - """Wait out a Chestnut brownout before tinygrad initializes the device. - - Initializing while Chestnut is power-cycling blocks forever inside tinygrad rather than - failing, so this gate is what keeps a startup brownout from wedging the whole load. - """ - if cancel is not None and cancel.is_set(): - raise BigModelLoadCancelled("cancelled before waiting for the external GPU supply") - - stable_since = None - wait_started = time.monotonic() - last_log = 0.0 - detail = "unavailable" - - while True: - if cancel is not None and cancel.is_set(): - raise BigModelLoadCancelled("cancelled while waiting for the external GPU supply") - now = time.monotonic() - - try: - voltage_mv, current_ma, fault = _read_external_gpu_supply() - enumerated = usbgpu_present() - detail = f"{voltage_mv / 1000:.2f} V, {current_ma} mA, fault {fault}, enumerated {enumerated}" - ready, stable_since = _external_gpu_supply_ready( - voltage_mv, current_ma, fault, enumerated, now, stable_since, - ) - if ready: - cloudlog.warning(f"external GPU supply stable at {voltage_mv / 1000:.2f} V; starting load") - return - except Exception as exc: - stable_since = None - detail = f"unreadable ({exc.__class__.__name__})" - - if now - wait_started >= EXTERNAL_GPU_LINK_WAIT_TIMEOUT_SECONDS: - raise TimeoutError(f"external GPU supply did not settle after " - f"{EXTERNAL_GPU_LINK_WAIT_TIMEOUT_SECONDS:.0f}s ({detail})") - - if now - last_log >= EXTERNAL_GPU_POWER_LOG_INTERVAL_SECONDS: - cloudlog.warning(f"external GPU load deferred: supply is {detail}") - last_log = now - - time.sleep(0.1) - - def get_lateral_smooth_seconds(v_ego: float, maximum: float = 0.0) -> float: return float(np.interp(v_ego, LAT_SMOOTH_BP, [maximum, 0.0])) @@ -1041,9 +956,6 @@ class BigModelLoader: set_core_affinity(sorted(BIG_MODEL_LOADER_CORES)) if not self.demo: wait_for_external_gpu_power_ready(self.CP, cancel=self._cancel) - # Chestnut can brown out and re-enumerate as the vehicle powers up; initializing - # across that window wedges tinygrad, so wait for its own supply to settle first. - wait_for_external_gpu_supply_ready(cancel=self._cancel) if self._cancel.is_set(): raise BigModelLoadCancelled("cancelled before the external GPU link check") wait_usbgpu_link() @@ -1603,6 +1515,9 @@ def main(demo=False): try: send_chestnut = ( chestnut_state is not None and + # Telemetry shares the USB device with the model transfer, so polling it while a + # background load is in flight stalls that transfer until it times out. + (big_loader is None or not big_loader.in_progress) and run_count % round(ModelConstants.MODEL_FREQ / SERVICE_LIST["chestnutState"].frequency) == 0 ) if model_lab_longitudinal is not None: diff --git a/selfdrive/modeld/tests/test_usbgpu_helpers.py b/selfdrive/modeld/tests/test_usbgpu_helpers.py index 9e1cf1d01d..7d395dbff5 100644 --- a/selfdrive/modeld/tests/test_usbgpu_helpers.py +++ b/selfdrive/modeld/tests/test_usbgpu_helpers.py @@ -287,8 +287,6 @@ def _stub_big_model_loader(monkeypatch, calls, *, uses_external_gpu=True, model_ monkeypatch.setattr(modeld, "wait_usbgpu_link", lambda: calls.append("link")) monkeypatch.setattr(modeld, "wait_for_external_gpu_power_ready", lambda CP, cancel=None: calls.append(("power", CP))) - monkeypatch.setattr(modeld, "wait_for_external_gpu_supply_ready", - lambda cancel=None: calls.append("supply")) monkeypatch.setattr(modeld, "_close_tinygrad_disk_cache_connection", lambda: calls.append("close_cache")) monkeypatch.setattr(modeld, "ModelState", FakeModelState) return FakeModelState @@ -316,7 +314,6 @@ def test_background_big_model_load_leaves_the_running_model_untouched(monkeypatc assert calls == [ ("affinity", tuple(sorted(modeld.BIG_MODEL_LOADER_CORES))), ("power", "car-params"), - "supply", "link", ("model", 1928, 1208, True, "big-model", False, "v15"), "warmup", @@ -366,41 +363,30 @@ def test_big_model_load_timeout_leaves_room_for_a_normal_load(): assert modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS > 60 -def test_external_gpu_supply_gate_rejects_a_brownout(): - # The e-GMP ECU-disable sequence needed for openpilot longitudinal cycles the car through - # ACC mode into READY, which power-cycles Chestnut. Measured on two such startups: its rail - # collapses to ~7 V with current flowing backwards out of its capacitors, while the vehicle - # rail still reads 11 V throughout. - ready, stable = modeld._external_gpu_supply_ready(7836, -1, True, True, 10.0, 5.0) - assert not ready and stable is None +def test_chestnut_telemetry_is_suppressed_while_a_background_load_runs(): + """Telemetry shares Chestnut's USB device with the model weight transfer. - # Negative current alone is a brownout even before the fault flag is latched. - ready, stable = modeld._external_gpu_supply_ready(11312, -10, False, True, 10.0, 5.0) - assert not ready and stable is None + ChestnutState._read_ina() issues USB control reads on the same device tinygrad streams + weights over. The old code loaded before the publish loop existed so the two never + overlapped; loading in the background makes them concurrent, which stalls the transfer + until it times out. Captured on four drives: the load never completed while telemetry + was polling at 10 Hz, and chestnutState first appeared only after the load on the one + drive that succeeded. + """ + import ast + from pathlib import Path - # A device that has not re-enumerated is not ready however healthy the rail looks. - ready, stable = modeld._external_gpu_supply_ready(13000, 1500, False, False, 10.0, 5.0) - assert not ready and stable is None - - -def test_external_gpu_supply_gate_waits_out_the_re_enumeration(): - # The supply recovers several seconds before the device is back on the PCIe bus, so a - # healthy reading must be held steady rather than acted on immediately. - ready, stable = modeld._external_gpu_supply_ready(13000, 1500, False, True, 100.0, None) - assert not ready and stable == 100.0 - - ready, _ = modeld._external_gpu_supply_ready(13000, 1500, False, True, 105.0, 100.0) - assert not ready, "must not release while still inside the settling window" - - ready, _ = modeld._external_gpu_supply_ready( - 13000, 1500, False, True, 100.0 + modeld.EXTERNAL_GPU_SUPPLY_STABLE_SECONDS, 100.0, + source = (Path(modeld.__file__).with_name("modeld.py")).read_text(encoding="utf-8") + main_fn = next(n for n in ast.parse(source).body + if isinstance(n, ast.FunctionDef) and n.name == "main") + assign = next( + node for node in ast.walk(main_fn) + if isinstance(node, ast.Assign) + and any(isinstance(t, ast.Name) and t.id == "send_chestnut" for t in node.targets) ) - assert ready - - -def test_external_gpu_supply_settling_outlasts_the_measured_re_enumeration(): - # PCIe reached L0 ~6 s after the supply recovered on both captured failures. - assert modeld.EXTERNAL_GPU_SUPPLY_STABLE_SECONDS >= 10.0 + guard = ast.dump(assign.value) + assert "big_loader" in guard and "in_progress" in guard, \ + "chestnutState polling must be gated on the background loader being idle" def test_background_big_model_loader_runs_off_modelds_realtime_core():