From ef24f97c4635e523e38729525f92b0c157b81684 Mon Sep 17 00:00:00 2001 From: whoisdomi Date: Mon, 14 Sep 2026 15:45:16 -0500 Subject: [PATCH] shmal --- selfdrive/modeld/modeld.py | 88 +++++++++++++++++++ selfdrive/modeld/tests/test_usbgpu_helpers.py | 40 +++++++++ 2 files changed, 128 insertions(+) diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 775782b724..ea3f290405 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -116,6 +116,13 @@ BIG_MODEL_LOADER_CORES = {6} # forever while the small model quietly keeps driving. BIG_MODEL_LOAD_TIMEOUT_SECONDS = 150.0 +# Chestnut's own supply must be healthy before tinygrad touches the device. The vehicle rail +# can read healthy while Chestnut browns out and re-enumerates, and initializing across that +# window blocks forever inside tinygrad instead of failing. Measured on two e-GMP startups: +# the supply recovers ~6 s before PCIe reaches L0, so hold it steady well past the recovery. +EXTERNAL_GPU_SUPPLY_STABLE_SECONDS = 10.0 +EXTERNAL_GPU_LINK_WAIT_TIMEOUT_SECONDS = 60.0 + class BigModelLoadCancelled(Exception): """Raised inside the background loader when modeld no longer wants the big model.""" @@ -219,6 +226,84 @@ def wait_for_external_gpu_power_ready(CP=None, cancel=None) -> None: last_log = now +def _external_gpu_supply_ready(voltage_mv: int, current_ma: int, fault: bool, enumerated: bool, + now: float, stable_since: float | None) -> tuple[bool, float | None]: + """Chestnut's own supply and USB enumeration must both be healthy. + + The vehicle rail does not tell us this: on an e-GMP startup it can read 11 V throughout + while Chestnut browns out to 7 V and re-enumerates. Negative current means its capacitors + are discharging, and the supply recovers seconds before the device is back on the bus, so + both signals have to hold steady together. + """ + if fault or current_ma < 0 or voltage_mv < EXTERNAL_GPU_POWER_READY_MV or not enumerated: + return False, None + + stable_since = now if stable_since is None else stable_since + return now - stable_since >= EXTERNAL_GPU_SUPPLY_STABLE_SECONDS, stable_since + + +def _read_external_gpu_supply() -> tuple[int, int, bool]: + """Read Chestnut's supply telemetry over USB, before tinygrad has opened the device.""" + context = usb1.USBContext() + try: + for vendor_id, product_id in CHESTNUT_USB_IDS: + handle = context.openByVendorIDAndProductID(vendor_id, product_id, skip_on_error=True) + if handle is None: + continue + try: + raw = handle.controlRead(0xC0, 0xC0, 0, 0, 5, timeout=100) + return struct.unpack(" None: + """Wait out a Chestnut brownout before tinygrad initializes the device. + + Initializing while Chestnut is power-cycling blocks forever inside tinygrad rather than + failing, so this gate is what keeps a startup brownout from wedging the whole load. + """ + if cancel is not None and cancel.is_set(): + raise BigModelLoadCancelled("cancelled before waiting for the external GPU supply") + + stable_since = None + wait_started = time.monotonic() + last_log = 0.0 + detail = "unavailable" + + while True: + if cancel is not None and cancel.is_set(): + raise BigModelLoadCancelled("cancelled while waiting for the external GPU supply") + now = time.monotonic() + + try: + voltage_mv, current_ma, fault = _read_external_gpu_supply() + enumerated = usbgpu_present() + detail = f"{voltage_mv / 1000:.2f} V, {current_ma} mA, fault {fault}, enumerated {enumerated}" + ready, stable_since = _external_gpu_supply_ready( + voltage_mv, current_ma, fault, enumerated, now, stable_since, + ) + if ready: + cloudlog.warning(f"external GPU supply stable at {voltage_mv / 1000:.2f} V; starting load") + return + except Exception as exc: + stable_since = None + detail = f"unreadable ({exc.__class__.__name__})" + + if now - wait_started >= EXTERNAL_GPU_LINK_WAIT_TIMEOUT_SECONDS: + raise TimeoutError(f"external GPU supply did not settle after " + f"{EXTERNAL_GPU_LINK_WAIT_TIMEOUT_SECONDS:.0f}s ({detail})") + + if now - last_log >= EXTERNAL_GPU_POWER_LOG_INTERVAL_SECONDS: + cloudlog.warning(f"external GPU load deferred: supply is {detail}") + last_log = now + + time.sleep(0.1) + + def get_lateral_smooth_seconds(v_ego: float, maximum: float = 0.0) -> float: return float(np.interp(v_ego, LAT_SMOOTH_BP, [maximum, 0.0])) @@ -956,6 +1041,9 @@ class BigModelLoader: set_core_affinity(sorted(BIG_MODEL_LOADER_CORES)) if not self.demo: wait_for_external_gpu_power_ready(self.CP, cancel=self._cancel) + # Chestnut can brown out and re-enumerate as the vehicle powers up; initializing + # across that window wedges tinygrad, so wait for its own supply to settle first. + wait_for_external_gpu_supply_ready(cancel=self._cancel) if self._cancel.is_set(): raise BigModelLoadCancelled("cancelled before the external GPU link check") wait_usbgpu_link() diff --git a/selfdrive/modeld/tests/test_usbgpu_helpers.py b/selfdrive/modeld/tests/test_usbgpu_helpers.py index f2a3d79d99..9e1cf1d01d 100644 --- a/selfdrive/modeld/tests/test_usbgpu_helpers.py +++ b/selfdrive/modeld/tests/test_usbgpu_helpers.py @@ -287,6 +287,8 @@ def _stub_big_model_loader(monkeypatch, calls, *, uses_external_gpu=True, model_ monkeypatch.setattr(modeld, "wait_usbgpu_link", lambda: calls.append("link")) monkeypatch.setattr(modeld, "wait_for_external_gpu_power_ready", lambda CP, cancel=None: calls.append(("power", CP))) + monkeypatch.setattr(modeld, "wait_for_external_gpu_supply_ready", + lambda cancel=None: calls.append("supply")) monkeypatch.setattr(modeld, "_close_tinygrad_disk_cache_connection", lambda: calls.append("close_cache")) monkeypatch.setattr(modeld, "ModelState", FakeModelState) return FakeModelState @@ -314,6 +316,7 @@ def test_background_big_model_load_leaves_the_running_model_untouched(monkeypatc assert calls == [ ("affinity", tuple(sorted(modeld.BIG_MODEL_LOADER_CORES))), ("power", "car-params"), + "supply", "link", ("model", 1928, 1208, True, "big-model", False, "v15"), "warmup", @@ -363,6 +366,43 @@ def test_big_model_load_timeout_leaves_room_for_a_normal_load(): assert modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS > 60 +def test_external_gpu_supply_gate_rejects_a_brownout(): + # The e-GMP ECU-disable sequence needed for openpilot longitudinal cycles the car through + # ACC mode into READY, which power-cycles Chestnut. Measured on two such startups: its rail + # collapses to ~7 V with current flowing backwards out of its capacitors, while the vehicle + # rail still reads 11 V throughout. + ready, stable = modeld._external_gpu_supply_ready(7836, -1, True, True, 10.0, 5.0) + assert not ready and stable is None + + # Negative current alone is a brownout even before the fault flag is latched. + ready, stable = modeld._external_gpu_supply_ready(11312, -10, False, True, 10.0, 5.0) + assert not ready and stable is None + + # A device that has not re-enumerated is not ready however healthy the rail looks. + ready, stable = modeld._external_gpu_supply_ready(13000, 1500, False, False, 10.0, 5.0) + assert not ready and stable is None + + +def test_external_gpu_supply_gate_waits_out_the_re_enumeration(): + # The supply recovers several seconds before the device is back on the PCIe bus, so a + # healthy reading must be held steady rather than acted on immediately. + ready, stable = modeld._external_gpu_supply_ready(13000, 1500, False, True, 100.0, None) + assert not ready and stable == 100.0 + + ready, _ = modeld._external_gpu_supply_ready(13000, 1500, False, True, 105.0, 100.0) + assert not ready, "must not release while still inside the settling window" + + ready, _ = modeld._external_gpu_supply_ready( + 13000, 1500, False, True, 100.0 + modeld.EXTERNAL_GPU_SUPPLY_STABLE_SECONDS, 100.0, + ) + assert ready + + +def test_external_gpu_supply_settling_outlasts_the_measured_re_enumeration(): + # PCIe reached L0 ~6 s after the supply recovered on both captured failures. + assert modeld.EXTERNAL_GPU_SUPPLY_STABLE_SECONDS >= 10.0 + + def test_background_big_model_loader_runs_off_modelds_realtime_core(): # modeld is SCHED_FIFO on core 7 and threads inherit its affinity, so a loader left there # would be starved behind the 20 Hz publish loop.