shmal

selfdrive/modeld/modeld.py

@@ -116,6 +116,13 @@ BIG_MODEL_LOADER_CORES = {6}
 # forever while the small model quietly keeps driving.
 BIG_MODEL_LOAD_TIMEOUT_SECONDS = 150.0
 
+# Chestnut's own supply must be healthy before tinygrad touches the device. The vehicle rail
+# can read healthy while Chestnut browns out and re-enumerates, and initializing across that
+# window blocks forever inside tinygrad instead of failing. Measured on two e-GMP startups:
+# the supply recovers ~6 s before PCIe reaches L0, so hold it steady well past the recovery.
+EXTERNAL_GPU_SUPPLY_STABLE_SECONDS = 10.0
+EXTERNAL_GPU_LINK_WAIT_TIMEOUT_SECONDS = 60.0
+
 
 class BigModelLoadCancelled(Exception):
   """Raised inside the background loader when modeld no longer wants the big model."""
@@ -219,6 +226,84 @@ def wait_for_external_gpu_power_ready(CP=None, cancel=None) -> None:
       last_log = now
 
 
+def _external_gpu_supply_ready(voltage_mv: int, current_ma: int, fault: bool, enumerated: bool,
+                               now: float, stable_since: float | None) -> tuple[bool, float | None]:
+  """Chestnut's own supply and USB enumeration must both be healthy.
+
+  The vehicle rail does not tell us this: on an e-GMP startup it can read 11 V throughout
+  while Chestnut browns out to 7 V and re-enumerates. Negative current means its capacitors
+  are discharging, and the supply recovers seconds before the device is back on the bus, so
+  both signals have to hold steady together.
+  """
+  if fault or current_ma < 0 or voltage_mv < EXTERNAL_GPU_POWER_READY_MV or not enumerated:
+    return False, None
+
+  stable_since = now if stable_since is None else stable_since
+  return now - stable_since >= EXTERNAL_GPU_SUPPLY_STABLE_SECONDS, stable_since
+
+
+def _read_external_gpu_supply() -> tuple[int, int, bool]:
+  """Read Chestnut's supply telemetry over USB, before tinygrad has opened the device."""
+  context = usb1.USBContext()
+  try:
+    for vendor_id, product_id in CHESTNUT_USB_IDS:
+      handle = context.openByVendorIDAndProductID(vendor_id, product_id, skip_on_error=True)
+      if handle is None:
+        continue
+      try:
+        raw = handle.controlRead(0xC0, 0xC0, 0, 0, 5, timeout=100)
+        return struct.unpack("<Hh?", bytes(raw))
+      finally:
+        handle.close()
+  finally:
+    context.close()
+  raise usb1.USBErrorNoDevice
+
+
+def wait_for_external_gpu_supply_ready(cancel=None) -> None:
+  """Wait out a Chestnut brownout before tinygrad initializes the device.
+
+  Initializing while Chestnut is power-cycling blocks forever inside tinygrad rather than
+  failing, so this gate is what keeps a startup brownout from wedging the whole load.
+  """
+  if cancel is not None and cancel.is_set():
+    raise BigModelLoadCancelled("cancelled before waiting for the external GPU supply")
+
+  stable_since = None
+  wait_started = time.monotonic()
+  last_log = 0.0
+  detail = "unavailable"
+
+  while True:
+    if cancel is not None and cancel.is_set():
+      raise BigModelLoadCancelled("cancelled while waiting for the external GPU supply")
+    now = time.monotonic()
+
+    try:
+      voltage_mv, current_ma, fault = _read_external_gpu_supply()
+      enumerated = usbgpu_present()
+      detail = f"{voltage_mv / 1000:.2f} V, {current_ma} mA, fault {fault}, enumerated {enumerated}"
+      ready, stable_since = _external_gpu_supply_ready(
+        voltage_mv, current_ma, fault, enumerated, now, stable_since,
+      )
+      if ready:
+        cloudlog.warning(f"external GPU supply stable at {voltage_mv / 1000:.2f} V; starting load")
+        return
+    except Exception as exc:
+      stable_since = None
+      detail = f"unreadable ({exc.__class__.__name__})"
+
+    if now - wait_started >= EXTERNAL_GPU_LINK_WAIT_TIMEOUT_SECONDS:
+      raise TimeoutError(f"external GPU supply did not settle after "
+                         f"{EXTERNAL_GPU_LINK_WAIT_TIMEOUT_SECONDS:.0f}s ({detail})")
+
+    if now - last_log >= EXTERNAL_GPU_POWER_LOG_INTERVAL_SECONDS:
+      cloudlog.warning(f"external GPU load deferred: supply is {detail}")
+      last_log = now
+
+    time.sleep(0.1)
+
+
 def get_lateral_smooth_seconds(v_ego: float, maximum: float = 0.0) -> float:
   return float(np.interp(v_ego, LAT_SMOOTH_BP, [maximum, 0.0]))
 
@@ -956,6 +1041,9 @@ class BigModelLoader:
       set_core_affinity(sorted(BIG_MODEL_LOADER_CORES))
       if not self.demo:
         wait_for_external_gpu_power_ready(self.CP, cancel=self._cancel)
+        # Chestnut can brown out and re-enumerate as the vehicle powers up; initializing
+        # across that window wedges tinygrad, so wait for its own supply to settle first.
+        wait_for_external_gpu_supply_ready(cancel=self._cancel)
       if self._cancel.is_set():
         raise BigModelLoadCancelled("cancelled before the external GPU link check")
       wait_usbgpu_link()

selfdrive/modeld/tests/test_usbgpu_helpers.py

@@ -287,6 +287,8 @@ def _stub_big_model_loader(monkeypatch, calls, *, uses_external_gpu=True, model_
   monkeypatch.setattr(modeld, "wait_usbgpu_link", lambda: calls.append("link"))
   monkeypatch.setattr(modeld, "wait_for_external_gpu_power_ready",
                       lambda CP, cancel=None: calls.append(("power", CP)))
+  monkeypatch.setattr(modeld, "wait_for_external_gpu_supply_ready",
+                      lambda cancel=None: calls.append("supply"))
   monkeypatch.setattr(modeld, "_close_tinygrad_disk_cache_connection", lambda: calls.append("close_cache"))
   monkeypatch.setattr(modeld, "ModelState", FakeModelState)
   return FakeModelState
@@ -314,6 +316,7 @@ def test_background_big_model_load_leaves_the_running_model_untouched(monkeypatc
   assert calls == [
     ("affinity", tuple(sorted(modeld.BIG_MODEL_LOADER_CORES))),
     ("power", "car-params"),
+    "supply",
     "link",
     ("model", 1928, 1208, True, "big-model", False, "v15"),
     "warmup",
@@ -363,6 +366,43 @@ def test_big_model_load_timeout_leaves_room_for_a_normal_load():
   assert modeld.BIG_MODEL_LOAD_TIMEOUT_SECONDS > 60
 
 
+def test_external_gpu_supply_gate_rejects_a_brownout():
+  # The e-GMP ECU-disable sequence needed for openpilot longitudinal cycles the car through
+  # ACC mode into READY, which power-cycles Chestnut. Measured on two such startups: its rail
+  # collapses to ~7 V with current flowing backwards out of its capacitors, while the vehicle
+  # rail still reads 11 V throughout.
+  ready, stable = modeld._external_gpu_supply_ready(7836, -1, True, True, 10.0, 5.0)
+  assert not ready and stable is None
+
+  # Negative current alone is a brownout even before the fault flag is latched.
+  ready, stable = modeld._external_gpu_supply_ready(11312, -10, False, True, 10.0, 5.0)
+  assert not ready and stable is None
+
+  # A device that has not re-enumerated is not ready however healthy the rail looks.
+  ready, stable = modeld._external_gpu_supply_ready(13000, 1500, False, False, 10.0, 5.0)
+  assert not ready and stable is None
+
+
+def test_external_gpu_supply_gate_waits_out_the_re_enumeration():
+  # The supply recovers several seconds before the device is back on the PCIe bus, so a
+  # healthy reading must be held steady rather than acted on immediately.
+  ready, stable = modeld._external_gpu_supply_ready(13000, 1500, False, True, 100.0, None)
+  assert not ready and stable == 100.0
+
+  ready, _ = modeld._external_gpu_supply_ready(13000, 1500, False, True, 105.0, 100.0)
+  assert not ready, "must not release while still inside the settling window"
+
+  ready, _ = modeld._external_gpu_supply_ready(
+    13000, 1500, False, True, 100.0 + modeld.EXTERNAL_GPU_SUPPLY_STABLE_SECONDS, 100.0,
+  )
+  assert ready
+
+
+def test_external_gpu_supply_settling_outlasts_the_measured_re_enumeration():
+  # PCIe reached L0 ~6 s after the supply recovered on both captured failures.
+  assert modeld.EXTERNAL_GPU_SUPPLY_STABLE_SECONDS >= 10.0
+
+
 def test_background_big_model_loader_runs_off_modelds_realtime_core():
   # modeld is SCHED_FIFO on core 7 and threads inherit its affinity, so a loader left there
   # would be starved behind the 20 Hz publish loop.