gpu

selfdrive/modeld/modeld.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+from collections.abc import Callable
+import ctypes
 from functools import cached_property
 import os
 import struct
@@ -159,8 +161,10 @@ class ChestnutState:
     if self.big and "AMD" in Device._opened_devices and self.sends % 100 == 1:
       try:
         smu = Device["AMD"].iface.dev_impl.smu
+        metrics_t = smu.smu_mod.SmuMetricsExternal_t
         smu._send_msg(smu.smu_mod.PPSMC_MSG_TransferTableSmu2Dram, smu.smu_mod.TABLE_SMU_METRICS, timeout=100)
-        metrics = smu.read_table(smu.smu_mod.SmuMetricsExternal_t, smu.smu_mod.TABLE_SMU_METRICS).SmuMetrics
+        metrics_buf = bytearray(smu.adev.vram.view(smu.driver_table_paddr, ctypes.sizeof(metrics_t))[:])
+        metrics = metrics_t.from_buffer(metrics_buf).SmuMetrics
         self.metrics = {
           "tempC": metrics.AvgTemperature[smu.smu_mod.TEMP_HOTSPOT],
           "memoryTempC": metrics.AvgTemperature[smu.smu_mod.TEMP_MEM],
@@ -569,7 +573,8 @@ class ModelState:
     self._reset_state()
 
   def run(self, bufs: dict[str, VisionBuf], transforms: dict[str, np.ndarray],
-          inputs: dict[str, np.ndarray], prepare_only: bool) -> dict[str, np.ndarray] | None:
+          inputs: dict[str, np.ndarray], prepare_only: bool,
+          after_enqueue: Callable[[], None] | None = None) -> dict[str, np.ndarray] | None:
     frames: dict[str, Tensor] = {}
     for key, buf in bufs.items():
       ptr = np.frombuffer(buf.data, dtype=np.uint8).ctypes.data
@@ -615,11 +620,12 @@ class ModelState:
         img=img,
         big_img=big_img,
       )
+    if after_enqueue is not None:
+      after_enqueue()
     outputs = [output.numpy().flatten() for output in output_tensors]
 
     if self.uses_external_gpu and any(not np.isfinite(output).all() for output in outputs):
-      cloudlog.error("external GPU model output not finite, dropping frame")
-      return None
+      raise RuntimeError("external GPU model output not finite")
 
     if self.model_type == "supercombo":
       model_output = outputs[0]
@@ -921,7 +927,17 @@ def main(demo=False):
 
     mt1 = time.perf_counter()
     try:
-      model_output = model.run(bufs, transforms, inputs, prepare_only)
+      send_chestnut = (
+        chestnut_state is not None and
+        run_count % round(ModelConstants.MODEL_FREQ / SERVICE_LIST["chestnutState"].frequency) == 0
+      )
+      model_output = model.run(
+        bufs,
+        transforms,
+        inputs,
+        prepare_only,
+        chestnut_state.send if send_chestnut else None,
+      )
     except Exception:
       if not external_gpu_active or small_model is None:
         raise
@@ -984,9 +1000,6 @@ def main(demo=False):
     if sm.updated['starpilotPlan']:
       starpilot_toggles = get_starpilot_toggles(sm)
 
-    if chestnut_state is not None and run_count % round(ModelConstants.MODEL_FREQ / SERVICE_LIST["chestnutState"].frequency) == 0:
-      chestnut_state.send()
-
 if __name__ == "__main__":
   try:
     import argparse

selfdrive/modeld/tests/test_usbgpu_helpers.py

@@ -3,6 +3,7 @@ from types import MethodType
 from types import SimpleNamespace
 
 import numpy as np
+import pytest
 
 from openpilot.selfdrive.modeld import modeld
 from openpilot.selfdrive.modeld.helpers import dump_oob, load_oob, tinygrad_dev_config
@@ -37,17 +38,18 @@ def test_external_gpu_uses_a_longer_load_watchdog():
   assert modeld.BIG_MODEL_RUN_WAIT_TIMEOUT_MS == 3000
 
 
-def test_external_gpu_signal_wait_matches_upstream_busy_poll():
+def test_external_gpu_signal_wait_yields_between_usb_polls(monkeypatch):
   from tinygrad.runtime import ops_amd
 
   sleeps = []
+  monkeypatch.setattr(ops_amd.time, "sleep", sleeps.append)
   signal = ops_amd.AMDSignal.__new__(ops_amd.AMDSignal)
   signal.should_return = False
-  signal.owner = SimpleNamespace(is_usb=lambda: True, iface=SimpleNamespace(sleep=sleeps.append))
+  signal.owner = SimpleNamespace(is_usb=lambda: True, iface=SimpleNamespace(sleep=lambda _: None))
 
   signal._sleep(0)
 
-  assert sleeps == []
+  assert sleeps == [ops_amd.AMD_USB_POLL_US / 1e6]
 
 
 def test_native_amd_signal_keeps_existing_short_wait_behavior():
@@ -201,7 +203,7 @@ def test_external_gpu_load_finishes_before_native_model_can_start(monkeypatch):
   ]
 
 
-def test_external_gpu_nonfinite_outputs_are_dropped_without_escalating(monkeypatch):
+def test_external_gpu_nonfinite_outputs_trigger_fallback(monkeypatch):
   class FakeTensor:
     @staticmethod
     def from_blob(*_args, **_kwargs):
@@ -235,13 +237,7 @@ def test_external_gpu_nonfinite_outputs_are_dropped_without_escalating(monkeypat
   state.image_history_pipeline = modeld.IMAGE_HISTORY_IN_POLICY
   state.warp_enqueue = lambda **_kwargs: object()
   state.run_policy = lambda **_kwargs: (FakeOutput(),)
-  state._reset_state = MethodType(
-    lambda self: (_ for _ in ()).throw(AssertionError("upstream does not reset or escalate transient non-finite output")),
-    state,
-  )
-
   monkeypatch.setattr(modeld, "Tensor", FakeTensor)
-  monkeypatch.setattr(modeld.cloudlog, "error", lambda *_args, **_kwargs: None)
   buffers = {
     "img": SimpleNamespace(data=bytearray(4)),
     "big_img": SimpleNamespace(data=bytearray(4)),
@@ -252,8 +248,10 @@ def test_external_gpu_nonfinite_outputs_are_dropped_without_escalating(monkeypat
   }
   inputs = {"desire_pulse": np.zeros(8, dtype=np.float32)}
 
-  for _ in range(10):
-    assert state.run(buffers, transforms, inputs, False) is None
+  callbacks = []
+  with pytest.raises(RuntimeError, match="external GPU model output not finite"):
+    state.run(buffers, transforms, inputs, False, lambda: callbacks.append("sent"))
+  assert callbacks == ["sent"]
 
 
 def test_out_of_band_artifact_round_trip():

selfdrive/ui/mici/onroad/hud_renderer.py

@@ -158,7 +158,7 @@ class HudRenderer(Widget):
     self._txt_exclamation_point: rl.Texture = gui_app.texture('icons_mici/exclamation_point.png', 44, 44)
     self._txt_egpu_loading: rl.Texture = gui_app.texture('icons_mici/egpu_loading.png', 60, 44)
     self._txt_egpu_green: rl.Texture = gui_app.texture('icons_mici/egpu_green.png', 60, 44)
-    self._txt_egpu_orange: rl.Texture = gui_app.texture('icons_mici/egpu_orange.png', 60, 44)
+    self._txt_egpu_orange: rl.Texture = gui_app.texture('icons_mici/egpu_orange.png', 75, 44)
     self._txt_egpu_crossed: rl.Texture = gui_app.texture('icons_mici/egpu_crossed.png', 60, 52)
     self._egpu_icon: rl.Texture | None = None
 

tinygrad_repo/tinygrad/runtime/ops_amd.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from typing import cast
-import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit
+import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit, time
 assert sys.platform != 'win32'
 from dataclasses import dataclass
 from tinygrad.runtime.support.hcq import HCQCompiled, HCQAllocator, HCQBuffer, HWQueue, CLikeArgsState, HCQSignal, HCQProgram, FileIOInterface
@@ -25,6 +25,7 @@ SQTT = ContextVar("SQTT", abs(VIZ.value)>=2)
 SQTT_ITRACE_SE_MASK, SQTT_LIMIT_SE, SQTT_SIMD_SEL, SQTT_TOKEN_EXCLUDE = \
   ContextVar("SQTT_ITRACE_SE_MASK", 0b11), ContextVar("SQTT_LIMIT_SE", 0), ContextVar("SQTT_SIMD_SEL", 0), ContextVar("SQTT_TOKEN_EXCLUDE", 0)
 PMC = ContextVar("PMC", abs(VIZ.value)>=2)
+AMD_USB_POLL_US = getenv("AMD_USB_POLL_US", 500) # microseconds to sleep between USB signal polls. 0 disables
 EVENT_INDEX_PARTIAL_FLUSH = 4 # based on a comment in nvd.h
 WAIT_REG_MEM_FUNCTION_EQ  = 3 # ==
 WAIT_REG_MEM_FUNCTION_NEQ = 4 # !=
@@ -45,8 +46,9 @@ class AMDSignal(HCQSignal):
   def __init__(self, *args, **kwargs): super().__init__(*args, **{**kwargs, 'timestamp_divider': 100})
 
   def _sleep(self, time_spent_since_last_sleep_ms:int):
-    # Reasonable to sleep for long workloads (which take more than 200ms) and only timeline signals.
-    if time_spent_since_last_sleep_ms > 200 and self.owner is not None: self.owner.iface.sleep(200)
+    # USB signals live in VRAM across the link, so yield between polls. Native AMD only blocks after 200 ms.
+    if self.owner is not None and self.owner.is_usb() and AMD_USB_POLL_US: time.sleep(AMD_USB_POLL_US / 1e6)
+    elif time_spent_since_last_sleep_ms > 200 and self.owner is not None: self.owner.iface.sleep(200)
 
 class AMDComputeQueue(HWQueue):
   def __init__(self, dev:AMDDevice):