diff --git a/selfdrive/assets/icons_mici/egpu_orange.png b/selfdrive/assets/icons_mici/egpu_orange.png index bdad854676..d8ad253f56 100644 Binary files a/selfdrive/assets/icons_mici/egpu_orange.png and b/selfdrive/assets/icons_mici/egpu_orange.png differ diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 20889f2677..89586aae52 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +from collections.abc import Callable +import ctypes from functools import cached_property import os import struct @@ -159,8 +161,10 @@ class ChestnutState: if self.big and "AMD" in Device._opened_devices and self.sends % 100 == 1: try: smu = Device["AMD"].iface.dev_impl.smu + metrics_t = smu.smu_mod.SmuMetricsExternal_t smu._send_msg(smu.smu_mod.PPSMC_MSG_TransferTableSmu2Dram, smu.smu_mod.TABLE_SMU_METRICS, timeout=100) - metrics = smu.read_table(smu.smu_mod.SmuMetricsExternal_t, smu.smu_mod.TABLE_SMU_METRICS).SmuMetrics + metrics_buf = bytearray(smu.adev.vram.view(smu.driver_table_paddr, ctypes.sizeof(metrics_t))[:]) + metrics = metrics_t.from_buffer(metrics_buf).SmuMetrics self.metrics = { "tempC": metrics.AvgTemperature[smu.smu_mod.TEMP_HOTSPOT], "memoryTempC": metrics.AvgTemperature[smu.smu_mod.TEMP_MEM], @@ -569,7 +573,8 @@ class ModelState: self._reset_state() def run(self, bufs: dict[str, VisionBuf], transforms: dict[str, np.ndarray], - inputs: dict[str, np.ndarray], prepare_only: bool) -> dict[str, np.ndarray] | None: + inputs: dict[str, np.ndarray], prepare_only: bool, + after_enqueue: Callable[[], None] | None = None) -> dict[str, np.ndarray] | None: frames: dict[str, Tensor] = {} for key, buf in bufs.items(): ptr = np.frombuffer(buf.data, dtype=np.uint8).ctypes.data @@ -615,11 +620,12 @@ class ModelState: img=img, big_img=big_img, ) + if after_enqueue is not None: + after_enqueue() outputs = [output.numpy().flatten() for output in output_tensors] if self.uses_external_gpu and any(not np.isfinite(output).all() for output in outputs): - cloudlog.error("external GPU model output not finite, dropping frame") - return None + raise RuntimeError("external GPU model output not finite") if self.model_type == "supercombo": model_output = outputs[0] @@ -921,7 +927,17 @@ def main(demo=False): mt1 = time.perf_counter() try: - model_output = model.run(bufs, transforms, inputs, prepare_only) + send_chestnut = ( + chestnut_state is not None and + run_count % round(ModelConstants.MODEL_FREQ / SERVICE_LIST["chestnutState"].frequency) == 0 + ) + model_output = model.run( + bufs, + transforms, + inputs, + prepare_only, + chestnut_state.send if send_chestnut else None, + ) except Exception: if not external_gpu_active or small_model is None: raise @@ -984,9 +1000,6 @@ def main(demo=False): if sm.updated['starpilotPlan']: starpilot_toggles = get_starpilot_toggles(sm) - if chestnut_state is not None and run_count % round(ModelConstants.MODEL_FREQ / SERVICE_LIST["chestnutState"].frequency) == 0: - chestnut_state.send() - if __name__ == "__main__": try: import argparse diff --git a/selfdrive/modeld/tests/test_usbgpu_helpers.py b/selfdrive/modeld/tests/test_usbgpu_helpers.py index cd3839dadd..0e643941c2 100644 --- a/selfdrive/modeld/tests/test_usbgpu_helpers.py +++ b/selfdrive/modeld/tests/test_usbgpu_helpers.py @@ -3,6 +3,7 @@ from types import MethodType from types import SimpleNamespace import numpy as np +import pytest from openpilot.selfdrive.modeld import modeld from openpilot.selfdrive.modeld.helpers import dump_oob, load_oob, tinygrad_dev_config @@ -37,17 +38,18 @@ def test_external_gpu_uses_a_longer_load_watchdog(): assert modeld.BIG_MODEL_RUN_WAIT_TIMEOUT_MS == 3000 -def test_external_gpu_signal_wait_matches_upstream_busy_poll(): +def test_external_gpu_signal_wait_yields_between_usb_polls(monkeypatch): from tinygrad.runtime import ops_amd sleeps = [] + monkeypatch.setattr(ops_amd.time, "sleep", sleeps.append) signal = ops_amd.AMDSignal.__new__(ops_amd.AMDSignal) signal.should_return = False - signal.owner = SimpleNamespace(is_usb=lambda: True, iface=SimpleNamespace(sleep=sleeps.append)) + signal.owner = SimpleNamespace(is_usb=lambda: True, iface=SimpleNamespace(sleep=lambda _: None)) signal._sleep(0) - assert sleeps == [] + assert sleeps == [ops_amd.AMD_USB_POLL_US / 1e6] def test_native_amd_signal_keeps_existing_short_wait_behavior(): @@ -201,7 +203,7 @@ def test_external_gpu_load_finishes_before_native_model_can_start(monkeypatch): ] -def test_external_gpu_nonfinite_outputs_are_dropped_without_escalating(monkeypatch): +def test_external_gpu_nonfinite_outputs_trigger_fallback(monkeypatch): class FakeTensor: @staticmethod def from_blob(*_args, **_kwargs): @@ -235,13 +237,7 @@ def test_external_gpu_nonfinite_outputs_are_dropped_without_escalating(monkeypat state.image_history_pipeline = modeld.IMAGE_HISTORY_IN_POLICY state.warp_enqueue = lambda **_kwargs: object() state.run_policy = lambda **_kwargs: (FakeOutput(),) - state._reset_state = MethodType( - lambda self: (_ for _ in ()).throw(AssertionError("upstream does not reset or escalate transient non-finite output")), - state, - ) - monkeypatch.setattr(modeld, "Tensor", FakeTensor) - monkeypatch.setattr(modeld.cloudlog, "error", lambda *_args, **_kwargs: None) buffers = { "img": SimpleNamespace(data=bytearray(4)), "big_img": SimpleNamespace(data=bytearray(4)), @@ -252,8 +248,10 @@ def test_external_gpu_nonfinite_outputs_are_dropped_without_escalating(monkeypat } inputs = {"desire_pulse": np.zeros(8, dtype=np.float32)} - for _ in range(10): - assert state.run(buffers, transforms, inputs, False) is None + callbacks = [] + with pytest.raises(RuntimeError, match="external GPU model output not finite"): + state.run(buffers, transforms, inputs, False, lambda: callbacks.append("sent")) + assert callbacks == ["sent"] def test_out_of_band_artifact_round_trip(): diff --git a/selfdrive/ui/mici/onroad/hud_renderer.py b/selfdrive/ui/mici/onroad/hud_renderer.py index 7ab54a152a..49dfd0e2fb 100644 --- a/selfdrive/ui/mici/onroad/hud_renderer.py +++ b/selfdrive/ui/mici/onroad/hud_renderer.py @@ -158,7 +158,7 @@ class HudRenderer(Widget): self._txt_exclamation_point: rl.Texture = gui_app.texture('icons_mici/exclamation_point.png', 44, 44) self._txt_egpu_loading: rl.Texture = gui_app.texture('icons_mici/egpu_loading.png', 60, 44) self._txt_egpu_green: rl.Texture = gui_app.texture('icons_mici/egpu_green.png', 60, 44) - self._txt_egpu_orange: rl.Texture = gui_app.texture('icons_mici/egpu_orange.png', 60, 44) + self._txt_egpu_orange: rl.Texture = gui_app.texture('icons_mici/egpu_orange.png', 75, 44) self._txt_egpu_crossed: rl.Texture = gui_app.texture('icons_mici/egpu_crossed.png', 60, 52) self._egpu_icon: rl.Texture | None = None diff --git a/tinygrad_repo/tinygrad/runtime/ops_amd.py b/tinygrad_repo/tinygrad/runtime/ops_amd.py index 7404a1fca3..a9248428b2 100644 --- a/tinygrad_repo/tinygrad/runtime/ops_amd.py +++ b/tinygrad_repo/tinygrad/runtime/ops_amd.py @@ -1,6 +1,6 @@ from __future__ import annotations from typing import cast -import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit +import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit, time assert sys.platform != 'win32' from dataclasses import dataclass from tinygrad.runtime.support.hcq import HCQCompiled, HCQAllocator, HCQBuffer, HWQueue, CLikeArgsState, HCQSignal, HCQProgram, FileIOInterface @@ -25,6 +25,7 @@ SQTT = ContextVar("SQTT", abs(VIZ.value)>=2) SQTT_ITRACE_SE_MASK, SQTT_LIMIT_SE, SQTT_SIMD_SEL, SQTT_TOKEN_EXCLUDE = \ ContextVar("SQTT_ITRACE_SE_MASK", 0b11), ContextVar("SQTT_LIMIT_SE", 0), ContextVar("SQTT_SIMD_SEL", 0), ContextVar("SQTT_TOKEN_EXCLUDE", 0) PMC = ContextVar("PMC", abs(VIZ.value)>=2) +AMD_USB_POLL_US = getenv("AMD_USB_POLL_US", 500) # microseconds to sleep between USB signal polls. 0 disables EVENT_INDEX_PARTIAL_FLUSH = 4 # based on a comment in nvd.h WAIT_REG_MEM_FUNCTION_EQ = 3 # == WAIT_REG_MEM_FUNCTION_NEQ = 4 # != @@ -45,8 +46,9 @@ class AMDSignal(HCQSignal): def __init__(self, *args, **kwargs): super().__init__(*args, **{**kwargs, 'timestamp_divider': 100}) def _sleep(self, time_spent_since_last_sleep_ms:int): - # Reasonable to sleep for long workloads (which take more than 200ms) and only timeline signals. - if time_spent_since_last_sleep_ms > 200 and self.owner is not None: self.owner.iface.sleep(200) + # USB signals live in VRAM across the link, so yield between polls. Native AMD only blocks after 200 ms. + if self.owner is not None and self.owner.is_usb() and AMD_USB_POLL_US: time.sleep(AMD_USB_POLL_US / 1e6) + elif time_spent_since_last_sleep_ms > 200 and self.owner is not None: self.owner.iface.sleep(200) class AMDComputeQueue(HWQueue): def __init__(self, dev:AMDDevice):