From 72d2d5381d6b6d924a4072f8acf25ca01d737ed8 Mon Sep 17 00:00:00 2001 From: royjr Date: Sat, 22 Aug 2026 17:07:39 -0400 Subject: [PATCH] fix 99 --- openpilot/common/params_keys.h | 1 + openpilot/selfdrive/modeld/load_progress.py | 48 ++++++++++++++++++++- openpilot/selfdrive/modeld/modeld.py | 5 ++- openpilot/sunnypilot/modeld_v2/modeld.py | 5 ++- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/openpilot/common/params_keys.h b/openpilot/common/params_keys.h index 4a29e58555..cec7d1ede4 100644 --- a/openpilot/common/params_keys.h +++ b/openpilot/common/params_keys.h @@ -133,6 +133,7 @@ inline static std::unordered_map keys = { {"UsbGpuActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}}, {"UsbGpuLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}}, {"UsbGpuLoadProgress", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, INT, "0"}}, + {"UsbGpuKernelTotal", {PERSISTENT, INT, "0"}}, {"Version", {PERSISTENT, STRING}}, // --- sunnypilot params --- // diff --git a/openpilot/selfdrive/modeld/load_progress.py b/openpilot/selfdrive/modeld/load_progress.py index 8c861f38ea..3dd344f166 100644 --- a/openpilot/selfdrive/modeld/load_progress.py +++ b/openpilot/selfdrive/modeld/load_progress.py @@ -1,8 +1,11 @@ import os +import contextlib from openpilot.common.file_chunker import open_file_chunked, get_existing_chunks from openpilot.common.params import Params PARAM = "UsbGpuLoadProgress" +KERNELS_PARAM = "UsbGpuKernelTotal" +BYTE_CEIL = 30 # byte read fills 0..BYTE_CEIL, warmup fills BYTE_CEIL..100 class ProgressReader: @@ -17,7 +20,7 @@ class ProgressReader: def _bump(self, n): self._read += n if self._total: - pct = min(100, self._read * 100 // self._total) + pct = min(BYTE_CEIL, self._read * BYTE_CEIL // self._total) if pct != self._pct: self._pct = pct self._params.put(PARAM, pct) @@ -42,3 +45,46 @@ class ProgressReader: def open_with_progress(pkl_path): total = sum(os.path.getsize(p) for p in get_existing_chunks(pkl_path)) return ProgressReader(open_file_chunked(pkl_path), total) + + +_warmup = {"on": False, "n": 0, "total": 0, "pct": -1, "params": None} + + +def _install_kernel_hook(): + # runtime patch (no tinygrad source edit, so no model recompile); best-effort, never break loading + # get_runtime runs once per kernel during graph build (each uploads a program to the eGPU = the slow warmup step) + try: + import tinygrad.engine.jit as jit + if getattr(jit, "_progress_hooked", False): + return + orig = jit.get_runtime + + def get_runtime(*args, **kwargs): + if _warmup["on"]: + _warmup["n"] += 1 + if _warmup["total"]: + pct = min(100, BYTE_CEIL + _warmup["n"] * (100 - BYTE_CEIL) // _warmup["total"]) + if pct != _warmup["pct"]: + _warmup["pct"] = pct + _warmup["params"].put(PARAM, pct) + return orig(*args, **kwargs) + + jit.get_runtime = get_runtime + jit._progress_hooked = True + except Exception: + pass + + +@contextlib.contextmanager +def warmup_progress(): + # smooth BYTE_CEIL..100 by counting eGPU kernels run during warmup; total self-calibrates across boots + _install_kernel_hook() + p = _warmup["params"] = Params() + _warmup.update(on=True, n=0, pct=-1, total=p.get(KERNELS_PARAM, return_default=True)) + try: + yield + finally: + _warmup["on"] = False + if _warmup["n"]: + p.put(KERNELS_PARAM, _warmup["n"]) + p.put(PARAM, 100) diff --git a/openpilot/selfdrive/modeld/modeld.py b/openpilot/selfdrive/modeld/modeld.py index 0f1b50b810..156d28fa2f 100755 --- a/openpilot/selfdrive/modeld/modeld.py +++ b/openpilot/selfdrive/modeld/modeld.py @@ -29,7 +29,7 @@ from openpilot.selfdrive.modeld.parse_model_outputs import Parser from openpilot.selfdrive.modeld.compile_modeld import make_input_queues, WARP_INPUTS, POLICY_INPUTS from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_driving_model_data, fill_pose_msg, PublishState from openpilot.common.file_chunker import open_file_chunked -from openpilot.selfdrive.modeld.load_progress import open_with_progress +from openpilot.selfdrive.modeld.load_progress import open_with_progress, warmup_progress from openpilot.selfdrive.modeld.constants import ModelConstants, Plan from openpilot.selfdrive.modeld.helpers import usbgpu_present, usbgpu_compiled, modeld_pkl_path, get_tg_input_devices, load_oob @@ -260,7 +260,8 @@ def main(demo=False): nonlocal big_model try: m = ModelState(vipc_client_main.width, vipc_client_main.height, True) - m.warmup() + with warmup_progress(): + m.warmup() big_model = m except Exception: cloudlog.exception("big model load failed") diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 001c30518d..d518ddcb6b 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -25,7 +25,7 @@ from opendbc.car.car_helpers import get_demo_car_params from tinygrad.tensor import Tensor from openpilot.common.file_chunker import open_file_chunked -from openpilot.selfdrive.modeld.load_progress import open_with_progress +from openpilot.selfdrive.modeld.load_progress import open_with_progress, warmup_progress from openpilot.common.swaglog import cloudlog from openpilot.common.params import Params from openpilot.common.filter_simple import FirstOrderFilter @@ -187,7 +187,8 @@ class ModelState(ModelStateBase): self.warp(**{k: self.input_queues[k] for k in WARP_INPUTS}, frame=frame_tensor, big_frame=big_frame_tensor) if self.usbgpu: - self.warmup() + with warmup_progress(): + self.warmup() def warmup(self) -> None: dummy_frames = {k: np.zeros(self.frame_buf_params[k][3], dtype=np.uint8) for k in self._vision_input_names}