fix 99

openpilot/common/params_keys.h

@@ -133,6 +133,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"UsbGpuActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
     {"UsbGpuLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
     {"UsbGpuLoadProgress", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, INT, "0"}},
+    {"UsbGpuKernelTotal", {PERSISTENT, INT, "0"}},
     {"Version", {PERSISTENT, STRING}},
 
     // --- sunnypilot params --- //

openpilot/selfdrive/modeld/load_progress.py

@@ -1,8 +1,11 @@
 import os
+import contextlib
 from openpilot.common.file_chunker import open_file_chunked, get_existing_chunks
 from openpilot.common.params import Params
 
 PARAM = "UsbGpuLoadProgress"
+KERNELS_PARAM = "UsbGpuKernelTotal"
+BYTE_CEIL = 30  # byte read fills 0..BYTE_CEIL, warmup fills BYTE_CEIL..100
 
 
 class ProgressReader:
@@ -17,7 +20,7 @@ class ProgressReader:
   def _bump(self, n):
     self._read += n
     if self._total:
-      pct = min(100, self._read * 100 // self._total)
+      pct = min(BYTE_CEIL, self._read * BYTE_CEIL // self._total)
       if pct != self._pct:
         self._pct = pct
         self._params.put(PARAM, pct)
@@ -42,3 +45,46 @@ class ProgressReader:
 def open_with_progress(pkl_path):
   total = sum(os.path.getsize(p) for p in get_existing_chunks(pkl_path))
   return ProgressReader(open_file_chunked(pkl_path), total)
+
+
+_warmup = {"on": False, "n": 0, "total": 0, "pct": -1, "params": None}
+
+
+def _install_kernel_hook():
+  # runtime patch (no tinygrad source edit, so no model recompile); best-effort, never break loading
+  # get_runtime runs once per kernel during graph build (each uploads a program to the eGPU = the slow warmup step)
+  try:
+    import tinygrad.engine.jit as jit
+    if getattr(jit, "_progress_hooked", False):
+      return
+    orig = jit.get_runtime
+
+    def get_runtime(*args, **kwargs):
+      if _warmup["on"]:
+        _warmup["n"] += 1
+        if _warmup["total"]:
+          pct = min(100, BYTE_CEIL + _warmup["n"] * (100 - BYTE_CEIL) // _warmup["total"])
+          if pct != _warmup["pct"]:
+            _warmup["pct"] = pct
+            _warmup["params"].put(PARAM, pct)
+      return orig(*args, **kwargs)
+
+    jit.get_runtime = get_runtime
+    jit._progress_hooked = True
+  except Exception:
+    pass
+
+
+@contextlib.contextmanager
+def warmup_progress():
+  # smooth BYTE_CEIL..100 by counting eGPU kernels run during warmup; total self-calibrates across boots
+  _install_kernel_hook()
+  p = _warmup["params"] = Params()
+  _warmup.update(on=True, n=0, pct=-1, total=p.get(KERNELS_PARAM, return_default=True))
+  try:
+    yield
+  finally:
+    _warmup["on"] = False
+    if _warmup["n"]:
+      p.put(KERNELS_PARAM, _warmup["n"])
+    p.put(PARAM, 100)

openpilot/selfdrive/modeld/modeld.py

@@ -29,7 +29,7 @@ from openpilot.selfdrive.modeld.parse_model_outputs import Parser
 from openpilot.selfdrive.modeld.compile_modeld import make_input_queues, WARP_INPUTS, POLICY_INPUTS
 from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_driving_model_data, fill_pose_msg, PublishState
 from openpilot.common.file_chunker import open_file_chunked
-from openpilot.selfdrive.modeld.load_progress import open_with_progress
+from openpilot.selfdrive.modeld.load_progress import open_with_progress, warmup_progress
 from openpilot.selfdrive.modeld.constants import ModelConstants, Plan
 from openpilot.selfdrive.modeld.helpers import usbgpu_present, usbgpu_compiled, modeld_pkl_path, get_tg_input_devices, load_oob
 
@@ -260,7 +260,8 @@ def main(demo=False):
       nonlocal big_model
       try:
         m = ModelState(vipc_client_main.width, vipc_client_main.height, True)
-        m.warmup()
+        with warmup_progress():
+          m.warmup()
         big_model = m
       except Exception:
         cloudlog.exception("big model load failed")

openpilot/sunnypilot/modeld_v2/modeld.py

@@ -25,7 +25,7 @@ from opendbc.car.car_helpers import get_demo_car_params
 from tinygrad.tensor import Tensor
 
 from openpilot.common.file_chunker import open_file_chunked
-from openpilot.selfdrive.modeld.load_progress import open_with_progress
+from openpilot.selfdrive.modeld.load_progress import open_with_progress, warmup_progress
 from openpilot.common.swaglog import cloudlog
 from openpilot.common.params import Params
 from openpilot.common.filter_simple import FirstOrderFilter
@@ -187,7 +187,8 @@ class ModelState(ModelStateBase):
       self.warp(**{k: self.input_queues[k] for k in WARP_INPUTS}, frame=frame_tensor, big_frame=big_frame_tensor)
 
     if self.usbgpu:
-      self.warmup()
+      with warmup_progress():
+        self.warmup()
 
   def warmup(self) -> None:
     dummy_frames = {k: np.zeros(self.frame_buf_params[k][3], dtype=np.uint8) for k in self._vision_input_names}