again

openpilot/cereal/log.capnp

@@ -132,8 +132,6 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     userBookmark @95;
     excessiveActuation @96;
     audioFeedback @97;
-    wgpuModeldLagging @100;
-
     soundsUnavailableDEPRECATED @47;
   }
 }

openpilot/common/params_keys.h

@@ -134,8 +134,8 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"UsbGpuPresent", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"UsbGpuCompiled", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"Version", {PERSISTENT, STRING}},
-    {"WgpuBenchMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON | DEVELOPMENT_ONLY, BOOL}},
     {"WgpuEnabled", {CLEAR_ON_MANAGER_START | DEVELOPMENT_ONLY, BOOL}},
+    {"WgpuModelName", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | DEVELOPMENT_ONLY, STRING}},
 
     // --- sunnypilot params --- //
     {"ApiCache_DriveStats", {PERSISTENT, JSON}},

openpilot/selfdrive/modeld/modeld.py

@@ -359,11 +359,10 @@ def main(demo=False, remote_addr: str | None = None, big_model: bool = False):
 
       fill_driving_model_data(drivingdata_send, modelv2_send)
       fill_pose_msg(posenet_send, model_output, meta_main.frame_id, vipc_dropped_frames, meta_main.timestamp_eof, live_calib_seen)
-      if remote_addr is not None or not params.get_bool("WgpuEnabled"):
-        pm.send('modelV2', modelv2_send)
-        pm.send('drivingModelData', drivingdata_send)
-        pm.send('cameraOdometry', posenet_send)
-        pm.send('modelDataV2SP', mdv2sp_send)
+      pm.send('modelV2', modelv2_send)
+      pm.send('drivingModelData', drivingdata_send)
+      pm.send('cameraOdometry', posenet_send)
+      pm.send('modelDataV2SP', mdv2sp_send)
     last_vipc_frame_id = meta_main.frame_id
 
 

openpilot/selfdrive/selfdrived/events.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 import math
 import os
-import time
 
 from openpilot.cereal import log
 from opendbc.car.structs import car
@@ -191,13 +190,6 @@ def modeld_lagging_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubM
   return NormalPermanentAlert("Driving Model Lagging", f"{sm['modelV2'].frameDropPerc:.1f}% frames dropped")
 
 
-def wgpu_modeld_lagging_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMaster, metric: bool,
-                              soft_disable_time: int, personality) -> Alert:
-  model_age_ms = max(0., (time.monotonic_ns() - sm['modelV2'].timestampEof) / 1e6)
-  return NormalPermanentAlert("WGPU Model Lagging",
-                              f"{model_age_ms:.0f} ms old · {sm['modelV2'].frameDropPerc:.1f}% frames dropped")
-
-
 def joystick_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMaster, metric: bool, soft_disable_time: int, personality) -> Alert:
   gb = sm['carControl'].actuators.accel / 4.
   steer = sm['carControl'].actuators.torque
@@ -730,10 +722,6 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
     ET.NO_ENTRY: NoEntryAlert("Driving Model Lagging"),
     ET.PERMANENT: modeld_lagging_alert,
   },
-  EventName.wgpuModeldLagging: {
-    ET.PERMANENT: wgpu_modeld_lagging_alert,
-  },
-
   # Besides predicting the path, lane lines and lead car data the model also
   # predicts the current velocity and rotation speed of the car. If the model is
   # very uncertain about the current velocity while the car is moving, this

openpilot/selfdrive/selfdrived/selfdrived.py

@@ -112,7 +112,6 @@ class SelfdriveD(CruiseHelper):
     self.is_ldw_enabled = self.params.get_bool("IsLdwEnabled")
     self.disengage_on_accelerator = self.params.get_bool("DisengageOnAccelerator")
     self.wgpu_enabled = self.params.get_bool("WgpuEnabled")
-    self.wgpu_bench_mode = self.params.get_bool("WgpuBenchMode")
 
     car_recognized = self.CP.brand != 'mock'
 
@@ -470,8 +469,8 @@ class SelfdriveD(CruiseHelper):
 
     # TODO: fix simulator
     if not SIMULATION or REPLAY:
-      if self.sm['modelV2'].frameDropPerc > 1:
-        self.events.add(EventName.wgpuModeldLagging if self.wgpu_enabled and self.wgpu_bench_mode else EventName.modeldLagging)
+      if self.sm['modelV2'].frameDropPerc > 1 and not self.wgpu_enabled:
+        self.events.add(EventName.modeldLagging)
 
     # mute canBusMissing event if in Park, as it sometimes may trigger a false alarm with MADS in Paused state
     if CS.gearShifter == car.CarState.GearShifter.park and self.mads.enabled:
@@ -628,7 +627,6 @@ class SelfdriveD(CruiseHelper):
       self.is_ldw_enabled = self.params.get_bool("IsLdwEnabled")
       self.disengage_on_accelerator = self.params.get_bool("DisengageOnAccelerator")
       self.wgpu_enabled = self.params.get_bool("WgpuEnabled")
-      self.wgpu_bench_mode = self.params.get_bool("WgpuBenchMode")
       self.experimental_mode = self.params.get_bool("ExperimentalMode") and self.CP.openpilotLongitudinalControl
       self.personality = self.params.get("LongitudinalPersonality", return_default=True)
 

openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py

@@ -1,3 +1,5 @@
+import time
+
 import numpy as np
 import pyray as rl
 from openpilot.cereal import log
@@ -160,6 +162,10 @@ class AugmentedRoadView(CameraView):
                                        text_color=rl.Color(255, 255, 255, int(255 * 0.9)),
                                        alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER,
                                        alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE)
+    self._model_status_label = UnifiedLabel("", 26, FontWeight.SEMI_BOLD,
+                                            alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER,
+                                            alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE,
+                                            wrap_text=False)
 
     self._fade_texture = gui_app.texture("icons_mici/onroad/onroad_fade.png")
 
@@ -230,6 +236,8 @@ class AugmentedRoadView(CameraView):
     self._driver_state_renderer.set_position(self._rect.x + 16, self._rect.y + 10)
     self._driver_state_renderer.render()
 
+    self._render_model_status()
+
     self._hud_renderer.set_can_draw_top_icons(alert_to_render is None)
     self._hud_renderer.set_wheel_critical_icon(alert_to_render is not None and not not_animating_out and
                                                alert_to_render.visual_alert == car.CarControl.HUDControl.VisualAlert.steerRequired)
@@ -248,6 +256,33 @@ class AugmentedRoadView(CameraView):
 
     self._bookmark_icon.render(self.rect)
 
+  def _render_model_status(self):
+    model = ui_state.sm["modelV2"]
+    if ui_state.sm.seen["modelV2"] and model.timestampEof:
+      model_age_ms = max(0., (time.monotonic_ns() - model.timestampEof) / 1e6)
+      lag_text = f"{model_age_ms:.0f} ms"
+    else:
+      model_age_ms = float("inf")
+      lag_text = "-- ms"
+
+    if ui_state.wgpu_enabled:
+      source, model_name = "WGPU", ui_state.wgpu_model_name
+    else:
+      source, model_name = "LOCAL", "DEVICE"
+
+    if model_age_ms < 200:
+      color = rl.Color(100, 255, 120, 230)
+    elif model_age_ms < 500:
+      color = rl.Color(255, 210, 80, 230)
+    else:
+      color = rl.Color(255, 120, 80, 230)
+
+    status_rect = rl.Rectangle(self._content_rect.x + 18, self._content_rect.y + self._content_rect.height - 58, 330, 42)
+    rl.draw_rectangle_rounded(status_rect, 0.35, 8, rl.Color(0, 0, 0, 165))
+    self._model_status_label.set_text(f"{source} · {model_name} · {lag_text}")
+    self._model_status_label.set_text_color(color)
+    self._model_status_label.render(status_rect)
+
   def _switch_stream_if_needed(self, sm):
     if sm['selfdriveState'].experimentalMode and WIDE_CAM in self.available_streams:
       v_ego = sm['carState'].vEgo

openpilot/selfdrive/ui/ui_state.py

@@ -83,6 +83,7 @@ class UIState(UIStateSP):
     self.usbgpu: bool = self.params.get_bool("UsbGpuPresent")
     self.usbgpu_compiled: bool = self.params.get_bool("UsbGpuCompiled")
     self.wgpu_enabled: bool = self.params.get_bool("WgpuEnabled")
+    self.wgpu_model_name: str = self.params.get("WgpuModelName") or "UNKNOWN"
     self.started: bool = False
     self.ignition: bool = False
     self.recording_audio: bool = False
@@ -215,6 +216,7 @@ class UIState(UIStateSP):
     self.usbgpu = self.params.get_bool("UsbGpuPresent")
     self.usbgpu_compiled = self.params.get_bool("UsbGpuCompiled")
     self.wgpu_enabled = self.params.get_bool("WgpuEnabled")
+    self.wgpu_model_name = self.params.get("WgpuModelName") or "UNKNOWN"
 
     UIStateSP.update_params(self)
 

openpilot/system/manager/process_config.py

@@ -95,6 +95,9 @@ def is_stock_model(started, params, CP: car.CarParams) -> bool:
   """Check if the active model runner is stock."""
   return bool(get_active_model_runner(params, not started) == custom.ModelManagerSP.Runner.stock)
 
+def not_wgpu(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return not params.get_bool("WgpuEnabled")
+
 def mapd_ready(started: bool, params: Params, CP: car.CarParams) -> bool:
   return bool(os.path.exists(Paths.mapd_root()))
 
@@ -128,7 +131,7 @@ procs = [
   PythonProcess("micd", "openpilot.system.micd", iscar),
   PythonProcess("timed", "openpilot.system.timed", always_run, enabled=not PC),
 
-  PythonProcess("modeld", "openpilot.selfdrive.modeld.modeld", and_(only_onroad, is_stock_model)),
+  PythonProcess("modeld", "openpilot.selfdrive.modeld.modeld", and_(and_(only_onroad, is_stock_model), not_wgpu)),
   PythonProcess("dmonitoringmodeld", "openpilot.selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(WEBCAM or not PC)),
 
   PythonProcess("sensord", "openpilot.system.sensord.sensord", only_onroad, enabled=not PC),
@@ -177,7 +180,7 @@ procs = [
 procs += [
   # Models
   PythonProcess("models_manager", "openpilot.sunnypilot.models.manager", only_offroad),
-  NativeProcess("modeld_tinygrad", "openpilot/sunnypilot/modeld_v2", ["./modeld"], and_(only_onroad, is_tinygrad_model)),
+  NativeProcess("modeld_tinygrad", "openpilot/sunnypilot/modeld_v2", ["./modeld"], and_(and_(only_onroad, is_tinygrad_model), not_wgpu)),
 
   # Backup
   PythonProcess("backup_manager", "openpilot.sunnypilot.sunnylink.backups.manager", and_(only_offroad, sunnylink_ready_shim)),

openpilot/tools/wgpu/README.md

@@ -5,9 +5,9 @@ device over the existing Wi-Fi network. It reuses the existing HEVC camera
 stream, VisionIPC decoder, and cereal ZMQ bridge.
 
 This is for controlled bench testing only. Wi-Fi has no deterministic latency
-or availability guarantee. Local `modeld` remains warm during a WGPU session;
-the device-side helper switches only after receiving a fresh remote model and
-restores local publication if the remote model is missing for 350 ms.
+or availability guarantee. The device-side helper switches only after receiving
+a fresh remote model and after manager has stopped the local model publisher.
+It restores local `modeld` if the remote model is missing for one second.
 
 ## Build
 
@@ -41,7 +41,8 @@ the small model on that class of laptop.
 
 Find the laptop's LAN IP address that the comma device can reach. The helper can
 start while onroad: it forwards camera/state while local `modeld` remains active,
-then switches publication after the laptop produces a fresh valid model:
+then performs an exclusive publisher handoff after the laptop produces a fresh
+valid model:
 
 ```sh
 cd /data/openpilot
@@ -58,23 +59,10 @@ python3 -m openpilot.tools.wgpu.host COMMA_IP
 Add `--big-model` after `COMMA_IP` to use the locally compiled big model.
 
 The first remote `carParams` packet can take up to 50 seconds. Stop either side
-with Ctrl+C. A host disconnect automatically restores the warm local publisher
-after a 350 ms timeout. Stop the device helper before changing branches or
-rebooting.
+with Ctrl+C. A host disconnect automatically stops remote publication and
+restores local `modeld` after a one-second timeout. Stop the device helper before
+changing branches or rebooting.
 
-For stationary bench testing only, model lag can be changed from a blocking
-event to a live warning showing model age and dropped frames. After ignition is
-on and manager is running, enable the temporary override on the device:
-
-```sh
-cd /data/openpilot
-python3 -c 'from openpilot.common.params import Params; Params().put_bool("WgpuBenchMode", True)'
-```
-
-The override requires `WgpuEnabled` and clears on manager restart or the next
-ignition-on transition. Disable it immediately with:
-
-```sh
-cd /data/openpilot
-python3 -c 'from openpilot.common.params import Params; Params().remove("WgpuBenchMode")'
-```
+While WGPU is active, model lag does not create an engagement-blocking alert.
+The mici onroad UI instead shows the active source (`LOCAL` or `WGPU`), remote
+model size (`SMALL` or `BIG`), and live model-frame age.

openpilot/tools/wgpu/device.py

@@ -7,12 +7,13 @@ from pathlib import Path
 
 import openpilot.cereal.messaging as messaging
 from openpilot.common.params import Params
-from openpilot.common.realtime import DT_MDL
 from openpilot.tools.wgpu.zmq import ZmqSubSocket
 
 
 MODEL_OUTPUTS = "modelV2,drivingModelData,cameraOdometry,modelDataV2SP"
-REMOTE_MODEL_TIMEOUT = 0.35
+MODEL_PROCESSES = {"modeld", "modeld_tinygrad"}
+REMOTE_MODEL_TIMEOUT = 1.0
+WGPU_STATUS = "wgpuStatus"
 ROOT = Path(__file__).resolve().parents[3]
 BRIDGE = ROOT / "openpilot/cereal/messaging/bridge"
 
@@ -30,15 +31,35 @@ def handle_sigterm(*_) -> None:
   raise KeyboardInterrupt
 
 
-def receive_fresh_model(sock: ZmqSubSocket) -> bool:
+def receive_model(sock: ZmqSubSocket) -> tuple[bool, float]:
   raw = sock.receive(non_blocking=True)
   if raw is None:
-    return False
+    return False, float("inf")
   event = messaging.log_from_bytes(raw)
   if event.which() != "modelV2" or not event.valid:
-    return False
+    return True, float("inf")
   model_age = (time.monotonic_ns() - event.modelV2.timestampEof) / 1e9
-  return 0 <= model_age < REMOTE_MODEL_TIMEOUT
+  return True, model_age
+
+
+def receive_model_name(sock: ZmqSubSocket) -> str | None:
+  raw = sock.receive(non_blocking=True)
+  if raw is None:
+    return None
+  model_name = raw.decode(errors="replace").upper()
+  return model_name if model_name in ("SMALL", "BIG") else None
+
+
+def wait_for_local_modeld_stop(timeout: float = 10.) -> None:
+  sm = messaging.SubMaster(["managerState"])
+  deadline = time.monotonic() + timeout
+  while time.monotonic() < deadline:
+    sm.update(100)
+    if sm.seen["managerState"]:
+      running = {p.name for p in sm["managerState"].processes if p.running}
+      if not running.intersection(MODEL_PROCESSES):
+        return
+  raise RuntimeError("timed out waiting for local modeld publisher to stop")
 
 
 def main() -> None:
@@ -57,22 +78,31 @@ def main() -> None:
     # Keep local modeld publishing while the remote model connects and warms up.
     forward = subprocess.Popen([str(BRIDGE)])
     remote_model = ZmqSubSocket("modelV2", args.host, conflate=True)
+    remote_status = ZmqSubSocket(WGPU_STATUS, args.host, conflate=True)
     print(f"forwarding camera/state to {args.host}; waiting for a fresh remote model")
-    while not receive_fresh_model(remote_model):
+    model_name = None
+    while True:
+      received, model_age = receive_model(remote_model)
+      model_name = receive_model_name(remote_status) or model_name
+      if received and 0 <= model_age < REMOTE_MODEL_TIMEOUT and model_name is not None:
+        break
       if forward.poll() is not None:
         raise RuntimeError(f"forward bridge exited with status {forward.returncode}")
       time.sleep(0.05)
 
-    # Local modeld remains warm but stops publishing when this flag changes.
+    # Stop the local publisher before attaching the reverse bridge. msgq permits
+    # only one publisher for each model service.
+    params.put("WgpuModelName", model_name, block=True)
     params.put_bool("WgpuEnabled", True, block=True)
     wgpu_enabled = True
-    time.sleep(2 * DT_MDL)
+    wait_for_local_modeld_stop()
     reverse = subprocess.Popen([str(BRIDGE), args.host, MODEL_OUTPUTS])
     print("wgpu active; Ctrl+C or loss of the remote model restores local modeld")
 
     last_remote_model = time.monotonic()
     while True:
-      if receive_fresh_model(remote_model):
+      received, _ = receive_model(remote_model)
+      if received:
         last_remote_model = time.monotonic()
       if forward.poll() is not None:
         raise RuntimeError(f"forward bridge exited with status {forward.returncode}")
@@ -82,11 +112,12 @@ def main() -> None:
         raise RuntimeError("remote model timed out; restoring local modeld")
       time.sleep(0.05)
   finally:
-    # Stop remote publication before allowing the warm local publisher to resume.
+    # Stop remote publication before allowing the local publisher to restart.
     if reverse is not None and reverse.poll() is None:
       stop_process(reverse)
     if wgpu_enabled:
       params.put_bool("WgpuEnabled", False, block=True)
+    params.remove("WgpuModelName")
     if forward is not None and forward.poll() is None:
       stop_process(forward)
     print("wgpu disabled; local modeld restored")

openpilot/tools/wgpu/host.py

@@ -7,9 +7,12 @@ import sys
 import time
 from pathlib import Path
 
+from openpilot.tools.wgpu.zmq import ZmqPubMaster
+
 
 ROOT = Path(__file__).resolve().parents[3]
 CAMERASTREAM = ROOT / "openpilot/tools/camerastream/compressed_vipc.py"
+WGPU_STATUS = "wgpuStatus"
 
 
 def stop_process(proc: subprocess.Popen) -> None:
@@ -36,10 +39,13 @@ def main() -> None:
   if args.big_model:
     model_args.append("--big-model")
   model = subprocess.Popen(model_args, cwd=ROOT, start_new_session=True)
+  status = ZmqPubMaster([WGPU_STATUS])
+  model_name = b"BIG" if args.big_model else b"SMALL"
 
   procs = {"camera bridge": camera, "modeld": model}
   try:
     while all(proc.poll() is None for proc in procs.values()):
+      status.send_raw(WGPU_STATUS, model_name)
       time.sleep(0.25)
     failed_name, failed = next((name, proc) for name, proc in procs.items() if proc.poll() is not None)
     raise RuntimeError(f"wgpu {failed_name} exited with status {failed.returncode}")

openpilot/tools/wgpu/zmq.py

@@ -88,3 +88,6 @@ class ZmqPubMaster:
 
   def send(self, service: str, message) -> None:
     self.sockets[service].send(message.to_bytes(), flags=zmq.NOBLOCK)
+
+  def send_raw(self, service: str, data: bytes) -> None:
+    self.sockets[service].send(data, flags=zmq.NOBLOCK)