diff --git a/openpilot/cereal/log.capnp b/openpilot/cereal/log.capnp index 9453a43742..eea519fd0f 100644 --- a/openpilot/cereal/log.capnp +++ b/openpilot/cereal/log.capnp @@ -132,8 +132,6 @@ struct OnroadEvent @0xc4fa6047f024e718 { userBookmark @95; excessiveActuation @96; audioFeedback @97; - wgpuModeldLagging @100; - soundsUnavailableDEPRECATED @47; } } diff --git a/openpilot/common/params_keys.h b/openpilot/common/params_keys.h index 335fbd0977..80925387ea 100644 --- a/openpilot/common/params_keys.h +++ b/openpilot/common/params_keys.h @@ -134,8 +134,8 @@ inline static std::unordered_map keys = { {"UsbGpuPresent", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}}, {"UsbGpuCompiled", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}}, {"Version", {PERSISTENT, STRING}}, - {"WgpuBenchMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON | DEVELOPMENT_ONLY, BOOL}}, {"WgpuEnabled", {CLEAR_ON_MANAGER_START | DEVELOPMENT_ONLY, BOOL}}, + {"WgpuModelName", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | DEVELOPMENT_ONLY, STRING}}, // --- sunnypilot params --- // {"ApiCache_DriveStats", {PERSISTENT, JSON}}, diff --git a/openpilot/selfdrive/modeld/modeld.py b/openpilot/selfdrive/modeld/modeld.py index 68f49b4113..3c7dd935a2 100755 --- a/openpilot/selfdrive/modeld/modeld.py +++ b/openpilot/selfdrive/modeld/modeld.py @@ -359,11 +359,10 @@ def main(demo=False, remote_addr: str | None = None, big_model: bool = False): fill_driving_model_data(drivingdata_send, modelv2_send) fill_pose_msg(posenet_send, model_output, meta_main.frame_id, vipc_dropped_frames, meta_main.timestamp_eof, live_calib_seen) - if remote_addr is not None or not params.get_bool("WgpuEnabled"): - pm.send('modelV2', modelv2_send) - pm.send('drivingModelData', drivingdata_send) - pm.send('cameraOdometry', posenet_send) - pm.send('modelDataV2SP', mdv2sp_send) + pm.send('modelV2', modelv2_send) + pm.send('drivingModelData', drivingdata_send) + pm.send('cameraOdometry', posenet_send) + pm.send('modelDataV2SP', mdv2sp_send) last_vipc_frame_id = meta_main.frame_id diff --git a/openpilot/selfdrive/selfdrived/events.py b/openpilot/selfdrive/selfdrived/events.py index 3d40db9f02..727482f528 100755 --- a/openpilot/selfdrive/selfdrived/events.py +++ b/openpilot/selfdrive/selfdrived/events.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import math import os -import time from openpilot.cereal import log from opendbc.car.structs import car @@ -191,13 +190,6 @@ def modeld_lagging_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubM return NormalPermanentAlert("Driving Model Lagging", f"{sm['modelV2'].frameDropPerc:.1f}% frames dropped") -def wgpu_modeld_lagging_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMaster, metric: bool, - soft_disable_time: int, personality) -> Alert: - model_age_ms = max(0., (time.monotonic_ns() - sm['modelV2'].timestampEof) / 1e6) - return NormalPermanentAlert("WGPU Model Lagging", - f"{model_age_ms:.0f} ms old · {sm['modelV2'].frameDropPerc:.1f}% frames dropped") - - def joystick_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMaster, metric: bool, soft_disable_time: int, personality) -> Alert: gb = sm['carControl'].actuators.accel / 4. steer = sm['carControl'].actuators.torque @@ -730,10 +722,6 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = { ET.NO_ENTRY: NoEntryAlert("Driving Model Lagging"), ET.PERMANENT: modeld_lagging_alert, }, - EventName.wgpuModeldLagging: { - ET.PERMANENT: wgpu_modeld_lagging_alert, - }, - # Besides predicting the path, lane lines and lead car data the model also # predicts the current velocity and rotation speed of the car. If the model is # very uncertain about the current velocity while the car is moving, this diff --git a/openpilot/selfdrive/selfdrived/selfdrived.py b/openpilot/selfdrive/selfdrived/selfdrived.py index 4dbae9abb6..7398ef4d36 100755 --- a/openpilot/selfdrive/selfdrived/selfdrived.py +++ b/openpilot/selfdrive/selfdrived/selfdrived.py @@ -112,7 +112,6 @@ class SelfdriveD(CruiseHelper): self.is_ldw_enabled = self.params.get_bool("IsLdwEnabled") self.disengage_on_accelerator = self.params.get_bool("DisengageOnAccelerator") self.wgpu_enabled = self.params.get_bool("WgpuEnabled") - self.wgpu_bench_mode = self.params.get_bool("WgpuBenchMode") car_recognized = self.CP.brand != 'mock' @@ -470,8 +469,8 @@ class SelfdriveD(CruiseHelper): # TODO: fix simulator if not SIMULATION or REPLAY: - if self.sm['modelV2'].frameDropPerc > 1: - self.events.add(EventName.wgpuModeldLagging if self.wgpu_enabled and self.wgpu_bench_mode else EventName.modeldLagging) + if self.sm['modelV2'].frameDropPerc > 1 and not self.wgpu_enabled: + self.events.add(EventName.modeldLagging) # mute canBusMissing event if in Park, as it sometimes may trigger a false alarm with MADS in Paused state if CS.gearShifter == car.CarState.GearShifter.park and self.mads.enabled: @@ -628,7 +627,6 @@ class SelfdriveD(CruiseHelper): self.is_ldw_enabled = self.params.get_bool("IsLdwEnabled") self.disengage_on_accelerator = self.params.get_bool("DisengageOnAccelerator") self.wgpu_enabled = self.params.get_bool("WgpuEnabled") - self.wgpu_bench_mode = self.params.get_bool("WgpuBenchMode") self.experimental_mode = self.params.get_bool("ExperimentalMode") and self.CP.openpilotLongitudinalControl self.personality = self.params.get("LongitudinalPersonality", return_default=True) diff --git a/openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py b/openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py index b5b9a54be8..4775426dc4 100644 --- a/openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py +++ b/openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py @@ -1,3 +1,5 @@ +import time + import numpy as np import pyray as rl from openpilot.cereal import log @@ -160,6 +162,10 @@ class AugmentedRoadView(CameraView): text_color=rl.Color(255, 255, 255, int(255 * 0.9)), alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER, alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) + self._model_status_label = UnifiedLabel("", 26, FontWeight.SEMI_BOLD, + alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER, + alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE, + wrap_text=False) self._fade_texture = gui_app.texture("icons_mici/onroad/onroad_fade.png") @@ -230,6 +236,8 @@ class AugmentedRoadView(CameraView): self._driver_state_renderer.set_position(self._rect.x + 16, self._rect.y + 10) self._driver_state_renderer.render() + self._render_model_status() + self._hud_renderer.set_can_draw_top_icons(alert_to_render is None) self._hud_renderer.set_wheel_critical_icon(alert_to_render is not None and not not_animating_out and alert_to_render.visual_alert == car.CarControl.HUDControl.VisualAlert.steerRequired) @@ -248,6 +256,33 @@ class AugmentedRoadView(CameraView): self._bookmark_icon.render(self.rect) + def _render_model_status(self): + model = ui_state.sm["modelV2"] + if ui_state.sm.seen["modelV2"] and model.timestampEof: + model_age_ms = max(0., (time.monotonic_ns() - model.timestampEof) / 1e6) + lag_text = f"{model_age_ms:.0f} ms" + else: + model_age_ms = float("inf") + lag_text = "-- ms" + + if ui_state.wgpu_enabled: + source, model_name = "WGPU", ui_state.wgpu_model_name + else: + source, model_name = "LOCAL", "DEVICE" + + if model_age_ms < 200: + color = rl.Color(100, 255, 120, 230) + elif model_age_ms < 500: + color = rl.Color(255, 210, 80, 230) + else: + color = rl.Color(255, 120, 80, 230) + + status_rect = rl.Rectangle(self._content_rect.x + 18, self._content_rect.y + self._content_rect.height - 58, 330, 42) + rl.draw_rectangle_rounded(status_rect, 0.35, 8, rl.Color(0, 0, 0, 165)) + self._model_status_label.set_text(f"{source} · {model_name} · {lag_text}") + self._model_status_label.set_text_color(color) + self._model_status_label.render(status_rect) + def _switch_stream_if_needed(self, sm): if sm['selfdriveState'].experimentalMode and WIDE_CAM in self.available_streams: v_ego = sm['carState'].vEgo diff --git a/openpilot/selfdrive/ui/ui_state.py b/openpilot/selfdrive/ui/ui_state.py index d2790a4df1..8a28aa0fea 100644 --- a/openpilot/selfdrive/ui/ui_state.py +++ b/openpilot/selfdrive/ui/ui_state.py @@ -83,6 +83,7 @@ class UIState(UIStateSP): self.usbgpu: bool = self.params.get_bool("UsbGpuPresent") self.usbgpu_compiled: bool = self.params.get_bool("UsbGpuCompiled") self.wgpu_enabled: bool = self.params.get_bool("WgpuEnabled") + self.wgpu_model_name: str = self.params.get("WgpuModelName") or "UNKNOWN" self.started: bool = False self.ignition: bool = False self.recording_audio: bool = False @@ -215,6 +216,7 @@ class UIState(UIStateSP): self.usbgpu = self.params.get_bool("UsbGpuPresent") self.usbgpu_compiled = self.params.get_bool("UsbGpuCompiled") self.wgpu_enabled = self.params.get_bool("WgpuEnabled") + self.wgpu_model_name = self.params.get("WgpuModelName") or "UNKNOWN" UIStateSP.update_params(self) diff --git a/openpilot/system/manager/process_config.py b/openpilot/system/manager/process_config.py index a72675385d..b8abc86031 100644 --- a/openpilot/system/manager/process_config.py +++ b/openpilot/system/manager/process_config.py @@ -95,6 +95,9 @@ def is_stock_model(started, params, CP: car.CarParams) -> bool: """Check if the active model runner is stock.""" return bool(get_active_model_runner(params, not started) == custom.ModelManagerSP.Runner.stock) +def not_wgpu(started: bool, params: Params, CP: car.CarParams) -> bool: + return not params.get_bool("WgpuEnabled") + def mapd_ready(started: bool, params: Params, CP: car.CarParams) -> bool: return bool(os.path.exists(Paths.mapd_root())) @@ -128,7 +131,7 @@ procs = [ PythonProcess("micd", "openpilot.system.micd", iscar), PythonProcess("timed", "openpilot.system.timed", always_run, enabled=not PC), - PythonProcess("modeld", "openpilot.selfdrive.modeld.modeld", and_(only_onroad, is_stock_model)), + PythonProcess("modeld", "openpilot.selfdrive.modeld.modeld", and_(and_(only_onroad, is_stock_model), not_wgpu)), PythonProcess("dmonitoringmodeld", "openpilot.selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(WEBCAM or not PC)), PythonProcess("sensord", "openpilot.system.sensord.sensord", only_onroad, enabled=not PC), @@ -177,7 +180,7 @@ procs = [ procs += [ # Models PythonProcess("models_manager", "openpilot.sunnypilot.models.manager", only_offroad), - NativeProcess("modeld_tinygrad", "openpilot/sunnypilot/modeld_v2", ["./modeld"], and_(only_onroad, is_tinygrad_model)), + NativeProcess("modeld_tinygrad", "openpilot/sunnypilot/modeld_v2", ["./modeld"], and_(and_(only_onroad, is_tinygrad_model), not_wgpu)), # Backup PythonProcess("backup_manager", "openpilot.sunnypilot.sunnylink.backups.manager", and_(only_offroad, sunnylink_ready_shim)), diff --git a/openpilot/tools/wgpu/README.md b/openpilot/tools/wgpu/README.md index 25208b026b..32b95a5a20 100644 --- a/openpilot/tools/wgpu/README.md +++ b/openpilot/tools/wgpu/README.md @@ -5,9 +5,9 @@ device over the existing Wi-Fi network. It reuses the existing HEVC camera stream, VisionIPC decoder, and cereal ZMQ bridge. This is for controlled bench testing only. Wi-Fi has no deterministic latency -or availability guarantee. Local `modeld` remains warm during a WGPU session; -the device-side helper switches only after receiving a fresh remote model and -restores local publication if the remote model is missing for 350 ms. +or availability guarantee. The device-side helper switches only after receiving +a fresh remote model and after manager has stopped the local model publisher. +It restores local `modeld` if the remote model is missing for one second. ## Build @@ -41,7 +41,8 @@ the small model on that class of laptop. Find the laptop's LAN IP address that the comma device can reach. The helper can start while onroad: it forwards camera/state while local `modeld` remains active, -then switches publication after the laptop produces a fresh valid model: +then performs an exclusive publisher handoff after the laptop produces a fresh +valid model: ```sh cd /data/openpilot @@ -58,23 +59,10 @@ python3 -m openpilot.tools.wgpu.host COMMA_IP Add `--big-model` after `COMMA_IP` to use the locally compiled big model. The first remote `carParams` packet can take up to 50 seconds. Stop either side -with Ctrl+C. A host disconnect automatically restores the warm local publisher -after a 350 ms timeout. Stop the device helper before changing branches or -rebooting. +with Ctrl+C. A host disconnect automatically stops remote publication and +restores local `modeld` after a one-second timeout. Stop the device helper before +changing branches or rebooting. -For stationary bench testing only, model lag can be changed from a blocking -event to a live warning showing model age and dropped frames. After ignition is -on and manager is running, enable the temporary override on the device: - -```sh -cd /data/openpilot -python3 -c 'from openpilot.common.params import Params; Params().put_bool("WgpuBenchMode", True)' -``` - -The override requires `WgpuEnabled` and clears on manager restart or the next -ignition-on transition. Disable it immediately with: - -```sh -cd /data/openpilot -python3 -c 'from openpilot.common.params import Params; Params().remove("WgpuBenchMode")' -``` +While WGPU is active, model lag does not create an engagement-blocking alert. +The mici onroad UI instead shows the active source (`LOCAL` or `WGPU`), remote +model size (`SMALL` or `BIG`), and live model-frame age. diff --git a/openpilot/tools/wgpu/device.py b/openpilot/tools/wgpu/device.py index 1d7e5820f2..36386fcd20 100644 --- a/openpilot/tools/wgpu/device.py +++ b/openpilot/tools/wgpu/device.py @@ -7,12 +7,13 @@ from pathlib import Path import openpilot.cereal.messaging as messaging from openpilot.common.params import Params -from openpilot.common.realtime import DT_MDL from openpilot.tools.wgpu.zmq import ZmqSubSocket MODEL_OUTPUTS = "modelV2,drivingModelData,cameraOdometry,modelDataV2SP" -REMOTE_MODEL_TIMEOUT = 0.35 +MODEL_PROCESSES = {"modeld", "modeld_tinygrad"} +REMOTE_MODEL_TIMEOUT = 1.0 +WGPU_STATUS = "wgpuStatus" ROOT = Path(__file__).resolve().parents[3] BRIDGE = ROOT / "openpilot/cereal/messaging/bridge" @@ -30,15 +31,35 @@ def handle_sigterm(*_) -> None: raise KeyboardInterrupt -def receive_fresh_model(sock: ZmqSubSocket) -> bool: +def receive_model(sock: ZmqSubSocket) -> tuple[bool, float]: raw = sock.receive(non_blocking=True) if raw is None: - return False + return False, float("inf") event = messaging.log_from_bytes(raw) if event.which() != "modelV2" or not event.valid: - return False + return True, float("inf") model_age = (time.monotonic_ns() - event.modelV2.timestampEof) / 1e9 - return 0 <= model_age < REMOTE_MODEL_TIMEOUT + return True, model_age + + +def receive_model_name(sock: ZmqSubSocket) -> str | None: + raw = sock.receive(non_blocking=True) + if raw is None: + return None + model_name = raw.decode(errors="replace").upper() + return model_name if model_name in ("SMALL", "BIG") else None + + +def wait_for_local_modeld_stop(timeout: float = 10.) -> None: + sm = messaging.SubMaster(["managerState"]) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + sm.update(100) + if sm.seen["managerState"]: + running = {p.name for p in sm["managerState"].processes if p.running} + if not running.intersection(MODEL_PROCESSES): + return + raise RuntimeError("timed out waiting for local modeld publisher to stop") def main() -> None: @@ -57,22 +78,31 @@ def main() -> None: # Keep local modeld publishing while the remote model connects and warms up. forward = subprocess.Popen([str(BRIDGE)]) remote_model = ZmqSubSocket("modelV2", args.host, conflate=True) + remote_status = ZmqSubSocket(WGPU_STATUS, args.host, conflate=True) print(f"forwarding camera/state to {args.host}; waiting for a fresh remote model") - while not receive_fresh_model(remote_model): + model_name = None + while True: + received, model_age = receive_model(remote_model) + model_name = receive_model_name(remote_status) or model_name + if received and 0 <= model_age < REMOTE_MODEL_TIMEOUT and model_name is not None: + break if forward.poll() is not None: raise RuntimeError(f"forward bridge exited with status {forward.returncode}") time.sleep(0.05) - # Local modeld remains warm but stops publishing when this flag changes. + # Stop the local publisher before attaching the reverse bridge. msgq permits + # only one publisher for each model service. + params.put("WgpuModelName", model_name, block=True) params.put_bool("WgpuEnabled", True, block=True) wgpu_enabled = True - time.sleep(2 * DT_MDL) + wait_for_local_modeld_stop() reverse = subprocess.Popen([str(BRIDGE), args.host, MODEL_OUTPUTS]) print("wgpu active; Ctrl+C or loss of the remote model restores local modeld") last_remote_model = time.monotonic() while True: - if receive_fresh_model(remote_model): + received, _ = receive_model(remote_model) + if received: last_remote_model = time.monotonic() if forward.poll() is not None: raise RuntimeError(f"forward bridge exited with status {forward.returncode}") @@ -82,11 +112,12 @@ def main() -> None: raise RuntimeError("remote model timed out; restoring local modeld") time.sleep(0.05) finally: - # Stop remote publication before allowing the warm local publisher to resume. + # Stop remote publication before allowing the local publisher to restart. if reverse is not None and reverse.poll() is None: stop_process(reverse) if wgpu_enabled: params.put_bool("WgpuEnabled", False, block=True) + params.remove("WgpuModelName") if forward is not None and forward.poll() is None: stop_process(forward) print("wgpu disabled; local modeld restored") diff --git a/openpilot/tools/wgpu/host.py b/openpilot/tools/wgpu/host.py index 7f9e731ae4..2d7fea4688 100644 --- a/openpilot/tools/wgpu/host.py +++ b/openpilot/tools/wgpu/host.py @@ -7,9 +7,12 @@ import sys import time from pathlib import Path +from openpilot.tools.wgpu.zmq import ZmqPubMaster + ROOT = Path(__file__).resolve().parents[3] CAMERASTREAM = ROOT / "openpilot/tools/camerastream/compressed_vipc.py" +WGPU_STATUS = "wgpuStatus" def stop_process(proc: subprocess.Popen) -> None: @@ -36,10 +39,13 @@ def main() -> None: if args.big_model: model_args.append("--big-model") model = subprocess.Popen(model_args, cwd=ROOT, start_new_session=True) + status = ZmqPubMaster([WGPU_STATUS]) + model_name = b"BIG" if args.big_model else b"SMALL" procs = {"camera bridge": camera, "modeld": model} try: while all(proc.poll() is None for proc in procs.values()): + status.send_raw(WGPU_STATUS, model_name) time.sleep(0.25) failed_name, failed = next((name, proc) for name, proc in procs.items() if proc.poll() is not None) raise RuntimeError(f"wgpu {failed_name} exited with status {failed.returncode}") diff --git a/openpilot/tools/wgpu/zmq.py b/openpilot/tools/wgpu/zmq.py index 0300dcb7cb..64c74d9acb 100644 --- a/openpilot/tools/wgpu/zmq.py +++ b/openpilot/tools/wgpu/zmq.py @@ -88,3 +88,6 @@ class ZmqPubMaster: def send(self, service: str, message) -> None: self.sockets[service].send(message.to_bytes(), flags=zmq.NOBLOCK) + + def send_raw(self, service: str, data: bytes) -> None: + self.sockets[service].send(data, flags=zmq.NOBLOCK)