diff --git a/common/watchdog.py b/common/watchdog.py index ddb6f744e..a60dc1d73 100644 --- a/common/watchdog.py +++ b/common/watchdog.py @@ -2,21 +2,27 @@ import os import time import struct from openpilot.system.hardware.hw import Paths +from openpilot.common.swaglog import cloudlog WATCHDOG_FN = f"{Paths.shm_path()}/wd_" _LAST_KICK = 0.0 +_LAST_ERROR_LOG = 0.0 def kick_watchdog(): - global _LAST_KICK + global _LAST_KICK, _LAST_ERROR_LOG current_time = time.monotonic() if current_time - _LAST_KICK < 1.0: - return + return True try: with open(f"{WATCHDOG_FN}{os.getpid()}", 'wb') as f: f.write(struct.pack('= 5.0: + cloudlog.error(f"watchdog kick failed for pid {os.getpid()}: {e}") + _LAST_ERROR_LOG = current_time + return False diff --git a/panda/board/obj/body_h7.bin.signed b/panda/board/obj/body_h7.bin.signed index 16301e3b0..491ce0d25 100644 Binary files a/panda/board/obj/body_h7.bin.signed and b/panda/board/obj/body_h7.bin.signed differ diff --git a/panda/board/obj/body_h7/bootstub.elf b/panda/board/obj/body_h7/bootstub.elf index 4d605163b..3495aa43f 100755 Binary files a/panda/board/obj/body_h7/bootstub.elf and b/panda/board/obj/body_h7/bootstub.elf differ diff --git a/panda/board/obj/body_h7/main.bin b/panda/board/obj/body_h7/main.bin index 4b0087749..06466228b 100755 Binary files a/panda/board/obj/body_h7/main.bin and b/panda/board/obj/body_h7/main.bin differ diff --git a/panda/board/obj/body_h7/main.elf b/panda/board/obj/body_h7/main.elf index ef342db39..6e7e82535 100755 Binary files a/panda/board/obj/body_h7/main.elf and b/panda/board/obj/body_h7/main.elf differ diff --git a/panda/board/obj/bootstub.body_h7.bin b/panda/board/obj/bootstub.body_h7.bin index a841786a1..42cc9a85c 100755 Binary files a/panda/board/obj/bootstub.body_h7.bin and b/panda/board/obj/bootstub.body_h7.bin differ diff --git a/panda/board/obj/bootstub.panda.bin b/panda/board/obj/bootstub.panda.bin index 58c1808f2..8b8d3c20b 100755 Binary files a/panda/board/obj/bootstub.panda.bin and b/panda/board/obj/bootstub.panda.bin differ diff --git a/panda/board/obj/bootstub.panda_h7.bin b/panda/board/obj/bootstub.panda_h7.bin index b73ac71e1..f1be2c4ef 100755 Binary files a/panda/board/obj/bootstub.panda_h7.bin and b/panda/board/obj/bootstub.panda_h7.bin differ diff --git a/panda/board/obj/bootstub.panda_h7_remote.bin b/panda/board/obj/bootstub.panda_h7_remote.bin index b73ac71e1..f1be2c4ef 100755 Binary files a/panda/board/obj/bootstub.panda_h7_remote.bin and b/panda/board/obj/bootstub.panda_h7_remote.bin differ diff --git a/panda/board/obj/bootstub.panda_jungle_h7.bin b/panda/board/obj/bootstub.panda_jungle_h7.bin index 7cdbfe8e6..e5b4971e5 100755 Binary files a/panda/board/obj/bootstub.panda_jungle_h7.bin and b/panda/board/obj/bootstub.panda_jungle_h7.bin differ diff --git a/panda/board/obj/bootstub.panda_remote.bin b/panda/board/obj/bootstub.panda_remote.bin index 58c1808f2..8b8d3c20b 100755 Binary files a/panda/board/obj/bootstub.panda_remote.bin and b/panda/board/obj/bootstub.panda_remote.bin differ diff --git a/panda/board/obj/gitversion.h b/panda/board/obj/gitversion.h index 409c55c92..4d9982d22 100644 --- a/panda/board/obj/gitversion.h +++ b/panda/board/obj/gitversion.h @@ -1,2 +1,2 @@ extern const uint8_t gitversion[19]; -const uint8_t gitversion[19] = "DEV-f1feaa2c-DEBUG"; +const uint8_t gitversion[19] = "DEV-ad948aa9-DEBUG"; diff --git a/panda/board/obj/panda.bin.signed b/panda/board/obj/panda.bin.signed index 60c4a4d64..ce406e43b 100644 Binary files a/panda/board/obj/panda.bin.signed and b/panda/board/obj/panda.bin.signed differ diff --git a/panda/board/obj/panda/bootstub.elf b/panda/board/obj/panda/bootstub.elf index 95af89862..349a5123a 100755 Binary files a/panda/board/obj/panda/bootstub.elf and b/panda/board/obj/panda/bootstub.elf differ diff --git a/panda/board/obj/panda/main.bin b/panda/board/obj/panda/main.bin index 58185363e..ed7299b7f 100755 Binary files a/panda/board/obj/panda/main.bin and b/panda/board/obj/panda/main.bin differ diff --git a/panda/board/obj/panda/main.elf b/panda/board/obj/panda/main.elf index ebdeb2574..62ba4f0d3 100755 Binary files a/panda/board/obj/panda/main.elf and b/panda/board/obj/panda/main.elf differ diff --git a/panda/board/obj/panda_h7.bin.signed b/panda/board/obj/panda_h7.bin.signed index cf14569c6..cc9665bba 100644 Binary files a/panda/board/obj/panda_h7.bin.signed and b/panda/board/obj/panda_h7.bin.signed differ diff --git a/panda/board/obj/panda_h7/bootstub.elf b/panda/board/obj/panda_h7/bootstub.elf index 463abce51..b28bc64bd 100755 Binary files a/panda/board/obj/panda_h7/bootstub.elf and b/panda/board/obj/panda_h7/bootstub.elf differ diff --git a/panda/board/obj/panda_h7/main.bin b/panda/board/obj/panda_h7/main.bin index 6a00ce87e..5ca120185 100755 Binary files a/panda/board/obj/panda_h7/main.bin and b/panda/board/obj/panda_h7/main.bin differ diff --git a/panda/board/obj/panda_h7/main.elf b/panda/board/obj/panda_h7/main.elf index 761399c94..f765122bf 100755 Binary files a/panda/board/obj/panda_h7/main.elf and b/panda/board/obj/panda_h7/main.elf differ diff --git a/panda/board/obj/panda_h7_remote.bin.signed b/panda/board/obj/panda_h7_remote.bin.signed index 0fcaf050c..3e013fa44 100644 Binary files a/panda/board/obj/panda_h7_remote.bin.signed and b/panda/board/obj/panda_h7_remote.bin.signed differ diff --git a/panda/board/obj/panda_h7_remote/bootstub.elf b/panda/board/obj/panda_h7_remote/bootstub.elf index 9201f9c8c..89c554dcd 100755 Binary files a/panda/board/obj/panda_h7_remote/bootstub.elf and b/panda/board/obj/panda_h7_remote/bootstub.elf differ diff --git a/panda/board/obj/panda_h7_remote/main.bin b/panda/board/obj/panda_h7_remote/main.bin index de8cb3036..2aaff35e8 100755 Binary files a/panda/board/obj/panda_h7_remote/main.bin and b/panda/board/obj/panda_h7_remote/main.bin differ diff --git a/panda/board/obj/panda_h7_remote/main.elf b/panda/board/obj/panda_h7_remote/main.elf index 453f15a23..228ef2b20 100755 Binary files a/panda/board/obj/panda_h7_remote/main.elf and b/panda/board/obj/panda_h7_remote/main.elf differ diff --git a/panda/board/obj/panda_jungle_h7.bin.signed b/panda/board/obj/panda_jungle_h7.bin.signed index d815b9f3a..5dbea9d98 100644 Binary files a/panda/board/obj/panda_jungle_h7.bin.signed and b/panda/board/obj/panda_jungle_h7.bin.signed differ diff --git a/panda/board/obj/panda_jungle_h7/bootstub.elf b/panda/board/obj/panda_jungle_h7/bootstub.elf index 7ebbef8a3..38bc47faa 100755 Binary files a/panda/board/obj/panda_jungle_h7/bootstub.elf and b/panda/board/obj/panda_jungle_h7/bootstub.elf differ diff --git a/panda/board/obj/panda_jungle_h7/main.bin b/panda/board/obj/panda_jungle_h7/main.bin index ba8df3e60..30a17132d 100755 Binary files a/panda/board/obj/panda_jungle_h7/main.bin and b/panda/board/obj/panda_jungle_h7/main.bin differ diff --git a/panda/board/obj/panda_jungle_h7/main.elf b/panda/board/obj/panda_jungle_h7/main.elf index 540bf0b9d..41d11117a 100755 Binary files a/panda/board/obj/panda_jungle_h7/main.elf and b/panda/board/obj/panda_jungle_h7/main.elf differ diff --git a/panda/board/obj/panda_remote.bin.signed b/panda/board/obj/panda_remote.bin.signed index df480c416..7805d4573 100644 Binary files a/panda/board/obj/panda_remote.bin.signed and b/panda/board/obj/panda_remote.bin.signed differ diff --git a/panda/board/obj/panda_remote/bootstub.elf b/panda/board/obj/panda_remote/bootstub.elf index daabcf07f..a0ede4b9d 100755 Binary files a/panda/board/obj/panda_remote/bootstub.elf and b/panda/board/obj/panda_remote/bootstub.elf differ diff --git a/panda/board/obj/panda_remote/main.bin b/panda/board/obj/panda_remote/main.bin index 86f494154..d5d55aac9 100755 Binary files a/panda/board/obj/panda_remote/main.bin and b/panda/board/obj/panda_remote/main.bin differ diff --git a/panda/board/obj/panda_remote/main.elf b/panda/board/obj/panda_remote/main.elf index b756c885e..308f84b98 100755 Binary files a/panda/board/obj/panda_remote/main.elf and b/panda/board/obj/panda_remote/main.elf differ diff --git a/panda/board/obj/version b/panda/board/obj/version index df1bd3e78..230301a37 100644 --- a/panda/board/obj/version +++ b/panda/board/obj/version @@ -1 +1 @@ -DEV-f1feaa2c-DEBUG \ No newline at end of file +DEV-ad948aa9-DEBUG \ No newline at end of file diff --git a/selfdrive/car/cruise.py b/selfdrive/car/cruise.py index 589bd36f4..76b338d11 100644 --- a/selfdrive/car/cruise.py +++ b/selfdrive/car/cruise.py @@ -50,14 +50,6 @@ class VCruiseHelper: def _get_cruise_delta_interval(interval: float | None) -> float: return interval if isinstance(interval, (int, float)) and interval > 0 else 1.0 - def _normalize_initialized_v_cruise(self, v_cruise_kph: float, starpilot_toggles: SimpleNamespace) -> float: - cruise_increase = self._get_cruise_delta_interval(starpilot_toggles.cruise_increase) - if cruise_increase % 5 != 0: - return v_cruise_kph - - v_cruise_delta = self._get_short_press_delta(starpilot_toggles.is_metric, starpilot_toggles) - return round(round(v_cruise_kph / v_cruise_delta) * v_cruise_delta, 1) - @property def v_cruise_initialized(self): return self.v_cruise_kph != V_CRUISE_UNSET @@ -162,7 +154,9 @@ class VCruiseHelper: and self.v_cruise_initialized or (self.gm_cc_only and resume_prev_button)): self.v_cruise_kph = self.v_cruise_kph_last elif desired_speed_limit > 0 and getattr(starpilot_toggles, "set_speed_limit", False): - initialized_speed_limit_kph = self._normalize_initialized_v_cruise(desired_speed_limit * CV.MS_TO_KPH, starpilot_toggles) + # Respect the exact SLC limit+offset on engage instead of snapping upward to + # the custom cruise-button interval. + initialized_speed_limit_kph = round(desired_speed_limit * CV.MS_TO_KPH, 1) self.v_cruise_kph = float(np.clip(initialized_speed_limit_kph, V_CRUISE_MIN, V_CRUISE_MAX)) else: self.v_cruise_kph = int(round(np.clip(CS.vEgo * CV.MS_TO_KPH, engage_floor_kph, V_CRUISE_MAX))) diff --git a/selfdrive/car/tests/test_cruise_speed.py b/selfdrive/car/tests/test_cruise_speed.py index c6a3b78ba..ab138f34c 100644 --- a/selfdrive/car/tests/test_cruise_speed.py +++ b/selfdrive/car/tests/test_cruise_speed.py @@ -207,6 +207,22 @@ class TestVCruiseHelper: assert self.v_cruise_helper.v_cruise_kph == pytest.approx(55 * CV.MPH_TO_KPH) + def test_initialize_v_cruise_keeps_exact_speed_limit_offset(self): + self.reset_cruise_speed_state() + self.starpilot_toggles.set_speed_limit = True + self.starpilot_toggles.cruise_increase = 5 + + desired_speed_limit = 38 * CV.MPH_TO_MS + self.v_cruise_helper.initialize_v_cruise( + car.CarState(vEgo=70 * CV.MPH_TO_MS), + experimental_mode=False, + resume_prev_button=False, + starpilot_toggles=self.starpilot_toggles, + desired_speed_limit=desired_speed_limit, + ) + + assert self.v_cruise_helper.v_cruise_kph == pytest.approx(38 * CV.MPH_TO_KPH) + def test_speed_limit_confirmation_does_not_adjust_cruise(self): self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False) initial_v_cruise_kph = self.v_cruise_helper.v_cruise_kph @@ -260,3 +276,31 @@ class TestVCruiseHelper: ) assert self.v_cruise_helper.v_cruise_kph == pytest.approx(initial_v_cruise_kph + IMPERIAL_INCREMENT) + + def test_zero_custom_cruise_toggles_fall_back_to_single_step(self): + self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False) + initial_v_cruise_kph = self.v_cruise_helper.v_cruise_kph + self.starpilot_toggles.cruise_increase = 0 + self.starpilot_toggles.cruise_increase_long = 0 + + pressed_cs = car.CarState(cruiseState={"available": True}) + pressed_cs.buttonEvents = [ButtonEvent(type=ButtonType.accelCruise, pressed=True)] + self.v_cruise_helper.update_v_cruise( + pressed_cs, + enabled=True, + is_metric=False, + speed_limit_changed=False, + starpilot_toggles=self.starpilot_toggles, + ) + + released_cs = car.CarState(cruiseState={"available": True}) + released_cs.buttonEvents = [ButtonEvent(type=ButtonType.accelCruise, pressed=False)] + self.v_cruise_helper.update_v_cruise( + released_cs, + enabled=True, + is_metric=False, + speed_limit_changed=False, + starpilot_toggles=self.starpilot_toggles, + ) + + assert self.v_cruise_helper.v_cruise_kph == pytest.approx(initial_v_cruise_kph + IMPERIAL_INCREMENT) diff --git a/selfdrive/ui/layouts/settings/starpilot/longitudinal.py b/selfdrive/ui/layouts/settings/starpilot/longitudinal.py index db4259d24..d2e492904 100644 --- a/selfdrive/ui/layouts/settings/starpilot/longitudinal.py +++ b/selfdrive/ui/layouts/settings/starpilot/longitudinal.py @@ -576,7 +576,7 @@ class StarPilotLongitudinalQOLLayout(StarPilotPanel): { "title": tr_noop("Cruise Interval"), "type": "value", - "get_value": lambda: f"{self._params.get_int('CustomCruise')} mph", + "get_value": lambda: f"{max(1, self._params.get_int('CustomCruise'))} mph", "on_click": lambda: self._show_speed_selector("CustomCruise"), "color": "#597497", "visible": lambda: self._params.get_bool("QOLLongitudinal"), @@ -584,7 +584,7 @@ class StarPilotLongitudinalQOLLayout(StarPilotPanel): { "title": tr_noop("Cruise Long"), "type": "value", - "get_value": lambda: f"{self._params.get_int('CustomCruiseLong')} mph", + "get_value": lambda: f"{max(1, self._params.get_int('CustomCruiseLong'))} mph", "on_click": lambda: self._show_speed_selector("CustomCruiseLong"), "color": "#597497", "visible": lambda: self._params.get_bool("QOLLongitudinal"), @@ -662,7 +662,8 @@ class StarPilotLongitudinalQOLLayout(StarPilotPanel): self._params.put_int(key, int(val)) self._rebuild_grid() - gui_app.set_modal_overlay(AetherSliderDialog(tr(key), 0, 100, 1, self._params.get_int(key), on_close, unit=" mph", color="#597497")) + current = max(1, self._params.get_int(key)) + gui_app.set_modal_overlay(AetherSliderDialog(tr(key), 1, 100, 1, current, on_close, unit=" mph", color="#597497")) def _show_int_selector(self, key, min_v, max_v, unit=""): def on_close(res, val): diff --git a/selfdrive/ui/stall_monitor.py b/selfdrive/ui/stall_monitor.py new file mode 100644 index 000000000..0e9ced8ca --- /dev/null +++ b/selfdrive/ui/stall_monitor.py @@ -0,0 +1,131 @@ +import os +import sys +import time +import traceback +import threading +from pathlib import Path + +from openpilot.common.swaglog import cloudlog + + +def _default_dump_dir() -> Path: + for candidate in ("/data/log", "/tmp"): + if os.path.isdir(candidate) and os.access(candidate, os.W_OK): + return Path(candidate) + return Path.cwd() + + +class UIStallMonitor: + def __init__(self, name: str): + self._name = name + self._threshold_s = float(os.getenv("UI_STALL_PROBE_MAX_DT", "5")) + self._poll_s = float(os.getenv("UI_STALL_PROBE_POLL_DT", "0.25")) + self._dump_dir = _default_dump_dir() + self._main_thread_id = threading.get_ident() + + now = time.monotonic() + self._last_progress = now + self._phase = "init" + self._phase_entered = now + self._stall_reported = False + self._stalled_since = now + self._stalled_phase = self._phase + + self._lock = threading.Lock() + self._stop_event = threading.Event() + self._thread = threading.Thread(target=self._run, name=f"{name}_stall_probe", daemon=True) + + def start(self) -> None: + if self._threshold_s <= 0.0: + return + self._thread.start() + + def stop(self) -> None: + if self._threshold_s <= 0.0: + return + self._stop_event.set() + self._thread.join(timeout=1.0) + + def progress(self, phase: str) -> None: + now = time.monotonic() + recovered = None + + with self._lock: + if phase != self._phase: + self._phase = phase + self._phase_entered = now + self._last_progress = now + + if self._stall_reported: + recovered = (now - self._stalled_since, self._stalled_phase, phase) + self._stall_reported = False + + if recovered is not None: + stalled_for_s, stalled_phase, current_phase = recovered + cloudlog.warning(f"{self._name} stall recovered after {stalled_for_s:.1f}s (stalled_phase={stalled_phase}, current_phase={current_phase})") + + def _run(self) -> None: + while not self._stop_event.wait(self._poll_s): + now = time.monotonic() + with self._lock: + stalled_for_s = now - self._last_progress + phase = self._phase + phase_for_s = now - self._phase_entered + already_reported = self._stall_reported + + if stalled_for_s < self._threshold_s or already_reported: + continue + + dump = self._build_dump(now, phase, stalled_for_s, phase_for_s) + dump_path = self._write_dump(dump) + with self._lock: + self._stall_reported = True + self._stalled_since = now + self._stalled_phase = phase + + preview = self._main_thread_preview() + path_s = str(dump_path) if dump_path is not None else "" + cloudlog.error(f"{self._name} main loop stalled for {stalled_for_s:.1f}s in phase={phase} (phase_for={phase_for_s:.1f}s) dump={path_s}\n{preview}") + + def _build_dump(self, now: float, phase: str, stalled_for_s: float, phase_for_s: float) -> str: + frames = sys._current_frames() + threads = {thread.ident: thread for thread in threading.enumerate()} + lines = [ + f"name={self._name}", + f"pid={os.getpid()}", + f"wall_time={time.strftime('%Y-%m-%dT%H:%M:%S%z')}", + f"monotonic={now:.6f}", + f"stalled_for_s={stalled_for_s:.3f}", + f"phase={phase}", + f"phase_for_s={phase_for_s:.3f}", + "", + ] + + ordered_idents = sorted(frames.keys(), key=lambda ident: ident != self._main_thread_id) + for ident in ordered_idents: + thread = threads.get(ident) + name = thread.name if thread is not None else "" + daemon = thread.daemon if thread is not None else None + lines.append(f"Thread {name} ident={ident} daemon={daemon}") + lines.extend(traceback.format_stack(frames[ident])) + lines.append("") + + return "".join(line if line.endswith("\n") else f"{line}\n" for line in lines) + + def _main_thread_preview(self) -> str: + frame = sys._current_frames().get(self._main_thread_id) + if frame is None: + return "main_thread_stack=" + stack_lines = traceback.format_stack(frame) + preview = "".join(stack_lines[-8:]).strip() + return f"main_thread_stack:\n{preview}" if preview else "main_thread_stack=" + + def _write_dump(self, dump: str) -> Path | None: + timestamp = time.strftime("%Y%m%d_%H%M%S") + path = self._dump_dir / f"{self._name}_stall_{os.getpid()}_{timestamp}.log" + try: + path.write_text(dump) + return path + except OSError as e: + cloudlog.error(f"failed to write {self._name} stall dump to {path}: {e}") + return None diff --git a/selfdrive/ui/ui.cc b/selfdrive/ui/ui.cc index d29752aa8..0b9601a08 100644 --- a/selfdrive/ui/ui.cc +++ b/selfdrive/ui/ui.cc @@ -1,7 +1,21 @@ #include "selfdrive/ui/ui.h" #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include @@ -14,6 +28,178 @@ #define BACKLIGHT_DT 0.05 #define BACKLIGHT_TS 10.00 +namespace { + +enum class UIStallPhase { + INIT = 0, + UPDATE_START, + AFTER_SOCKETS, + AFTER_STATE, + AFTER_STATUS, + AFTER_WATCHDOG, + AFTER_EMIT, + AFTER_FS_UPDATE, + IDLE, +}; + +std::atomic ui_stall_last_progress_ns{0}; +std::atomic ui_stall_phase{static_cast(UIStallPhase::INIT)}; +std::atomic ui_stall_frame{0}; +std::atomic ui_stall_reported{false}; +std::atomic ui_stall_reported_ns{0}; +std::atomic ui_stall_reported_phase{static_cast(UIStallPhase::INIT)}; +std::atomic ui_stall_dump_fd{-1}; +pthread_t ui_main_thread{}; + +double read_env_double(const char *name, double default_value) { + const char *value = std::getenv(name); + if (value == nullptr || *value == '\0') { + return default_value; + } + + char *end = nullptr; + double parsed = std::strtod(value, &end); + if (end == value || (end != nullptr && *end != '\0') || parsed <= 0.0) { + return default_value; + } + return parsed; +} + +const char *ui_stall_phase_name(UIStallPhase phase) { + switch (phase) { + case UIStallPhase::INIT: return "init"; + case UIStallPhase::UPDATE_START: return "update_start"; + case UIStallPhase::AFTER_SOCKETS: return "after_sockets"; + case UIStallPhase::AFTER_STATE: return "after_state"; + case UIStallPhase::AFTER_STATUS: return "after_status"; + case UIStallPhase::AFTER_WATCHDOG: return "after_watchdog"; + case UIStallPhase::AFTER_EMIT: return "after_emit"; + case UIStallPhase::AFTER_FS_UPDATE: return "after_fs_update"; + case UIStallPhase::IDLE: return "idle"; + } + return "unknown"; +} + +std::string ui_stall_dump_dir() { + return access("/data/log", W_OK) == 0 ? "/data/log" : "/tmp"; +} + +void ui_stall_signal_handler(int sig) { + const int fd = ui_stall_dump_fd.load(std::memory_order_relaxed); + if (fd < 0) { + return; + } + + char header[256]; + const pid_t tid = static_cast(syscall(SYS_gettid)); + const int header_len = std::snprintf(header, sizeof(header), + "=== UI stall backtrace (signal=%d pid=%d tid=%d) ===\n", + sig, getpid(), tid); + if (header_len > 0) { + write(fd, header, header_len); + } + + void *frames[128]; + const int frame_count = backtrace(frames, 128); + backtrace_symbols_fd(frames, frame_count, fd); + write(fd, "\n", 1); +} + +void ui_stall_progress(UIStallPhase phase, uint64_t frame = 0) { + const uint64_t now = nanos_since_boot(); + ui_stall_phase.store(static_cast(phase), std::memory_order_relaxed); + ui_stall_frame.store(frame, std::memory_order_relaxed); + ui_stall_last_progress_ns.store(now, std::memory_order_relaxed); + + if (ui_stall_reported.exchange(false, std::memory_order_relaxed)) { + const uint64_t stall_started = ui_stall_reported_ns.load(std::memory_order_relaxed); + const UIStallPhase stalled_phase = static_cast(ui_stall_reported_phase.load(std::memory_order_relaxed)); + const double stalled_for_s = stall_started == 0 ? 0.0 : (now - stall_started) / 1e9; + LOGW("UI stall recovered after %.1fs (stalled_phase=%s current_phase=%s frame=%llu)", + stalled_for_s, + ui_stall_phase_name(stalled_phase), + ui_stall_phase_name(phase), + static_cast(frame)); + } +} + +void start_ui_stall_monitor() { + static std::once_flag once; + std::call_once(once, [] { + ui_main_thread = pthread_self(); + std::signal(SIGUSR1, ui_stall_signal_handler); + ui_stall_progress(UIStallPhase::INIT, 0); + + const double stall_probe_dt = read_env_double("UI_STALL_PROBE_MAX_DT", 5.0); + if (stall_probe_dt <= 0.0) { + return; + } + + std::thread([stall_probe_dt]() { + using namespace std::chrono_literals; + constexpr auto poll_interval = 250ms; + + while (true) { + std::this_thread::sleep_for(poll_interval); + + const uint64_t now = nanos_since_boot(); + const uint64_t last_progress = ui_stall_last_progress_ns.load(std::memory_order_relaxed); + if (last_progress == 0) { + continue; + } + + const double stalled_for_s = (now - last_progress) / 1e9; + if (stalled_for_s < stall_probe_dt) { + continue; + } + + bool expected = false; + if (!ui_stall_reported.compare_exchange_strong(expected, true, std::memory_order_relaxed)) { + continue; + } + + const UIStallPhase phase = static_cast(ui_stall_phase.load(std::memory_order_relaxed)); + const uint64_t frame = ui_stall_frame.load(std::memory_order_relaxed); + ui_stall_reported_ns.store(now, std::memory_order_relaxed); + ui_stall_reported_phase.store(static_cast(phase), std::memory_order_relaxed); + + const std::string path = ui_stall_dump_dir() + "/qt_ui_stall_" + std::to_string(getpid()) + "_" + std::to_string(now) + ".log"; + int fd = open(path.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0644); + if (fd >= 0) { + char header[256]; + const int header_len = std::snprintf(header, sizeof(header), + "phase=%s frame=%llu stalled_for_s=%.3f\n", + ui_stall_phase_name(phase), + static_cast(frame), + stalled_for_s); + if (header_len > 0) { + write(fd, header, header_len); + } + + ui_stall_dump_fd.store(fd, std::memory_order_relaxed); + pthread_kill(ui_main_thread, SIGUSR1); + std::this_thread::sleep_for(50ms); + ui_stall_dump_fd.store(-1, std::memory_order_relaxed); + close(fd); + LOGE("UI main thread stalled for %.1fs (phase=%s frame=%llu dump=%s)", + stalled_for_s, + ui_stall_phase_name(phase), + static_cast(frame), + path.c_str()); + } else { + LOGE("UI main thread stalled for %.1fs (phase=%s frame=%llu dump_open_failed errno=%d)", + stalled_for_s, + ui_stall_phase_name(phase), + static_cast(frame), + errno); + } + } + }).detach(); + }); +} + +} // namespace + static void update_sockets(UIState *s) { s->sm->update(0); } @@ -144,6 +330,7 @@ void UIState::updateStatus(StarPilotUIState *fs) { } UIState::UIState(QObject *parent) : QObject(parent) { + start_ui_stall_monitor(); sm = std::make_unique(std::vector{ "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState", "pandaStates", "carParams", "driverMonitoringState", "carState", "driverStateV2", @@ -156,17 +343,26 @@ UIState::UIState(QObject *parent) : QObject(parent) { timer = new QTimer(this); QObject::connect(timer, &QTimer::timeout, this, &UIState::update); timer->start(1000 / UI_FREQ); + ui_stall_progress(UIStallPhase::IDLE, sm->frame); } void UIState::update() { + ui_stall_progress(UIStallPhase::UPDATE_START, sm->frame); update_sockets(this); + ui_stall_progress(UIStallPhase::AFTER_SOCKETS, sm->frame); update_state(this, starpilotUIState()); + ui_stall_progress(UIStallPhase::AFTER_STATE, sm->frame); updateStatus(starpilotUIState()); + ui_stall_progress(UIStallPhase::AFTER_STATUS, sm->frame); if (sm->frame % UI_FREQ == 0) { - watchdog_kick(nanos_since_boot()); + if (!watchdog_kick(nanos_since_boot())) { + LOGE("UI watchdog kick failed at frame %llu", static_cast(sm->frame)); + } } + ui_stall_progress(UIStallPhase::AFTER_WATCHDOG, sm->frame); emit uiUpdate(*this, *starpilotUIState()); + ui_stall_progress(UIStallPhase::AFTER_EMIT, sm->frame); StarPilotUIState *fs = starpilotUIState(); StarPilotUIScene &starpilot_scene = fs->starpilot_scene; @@ -177,6 +373,8 @@ void UIState::update() { } fs->update(); + ui_stall_progress(UIStallPhase::AFTER_FS_UPDATE, sm->frame); + ui_stall_progress(UIStallPhase::IDLE, sm->frame); } Device::Device(QObject *parent) : brightness_filter(BACKLIGHT_OFFROAD, BACKLIGHT_TS, BACKLIGHT_DT), QObject(parent) { diff --git a/selfdrive/ui/ui.py b/selfdrive/ui/ui.py index ea22b5399..78651d7b2 100755 --- a/selfdrive/ui/ui.py +++ b/selfdrive/ui/ui.py @@ -6,6 +6,7 @@ from openpilot.system.hardware import TICI from openpilot.common.realtime import config_realtime_process, set_core_affinity from openpilot.common.watchdog import kick_watchdog from openpilot.system.ui.lib.application import gui_app +from openpilot.selfdrive.ui.stall_monitor import UIStallMonitor from openpilot.selfdrive.ui.layouts.main import MainLayout from openpilot.selfdrive.ui.mici.layouts.main import MiciMainLayout from openpilot.selfdrive.ui.ui_state import ui_state @@ -21,18 +22,33 @@ def main(): else: main_layout = MiciMainLayout() main_layout.set_rect(rl.Rectangle(0, 0, gui_app.width, gui_app.height)) - for should_render in gui_app.render(): - kick_watchdog() - ui_state.update() - if should_render: - main_layout.render() + stall_monitor = UIStallMonitor("raylib_ui") + gui_app.set_progress_hook(stall_monitor.progress) + stall_monitor.progress("ui.loop_ready") + stall_monitor.start() - # reaffine after power save offlines our core - if TICI and os.sched_getaffinity(0) != cores: - try: - set_core_affinity(list(cores)) - except OSError: - pass + try: + for should_render in gui_app.render(): + stall_monitor.progress("ui.loop_iteration") + kick_watchdog() + stall_monitor.progress("ui.after_watchdog") + ui_state.update() + stall_monitor.progress("ui.after_state_update") + if should_render: + stall_monitor.progress("ui.before_layout_render") + main_layout.render() + stall_monitor.progress("ui.after_layout_render") + + # reaffine after power save offlines our core + if TICI and os.sched_getaffinity(0) != cores: + try: + set_core_affinity(list(cores)) + except OSError: + pass + stall_monitor.progress("ui.loop_idle") + finally: + gui_app.set_progress_hook(None) + stall_monitor.stop() if __name__ == "__main__": diff --git a/system/ui/lib/application.py b/system/ui/lib/application.py index 6a6648ea6..90ec25a83 100644 --- a/system/ui/lib/application.py +++ b/system/ui/lib/application.py @@ -224,6 +224,7 @@ class GuiApplication: self._frame = 0 self._window_close_requested = False self._trace_log_callback = None + self._progress_hook: Callable[[str], None] | None = None self._modal_overlay = ModalOverlay() self._modal_overlay_shown = False self._modal_overlay_tick: Callable[[], None] | None = None @@ -261,6 +262,18 @@ class GuiApplication: def request_close(self): self._window_close_requested = True + def set_progress_hook(self, hook: Callable[[str], None] | None): + self._progress_hook = hook + + def _mark_progress(self, phase: str): + if self._progress_hook is None: + return + + try: + self._progress_hook(phase) + except Exception: + pass + def init_window(self, title: str, fps: int = _DEFAULT_FPS): with self._startup_profile_context(): @@ -536,6 +549,7 @@ class GuiApplication: self._render_profiler.enable() while not (self._window_close_requested or rl.window_should_close()): + self._mark_progress("gui_app.loop_start") if PC: # Thread is not used on PC, need to manually add mouse events self._mouse._handle_mouse_event() @@ -547,6 +561,7 @@ class GuiApplication: # Skip rendering when screen is off if not self._should_render: + self._mark_progress("gui_app.skip_render") if PC: rl.poll_input_events() time.sleep(1 / self._target_fps) @@ -554,25 +569,32 @@ class GuiApplication: continue if self._render_texture: + self._mark_progress("gui_app.begin_texture_mode") rl.begin_texture_mode(self._render_texture) rl.clear_background(rl.BLACK) else: + self._mark_progress("gui_app.begin_drawing") rl.begin_drawing() rl.clear_background(rl.BLACK) # Handle modal overlay rendering and input processing if self._render_nav_stack(): + self._mark_progress("gui_app.nav_stack") yield False elif self._handle_modal_overlay(): # Allow a Widget to still run a function while overlay is shown if self._modal_overlay_tick is not None: self._modal_overlay_tick() + self._mark_progress("gui_app.modal_overlay") yield False else: + self._mark_progress("gui_app.frame_ready") yield True if self._render_texture: + self._mark_progress("gui_app.end_texture_mode") rl.end_texture_mode() + self._mark_progress("gui_app.begin_present") rl.begin_drawing() rl.clear_background(rl.BLACK) src_rect = rl.Rectangle(0, 0, float(self._width), -float(self._height)) @@ -595,7 +617,9 @@ class GuiApplication: if self._grid_size > 0: self._draw_grid() + self._mark_progress("gui_app.end_drawing") rl.end_drawing() + self._mark_progress("gui_app.after_end_drawing") if RECORD: image = rl.load_image_from_texture(self._render_texture.texture) @@ -607,6 +631,7 @@ class GuiApplication: self._monitor_fps() self._frame += 1 + self._mark_progress("gui_app.frame_complete") if self._profile_render_frames > 0 and self._frame >= self._profile_render_frames: self._output_render_profile()