cruise speed and ui test

common/watchdog.py

@@ -2,21 +2,27 @@ import os
 import time
 import struct
 from openpilot.system.hardware.hw import Paths
+from openpilot.common.swaglog import cloudlog
 
 WATCHDOG_FN = f"{Paths.shm_path()}/wd_"
 _LAST_KICK = 0.0
+_LAST_ERROR_LOG = 0.0
 
 def kick_watchdog():
-  global _LAST_KICK
+  global _LAST_KICK, _LAST_ERROR_LOG
   current_time = time.monotonic()
 
   if current_time - _LAST_KICK < 1.0:
-    return
+    return True
 
   try:
     with open(f"{WATCHDOG_FN}{os.getpid()}", 'wb') as f:
       f.write(struct.pack('<Q', int(current_time * 1e9)))
       f.flush()
     _LAST_KICK = current_time
-  except OSError:
-    pass
+    return True
+  except OSError as e:
+    if current_time - _LAST_ERROR_LOG >= 5.0:
+      cloudlog.error(f"watchdog kick failed for pid {os.getpid()}: {e}")
+      _LAST_ERROR_LOG = current_time
+    return False

panda/board/obj/gitversion.h

@@ -1,2 +1,2 @@
 extern const uint8_t gitversion[19];
-const uint8_t gitversion[19] = "DEV-f1feaa2c-DEBUG";
+const uint8_t gitversion[19] = "DEV-ad948aa9-DEBUG";

panda/board/obj/version

@@ -1 +1 @@
-DEV-f1feaa2c-DEBUG
+DEV-ad948aa9-DEBUG

selfdrive/car/cruise.py

@@ -50,14 +50,6 @@ class VCruiseHelper:
   def _get_cruise_delta_interval(interval: float | None) -> float:
     return interval if isinstance(interval, (int, float)) and interval > 0 else 1.0
 
-  def _normalize_initialized_v_cruise(self, v_cruise_kph: float, starpilot_toggles: SimpleNamespace) -> float:
-    cruise_increase = self._get_cruise_delta_interval(starpilot_toggles.cruise_increase)
-    if cruise_increase % 5 != 0:
-      return v_cruise_kph
-
-    v_cruise_delta = self._get_short_press_delta(starpilot_toggles.is_metric, starpilot_toggles)
-    return round(round(v_cruise_kph / v_cruise_delta) * v_cruise_delta, 1)
-
   @property
   def v_cruise_initialized(self):
     return self.v_cruise_kph != V_CRUISE_UNSET
@@ -162,7 +154,9 @@ class VCruiseHelper:
       and self.v_cruise_initialized or (self.gm_cc_only and resume_prev_button)):
       self.v_cruise_kph = self.v_cruise_kph_last
     elif desired_speed_limit > 0 and getattr(starpilot_toggles, "set_speed_limit", False):
-      initialized_speed_limit_kph = self._normalize_initialized_v_cruise(desired_speed_limit * CV.MS_TO_KPH, starpilot_toggles)
+      # Respect the exact SLC limit+offset on engage instead of snapping upward to
+      # the custom cruise-button interval.
+      initialized_speed_limit_kph = round(desired_speed_limit * CV.MS_TO_KPH, 1)
       self.v_cruise_kph = float(np.clip(initialized_speed_limit_kph, V_CRUISE_MIN, V_CRUISE_MAX))
     else:
       self.v_cruise_kph = int(round(np.clip(CS.vEgo * CV.MS_TO_KPH, engage_floor_kph, V_CRUISE_MAX)))

selfdrive/car/tests/test_cruise_speed.py

@@ -207,6 +207,22 @@ class TestVCruiseHelper:
 
     assert self.v_cruise_helper.v_cruise_kph == pytest.approx(55 * CV.MPH_TO_KPH)
 
+  def test_initialize_v_cruise_keeps_exact_speed_limit_offset(self):
+    self.reset_cruise_speed_state()
+    self.starpilot_toggles.set_speed_limit = True
+    self.starpilot_toggles.cruise_increase = 5
+
+    desired_speed_limit = 38 * CV.MPH_TO_MS
+    self.v_cruise_helper.initialize_v_cruise(
+      car.CarState(vEgo=70 * CV.MPH_TO_MS),
+      experimental_mode=False,
+      resume_prev_button=False,
+      starpilot_toggles=self.starpilot_toggles,
+      desired_speed_limit=desired_speed_limit,
+    )
+
+    assert self.v_cruise_helper.v_cruise_kph == pytest.approx(38 * CV.MPH_TO_KPH)
+
   def test_speed_limit_confirmation_does_not_adjust_cruise(self):
     self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False)
     initial_v_cruise_kph = self.v_cruise_helper.v_cruise_kph
@@ -260,3 +276,31 @@ class TestVCruiseHelper:
     )
 
     assert self.v_cruise_helper.v_cruise_kph == pytest.approx(initial_v_cruise_kph + IMPERIAL_INCREMENT)
+
+  def test_zero_custom_cruise_toggles_fall_back_to_single_step(self):
+    self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False)
+    initial_v_cruise_kph = self.v_cruise_helper.v_cruise_kph
+    self.starpilot_toggles.cruise_increase = 0
+    self.starpilot_toggles.cruise_increase_long = 0
+
+    pressed_cs = car.CarState(cruiseState={"available": True})
+    pressed_cs.buttonEvents = [ButtonEvent(type=ButtonType.accelCruise, pressed=True)]
+    self.v_cruise_helper.update_v_cruise(
+      pressed_cs,
+      enabled=True,
+      is_metric=False,
+      speed_limit_changed=False,
+      starpilot_toggles=self.starpilot_toggles,
+    )
+
+    released_cs = car.CarState(cruiseState={"available": True})
+    released_cs.buttonEvents = [ButtonEvent(type=ButtonType.accelCruise, pressed=False)]
+    self.v_cruise_helper.update_v_cruise(
+      released_cs,
+      enabled=True,
+      is_metric=False,
+      speed_limit_changed=False,
+      starpilot_toggles=self.starpilot_toggles,
+    )
+
+    assert self.v_cruise_helper.v_cruise_kph == pytest.approx(initial_v_cruise_kph + IMPERIAL_INCREMENT)

selfdrive/ui/layouts/settings/starpilot/longitudinal.py

@@ -576,7 +576,7 @@ class StarPilotLongitudinalQOLLayout(StarPilotPanel):
       {
         "title": tr_noop("Cruise Interval"),
         "type": "value",
-        "get_value": lambda: f"{self._params.get_int('CustomCruise')} mph",
+        "get_value": lambda: f"{max(1, self._params.get_int('CustomCruise'))} mph",
         "on_click": lambda: self._show_speed_selector("CustomCruise"),
         "color": "#597497",
         "visible": lambda: self._params.get_bool("QOLLongitudinal"),
@@ -584,7 +584,7 @@ class StarPilotLongitudinalQOLLayout(StarPilotPanel):
       {
         "title": tr_noop("Cruise Long"),
         "type": "value",
-        "get_value": lambda: f"{self._params.get_int('CustomCruiseLong')} mph",
+        "get_value": lambda: f"{max(1, self._params.get_int('CustomCruiseLong'))} mph",
         "on_click": lambda: self._show_speed_selector("CustomCruiseLong"),
         "color": "#597497",
         "visible": lambda: self._params.get_bool("QOLLongitudinal"),
@@ -662,7 +662,8 @@ class StarPilotLongitudinalQOLLayout(StarPilotPanel):
         self._params.put_int(key, int(val))
         self._rebuild_grid()
 
-    gui_app.set_modal_overlay(AetherSliderDialog(tr(key), 0, 100, 1, self._params.get_int(key), on_close, unit=" mph", color="#597497"))
+    current = max(1, self._params.get_int(key))
+    gui_app.set_modal_overlay(AetherSliderDialog(tr(key), 1, 100, 1, current, on_close, unit=" mph", color="#597497"))
 
   def _show_int_selector(self, key, min_v, max_v, unit=""):
     def on_close(res, val):

selfdrive/ui/stall_monitor.py

@@ -0,0 +1,131 @@
+import os
+import sys
+import time
+import traceback
+import threading
+from pathlib import Path
+
+from openpilot.common.swaglog import cloudlog
+
+
+def _default_dump_dir() -> Path:
+  for candidate in ("/data/log", "/tmp"):
+    if os.path.isdir(candidate) and os.access(candidate, os.W_OK):
+      return Path(candidate)
+  return Path.cwd()
+
+
+class UIStallMonitor:
+  def __init__(self, name: str):
+    self._name = name
+    self._threshold_s = float(os.getenv("UI_STALL_PROBE_MAX_DT", "5"))
+    self._poll_s = float(os.getenv("UI_STALL_PROBE_POLL_DT", "0.25"))
+    self._dump_dir = _default_dump_dir()
+    self._main_thread_id = threading.get_ident()
+
+    now = time.monotonic()
+    self._last_progress = now
+    self._phase = "init"
+    self._phase_entered = now
+    self._stall_reported = False
+    self._stalled_since = now
+    self._stalled_phase = self._phase
+
+    self._lock = threading.Lock()
+    self._stop_event = threading.Event()
+    self._thread = threading.Thread(target=self._run, name=f"{name}_stall_probe", daemon=True)
+
+  def start(self) -> None:
+    if self._threshold_s <= 0.0:
+      return
+    self._thread.start()
+
+  def stop(self) -> None:
+    if self._threshold_s <= 0.0:
+      return
+    self._stop_event.set()
+    self._thread.join(timeout=1.0)
+
+  def progress(self, phase: str) -> None:
+    now = time.monotonic()
+    recovered = None
+
+    with self._lock:
+      if phase != self._phase:
+        self._phase = phase
+        self._phase_entered = now
+      self._last_progress = now
+
+      if self._stall_reported:
+        recovered = (now - self._stalled_since, self._stalled_phase, phase)
+        self._stall_reported = False
+
+    if recovered is not None:
+      stalled_for_s, stalled_phase, current_phase = recovered
+      cloudlog.warning(f"{self._name} stall recovered after {stalled_for_s:.1f}s (stalled_phase={stalled_phase}, current_phase={current_phase})")
+
+  def _run(self) -> None:
+    while not self._stop_event.wait(self._poll_s):
+      now = time.monotonic()
+      with self._lock:
+        stalled_for_s = now - self._last_progress
+        phase = self._phase
+        phase_for_s = now - self._phase_entered
+        already_reported = self._stall_reported
+
+      if stalled_for_s < self._threshold_s or already_reported:
+        continue
+
+      dump = self._build_dump(now, phase, stalled_for_s, phase_for_s)
+      dump_path = self._write_dump(dump)
+      with self._lock:
+        self._stall_reported = True
+        self._stalled_since = now
+        self._stalled_phase = phase
+
+      preview = self._main_thread_preview()
+      path_s = str(dump_path) if dump_path is not None else "<write_failed>"
+      cloudlog.error(f"{self._name} main loop stalled for {stalled_for_s:.1f}s in phase={phase} (phase_for={phase_for_s:.1f}s) dump={path_s}\n{preview}")
+
+  def _build_dump(self, now: float, phase: str, stalled_for_s: float, phase_for_s: float) -> str:
+    frames = sys._current_frames()
+    threads = {thread.ident: thread for thread in threading.enumerate()}
+    lines = [
+      f"name={self._name}",
+      f"pid={os.getpid()}",
+      f"wall_time={time.strftime('%Y-%m-%dT%H:%M:%S%z')}",
+      f"monotonic={now:.6f}",
+      f"stalled_for_s={stalled_for_s:.3f}",
+      f"phase={phase}",
+      f"phase_for_s={phase_for_s:.3f}",
+      "",
+    ]
+
+    ordered_idents = sorted(frames.keys(), key=lambda ident: ident != self._main_thread_id)
+    for ident in ordered_idents:
+      thread = threads.get(ident)
+      name = thread.name if thread is not None else "<unknown>"
+      daemon = thread.daemon if thread is not None else None
+      lines.append(f"Thread {name} ident={ident} daemon={daemon}")
+      lines.extend(traceback.format_stack(frames[ident]))
+      lines.append("")
+
+    return "".join(line if line.endswith("\n") else f"{line}\n" for line in lines)
+
+  def _main_thread_preview(self) -> str:
+    frame = sys._current_frames().get(self._main_thread_id)
+    if frame is None:
+      return "main_thread_stack=<unavailable>"
+    stack_lines = traceback.format_stack(frame)
+    preview = "".join(stack_lines[-8:]).strip()
+    return f"main_thread_stack:\n{preview}" if preview else "main_thread_stack=<empty>"
+
+  def _write_dump(self, dump: str) -> Path | None:
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
+    path = self._dump_dir / f"{self._name}_stall_{os.getpid()}_{timestamp}.log"
+    try:
+      path.write_text(dump)
+      return path
+    except OSError as e:
+      cloudlog.error(f"failed to write {self._name} stall dump to {path}: {e}")
+      return None

selfdrive/ui/ui.cc

@@ -1,7 +1,21 @@
 #include "selfdrive/ui/ui.h"
 
 #include <algorithm>
+#include <atomic>
+#include <cerrno>
 #include <cmath>
+#include <chrono>
+#include <csignal>
+#include <cstdio>
+#include <cstdlib>
+#include <execinfo.h>
+#include <fcntl.h>
+#include <mutex>
+#include <pthread.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <thread>
+#include <unistd.h>
 
 #include <QtConcurrent>
 
@@ -14,6 +28,178 @@
 #define BACKLIGHT_DT 0.05
 #define BACKLIGHT_TS 10.00
 
+namespace {
+
+enum class UIStallPhase {
+  INIT = 0,
+  UPDATE_START,
+  AFTER_SOCKETS,
+  AFTER_STATE,
+  AFTER_STATUS,
+  AFTER_WATCHDOG,
+  AFTER_EMIT,
+  AFTER_FS_UPDATE,
+  IDLE,
+};
+
+std::atomic<uint64_t> ui_stall_last_progress_ns{0};
+std::atomic<int> ui_stall_phase{static_cast<int>(UIStallPhase::INIT)};
+std::atomic<uint64_t> ui_stall_frame{0};
+std::atomic<bool> ui_stall_reported{false};
+std::atomic<uint64_t> ui_stall_reported_ns{0};
+std::atomic<int> ui_stall_reported_phase{static_cast<int>(UIStallPhase::INIT)};
+std::atomic<int> ui_stall_dump_fd{-1};
+pthread_t ui_main_thread{};
+
+double read_env_double(const char *name, double default_value) {
+  const char *value = std::getenv(name);
+  if (value == nullptr || *value == '\0') {
+    return default_value;
+  }
+
+  char *end = nullptr;
+  double parsed = std::strtod(value, &end);
+  if (end == value || (end != nullptr && *end != '\0') || parsed <= 0.0) {
+    return default_value;
+  }
+  return parsed;
+}
+
+const char *ui_stall_phase_name(UIStallPhase phase) {
+  switch (phase) {
+    case UIStallPhase::INIT: return "init";
+    case UIStallPhase::UPDATE_START: return "update_start";
+    case UIStallPhase::AFTER_SOCKETS: return "after_sockets";
+    case UIStallPhase::AFTER_STATE: return "after_state";
+    case UIStallPhase::AFTER_STATUS: return "after_status";
+    case UIStallPhase::AFTER_WATCHDOG: return "after_watchdog";
+    case UIStallPhase::AFTER_EMIT: return "after_emit";
+    case UIStallPhase::AFTER_FS_UPDATE: return "after_fs_update";
+    case UIStallPhase::IDLE: return "idle";
+  }
+  return "unknown";
+}
+
+std::string ui_stall_dump_dir() {
+  return access("/data/log", W_OK) == 0 ? "/data/log" : "/tmp";
+}
+
+void ui_stall_signal_handler(int sig) {
+  const int fd = ui_stall_dump_fd.load(std::memory_order_relaxed);
+  if (fd < 0) {
+    return;
+  }
+
+  char header[256];
+  const pid_t tid = static_cast<pid_t>(syscall(SYS_gettid));
+  const int header_len = std::snprintf(header, sizeof(header),
+                                       "=== UI stall backtrace (signal=%d pid=%d tid=%d) ===\n",
+                                       sig, getpid(), tid);
+  if (header_len > 0) {
+    write(fd, header, header_len);
+  }
+
+  void *frames[128];
+  const int frame_count = backtrace(frames, 128);
+  backtrace_symbols_fd(frames, frame_count, fd);
+  write(fd, "\n", 1);
+}
+
+void ui_stall_progress(UIStallPhase phase, uint64_t frame = 0) {
+  const uint64_t now = nanos_since_boot();
+  ui_stall_phase.store(static_cast<int>(phase), std::memory_order_relaxed);
+  ui_stall_frame.store(frame, std::memory_order_relaxed);
+  ui_stall_last_progress_ns.store(now, std::memory_order_relaxed);
+
+  if (ui_stall_reported.exchange(false, std::memory_order_relaxed)) {
+    const uint64_t stall_started = ui_stall_reported_ns.load(std::memory_order_relaxed);
+    const UIStallPhase stalled_phase = static_cast<UIStallPhase>(ui_stall_reported_phase.load(std::memory_order_relaxed));
+    const double stalled_for_s = stall_started == 0 ? 0.0 : (now - stall_started) / 1e9;
+    LOGW("UI stall recovered after %.1fs (stalled_phase=%s current_phase=%s frame=%llu)",
+         stalled_for_s,
+         ui_stall_phase_name(stalled_phase),
+         ui_stall_phase_name(phase),
+         static_cast<unsigned long long>(frame));
+  }
+}
+
+void start_ui_stall_monitor() {
+  static std::once_flag once;
+  std::call_once(once, [] {
+    ui_main_thread = pthread_self();
+    std::signal(SIGUSR1, ui_stall_signal_handler);
+    ui_stall_progress(UIStallPhase::INIT, 0);
+
+    const double stall_probe_dt = read_env_double("UI_STALL_PROBE_MAX_DT", 5.0);
+    if (stall_probe_dt <= 0.0) {
+      return;
+    }
+
+    std::thread([stall_probe_dt]() {
+      using namespace std::chrono_literals;
+      constexpr auto poll_interval = 250ms;
+
+      while (true) {
+        std::this_thread::sleep_for(poll_interval);
+
+        const uint64_t now = nanos_since_boot();
+        const uint64_t last_progress = ui_stall_last_progress_ns.load(std::memory_order_relaxed);
+        if (last_progress == 0) {
+          continue;
+        }
+
+        const double stalled_for_s = (now - last_progress) / 1e9;
+        if (stalled_for_s < stall_probe_dt) {
+          continue;
+        }
+
+        bool expected = false;
+        if (!ui_stall_reported.compare_exchange_strong(expected, true, std::memory_order_relaxed)) {
+          continue;
+        }
+
+        const UIStallPhase phase = static_cast<UIStallPhase>(ui_stall_phase.load(std::memory_order_relaxed));
+        const uint64_t frame = ui_stall_frame.load(std::memory_order_relaxed);
+        ui_stall_reported_ns.store(now, std::memory_order_relaxed);
+        ui_stall_reported_phase.store(static_cast<int>(phase), std::memory_order_relaxed);
+
+        const std::string path = ui_stall_dump_dir() + "/qt_ui_stall_" + std::to_string(getpid()) + "_" + std::to_string(now) + ".log";
+        int fd = open(path.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0644);
+        if (fd >= 0) {
+          char header[256];
+          const int header_len = std::snprintf(header, sizeof(header),
+                                               "phase=%s frame=%llu stalled_for_s=%.3f\n",
+                                               ui_stall_phase_name(phase),
+                                               static_cast<unsigned long long>(frame),
+                                               stalled_for_s);
+          if (header_len > 0) {
+            write(fd, header, header_len);
+          }
+
+          ui_stall_dump_fd.store(fd, std::memory_order_relaxed);
+          pthread_kill(ui_main_thread, SIGUSR1);
+          std::this_thread::sleep_for(50ms);
+          ui_stall_dump_fd.store(-1, std::memory_order_relaxed);
+          close(fd);
+          LOGE("UI main thread stalled for %.1fs (phase=%s frame=%llu dump=%s)",
+               stalled_for_s,
+               ui_stall_phase_name(phase),
+               static_cast<unsigned long long>(frame),
+               path.c_str());
+        } else {
+          LOGE("UI main thread stalled for %.1fs (phase=%s frame=%llu dump_open_failed errno=%d)",
+               stalled_for_s,
+               ui_stall_phase_name(phase),
+               static_cast<unsigned long long>(frame),
+               errno);
+        }
+      }
+    }).detach();
+  });
+}
+
+}  // namespace
+
 static void update_sockets(UIState *s) {
   s->sm->update(0);
 }
@@ -144,6 +330,7 @@ void UIState::updateStatus(StarPilotUIState *fs) {
 }
 
 UIState::UIState(QObject *parent) : QObject(parent) {
+  start_ui_stall_monitor();
   sm = std::make_unique<SubMaster>(std::vector<const char*>{
     "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState",
     "pandaStates", "carParams", "driverMonitoringState", "carState", "driverStateV2",
@@ -156,17 +343,26 @@ UIState::UIState(QObject *parent) : QObject(parent) {
   timer = new QTimer(this);
   QObject::connect(timer, &QTimer::timeout, this, &UIState::update);
   timer->start(1000 / UI_FREQ);
+  ui_stall_progress(UIStallPhase::IDLE, sm->frame);
 }
 
 void UIState::update() {
+  ui_stall_progress(UIStallPhase::UPDATE_START, sm->frame);
   update_sockets(this);
+  ui_stall_progress(UIStallPhase::AFTER_SOCKETS, sm->frame);
   update_state(this, starpilotUIState());
+  ui_stall_progress(UIStallPhase::AFTER_STATE, sm->frame);
   updateStatus(starpilotUIState());
+  ui_stall_progress(UIStallPhase::AFTER_STATUS, sm->frame);
 
   if (sm->frame % UI_FREQ == 0) {
-    watchdog_kick(nanos_since_boot());
+    if (!watchdog_kick(nanos_since_boot())) {
+      LOGE("UI watchdog kick failed at frame %llu", static_cast<unsigned long long>(sm->frame));
+    }
   }
+  ui_stall_progress(UIStallPhase::AFTER_WATCHDOG, sm->frame);
   emit uiUpdate(*this, *starpilotUIState());
+  ui_stall_progress(UIStallPhase::AFTER_EMIT, sm->frame);
 
   StarPilotUIState *fs = starpilotUIState();
   StarPilotUIScene &starpilot_scene = fs->starpilot_scene;
@@ -177,6 +373,8 @@ void UIState::update() {
   }
 
   fs->update();
+  ui_stall_progress(UIStallPhase::AFTER_FS_UPDATE, sm->frame);
+  ui_stall_progress(UIStallPhase::IDLE, sm->frame);
 }
 
 Device::Device(QObject *parent) : brightness_filter(BACKLIGHT_OFFROAD, BACKLIGHT_TS, BACKLIGHT_DT), QObject(parent) {

selfdrive/ui/ui.py

@@ -6,6 +6,7 @@ from openpilot.system.hardware import TICI
 from openpilot.common.realtime import config_realtime_process, set_core_affinity
 from openpilot.common.watchdog import kick_watchdog
 from openpilot.system.ui.lib.application import gui_app
+from openpilot.selfdrive.ui.stall_monitor import UIStallMonitor
 from openpilot.selfdrive.ui.layouts.main import MainLayout
 from openpilot.selfdrive.ui.mici.layouts.main import MiciMainLayout
 from openpilot.selfdrive.ui.ui_state import ui_state
@@ -21,18 +22,33 @@ def main():
   else:
     main_layout = MiciMainLayout()
   main_layout.set_rect(rl.Rectangle(0, 0, gui_app.width, gui_app.height))
-  for should_render in gui_app.render():
-    kick_watchdog()
-    ui_state.update()
-    if should_render:
-      main_layout.render()
+  stall_monitor = UIStallMonitor("raylib_ui")
+  gui_app.set_progress_hook(stall_monitor.progress)
+  stall_monitor.progress("ui.loop_ready")
+  stall_monitor.start()
 
-      # reaffine after power save offlines our core
-      if TICI and os.sched_getaffinity(0) != cores:
-        try:
-          set_core_affinity(list(cores))
-        except OSError:
-          pass
+  try:
+    for should_render in gui_app.render():
+      stall_monitor.progress("ui.loop_iteration")
+      kick_watchdog()
+      stall_monitor.progress("ui.after_watchdog")
+      ui_state.update()
+      stall_monitor.progress("ui.after_state_update")
+      if should_render:
+        stall_monitor.progress("ui.before_layout_render")
+        main_layout.render()
+        stall_monitor.progress("ui.after_layout_render")
+
+        # reaffine after power save offlines our core
+        if TICI and os.sched_getaffinity(0) != cores:
+          try:
+            set_core_affinity(list(cores))
+          except OSError:
+            pass
+      stall_monitor.progress("ui.loop_idle")
+  finally:
+    gui_app.set_progress_hook(None)
+    stall_monitor.stop()
 
 
 if __name__ == "__main__":

system/ui/lib/application.py

@@ -224,6 +224,7 @@ class GuiApplication:
     self._frame = 0
     self._window_close_requested = False
     self._trace_log_callback = None
+    self._progress_hook: Callable[[str], None] | None = None
     self._modal_overlay = ModalOverlay()
     self._modal_overlay_shown = False
     self._modal_overlay_tick: Callable[[], None] | None = None
@@ -261,6 +262,18 @@ class GuiApplication:
   def request_close(self):
     self._window_close_requested = True
 
+  def set_progress_hook(self, hook: Callable[[str], None] | None):
+    self._progress_hook = hook
+
+  def _mark_progress(self, phase: str):
+    if self._progress_hook is None:
+      return
+
+    try:
+      self._progress_hook(phase)
+    except Exception:
+      pass
+
   def init_window(self, title: str, fps: int = _DEFAULT_FPS):
     with self._startup_profile_context():
 
@@ -536,6 +549,7 @@ class GuiApplication:
         self._render_profiler.enable()
 
       while not (self._window_close_requested or rl.window_should_close()):
+        self._mark_progress("gui_app.loop_start")
         if PC:
           # Thread is not used on PC, need to manually add mouse events
           self._mouse._handle_mouse_event()
@@ -547,6 +561,7 @@ class GuiApplication:
 
         # Skip rendering when screen is off
         if not self._should_render:
+          self._mark_progress("gui_app.skip_render")
           if PC:
             rl.poll_input_events()
           time.sleep(1 / self._target_fps)
@@ -554,25 +569,32 @@ class GuiApplication:
           continue
 
         if self._render_texture:
+          self._mark_progress("gui_app.begin_texture_mode")
           rl.begin_texture_mode(self._render_texture)
           rl.clear_background(rl.BLACK)
         else:
+          self._mark_progress("gui_app.begin_drawing")
           rl.begin_drawing()
           rl.clear_background(rl.BLACK)
 
         # Handle modal overlay rendering and input processing
         if self._render_nav_stack():
+          self._mark_progress("gui_app.nav_stack")
           yield False
         elif self._handle_modal_overlay():
           # Allow a Widget to still run a function while overlay is shown
           if self._modal_overlay_tick is not None:
             self._modal_overlay_tick()
+          self._mark_progress("gui_app.modal_overlay")
           yield False
         else:
+          self._mark_progress("gui_app.frame_ready")
           yield True
 
         if self._render_texture:
+          self._mark_progress("gui_app.end_texture_mode")
           rl.end_texture_mode()
+          self._mark_progress("gui_app.begin_present")
           rl.begin_drawing()
           rl.clear_background(rl.BLACK)
           src_rect = rl.Rectangle(0, 0, float(self._width), -float(self._height))
@@ -595,7 +617,9 @@ class GuiApplication:
         if self._grid_size > 0:
           self._draw_grid()
 
+        self._mark_progress("gui_app.end_drawing")
         rl.end_drawing()
+        self._mark_progress("gui_app.after_end_drawing")
 
         if RECORD:
           image = rl.load_image_from_texture(self._render_texture.texture)
@@ -607,6 +631,7 @@ class GuiApplication:
 
         self._monitor_fps()
         self._frame += 1
+        self._mark_progress("gui_app.frame_complete")
 
         if self._profile_render_frames > 0 and self._frame >= self._profile_render_frames:
           self._output_render_profile()