Merge branch 'master-new' into feature/external-storage

selfdrive/locationd/lagd.py

@@ -157,7 +157,8 @@ class LateralLagEstimator:
                block_count: int = BLOCK_NUM, min_valid_block_count: int = BLOCK_NUM_NEEDED, block_size: int = BLOCK_SIZE,
                window_sec: float = MOVING_WINDOW_SEC, okay_window_sec: float = MIN_OKAY_WINDOW_SEC, min_recovery_buffer_sec: float = MIN_RECOVERY_BUFFER_SEC,
                min_vego: float = MIN_VEGO, min_yr: float = MIN_ABS_YAW_RATE, min_ncc: float = MIN_NCC,
-               max_lat_accel: float = MAX_LAT_ACCEL, max_lat_accel_diff: float = MAX_LAT_ACCEL_DIFF, min_confidence: float = MIN_CONFIDENCE):
+               max_lat_accel: float = MAX_LAT_ACCEL, max_lat_accel_diff: float = MAX_LAT_ACCEL_DIFF, min_confidence: float = MIN_CONFIDENCE,
+               enabled: bool = True):
     self.dt = dt
     self.window_sec = window_sec
     self.okay_window_sec = okay_window_sec
@@ -172,6 +173,7 @@ class LateralLagEstimator:
     self.min_confidence = min_confidence
     self.max_lat_accel = max_lat_accel
     self.max_lat_accel_diff = max_lat_accel_diff
+    self.enabled = enabled
 
     self.t = 0.0
     self.lat_active = False
@@ -206,7 +208,7 @@ class LateralLagEstimator:
     liveDelay = msg.liveDelay
 
     valid_mean_lag, valid_std, current_mean_lag, current_std = self.block_avg.get()
-    if self.block_avg.valid_blocks >= self.min_valid_block_count and not np.isnan(valid_mean_lag) and not np.isnan(valid_std):
+    if self.enabled and self.block_avg.valid_blocks >= self.min_valid_block_count and not np.isnan(valid_mean_lag) and not np.isnan(valid_std):
       if valid_std > MAX_LAG_STD:
         liveDelay.status = log.LiveDelayData.Status.invalid
       else:
@@ -303,7 +305,8 @@ class LateralLagEstimator:
     self.block_avg.update(delay)
     self.last_estimate_t = self.t
 
-  def actuator_delay(self, expected_sig: np.ndarray, actual_sig: np.ndarray, mask: np.ndarray, dt: float, max_lag: float) -> tuple[float, float, float]:
+  @staticmethod
+  def actuator_delay(expected_sig: np.ndarray, actual_sig: np.ndarray, mask: np.ndarray, dt: float, max_lag: float) -> tuple[float, float, float]:
     assert len(expected_sig) == len(actual_sig)
     max_lag_samples = int(max_lag / dt)
     padded_size = fft_next_good_size(len(expected_sig) + max_lag_samples)
@@ -366,7 +369,10 @@ def main():
   params = Params()
   CP = messaging.log_from_bytes(params.get("CarParams", block=True), car.CarParams)
 
-  lag_learner = LateralLagEstimator(CP, 1. / SERVICE_LIST['livePose'].frequency)
+  # TODO: remove me, lagd is in shadow mode on release
+  is_release = params.get_bool("IsReleaseBranch")
+
+  lag_learner = LateralLagEstimator(CP, 1. / SERVICE_LIST['livePose'].frequency, enabled=not is_release)
   if (initial_lag_params := retrieve_initial_lag(params, CP)) is not None:
     lag, valid_blocks = initial_lag_params
     lag_learner.reset(lag, valid_blocks)

selfdrive/locationd/test/test_lagd.py

@@ -3,7 +3,7 @@ import numpy as np
 import time
 import pytest
 
-from cereal import messaging, log
+from cereal import messaging, log, car
 from openpilot.selfdrive.locationd.lagd import LateralLagEstimator, retrieve_initial_lag, masked_normalized_cross_correlation, \
                                                BLOCK_NUM_NEEDED, BLOCK_SIZE, MIN_OKAY_WINDOW_SEC
 from openpilot.selfdrive.test.process_replay.migration import migrate, migrate_carParams
@@ -16,11 +16,7 @@ MAX_ERR_FRAMES = 1
 DT = 0.05
 
 
-def process_messages(mocker, estimator, lag_frames, n_frames, vego=20.0, rejection_threshold=0.0):
-  class ZeroMock(mocker.Mock):
-    def __getattr__(self, *args):
-      return 0
-
+def process_messages(estimator, lag_frames, n_frames, vego=20.0, rejection_threshold=0.0):
   for i in range(n_frames):
     t = i * estimator.dt
     desired_la = np.cos(10 * t) * 0.1
@@ -31,19 +27,15 @@ def process_messages(mocker, estimator, lag_frames, n_frames, vego=20.0, rejecti
     if rejected:
       actual_la = desired_la
 
-    desired_cuvature = desired_la / (vego ** 2)
-    actual_yr = actual_la / vego
+    desired_cuvature = float(desired_la / (vego ** 2))
+    actual_yr = float(actual_la / vego)
     msgs = [
-      (t, "carControl", mocker.Mock(latActive=not rejected)),
-      (t, "carState", mocker.Mock(vEgo=vego, steeringPressed=False)),
-      (t, "controlsState", mocker.Mock(desiredCurvature=desired_cuvature,
-                                        lateralControlState=mocker.Mock(which=mocker.Mock(return_value='debugControlState'), debugControlState=ZeroMock()))),
-      (t, "livePose", mocker.Mock(orientationNED=ZeroMock(),
-                                velocityDevice=ZeroMock(),
-                                accelerationDevice=ZeroMock(),
-                                angularVelocityDevice=ZeroMock(z=actual_yr, valid=True),
-                                posenetOK=True, inputsOK=True)),
-      (t, "liveCalibration", mocker.Mock(rpyCalib=[0, 0, 0], calStatus=log.LiveCalibrationData.Status.calibrated)),
+      (t, "carControl", car.CarControl(latActive=not rejected)),
+      (t, "carState", car.CarState(vEgo=vego, steeringPressed=False)),
+      (t, "controlsState", log.ControlsState(desiredCurvature=desired_cuvature)),
+      (t, "livePose", log.LivePose(angularVelocityDevice=log.LivePose.XYZMeasurement(z=actual_yr, valid=True),
+                                   posenetOK=True, inputsOK=True)),
+      (t, "liveCalibration", log.LiveCalibrationData(rpyCalib=[0, 0, 0], calStatus=log.LiveCalibrationData.Status.calibrated)),
     ]
     for t, w, m in msgs:
       estimator.handle_log(t, w, m)
@@ -94,8 +86,8 @@ class TestLagd:
     corr = masked_normalized_cross_correlation(desired_sig, actual_sig, mask, 200)[len(desired_sig) - 1:len(desired_sig) + 20]
     assert np.argmax(corr) in range(lag_frames - MAX_ERR_FRAMES, lag_frames + MAX_ERR_FRAMES + 1)
 
-  def test_empty_estimator(self, mocker):
-    mocked_CP = mocker.Mock(steerActuatorDelay=0.8)
+  def test_empty_estimator(self):
+    mocked_CP = car.CarParams(steerActuatorDelay=0.8)
     estimator = LateralLagEstimator(mocked_CP, DT)
     msg = estimator.get_msg(True)
     assert msg.liveDelay.status == 'unestimated'
@@ -103,12 +95,12 @@ class TestLagd:
     assert np.allclose(msg.liveDelay.lateralDelayEstimate, estimator.initial_lag)
     assert msg.liveDelay.validBlocks == 0
 
-  def test_estimator_basics(self, mocker, subtests):
+  def test_estimator_basics(self, subtests):
     for lag_frames in range(5):
       with subtests.test(msg=f"lag_frames={lag_frames}"):
-        mocked_CP = mocker.Mock(steerActuatorDelay=0.8)
+        mocked_CP = car.CarParams(steerActuatorDelay=0.8)
         estimator = LateralLagEstimator(mocked_CP, DT, min_recovery_buffer_sec=0.0, min_yr=0.0)
-        process_messages(mocker, estimator, lag_frames, int(MIN_OKAY_WINDOW_SEC / DT) + BLOCK_NUM_NEEDED * BLOCK_SIZE)
+        process_messages(estimator, lag_frames, int(MIN_OKAY_WINDOW_SEC / DT) + BLOCK_NUM_NEEDED * BLOCK_SIZE)
         msg = estimator.get_msg(True)
         assert msg.liveDelay.status == 'estimated'
         assert np.allclose(msg.liveDelay.lateralDelay, lag_frames * DT, atol=0.01)
@@ -116,18 +108,30 @@ class TestLagd:
         assert np.allclose(msg.liveDelay.lateralDelayEstimateStd, 0.0, atol=0.01)
         assert msg.liveDelay.validBlocks == BLOCK_NUM_NEEDED
 
-  def test_estimator_masking(self, mocker):
-    mocked_CP, lag_frames = mocker.Mock(steerActuatorDelay=0.8), random.randint(1, 19)
+  def test_disabled_estimator(self):
+    mocked_CP = car.CarParams(steerActuatorDelay=0.8)
+    estimator = LateralLagEstimator(mocked_CP, DT, min_recovery_buffer_sec=0.0, min_yr=0.0, enabled=False)
+    lag_frames = 5
+    process_messages(estimator, lag_frames, int(MIN_OKAY_WINDOW_SEC / DT) + BLOCK_NUM_NEEDED * BLOCK_SIZE)
+    msg = estimator.get_msg(True)
+    assert msg.liveDelay.status == 'unestimated'
+    assert np.allclose(msg.liveDelay.lateralDelay, 1.0, atol=0.01)
+    assert np.allclose(msg.liveDelay.lateralDelayEstimate, lag_frames * DT, atol=0.01)
+    assert np.allclose(msg.liveDelay.lateralDelayEstimateStd, 0.0, atol=0.01)
+    assert msg.liveDelay.validBlocks == BLOCK_NUM_NEEDED
+
+  def test_estimator_masking(self):
+    mocked_CP, lag_frames = car.CarParams(steerActuatorDelay=0.8), random.randint(1, 19)
     estimator = LateralLagEstimator(mocked_CP, DT, min_recovery_buffer_sec=0.0, min_yr=0.0, min_valid_block_count=1)
-    process_messages(mocker, estimator, lag_frames, (int(MIN_OKAY_WINDOW_SEC / DT) + BLOCK_SIZE) * 2, rejection_threshold=0.4)
+    process_messages(estimator, lag_frames, (int(MIN_OKAY_WINDOW_SEC / DT) + BLOCK_SIZE) * 2, rejection_threshold=0.4)
     msg = estimator.get_msg(True)
     assert np.allclose(msg.liveDelay.lateralDelayEstimate, lag_frames * DT, atol=0.01)
     assert np.allclose(msg.liveDelay.lateralDelayEstimateStd, 0.0, atol=0.01)
 
   @pytest.mark.skipif(PC, reason="only on device")
   @pytest.mark.timeout(60)
-  def test_estimator_performance(self, mocker):
-    mocked_CP = mocker.Mock(steerActuatorDelay=0.8)
+  def test_estimator_performance(self):
+    mocked_CP = car.CarParams(steerActuatorDelay=0.8)
     estimator = LateralLagEstimator(mocked_CP, DT)
 
     ds = []

selfdrive/test/process_replay/model_replay.py

@@ -15,15 +15,15 @@ from openpilot.system.hardware import PC
 from openpilot.tools.lib.openpilotci import get_url
 from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
 from openpilot.selfdrive.test.process_replay.process_replay import get_process_config, replay_process
-from openpilot.tools.lib.framereader import FrameReader, NumpyFrameReader
+from openpilot.tools.lib.framereader import FrameReader
 from openpilot.tools.lib.logreader import LogReader, save_log
 from openpilot.tools.lib.github_utils import GithubUtils
 
 TEST_ROUTE = "8494c69d3c710e81|000001d4--2648a9a404"
 SEGMENT = 4
-MAX_FRAMES = 100 if PC else 400
+START_FRAME = 0
+END_FRAME = 60
 
-NO_MODEL = "NO_MODEL" in os.environ
 SEND_EXTRA_INPUTS = bool(int(os.getenv("SEND_EXTRA_INPUTS", "0")))
 
 DATA_TOKEN = os.getenv("CI_ARTIFACTS_TOKEN","")
@@ -125,16 +125,15 @@ def comment_replay_report(proposed, master, full_logs):
     comment = f"ref for commit {commit}: {link}/{log_name}" + diff_plots + all_plots
     GITHUB.comment_on_pr(comment, PR_BRANCH, "commaci-public", True)
 
-def trim_logs_to_max_frames(logs, max_frames, frs_types, include_all_types):
+def trim_logs(logs, start_frame, end_frame, frs_types, include_all_types):
   all_msgs = []
   cam_state_counts = defaultdict(int)
-  # keep adding messages until cam states are equal to MAX_FRAMES
   for msg in sorted(logs, key=lambda m: m.logMonoTime):
-    all_msgs.append(msg)
     if msg.which() in frs_types:
       cam_state_counts[msg.which()] += 1
-
-    if all(cam_state_counts[state] == max_frames for state in frs_types):
+    if any(cam_state_counts[state]  >= start_frame for state in frs_types):
+      all_msgs.append(msg)
+    if all(cam_state_counts[state] == end_frame for state in frs_types):
       break
 
   if len(include_all_types) != 0:
@@ -146,9 +145,9 @@ def trim_logs_to_max_frames(logs, max_frames, frs_types, include_all_types):
 
 def model_replay(lr, frs):
   # modeld is using frame pairs
-  modeld_logs = trim_logs_to_max_frames(lr, MAX_FRAMES, {"roadCameraState", "wideRoadCameraState"},
-                                                                         {"roadEncodeIdx", "wideRoadEncodeIdx", "carParams", "carState", "carControl"})
-  dmodeld_logs = trim_logs_to_max_frames(lr, MAX_FRAMES, {"driverCameraState"}, {"driverEncodeIdx", "carParams"})
+  modeld_logs = trim_logs(lr, START_FRAME, END_FRAME, {"roadCameraState", "wideRoadCameraState"},
+                                                                         {"roadEncodeIdx", "wideRoadEncodeIdx", "carParams", "carState", "carControl", "can"})
+  dmodeld_logs = trim_logs(lr, START_FRAME, END_FRAME, {"driverCameraState"}, {"driverEncodeIdx", "carParams", "can"})
 
   if not SEND_EXTRA_INPUTS:
     modeld_logs = [msg for msg in modeld_logs if msg.which() != 'liveCalibration']
@@ -165,9 +164,6 @@ def model_replay(lr, frs):
   dmonitoringmodeld = get_process_config("dmonitoringmodeld")
 
   modeld_msgs = replay_process(modeld, modeld_logs, frs)
-  if isinstance(frs['roadCameraState'], NumpyFrameReader):
-    del frs['roadCameraState'].frames
-    del frs['wideRoadCameraState'].frames
   dmonitoringmodeld_msgs = replay_process(dmonitoringmodeld, dmodeld_logs, frs)
 
   msgs = modeld_msgs + dmonitoringmodeld_msgs
@@ -198,42 +194,21 @@ def model_replay(lr, frs):
   return msgs
 
 
-def get_frames():
-  regen_cache = "--regen-cache" in sys.argv
-  cache = "--cache" in sys.argv or not PC or regen_cache
-  videos = ('fcamera.hevc', 'dcamera.hevc', 'ecamera.hevc')
-  cams = ('roadCameraState', 'driverCameraState', 'wideRoadCameraState')
-
-  if cache:
-    frames_cache = '/tmp/model_replay_cache' if PC else '/data/model_replay_cache'
-    os.makedirs(frames_cache, exist_ok=True)
-
-    cache_size = 200
-    for v in videos:
-      if not all(os.path.isfile(f'{frames_cache}/{TEST_ROUTE}_{v}_{i}.npy') for i in range(MAX_FRAMES//cache_size)) or regen_cache:
-        f = FrameReader(get_url(TEST_ROUTE, SEGMENT, v)).get(0, MAX_FRAMES + 1, pix_fmt="nv12")
-        print(f'Caching {v}...')
-        for i in range(MAX_FRAMES//cache_size):
-          np.save(f'{frames_cache}/{TEST_ROUTE}_{v}_{i}', f[(i * cache_size) + 1:((i + 1) * cache_size) + 1])
-        del f
-
-    return {c : NumpyFrameReader(f"{frames_cache}/{TEST_ROUTE}_{v}", 1928, 1208, cache_size) for c,v in zip(cams, videos, strict=True)}
-  else:
-    return {c : FrameReader(get_url(TEST_ROUTE, SEGMENT, v), readahead=True) for c,v in zip(cams, videos, strict=True)}
-
-
 if __name__ == "__main__":
   update = "--update" in sys.argv or (os.getenv("GIT_BRANCH", "") == 'master')
   replay_dir = os.path.dirname(os.path.abspath(__file__))
 
   # load logs
   lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT, "rlog.zst")))
-  frs = get_frames()
+  frs = {
+    'roadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "fcamera.hevc"), readahead=True),
+    'driverCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "dcamera.hevc"), readahead=True),
+    'wideRoadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "ecamera.hevc"), readahead=True)
+  }
 
   log_msgs = []
   # run replays
-  if not NO_MODEL:
-    log_msgs += model_replay(lr, frs)
+  log_msgs += model_replay(lr, frs)
 
   # get diff
   failed = False
@@ -242,13 +217,10 @@ if __name__ == "__main__":
     try:
       all_logs = list(LogReader(GITHUB.get_file_url(MODEL_REPLAY_BUCKET, log_fn)))
       cmp_log = []
-
-      # logs are ordered based on type: modelV2, drivingModelData, driverStateV2
-      if not NO_MODEL:
-        model_start_index = next(i for i, m in enumerate(all_logs) if m.which() in ("modelV2", "drivingModelData", "cameraOdometry"))
-        cmp_log += all_logs[model_start_index:model_start_index + MAX_FRAMES*3]
-        dmon_start_index = next(i for i, m in enumerate(all_logs) if m.which() == "driverStateV2")
-        cmp_log += all_logs[dmon_start_index:dmon_start_index + MAX_FRAMES]
+      model_start_index = next(i for i, m in enumerate(all_logs) if m.which() in ("modelV2", "drivingModelData", "cameraOdometry"))
+      cmp_log += all_logs[model_start_index+START_FRAME*3:model_start_index + END_FRAME*3]
+      dmon_start_index = next(i for i, m in enumerate(all_logs) if m.which() == "driverStateV2")
+      cmp_log += all_logs[dmon_start_index+START_FRAME:dmon_start_index + END_FRAME]
 
       ignore = [
         'logMonoTime',

selfdrive/ui/layouts/home.py

@@ -0,0 +1,17 @@
+import pyray as rl
+from openpilot.system.ui.lib.label import gui_text_box
+
+
+class HomeLayout:
+  def __init__(self):
+    pass
+
+  def render(self, rect: rl.Rectangle):
+    gui_text_box(
+      rect,
+      "Demo Home Layout",
+      font_size=170,
+      color=rl.WHITE,
+      alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER,
+      alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE,
+    )

selfdrive/ui/layouts/main.py

@@ -0,0 +1,92 @@
+import pyray as rl
+from enum import IntEnum
+from openpilot.selfdrive.ui.layouts.sidebar import Sidebar, SIDEBAR_WIDTH
+from openpilot.selfdrive.ui.layouts.home import HomeLayout
+from openpilot.selfdrive.ui.layouts.settings.settings import SettingsLayout
+from openpilot.selfdrive.ui.ui_state import ui_state
+from openpilot.selfdrive.ui.onroad.augmented_road_view import AugmentedRoadView
+
+
+class MainState(IntEnum):
+  HOME = 0
+  SETTINGS = 1
+  ONROAD = 2
+
+
+class MainLayout:
+  def __init__(self):
+    self._sidebar = Sidebar()
+    self._sidebar_visible = True
+    self._current_mode = MainState.HOME
+    self._prev_onroad = False
+    self._window_rect = None
+    self._current_callback: callable | None = None
+
+    # Initialize layouts
+    self._layouts = {MainState.HOME: HomeLayout(), MainState.SETTINGS: SettingsLayout(), MainState.ONROAD: AugmentedRoadView()}
+
+    self._sidebar_rect = rl.Rectangle(0, 0, 0, 0)
+    self._content_rect = rl.Rectangle(0, 0, 0, 0)
+
+    # Set callbacks
+    self._setup_callbacks()
+
+  def render(self, rect):
+    self._current_callback = None
+
+    self._update_layout_rects(rect)
+    self._render_main_content()
+    self._handle_input()
+
+    if self._current_callback:
+      self._current_callback()
+
+  def _setup_callbacks(self):
+    self._sidebar.set_callbacks(
+      on_settings=lambda: setattr(self, '_current_callback', self._on_settings_clicked),
+      on_flag=lambda: setattr(self, '_current_callback', self._on_flag_clicked),
+    )
+    self._layouts[MainState.SETTINGS].set_callbacks(
+      on_close=lambda: setattr(self, '_current_callback', self._on_settings_closed)
+    )
+
+  def _update_layout_rects(self, rect):
+    self._window_rect = rect
+    self._sidebar_rect = rl.Rectangle(rect.x, rect.y, SIDEBAR_WIDTH, rect.height)
+
+    x_offset = SIDEBAR_WIDTH if self._sidebar_visible else 0
+    self._content_rect = rl.Rectangle(rect.y + x_offset, rect.y, rect.width - x_offset, rect.height)
+
+  def _on_settings_clicked(self):
+    self._current_mode = MainState.SETTINGS
+    self._sidebar_visible = False
+
+  def _on_settings_closed(self):
+    self._current_mode = MainState.HOME if not ui_state.started else MainState.ONROAD
+    self._sidebar_visible = True
+
+  def _on_flag_clicked(self):
+    pass
+
+  def _render_main_content(self):
+    # Render sidebar
+    if self._sidebar_visible:
+      self._sidebar.render(self._sidebar_rect)
+
+    if ui_state.started != self._prev_onroad:
+      self._prev_onroad = ui_state.started
+      if ui_state.started:
+        self._current_mode = MainState.ONROAD
+      else:
+        self._current_mode = MainState.HOME
+
+    content_rect = self._content_rect if self._sidebar_visible else self._window_rect
+    self._layouts[self._current_mode].render(content_rect)
+
+  def _handle_input(self):
+    if self._current_mode != MainState.ONROAD or not rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT):
+      return
+
+    mouse_pos = rl.get_mouse_position()
+    if rl.check_collision_point_rec(mouse_pos, self._content_rect):
+      self._sidebar_visible = not self._sidebar_visible

selfdrive/ui/layouts/settings/settings.py

@@ -0,0 +1,174 @@
+import pyray as rl
+from dataclasses import dataclass
+from enum import IntEnum
+from collections.abc import Callable
+from openpilot.common.params import Params
+from openpilot.system.ui.lib.application import gui_app, FontWeight
+from openpilot.system.ui.lib.label import gui_text_box
+
+# Import individual panels
+
+SETTINGS_CLOSE_TEXT = "X"
+# Constants
+SIDEBAR_WIDTH = 500
+CLOSE_BTN_SIZE = 200
+NAV_BTN_HEIGHT = 80
+PANEL_MARGIN = 50
+SCROLL_SPEED = 30
+
+# Colors
+SIDEBAR_COLOR = rl.BLACK
+PANEL_COLOR = rl.Color(41, 41, 41, 255)
+CLOSE_BTN_COLOR = rl.Color(41, 41, 41, 255)
+CLOSE_BTN_PRESSED = rl.Color(59, 59, 59, 255)
+TEXT_NORMAL = rl.Color(128, 128, 128, 255)
+TEXT_SELECTED = rl.Color(255, 255, 255, 255)
+TEXT_PRESSED = rl.Color(173, 173, 173, 255)
+
+
+class PanelType(IntEnum):
+  DEVICE = 0
+  NETWORK = 1
+  TOGGLES = 2
+  SOFTWARE = 3
+  FIREHOSE = 4
+  DEVELOPER = 5
+
+
+@dataclass
+class PanelInfo:
+  name: str
+  instance: object
+  button_rect: rl.Rectangle
+
+
+class SettingsLayout:
+  def __init__(self):
+    self._params = Params()
+    self._current_panel = PanelType.DEVICE
+    self._close_btn_pressed = False
+    self._scroll_offset = 0.0
+    self._max_scroll = 0.0
+
+    # Panel configuration
+    self._panels = {
+      PanelType.DEVICE: PanelInfo("Device", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.TOGGLES: PanelInfo("Toggles", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.SOFTWARE: PanelInfo("Software", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.FIREHOSE: PanelInfo("Firehose", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.NETWORK: PanelInfo("Network", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.DEVELOPER: PanelInfo("Developer", None, rl.Rectangle(0, 0, 0, 0)),
+    }
+
+    self._font_medium = gui_app.font(FontWeight.MEDIUM)
+    self._font_bold = gui_app.font(FontWeight.SEMI_BOLD)
+
+    # Callbacks
+    self._close_callback: Callable | None = None
+
+  def set_callbacks(self, on_close: Callable):
+    self._close_callback = on_close
+
+  def render(self, rect: rl.Rectangle):
+    # Calculate layout
+    sidebar_rect = rl.Rectangle(rect.x, rect.y, SIDEBAR_WIDTH, rect.height)
+    panel_rect = rl.Rectangle(rect.x + SIDEBAR_WIDTH, rect.y, rect.width - SIDEBAR_WIDTH, rect.height)
+
+    # Draw components
+    self._draw_sidebar(sidebar_rect)
+    self._draw_current_panel(panel_rect)
+
+    if rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT):
+      self.handle_mouse_release(rl.get_mouse_position())
+
+  def _draw_sidebar(self, rect: rl.Rectangle):
+    rl.draw_rectangle_rec(rect, SIDEBAR_COLOR)
+
+    # Close button
+    close_btn_rect = rl.Rectangle(
+      rect.x + (rect.width - CLOSE_BTN_SIZE) / 2, rect.y + 45, CLOSE_BTN_SIZE, CLOSE_BTN_SIZE
+    )
+
+    close_color = CLOSE_BTN_PRESSED if self._close_btn_pressed else CLOSE_BTN_COLOR
+    rl.draw_rectangle_rounded(close_btn_rect, 0.5, 20, close_color)
+    close_text_size = rl.measure_text_ex(self._font_bold, SETTINGS_CLOSE_TEXT, 140, 0)
+    close_text_pos = rl.Vector2(
+      close_btn_rect.x + (close_btn_rect.width - close_text_size.x) / 2,
+      close_btn_rect.y + (close_btn_rect.height - close_text_size.y) / 2 - 20,
+    )
+    rl.draw_text_ex(self._font_bold, SETTINGS_CLOSE_TEXT, close_text_pos, 140, 0, TEXT_SELECTED)
+
+    # Store close button rect for click detection
+    self._close_btn_rect = close_btn_rect
+
+    # Navigation buttons
+    nav_start_y = rect.y + 300
+    button_spacing = 20
+
+    i = 0
+    for panel_type, panel_info in self._panels.items():
+      button_rect = rl.Rectangle(
+        rect.x + 50,
+        nav_start_y + i * (NAV_BTN_HEIGHT + button_spacing),
+        rect.width - 150,  # Right-aligned with margin
+        NAV_BTN_HEIGHT,
+      )
+
+      # Button styling
+      is_selected = panel_type == self._current_panel
+      text_color = TEXT_SELECTED if is_selected else TEXT_NORMAL
+
+      # Draw button text (right-aligned)
+      text_size = rl.measure_text_ex(self._font_medium, panel_info.name, 65, 0)
+      text_pos = rl.Vector2(
+        button_rect.x + button_rect.width - text_size.x, button_rect.y + (button_rect.height - text_size.y) / 2
+      )
+      rl.draw_text_ex(self._font_medium, panel_info.name, text_pos, 65, 0, text_color)
+
+      # Store button rect for click detection
+      panel_info.button_rect = button_rect
+      i += 1
+
+  def _draw_current_panel(self, rect: rl.Rectangle):
+    content_rect = rl.Rectangle(rect.x + PANEL_MARGIN, rect.y + 25, rect.width - (PANEL_MARGIN * 2), rect.height - 50)
+    rl.draw_rectangle_rounded(content_rect, 0.03, 30, PANEL_COLOR)
+    gui_text_box(
+      content_rect,
+      f"Demo {self._panels[self._current_panel].name} Panel",
+      font_size=170,
+      color=rl.WHITE,
+      alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER,
+      alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE,
+    )
+
+  def handle_mouse_release(self, mouse_pos: rl.Vector2) -> bool:
+    # Check close button
+    if rl.check_collision_point_rec(mouse_pos, self._close_btn_rect):
+      self._close_btn_pressed = True
+      if self._close_callback:
+        self._close_callback()
+      return True
+
+    # Check navigation buttons
+    for panel_type, panel_info in self._panels.items():
+      if rl.check_collision_point_rec(mouse_pos, panel_info.button_rect):
+        self._switch_to_panel(panel_type)
+        return True
+
+    return False
+
+  def _switch_to_panel(self, panel_type: PanelType):
+    if panel_type != self._current_panel:
+      self._current_panel = panel_type
+      self._scroll_offset = 0.0  # Reset scroll when switching panels
+      self._transition_progress = 0.0
+      self._transitioning = True
+
+  def set_current_panel(self, index: int, param: str = ""):
+    panel_types = list(self._panels.keys())
+    if 0 <= index < len(panel_types):
+      self._switch_to_panel(panel_types[index])
+
+  def close_settings(self):
+    if self._close_callback:
+      self._close_callback()

selfdrive/ui/layouts/sidebar.py

@@ -0,0 +1,207 @@
+import pyray as rl
+import time
+from dataclasses import dataclass
+from collections.abc import Callable
+from cereal import log
+from openpilot.selfdrive.ui.ui_state import ui_state
+from openpilot.system.ui.lib.application import gui_app, FontWeight
+
+SIDEBAR_WIDTH = 300
+METRIC_HEIGHT = 126
+METRIC_WIDTH = 240
+METRIC_MARGIN = 30
+
+SETTINGS_BTN = rl.Rectangle(50, 35, 200, 117)
+HOME_BTN = rl.Rectangle(60, 860, 180, 180)
+
+ThermalStatus = log.DeviceState.ThermalStatus
+NetworkType = log.DeviceState.NetworkType
+
+# Color scheme
+class Colors:
+  SIDEBAR_BG = rl.Color(57, 57, 57, 255)
+  WHITE = rl.Color(255, 255, 255, 255)
+  WHITE_DIM = rl.Color(255, 255, 255, 85)
+  GRAY = rl.Color(84, 84, 84, 255)
+
+  # Status colors
+  GOOD = rl.Color(255, 255, 255, 255)
+  WARNING = rl.Color(218, 202, 37, 255)
+  DANGER = rl.Color(201, 34, 49, 255)
+
+  # UI elements
+  METRIC_BORDER = rl.Color(255, 255, 255, 85)
+  BUTTON_NORMAL = rl.Color(255, 255, 255, 255)
+  BUTTON_PRESSED = rl.Color(255, 255, 255, 166)
+
+NETWORK_TYPES = {
+  NetworkType.none: "Offline",
+  NetworkType.wifi: "WiFi",
+  NetworkType.cell2G: "2G",
+  NetworkType.cell3G: "3G",
+  NetworkType.cell4G: "LTE",
+  NetworkType.cell5G: "5G",
+  NetworkType.ethernet: "Ethernet",
+}
+
+
+@dataclass(slots=True)
+class MetricData:
+  label: str
+  value: str
+  color: rl.Color
+
+  def update(self, label: str, value: str, color: rl.Color):
+    self.label = label
+    self.value = value
+    self.color = color
+
+class Sidebar:
+  def __init__(self):
+    self._net_type = NETWORK_TYPES.get(NetworkType.none)
+    self._net_strength = 0
+
+    self._temp_status = MetricData("TEMP", "GOOD", Colors.GOOD)
+    self._panda_status = MetricData("VEHICLE", "ONLINE", Colors.GOOD)
+    self._connect_status = MetricData("CONNECT", "OFFLINE", Colors.WARNING)
+
+    self._home_img = gui_app.texture("images/button_home.png", HOME_BTN.width, HOME_BTN.height)
+    self._flag_img = gui_app.texture("images/button_flag.png", HOME_BTN.width, HOME_BTN.height)
+    self._settings_img = gui_app.texture("images/button_settings.png", SETTINGS_BTN.width, SETTINGS_BTN.height)
+    self._font_regular = gui_app.font(FontWeight.NORMAL)
+    self._font_bold = gui_app.font(FontWeight.SEMI_BOLD)
+
+     # Callbacks
+    self._on_settings_click: Callable | None = None
+    self._on_flag_click: Callable | None = None
+
+  def set_callbacks(self, on_settings: Callable | None = None, on_flag: Callable | None = None):
+    self._on_settings_click = on_settings
+    self._on_flag_click = on_flag
+
+  def render(self, rect: rl.Rectangle):
+    self.update_state()
+
+    # Background
+    rl.draw_rectangle_rec(rect, Colors.SIDEBAR_BG)
+
+    self._draw_buttons(rect)
+    self._draw_network_indicator(rect)
+    self._draw_metrics(rect)
+
+    self._handle_mouse_release()
+
+  def update_state(self):
+    sm = ui_state.sm
+    if not sm.updated['deviceState']:
+      return
+
+    device_state = sm['deviceState']
+
+    self._update_network_status(device_state)
+    self._update_temperature_status(device_state)
+    self._update_connection_status(device_state)
+    self._update_panda_status()
+
+  def _update_network_status(self, device_state):
+    self._net_type = NETWORK_TYPES.get(device_state.networkType.raw, "Unknown")
+    strength = device_state.networkStrength
+    self._net_strength = max(0, min(5, strength.raw + 1)) if strength > 0 else 0
+
+  def _update_temperature_status(self, device_state):
+    thermal_status = device_state.thermalStatus
+
+    if thermal_status == ThermalStatus.green:
+      self._temp_status.update("TEMP", "GOOD", Colors.GOOD)
+    elif thermal_status == ThermalStatus.yellow:
+      self._temp_status.update("TEMP", "OK", Colors.WARNING)
+    else:
+      self._temp_status.update("TEMP", "HIGH", Colors.DANGER)
+
+  def _update_connection_status(self, device_state):
+    last_ping = device_state.lastAthenaPingTime
+    if last_ping == 0:
+      self._connect_status.update("CONNECT", "OFFLINE", Colors.WARNING)
+    elif time.monotonic_ns() - last_ping < 80_000_000_000:  # 80 seconds in nanoseconds
+      self._connect_status.update("CONNECT", "ONLINE", Colors.GOOD)
+    else:
+      self._connect_status.update("CONNECT", "ERROR", Colors.DANGER)
+
+  def _update_panda_status(self):
+    if ui_state.panda_type == log.PandaState.PandaType.unknown:
+      self._panda_status.update("NO", "PANDA", Colors.DANGER)
+    else:
+      self._panda_status.update("VEHICLE", "ONLINE", Colors.GOOD)
+
+  def _handle_mouse_release(self):
+    if not rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT):
+      return
+
+    mouse_pos = rl.get_mouse_position()
+    if rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN):
+      if self._on_settings_click:
+        self._on_settings_click()
+    elif rl.check_collision_point_rec(mouse_pos, HOME_BTN) and ui_state.started:
+      if self._on_flag_click:
+        self._on_flag_click()
+
+  def _draw_buttons(self, rect: rl.Rectangle):
+    mouse_pos = rl.get_mouse_position()
+    mouse_down = rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT)
+
+
+    # Settings button
+    settings_down = mouse_down and rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN)
+    tint = Colors.BUTTON_PRESSED if settings_down else Colors.BUTTON_NORMAL
+    rl.draw_texture(self._settings_img, int(SETTINGS_BTN.x), int(SETTINGS_BTN.y), tint)
+
+    # Home/Flag button
+    flag_pressed = mouse_down and rl.check_collision_point_rec(mouse_pos, HOME_BTN)
+    button_img = self._flag_img if ui_state.started else self._home_img
+
+    tint = Colors.BUTTON_PRESSED if (ui_state.started and flag_pressed) else Colors.BUTTON_NORMAL
+    rl.draw_texture(button_img, int(HOME_BTN.x), int(HOME_BTN.y), tint)
+
+  def _draw_network_indicator(self, rect: rl.Rectangle):
+    # Signal strength dots
+    x_start = rect.x + 58
+    y_pos = rect.y + 196
+    dot_size = 27
+    dot_spacing = 37
+
+    for i in range(5):
+      color = Colors.WHITE if i < self._net_strength else Colors.GRAY
+      x = int(x_start + i * dot_spacing + dot_size // 2)
+      y = int(y_pos + dot_size // 2)
+      rl.draw_circle(x, y, dot_size // 2, color)
+
+    # Network type text
+    text_y = rect.y + 247
+    text_pos = rl.Vector2(rect.x + 58, text_y)
+    rl.draw_text_ex(self._font_regular, self._net_type, text_pos, 35, 0, Colors.WHITE)
+
+  def _draw_metrics(self, rect: rl.Rectangle):
+    metrics = [(self._temp_status, 338), (self._panda_status, 496), (self._connect_status, 654)]
+
+    for metric, y_offset in metrics:
+      self._draw_metric(rect, metric, rect.y + y_offset)
+
+  def _draw_metric(self, rect: rl.Rectangle, metric: MetricData, y: float):
+    metric_rect = rl.Rectangle(rect.x + METRIC_MARGIN, y, METRIC_WIDTH, METRIC_HEIGHT)
+    # Draw colored left edge (clipped rounded rectangle)
+    edge_rect = rl.Rectangle(metric_rect.x + 4, metric_rect.y + 4, 100, 118)
+    rl.begin_scissor_mode(int(metric_rect.x + 4), int(metric_rect.y), 18, int(metric_rect.height))
+    rl.draw_rectangle_rounded(edge_rect, 0.18, 10, metric.color)
+    rl.end_scissor_mode()
+
+    # Draw border
+    rl.draw_rectangle_rounded_lines_ex(metric_rect, 0.15, 10, 2, Colors.METRIC_BORDER)
+
+    # Draw text
+    text = f"{metric.label}\n{metric.value}"
+    text_size = rl.measure_text_ex(self._font_bold, text, 35, 0)
+    text_pos = rl.Vector2(
+      metric_rect.x + 22 + (metric_rect.width - 22 - text_size.x) / 2,
+      metric_rect.y + (metric_rect.height - text_size.y) / 2
+    )
+    rl.draw_text_ex(self._font_bold, text, text_pos, 35, 0, Colors.WHITE)

selfdrive/ui/onroad/alert_renderer.py

@@ -0,0 +1,159 @@
+import time
+import pyray as rl
+from dataclasses import dataclass
+from cereal import messaging, log
+from openpilot.system.hardware import TICI
+from openpilot.system.ui.lib.application import gui_app, FontWeight, DEFAULT_FPS
+from openpilot.system.ui.lib.label import gui_text_box
+from openpilot.system.ui.lib.text_measure import measure_text_cached
+from openpilot.selfdrive.ui.ui_state import ui_state
+
+
+ALERT_MARGIN = 40
+ALERT_PADDING = 60
+ALERT_LINE_SPACING = 45
+ALERT_BORDER_RADIUS = 30
+
+ALERT_FONT_SMALL = 66
+ALERT_FONT_MEDIUM = 74
+ALERT_FONT_BIG = 88
+
+SELFDRIVE_STATE_TIMEOUT = 5  # Seconds
+SELFDRIVE_UNRESPONSIVE_TIMEOUT = 10  # Seconds
+
+
+# Constants
+ALERT_COLORS = {
+  log.SelfdriveState.AlertStatus.normal: rl.Color(0, 0, 0, 235),  # Black
+  log.SelfdriveState.AlertStatus.userPrompt: rl.Color(0xFE, 0x8C, 0x34, 235),  # Orange
+  log.SelfdriveState.AlertStatus.critical: rl.Color(0xC9, 0x22, 0x31, 235),  # Red
+}
+
+
+@dataclass
+class Alert:
+  text1: str = ""
+  text2: str = ""
+  size: int = 0
+  status: int = 0
+
+
+# Pre-defined alert instances
+ALERT_STARTUP_PENDING = Alert(
+  text1="openpilot Unavailable",
+  text2="Waiting to start",
+  size=log.SelfdriveState.AlertSize.mid,
+  status=log.SelfdriveState.AlertStatus.normal,
+)
+
+ALERT_CRITICAL_TIMEOUT = Alert(
+  text1="TAKE CONTROL IMMEDIATELY",
+  text2="System Unresponsive",
+  size=log.SelfdriveState.AlertSize.full,
+  status=log.SelfdriveState.AlertStatus.critical,
+)
+
+ALERT_CRITICAL_REBOOT = Alert(
+  text1="System Unresponsive",
+  text2="Reboot Device",
+  size=log.SelfdriveState.AlertSize.full,
+  status=log.SelfdriveState.AlertStatus.critical,
+)
+
+
+class AlertRenderer:
+  def __init__(self):
+    self.font_regular: rl.Font = gui_app.font(FontWeight.NORMAL)
+    self.font_bold: rl.Font = gui_app.font(FontWeight.BOLD)
+
+  def get_alert(self, sm: messaging.SubMaster) -> Alert | None:
+    """Generate the current alert based on selfdrive state."""
+    ss = sm['selfdriveState']
+
+    # Check if selfdriveState messages have stopped arriving
+    if not sm.updated['selfdriveState']:
+      recv_frame = sm.recv_frame['selfdriveState']
+      if (sm.frame - recv_frame) > 5 * DEFAULT_FPS:
+        # Check if waiting to start
+        if recv_frame < ui_state.started_frame:
+          return ALERT_STARTUP_PENDING
+
+        # Handle selfdrive timeout
+        if TICI:
+          ss_missing = time.monotonic() - sm.recv_time['selfdriveState']
+          if ss_missing > SELFDRIVE_STATE_TIMEOUT:
+            if ss.enabled and (ss_missing - SELFDRIVE_STATE_TIMEOUT) < SELFDRIVE_UNRESPONSIVE_TIMEOUT:
+              return ALERT_CRITICAL_TIMEOUT
+            return ALERT_CRITICAL_REBOOT
+
+    # No alert if size is none
+    if ss.alertSize == 0:
+      return None
+
+    # Return current alert
+    return Alert(text1=ss.alertText1, text2=ss.alertText2, size=ss.alertSize, status=ss.alertStatus)
+
+  def draw(self, rect: rl.Rectangle, sm: messaging.SubMaster) -> None:
+    alert = self.get_alert(sm)
+    if not alert:
+      return
+
+    alert_rect = self._get_alert_rect(rect, alert.size)
+    self._draw_background(alert_rect, alert)
+
+    text_rect = rl.Rectangle(
+      alert_rect.x + ALERT_PADDING,
+      alert_rect.y + ALERT_PADDING,
+      alert_rect.width - 2 * ALERT_PADDING,
+      alert_rect.height - 2 * ALERT_PADDING
+    )
+    self._draw_text(text_rect, alert)
+
+  def _get_alert_rect(self, rect: rl.Rectangle, size: int) -> rl.Rectangle:
+    if size == log.SelfdriveState.AlertSize.full:
+      return rect
+
+    height = (ALERT_FONT_MEDIUM + 2 * ALERT_PADDING if size == log.SelfdriveState.AlertSize.small else
+             ALERT_FONT_BIG + ALERT_LINE_SPACING + ALERT_FONT_SMALL + 2 * ALERT_PADDING)
+
+    return rl.Rectangle(
+      rect.x + ALERT_MARGIN,
+      rect.y + rect.height - ALERT_MARGIN - height,
+      rect.width - 2 * ALERT_MARGIN,
+      height
+    )
+
+  def _draw_background(self, rect: rl.Rectangle, alert: Alert) -> None:
+    color = ALERT_COLORS.get(alert.status, ALERT_COLORS[log.SelfdriveState.AlertStatus.normal])
+
+    if alert.size != log.SelfdriveState.AlertSize.full:
+      roundness = ALERT_BORDER_RADIUS / (min(rect.width, rect.height) / 2)
+      rl.draw_rectangle_rounded(rect, roundness, 10, color)
+    else:
+      rl.draw_rectangle_rec(rect, color)
+
+  def _draw_text(self, rect: rl.Rectangle, alert: Alert) -> None:
+    if alert.size == log.SelfdriveState.AlertSize.small:
+      self._draw_centered(alert.text1, rect, self.font_bold, ALERT_FONT_MEDIUM)
+
+    elif alert.size == log.SelfdriveState.AlertSize.mid:
+      self._draw_centered(alert.text1, rect, self.font_bold, ALERT_FONT_BIG, center_y=False)
+      rect.y += ALERT_FONT_BIG + ALERT_LINE_SPACING
+      self._draw_centered(alert.text2, rect, self.font_regular, ALERT_FONT_SMALL, center_y=False)
+
+    else:
+      is_long = len(alert.text1) > 15
+      font_size1 = 132 if is_long else 177
+      align_ment = rl.GuiTextAlignment.TEXT_ALIGN_CENTER
+      vertical_align = rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE
+      text_rect = rl.Rectangle(rect.x, rect.y, rect.width, rect.height // 2)
+
+      gui_text_box(text_rect, alert.text1, font_size1, alignment=align_ment, alignment_vertical=vertical_align, font_weight=FontWeight.BOLD)
+      text_rect.y = rect.y + rect.height // 2
+      gui_text_box(text_rect, alert.text2, ALERT_FONT_BIG, alignment=align_ment)
+
+  def _draw_centered(self, text, rect, font, font_size, center_y=True, color=rl.WHITE) -> None:
+    text_size = measure_text_cached(font, text, font_size)
+    x = rect.x + (rect.width - text_size.x) / 2
+    y = rect.y + ((rect.height - text_size.y) / 2 if center_y else 0)
+    rl.draw_text_ex(font, text, rl.Vector2(x, y), font_size, 0, color)

selfdrive/ui/onroad/augmented_road_view.py

@@ -0,0 +1,188 @@
+import numpy as np
+import pyray as rl
+
+from cereal import log
+from msgq.visionipc import VisionStreamType
+from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus, UI_BORDER_SIZE
+from openpilot.selfdrive.ui.onroad.alert_renderer import AlertRenderer
+from openpilot.selfdrive.ui.onroad.driver_state import DriverStateRenderer
+from openpilot.selfdrive.ui.onroad.hud_renderer import HudRenderer
+from openpilot.selfdrive.ui.onroad.model_renderer import ModelRenderer
+from openpilot.selfdrive.ui.onroad.cameraview import CameraView
+from openpilot.system.ui.lib.application import gui_app
+from openpilot.common.transformations.camera import DEVICE_CAMERAS, DeviceCameraConfig, view_frame_from_device_frame
+from openpilot.common.transformations.orientation import rot_from_euler
+
+
+OpState = log.SelfdriveState.OpenpilotState
+CALIBRATED = log.LiveCalibrationData.Status.calibrated
+DEFAULT_DEVICE_CAMERA = DEVICE_CAMERAS["tici", "ar0231"]
+
+BORDER_COLORS = {
+  UIStatus.DISENGAGED: rl.Color(0x17, 0x33, 0x49, 0xC8),   # Blue for disengaged state
+  UIStatus.OVERRIDE: rl.Color(0x91, 0x9B, 0x95, 0xF1),     # Gray for override state
+  UIStatus.ENGAGED: rl.Color(0x17, 0x86, 0x44, 0xF1),      # Green for engaged state
+}
+
+
+class AugmentedRoadView(CameraView):
+  def __init__(self, stream_type: VisionStreamType = VisionStreamType.VISION_STREAM_ROAD):
+    super().__init__("camerad", stream_type)
+
+    self.device_camera: DeviceCameraConfig | None = None
+    self.view_from_calib = view_frame_from_device_frame.copy()
+    self.view_from_wide_calib = view_frame_from_device_frame.copy()
+
+    self._last_calib_time: float = 0
+    self._last_rect_dims = (0.0, 0.0)
+    self._cached_matrix: np.ndarray | None = None
+    self._content_rect = rl.Rectangle()
+
+    self.model_renderer = ModelRenderer()
+    self._hud_renderer = HudRenderer()
+    self.alert_renderer = AlertRenderer()
+    self.driver_state_renderer = DriverStateRenderer()
+
+  def render(self, rect):
+    # Only render when system is started to avoid invalid data access
+    if not ui_state.started:
+      return
+
+    # Update calibration before rendering
+    self._update_calibration()
+
+    # Create inner content area with border padding
+    self._content_rect = rl.Rectangle(
+      rect.x + UI_BORDER_SIZE,
+      rect.y + UI_BORDER_SIZE,
+      rect.width - 2 * UI_BORDER_SIZE,
+      rect.height - 2 * UI_BORDER_SIZE,
+    )
+
+    # Draw colored border based on driving state
+    self._draw_border(rect)
+
+    # Enable scissor mode to clip all rendering within content rectangle boundaries
+    # This creates a rendering viewport that prevents graphics from drawing outside the border
+    rl.begin_scissor_mode(
+      int(self._content_rect.x),
+      int(self._content_rect.y),
+      int(self._content_rect.width),
+      int(self._content_rect.height)
+    )
+
+    # Render the base camera view
+    super().render(rect)
+
+    # Draw all UI overlays
+    self.model_renderer.draw(self._content_rect, ui_state.sm)
+    self._hud_renderer.draw(self._content_rect, ui_state.sm)
+    self.alert_renderer.draw(self._content_rect, ui_state.sm)
+    self.driver_state_renderer.draw(self._content_rect, ui_state.sm)
+
+    # Custom UI extension point - add custom overlays here
+    # Use self._content_rect for positioning within camera bounds
+
+    # End clipping region
+    rl.end_scissor_mode()
+
+  def _draw_border(self, rect: rl.Rectangle):
+    border_color = BORDER_COLORS.get(ui_state.status, BORDER_COLORS[UIStatus.DISENGAGED])
+    rl.draw_rectangle_lines_ex(rect, UI_BORDER_SIZE, border_color)
+
+  def _update_calibration(self):
+    # Update device camera if not already set
+    sm = ui_state.sm
+    if not self.device_camera and sm.seen['roadCameraState'] and sm.seen['deviceState']:
+      self.device_camera = DEVICE_CAMERAS[(str(sm['deviceState'].deviceType), str(sm['roadCameraState'].sensor))]
+
+    # Check if live calibration data is available and valid
+    if not (sm.updated["liveCalibration"] and sm.valid['liveCalibration']):
+      return
+
+    calib = sm['liveCalibration']
+    if len(calib.rpyCalib) != 3 or calib.calStatus != CALIBRATED:
+      return
+
+    # Update view_from_calib matrix
+    device_from_calib = rot_from_euler(calib.rpyCalib)
+    self.view_from_calib = view_frame_from_device_frame @ device_from_calib
+
+    # Update wide calibration if available
+    if hasattr(calib, 'wideFromDeviceEuler') and len(calib.wideFromDeviceEuler) == 3:
+      wide_from_device = rot_from_euler(calib.wideFromDeviceEuler)
+      self.view_from_wide_calib = view_frame_from_device_frame @ wide_from_device @ device_from_calib
+
+  def _calc_frame_matrix(self, rect: rl.Rectangle) -> np.ndarray:
+    # Check if we can use cached matrix
+    calib_time = ui_state.sm.recv_frame['liveCalibration']
+    current_dims = (self._content_rect.width, self._content_rect.height)
+    if (self._last_calib_time == calib_time and
+        self._last_rect_dims == current_dims and
+        self._cached_matrix is not None):
+      return self._cached_matrix
+
+    # Get camera configuration
+    device_camera = self.device_camera or DEFAULT_DEVICE_CAMERA
+    is_wide_camera = self.stream_type == VisionStreamType.VISION_STREAM_WIDE_ROAD
+    intrinsic = device_camera.ecam.intrinsics if is_wide_camera else device_camera.fcam.intrinsics
+    calibration = self.view_from_wide_calib if is_wide_camera else self.view_from_calib
+    zoom = 2.0 if is_wide_camera else 1.1
+
+    # Calculate transforms for vanishing point
+    inf_point = np.array([1000.0, 0.0, 0.0])
+    calib_transform = intrinsic @ calibration
+    kep = calib_transform @ inf_point
+
+    # Calculate center points and dimensions
+    x, y = self._content_rect.x, self._content_rect.y
+    w, h = self._content_rect.width, self._content_rect.height
+    cx, cy = intrinsic[0, 2], intrinsic[1, 2]
+
+    # Calculate max allowed offsets with margins
+    margin = 5
+    max_x_offset = cx * zoom - w / 2 - margin
+    max_y_offset = cy * zoom - h / 2 - margin
+
+    # Calculate and clamp offsets to prevent out-of-bounds issues
+    try:
+      if abs(kep[2]) > 1e-6:
+        x_offset = np.clip((kep[0] / kep[2] - cx) * zoom, -max_x_offset, max_x_offset)
+        y_offset = np.clip((kep[1] / kep[2] - cy) * zoom, -max_y_offset, max_y_offset)
+      else:
+        x_offset, y_offset = 0, 0
+    except (ZeroDivisionError, OverflowError):
+      x_offset, y_offset = 0, 0
+
+    # Update cache values
+    self._last_calib_time = calib_time
+    self._last_rect_dims = current_dims
+    self._cached_matrix = np.array([
+      [zoom * 2 * cx / w, 0, -x_offset / w * 2],
+      [0, zoom * 2 * cy / h, -y_offset / h * 2],
+      [0, 0, 1.0]
+    ])
+
+    video_transform = np.array([
+        [zoom, 0.0, (w / 2 + x - x_offset) - (cx * zoom)],
+        [0.0, zoom, (h / 2 + y - y_offset) - (cy * zoom)],
+        [0.0, 0.0, 1.0]
+    ])
+    self.model_renderer.set_transform(video_transform @ calib_transform)
+
+    return self._cached_matrix
+
+
+if __name__ == "__main__":
+  gui_app.init_window("OnRoad Camera View")
+  road_camera_view = AugmentedRoadView(VisionStreamType.VISION_STREAM_ROAD)
+  print("***press space to switch camera view***")
+  try:
+    for _ in gui_app.render():
+      ui_state.update()
+      if rl.is_key_released(rl.KeyboardKey.KEY_SPACE):
+        is_wide = road_camera_view.stream_type == VisionStreamType.VISION_STREAM_WIDE_ROAD
+        road_camera_view.switch_stream(VisionStreamType.VISION_STREAM_ROAD if is_wide else VisionStreamType.VISION_STREAM_WIDE_ROAD)
+      road_camera_view.render(rl.Rectangle(0, 0, gui_app.width, gui_app.height))
+  finally:
+    road_camera_view.close()

selfdrive/ui/onroad/cameraview.py

@@ -0,0 +1,267 @@
+import numpy as np
+import pyray as rl
+
+from openpilot.system.hardware import TICI
+from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.ui.lib.application import gui_app
+from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage
+
+
+CONNECTION_RETRY_INTERVAL = 0.2  # seconds between connection attempts
+
+VERTEX_SHADER = """
+#version 300 es
+in vec3 vertexPosition;
+in vec2 vertexTexCoord;
+in vec3 vertexNormal;
+in vec4 vertexColor;
+uniform mat4 mvp;
+out vec2 fragTexCoord;
+out vec4 fragColor;
+void main() {
+  fragTexCoord = vertexTexCoord;
+  fragColor = vertexColor;
+  gl_Position = mvp * vec4(vertexPosition, 1.0);
+}
+"""
+
+# Choose fragment shader based on platform capabilities
+if TICI:
+  FRAME_FRAGMENT_SHADER = """
+    #version 300 es
+    #extension GL_OES_EGL_image_external_essl3 : enable
+    precision mediump float;
+    in vec2 fragTexCoord;
+    uniform samplerExternalOES texture0;
+    out vec4 fragColor;
+    void main() {
+      vec4 color = texture(texture0, fragTexCoord);
+      fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a);
+    }
+    """
+else:
+  FRAME_FRAGMENT_SHADER = """
+    #version 300 es
+    precision mediump float;
+    in vec2 fragTexCoord;
+    uniform sampler2D texture0;
+    uniform sampler2D texture1;
+    out vec4 fragColor;
+    void main() {
+      float y = texture(texture0, fragTexCoord).r;
+      vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
+      fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0);
+    }
+    """
+
+class CameraView:
+  def __init__(self, name: str, stream_type: VisionStreamType):
+    self.client = VisionIpcClient(name, stream_type, conflate=True)
+    self._name = name
+    self._stream_type = stream_type
+
+    self._texture_needs_update = True
+    self.last_connection_attempt: float = 0.0
+    self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER)
+    self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1
+
+    self.frame: VisionBuf | None = None
+    self.texture_y: rl.Texture | None = None
+    self.texture_uv: rl.Texture | None = None
+
+    # EGL resources
+    self.egl_images: dict[int, EGLImage] = {}
+    self.egl_texture: rl.Texture | None = None
+
+    # Initialize EGL for zero-copy rendering on TICI
+    if TICI:
+      if not init_egl():
+        raise RuntimeError("Failed to initialize EGL")
+
+      # Create a 1x1 pixel placeholder texture for EGL image binding
+      temp_image = rl.gen_image_color(1, 1, rl.BLACK)
+      self.egl_texture = rl.load_texture_from_image(temp_image)
+      rl.unload_image(temp_image)
+
+  def switch_stream(self, stream_type: VisionStreamType) -> None:
+    if self._stream_type != stream_type:
+      cloudlog.debug(f'switching stream from {self._stream_type} to {stream_type}')
+      self._clear_textures()
+      self.frame = None
+      self._stream_type = stream_type
+      self.client = VisionIpcClient(self._name, stream_type, conflate=True)
+
+  @property
+  def stream_type(self) -> VisionStreamType:
+    return self._stream_type
+
+  def close(self) -> None:
+    self._clear_textures()
+
+    # Clean up EGL texture
+    if TICI and self.egl_texture:
+      rl.unload_texture(self.egl_texture)
+      self.egl_texture = None
+
+    # Clean up shader
+    if self.shader and self.shader.id:
+      rl.unload_shader(self.shader)
+
+    self.client = None
+
+  def _calc_frame_matrix(self, rect: rl.Rectangle) -> np.ndarray:
+    if not self.frame:
+      return np.eye(3)
+
+    # Calculate aspect ratios
+    widget_aspect_ratio = rect.width / rect.height
+    frame_aspect_ratio = self.frame.width / self.frame.height
+
+    # Calculate scaling factors to maintain aspect ratio
+    zx = min(frame_aspect_ratio / widget_aspect_ratio, 1.0)
+    zy = min(widget_aspect_ratio / frame_aspect_ratio, 1.0)
+
+    return np.array([
+        [zx, 0.0, 0.0],
+        [0.0, zy, 0.0],
+        [0.0, 0.0, 1.0]
+    ])
+
+  def render(self, rect: rl.Rectangle):
+    if not self._ensure_connection():
+      return
+
+    # Try to get a new buffer without blocking
+    buffer = self.client.recv(timeout_ms=0)
+    if buffer:
+      self._texture_needs_update = True
+      self.frame = buffer
+
+    if not self.frame:
+      return
+
+    transform = self._calc_frame_matrix(rect)
+    src_rect = rl.Rectangle(0, 0, float(self.frame.width), float(self.frame.height))
+
+    # Calculate scale
+    scale_x = rect.width * transform[0, 0]  # zx
+    scale_y = rect.height * transform[1, 1]  # zy
+
+    # Calculate base position (centered)
+    x_offset = rect.x + (rect.width - scale_x) / 2
+    y_offset = rect.y + (rect.height - scale_y) / 2
+
+    x_offset += transform[0, 2] * rect.width / 2
+    y_offset += transform[1, 2] * rect.height / 2
+
+    dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y)
+
+    # Render with appropriate method
+    if TICI:
+      self._render_egl(src_rect, dst_rect)
+    else:
+      self._render_textures(src_rect, dst_rect)
+
+  def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
+    """Render using EGL for direct buffer access"""
+    if self.frame is None or self.egl_texture is None:
+      return
+
+    idx = self.frame.idx
+    egl_image = self.egl_images.get(idx)
+
+    # Create EGL image if needed
+    if egl_image is None:
+      egl_image = create_egl_image(self.frame.width, self.frame.height, self.frame.stride, self.frame.fd, self.frame.uv_offset)
+      if egl_image:
+        self.egl_images[idx] = egl_image
+      else:
+        return
+
+    # Update texture dimensions to match current frame
+    self.egl_texture.width = self.frame.width
+    self.egl_texture.height = self.frame.height
+
+    # Bind the EGL image to our texture
+    bind_egl_image_to_texture(self.egl_texture.id, egl_image)
+
+    # Render with shader
+    rl.begin_shader_mode(self.shader)
+    rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
+    rl.end_shader_mode()
+
+  def _render_textures(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
+    """Render using texture copies"""
+    if not self.texture_y or not self.texture_uv or self.frame is None:
+      return
+
+    # Update textures with new frame data
+    if self._texture_needs_update:
+      y_data = self.frame.data[: self.frame.uv_offset]
+      uv_data = self.frame.data[self.frame.uv_offset :]
+
+      rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data))
+      rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data))
+      self._texture_needs_update = False
+
+    # Render with shader
+    rl.begin_shader_mode(self.shader)
+    rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv)
+    rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
+    rl.end_shader_mode()
+
+  def _ensure_connection(self) -> bool:
+    if not self.client.is_connected():
+      self.frame = None
+
+      # Throttle connection attempts
+      current_time = rl.get_time()
+      if current_time - self.last_connection_attempt < CONNECTION_RETRY_INTERVAL:
+        return False
+      self.last_connection_attempt = current_time
+
+      if not self.client.connect(False) or not self.client.num_buffers:
+        return False
+
+      cloudlog.debug(f"Connected to {self._name} stream: {self._stream_type}, buffers: {self.client.num_buffers}")
+      self._clear_textures()
+
+      if not TICI:
+        self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
+          int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
+        self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
+          int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
+
+    return True
+
+  def _clear_textures(self):
+    if self.texture_y and self.texture_y.id:
+      rl.unload_texture(self.texture_y)
+      self.texture_y = None
+
+    if self.texture_uv and self.texture_uv.id:
+      rl.unload_texture(self.texture_uv)
+      self.texture_uv = None
+
+    # Clean up EGL resources
+    if TICI:
+      for data in self.egl_images.values():
+        destroy_egl_image(data)
+      self.egl_images = {}
+
+
+if __name__ == "__main__":
+  gui_app.init_window("watch3")
+  road_camera_view = CameraView("camerad", VisionStreamType.VISION_STREAM_ROAD)
+  driver_camera_view = CameraView("camerad", VisionStreamType.VISION_STREAM_DRIVER)
+  wide_road_camera_view = CameraView("camerad", VisionStreamType.VISION_STREAM_WIDE_ROAD)
+  try:
+    for _ in gui_app.render():
+      road_camera_view.render(rl.Rectangle(gui_app.width // 4, 0, gui_app.width // 2, gui_app.height // 2))
+      driver_camera_view.render(rl.Rectangle(0, gui_app.height // 2, gui_app.width // 2, gui_app.height // 2))
+      wide_road_camera_view.render(rl.Rectangle(gui_app.width // 2, gui_app.height // 2, gui_app.width // 2, gui_app.height // 2))
+  finally:
+    road_camera_view.close()
+    driver_camera_view.close()
+    wide_road_camera_view.close()

selfdrive/ui/onroad/driver_camera_view.py

@@ -0,0 +1,94 @@
+import numpy as np
+import pyray as rl
+from cereal import messaging
+from msgq.visionipc import VisionStreamType
+from openpilot.selfdrive.ui.onroad.cameraview import CameraView
+from openpilot.selfdrive.ui.onroad.driver_state import DriverStateRenderer
+from openpilot.system.ui.lib.application import gui_app, FontWeight
+from openpilot.system.ui.lib.label import gui_label
+
+
+class DriverCameraView(CameraView):
+  def __init__(self, stream_type: VisionStreamType):
+    super().__init__("camerad", stream_type)
+    self.driver_state_renderer = DriverStateRenderer()
+
+  def render(self, rect, sm):
+    super().render(rect)
+
+    if not self.frame:
+      gui_label(
+        rect,
+        "camera starting",
+        font_size=100,
+        font_weight=FontWeight.BOLD,
+        alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER,
+      )
+      return
+
+    self._draw_face_detection(rect, sm)
+    self.driver_state_renderer.draw(rect, sm)
+
+  def _draw_face_detection(self, rect: rl.Rectangle, sm) -> None:
+    driver_state = sm["driverStateV2"]
+    is_rhd = driver_state.wheelOnRightProb > 0.5
+    driver_data = driver_state.rightDriverData if is_rhd else driver_state.leftDriverData
+    face_detect = driver_data.faceProb > 0.7
+    if not face_detect:
+      return
+
+    # Get face position and orientation
+    face_x, face_y = driver_data.facePosition
+    face_std = max(driver_data.faceOrientationStd[0], driver_data.faceOrientationStd[1])
+    alpha = 0.7
+    if face_std > 0.15:
+      alpha = max(0.7 - (face_std - 0.15) * 3.5, 0.0)
+
+    # use approx instead of distort_points
+    # TODO: replace with distort_points
+    fbox_x = int(1080.0 - 1714.0 * face_x)
+    fbox_y = int(-135.0 + (504.0 + abs(face_x) * 112.0) + (1205.0 - abs(face_x) * 724.0) * face_y)
+    box_size = 220
+
+    line_color = rl.Color(255, 255, 255, int(alpha * 255))
+    rl.draw_rectangle_rounded_lines_ex(
+      rl.Rectangle(fbox_x - box_size / 2, fbox_y - box_size / 2, box_size, box_size),
+      35.0 / box_size / 2,
+      10,
+      10,
+      line_color,
+    )
+
+  def _calc_frame_matrix(self, rect: rl.Rectangle) -> np.ndarray:
+    driver_view_ratio = 2.0
+
+    # Get stream dimensions
+    if self.frame:
+      stream_width = self.frame.width
+      stream_height = self.frame.height
+    else:
+      # Default values if frame not available
+      stream_width = 1928
+      stream_height = 1208
+
+    yscale = stream_height * driver_view_ratio / stream_width
+    xscale = yscale * rect.height / rect.width * stream_width / stream_height
+
+    return np.array([
+      [xscale, 0.0, 0.0],
+      [0.0, yscale, 0.0],
+      [0.0, 0.0, 1.0]
+    ])
+
+
+if __name__ == "__main__":
+  gui_app.init_window("Driver Camera View")
+  sm = messaging.SubMaster(["selfdriveState", "driverStateV2", "driverMonitoringState"])
+
+  driver_camera_view = DriverCameraView(VisionStreamType.VISION_STREAM_DRIVER)
+  try:
+    for _ in gui_app.render():
+      sm.update()
+      driver_camera_view.render(rl.Rectangle(0, 0, gui_app.width, gui_app.height), sm)
+  finally:
+    driver_camera_view.close()

selfdrive/ui/onroad/driver_state.py

@@ -0,0 +1,239 @@
+import numpy as np
+import pyray as rl
+from dataclasses import dataclass
+from openpilot.selfdrive.ui.ui_state import ui_state, UI_BORDER_SIZE
+from openpilot.system.ui.lib.application import gui_app
+
+
+# Default 3D coordinates for face keypoints as a NumPy array
+DEFAULT_FACE_KPTS_3D = np.array([
+    [-5.98, -51.20, 8.00], [-17.64, -49.14, 8.00], [-23.81, -46.40, 8.00], [-29.98, -40.91, 8.00],
+    [-32.04, -37.49, 8.00], [-34.10, -32.00, 8.00], [-36.16, -21.03, 8.00], [-36.16, 6.40, 8.00],
+    [-35.47, 10.51, 8.00], [-32.73, 19.43, 8.00], [-29.30, 26.29, 8.00], [-24.50, 33.83, 8.00],
+    [-19.01, 41.37, 8.00], [-14.21, 46.17, 8.00], [-12.16, 47.54, 8.00], [-4.61, 49.60, 8.00],
+    [4.99, 49.60, 8.00], [12.53, 47.54, 8.00], [14.59, 46.17, 8.00], [19.39, 41.37, 8.00],
+    [24.87, 33.83, 8.00], [29.67, 26.29, 8.00], [33.10, 19.43, 8.00], [35.84, 10.51, 8.00],
+    [36.53, 6.40, 8.00], [36.53, -21.03, 8.00], [34.47, -32.00, 8.00], [32.42, -37.49, 8.00],
+    [30.36, -40.91, 8.00], [24.19, -46.40, 8.00], [18.02, -49.14, 8.00], [6.36, -51.20, 8.00],
+    [-5.98, -51.20, 8.00],
+], dtype=np.float32)
+
+# UI constants
+BTN_SIZE = 192
+IMG_SIZE = 144
+ARC_LENGTH = 133
+ARC_THICKNESS_DEFAULT = 6.7
+ARC_THICKNESS_EXTEND = 12.0
+
+SCALES_POS = np.array([0.9, 0.4, 0.4], dtype=np.float32)
+SCALES_NEG = np.array([0.7, 0.4, 0.4], dtype=np.float32)
+
+ARC_POINT_COUNT = 37  # Number of points in the arc
+ARC_ANGLES = np.linspace(0.0, np.pi, ARC_POINT_COUNT, dtype=np.float32)
+
+@dataclass
+class ArcData:
+  """Data structure for arc rendering parameters."""
+  x: float
+  y: float
+  width: float
+  height: float
+  thickness: float
+
+class DriverStateRenderer:
+  def __init__(self):
+    # Initial state with NumPy arrays
+    self.face_kpts_draw = DEFAULT_FACE_KPTS_3D.copy()
+    self.is_active = False
+    self.is_rhd = False
+    self.dm_fade_state = 0.0
+    self.state_updated = False
+    self.last_rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0)
+    self.driver_pose_vals = np.zeros(3, dtype=np.float32)
+    self.driver_pose_diff = np.zeros(3, dtype=np.float32)
+    self.driver_pose_sins = np.zeros(3, dtype=np.float32)
+    self.driver_pose_coss = np.zeros(3, dtype=np.float32)
+    self.face_keypoints_transformed = np.zeros((DEFAULT_FACE_KPTS_3D.shape[0], 2), dtype=np.float32)
+    self.position_x: float = 0.0
+    self.position_y: float = 0.0
+    self.h_arc_data = None
+    self.v_arc_data = None
+
+    # Pre-allocate drawing arrays
+    self.face_lines = [rl.Vector2(0, 0) for _ in range(len(DEFAULT_FACE_KPTS_3D))]
+    self.h_arc_lines = [rl.Vector2(0, 0) for _ in range(ARC_POINT_COUNT)]
+    self.v_arc_lines = [rl.Vector2(0, 0) for _ in range(ARC_POINT_COUNT)]
+
+    # Load the driver face icon
+    self.dm_img = gui_app.texture("icons/driver_face.png", IMG_SIZE, IMG_SIZE)
+
+    # Colors
+    self.white_color = rl.Color(255, 255, 255, 255)
+    self.arc_color = rl.Color(26, 242, 66, 255)
+    self.engaged_color = rl.Color(26, 242, 66, 255)
+    self.disengaged_color = rl.Color(139, 139, 139, 255)
+
+  def draw(self, rect, sm):
+    if not self._is_visible(sm):
+      return
+
+    self._update_state(sm, rect)
+    if not self.state_updated:
+      return
+
+    # Set opacity based on active state
+    opacity = 0.65 if self.is_active else 0.2
+
+    # Draw background circle
+    rl.draw_circle(int(self.position_x), int(self.position_y), BTN_SIZE // 2, rl.Color(0, 0, 0, 70))
+
+    # Draw face icon
+    icon_pos = rl.Vector2(self.position_x - self.dm_img.width // 2, self.position_y - self.dm_img.height // 2)
+    rl.draw_texture_v(self.dm_img, icon_pos, rl.Color(255, 255, 255, int(255 * opacity)))
+
+    # Draw face outline
+    self.white_color.a = int(255 * opacity)
+    rl.draw_spline_linear(self.face_lines, len(self.face_lines), 5.2, self.white_color)
+
+    # Set arc color based on engaged state
+    self.arc_color = self.engaged_color if ui_state.engaged else self.disengaged_color
+    self.arc_color.a = int(0.4 * 255 * (1.0 - self.dm_fade_state))  # Fade out when inactive
+
+    # Draw arcs
+    if self.h_arc_data:
+      rl.draw_spline_linear(self.h_arc_lines, len(self.h_arc_lines), self.h_arc_data.thickness, self.arc_color)
+    if self.v_arc_data:
+      rl.draw_spline_linear(self.v_arc_lines, len(self.v_arc_lines), self.v_arc_data.thickness, self.arc_color)
+
+  def _is_visible(self, sm):
+    """Check if the visualization should be rendered."""
+    return (sm.recv_frame['driverStateV2'] > ui_state.started_frame and
+            sm.seen['driverMonitoringState'] and
+            sm['selfdriveState'].alertSize == 0)
+
+  def _update_state(self, sm, rect):
+    """Update the driver monitoring state based on model data"""
+    if  not sm.updated["driverMonitoringState"]:
+      if self.state_updated and (rect.x != self.last_rect.x or rect.y != self.last_rect.y or \
+         rect.width != self.last_rect.width or rect.height != self.last_rect.height):
+        self._pre_calculate_drawing_elements(rect)
+      return
+
+    # Get monitoring state
+    dm_state = sm["driverMonitoringState"]
+    self.is_active = dm_state.isActiveMode
+    self.is_rhd = dm_state.isRHD
+
+    # Update fade state (smoother transition between active/inactive)
+    fade_target = 0.0 if self.is_active else 0.5
+    self.dm_fade_state = np.clip(self.dm_fade_state + 0.2 * (fade_target - self.dm_fade_state), 0.0, 1.0)
+
+    # Get driver orientation data from appropriate camera
+    driverstate = sm["driverStateV2"]
+    driver_data = driverstate.rightDriverData if self.is_rhd else driverstate.leftDriverData
+    driver_orient = driver_data.faceOrientation
+
+    # Update pose values with scaling and smoothing
+    driver_orient = np.array(driver_orient)
+    scales = np.where(driver_orient < 0, SCALES_NEG, SCALES_POS)
+    v_this = driver_orient * scales
+    self.driver_pose_diff = np.abs(self.driver_pose_vals - v_this)
+    self.driver_pose_vals = 0.8 * v_this + 0.2 * self.driver_pose_vals  # Smooth changes
+
+    # Apply fade to rotation and compute sin/cos
+    rotation_amount = self.driver_pose_vals * (1.0 - self.dm_fade_state)
+    self.driver_pose_sins = np.sin(rotation_amount)
+    self.driver_pose_coss = np.cos(rotation_amount)
+
+    # Create rotation matrix for 3D face model
+    sin_y, sin_x, sin_z = self.driver_pose_sins
+    cos_y, cos_x, cos_z = self.driver_pose_coss
+    r_xyz = np.array(
+      [
+        [cos_x * cos_z, cos_x * sin_z, -sin_x],
+        [-sin_y * sin_x * cos_z - cos_y * sin_z, -sin_y * sin_x * sin_z + cos_y * cos_z, -sin_y * cos_x],
+        [cos_y * sin_x * cos_z - sin_y * sin_z, cos_y * sin_x * sin_z + sin_y * cos_z, cos_y * cos_x],
+      ]
+    )
+
+    # Transform face keypoints using vectorized matrix multiplication
+    self.face_kpts_draw = DEFAULT_FACE_KPTS_3D @ r_xyz.T
+    self.face_kpts_draw[:, 2] = self.face_kpts_draw[:, 2] * (1.0 - self.dm_fade_state) + 8 * self.dm_fade_state
+
+    # Pre-calculate the transformed keypoints
+    kp_depth = (self.face_kpts_draw[:, 2] - 8) / 120.0 + 1.0
+    self.face_keypoints_transformed = self.face_kpts_draw[:, :2] * kp_depth[:, None]
+
+    # Pre-calculate all drawing elements
+    self._pre_calculate_drawing_elements(rect)
+    self.state_updated = True
+
+  def _pre_calculate_drawing_elements(self, rect):
+    """Pre-calculate all drawing elements based on the current rectangle"""
+    # Calculate icon position (bottom-left or bottom-right)
+    width, height = rect.width, rect.height
+    offset = UI_BORDER_SIZE + BTN_SIZE // 2
+    self.position_x = rect.x + (width - offset if self.is_rhd else offset)
+    self.position_y = rect.y + height - offset
+
+    # Pre-calculate the face lines positions
+    positioned_keypoints = self.face_keypoints_transformed + np.array([self.position_x, self.position_y])
+    for i in range(len(positioned_keypoints)):
+      self.face_lines[i].x = positioned_keypoints[i][0]
+      self.face_lines[i].y = positioned_keypoints[i][1]
+
+    # Calculate arc dimensions based on head rotation
+    delta_x = -self.driver_pose_sins[1] * ARC_LENGTH / 2.0  # Horizontal movement
+    delta_y = -self.driver_pose_sins[0] * ARC_LENGTH / 2.0  # Vertical movement
+
+    # Horizontal arc
+    h_width = abs(delta_x)
+    self.h_arc_data = self._calculate_arc_data(
+        delta_x, h_width, self.position_x, self.position_y - ARC_LENGTH / 2,
+        self.driver_pose_sins[1], self.driver_pose_diff[1], is_horizontal=True
+    )
+
+    # Vertical arc
+    v_height = abs(delta_y)
+    self.v_arc_data = self._calculate_arc_data(
+        delta_y, v_height, self.position_x - ARC_LENGTH / 2, self.position_y,
+        self.driver_pose_sins[0], self.driver_pose_diff[0], is_horizontal=False
+    )
+
+  def _calculate_arc_data(
+    self, delta: float, size: float, x: float, y: float, sin_val: float, diff_val: float, is_horizontal: bool
+  ):
+    """Calculate arc data and pre-compute arc points."""
+    if size <= 0:
+      return None
+
+    thickness = ARC_THICKNESS_DEFAULT + ARC_THICKNESS_EXTEND * min(1.0, diff_val * 5.0)
+    start_angle = (90 if sin_val > 0 else -90) if is_horizontal else (0 if sin_val > 0 else 180)
+    x = min(x + delta, x) if is_horizontal else x
+    y = y if is_horizontal else min(y + delta, y)
+
+    arc_data = ArcData(
+      x=x,
+      y=y,
+      width=size if is_horizontal else ARC_LENGTH,
+      height=ARC_LENGTH if is_horizontal else size,
+      thickness=thickness,
+    )
+
+    # Pre-calculate arc points
+    angles = ARC_ANGLES + np.deg2rad(start_angle)
+
+    center_x = x + arc_data.width / 2
+    center_y = y + arc_data.height / 2
+    radius_x = arc_data.width / 2
+    radius_y = arc_data.height / 2
+
+    x_coords = center_x + np.cos(angles) * radius_x
+    y_coords = center_y + np.sin(angles) * radius_y
+
+    arc_lines = self.h_arc_lines if is_horizontal else self.v_arc_lines
+    for i, (x_coord, y_coord) in enumerate(zip(x_coords, y_coords, strict=True)):
+      arc_lines[i].x = x_coord
+      arc_lines[i].y = y_coord
+
+    return arc_data

selfdrive/ui/onroad/hud_renderer.py

@@ -0,0 +1,178 @@
+import pyray as rl
+from dataclasses import dataclass
+from cereal.messaging import SubMaster
+from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus
+from openpilot.system.ui.lib.application import gui_app, FontWeight
+from openpilot.system.ui.lib.text_measure import measure_text_cached
+from openpilot.common.conversions import Conversions as CV
+
+# Constants
+SET_SPEED_NA = 255
+KM_TO_MILE = 0.621371
+CRUISE_DISABLED_CHAR = '–'
+
+
+@dataclass(frozen=True)
+class UIConfig:
+  header_height: int = 300
+  border_size: int = 30
+  button_size: int = 192
+  set_speed_width_metric: int = 200
+  set_speed_width_imperial: int = 172
+  set_speed_height: int = 204
+  wheel_icon_size: int = 144
+
+
+@dataclass(frozen=True)
+class FontSizes:
+  current_speed: int = 176
+  speed_unit: int = 66
+  max_speed: int = 40
+  set_speed: int = 90
+
+
+@dataclass(frozen=True)
+class Colors:
+  white: rl.Color = rl.Color(255, 255, 255, 255)
+  disengaged: rl.Color = rl.Color(145, 155, 149, 255)
+  override: rl.Color = rl.Color(145, 155, 149, 255)  # Added
+  engaged: rl.Color = rl.Color(128, 216, 166, 255)
+  disengaged_bg: rl.Color = rl.Color(0, 0, 0, 153)
+  override_bg: rl.Color = rl.Color(145, 155, 149, 204)
+  engaged_bg: rl.Color = rl.Color(128, 216, 166, 204)
+  grey: rl.Color = rl.Color(166, 166, 166, 255)
+  dark_grey: rl.Color = rl.Color(114, 114, 114, 255)
+  black_translucent: rl.Color = rl.Color(0, 0, 0, 166)
+  white_translucent: rl.Color = rl.Color(255, 255, 255, 200)
+  border_translucent: rl.Color = rl.Color(255, 255, 255, 75)
+  header_gradient_start: rl.Color = rl.Color(0, 0, 0, 114)
+  header_gradient_end: rl.Color = rl.Color(0, 0, 0, 0)
+
+
+UI_CONFIG = UIConfig()
+FONT_SIZES = FontSizes()
+COLORS = Colors()
+
+
+class HudRenderer:
+  def __init__(self):
+    """Initialize the HUD renderer."""
+    self.is_cruise_set: bool = False
+    self.is_cruise_available: bool = False
+    self.set_speed: float = SET_SPEED_NA
+    self.speed: float = 0.0
+    self.v_ego_cluster_seen: bool = False
+    self._wheel_texture: rl.Texture = gui_app.texture('icons/chffr_wheel.png', UI_CONFIG.wheel_icon_size, UI_CONFIG.wheel_icon_size)
+    self._font_semi_bold: rl.Font = gui_app.font(FontWeight.SEMI_BOLD)
+    self._font_bold: rl.Font = gui_app.font(FontWeight.BOLD)
+    self._font_medium: rl.Font = gui_app.font(FontWeight.MEDIUM)
+
+  def _update_state(self, sm: SubMaster) -> None:
+    """Update HUD state based on car state and controls state."""
+    if sm.recv_frame["carState"] < ui_state.started_frame:
+      self.is_cruise_set = False
+      self.set_speed = SET_SPEED_NA
+      self.speed = 0.0
+      return
+
+    controls_state = sm['controlsState']
+    car_state = sm['carState']
+
+    v_cruise_cluster = car_state.vCruiseCluster
+    self.set_speed = (
+      controls_state.vCruiseDEPRECATED if v_cruise_cluster == 0.0 else v_cruise_cluster
+    )
+    self.is_cruise_set = 0 < self.set_speed < SET_SPEED_NA
+    self.is_cruise_available = self.set_speed != -1
+
+    if self.is_cruise_set and not ui_state.is_metric:
+      self.set_speed *= KM_TO_MILE
+
+    v_ego_cluster = car_state.vEgoCluster
+    self.v_ego_cluster_seen = self.v_ego_cluster_seen or v_ego_cluster != 0.0
+    v_ego = v_ego_cluster if self.v_ego_cluster_seen else car_state.vEgo
+    speed_conversion = CV.MS_TO_KPH if ui_state.is_metric else CV.MS_TO_MPH
+    self.speed = max(0.0, v_ego * speed_conversion)
+
+  def draw(self, rect: rl.Rectangle, sm: SubMaster) -> None:
+    """Render HUD elements to the screen."""
+    self._update_state(sm)
+    rl.draw_rectangle_gradient_v(
+      int(rect.x),
+      int(rect.y),
+      int(rect.width),
+      UI_CONFIG.header_height,
+      COLORS.header_gradient_start,
+      COLORS.header_gradient_end,
+    )
+
+    if self.is_cruise_available:
+      self._draw_set_speed(rect)
+
+    self._draw_current_speed(rect)
+    self._draw_wheel_icon(rect)
+
+  def _draw_set_speed(self, rect: rl.Rectangle) -> None:
+    """Draw the MAX speed indicator box."""
+    set_speed_width = UI_CONFIG.set_speed_width_metric if ui_state.is_metric else UI_CONFIG.set_speed_width_imperial
+    x = rect.x + 60 + (UI_CONFIG.set_speed_width_imperial - set_speed_width) // 2
+    y = rect.y + 45
+
+    set_speed_rect = rl.Rectangle(x, y, set_speed_width, UI_CONFIG.set_speed_height)
+    rl.draw_rectangle_rounded(set_speed_rect, 0.2, 30, COLORS.black_translucent)
+    rl.draw_rectangle_rounded_lines_ex(set_speed_rect, 0.2, 30, 6, COLORS.border_translucent)
+
+    max_color = COLORS.grey
+    set_speed_color = COLORS.dark_grey
+    if self.is_cruise_set:
+      set_speed_color = COLORS.white
+      if ui_state.status == UIStatus.ENGAGED:
+        max_color = COLORS.engaged
+      elif ui_state.status == UIStatus.DISENGAGED:
+        max_color = COLORS.disengaged
+      elif ui_state.status == UIStatus.OVERRIDE:
+        max_color = COLORS.override
+
+    max_text = "MAX"
+    max_text_width = measure_text_cached(self._font_semi_bold, max_text, FONT_SIZES.max_speed).x
+    rl.draw_text_ex(
+      self._font_semi_bold,
+      max_text,
+      rl.Vector2(x + (set_speed_width - max_text_width) / 2, y + 27),
+      FONT_SIZES.max_speed,
+      0,
+      max_color,
+    )
+
+    set_speed_text = CRUISE_DISABLED_CHAR if not self.is_cruise_set else str(round(self.set_speed))
+    speed_text_width = measure_text_cached(self._font_bold, set_speed_text, FONT_SIZES.set_speed).x
+    rl.draw_text_ex(
+      self._font_bold,
+      set_speed_text,
+      rl.Vector2(x + (set_speed_width - speed_text_width) / 2, y + 77),
+      FONT_SIZES.set_speed,
+      0,
+      set_speed_color,
+    )
+
+  def _draw_current_speed(self, rect: rl.Rectangle) -> None:
+    """Draw the current vehicle speed and unit."""
+    speed_text = str(round(self.speed))
+    speed_text_size = measure_text_cached(self._font_bold, speed_text, FONT_SIZES.current_speed)
+    speed_pos = rl.Vector2(rect.x + rect.width / 2 - speed_text_size.x / 2, 180 - speed_text_size.y / 2)
+    rl.draw_text_ex(self._font_bold, speed_text, speed_pos, FONT_SIZES.current_speed, 0, COLORS.white)
+
+    unit_text = "km/h" if ui_state.is_metric else "mph"
+    unit_text_size = measure_text_cached(self._font_medium, unit_text, FONT_SIZES.speed_unit)
+    unit_pos = rl.Vector2(rect.x + rect.width / 2 - unit_text_size.x / 2, 290 - unit_text_size.y / 2)
+    rl.draw_text_ex(self._font_medium, unit_text, unit_pos, FONT_SIZES.speed_unit, 0, COLORS.white_translucent)
+
+  def _draw_wheel_icon(self, rect: rl.Rectangle) -> None:
+    """Draw the steering wheel icon with status-based opacity."""
+    center_x = int(rect.x + rect.width - UI_CONFIG.border_size - UI_CONFIG.button_size / 2)
+    center_y = int(rect.y + UI_CONFIG.border_size + UI_CONFIG.button_size / 2)
+    rl.draw_circle(center_x, center_y, UI_CONFIG.button_size / 2, COLORS.black_translucent)
+
+    opacity = 0.7 if ui_state.status == UIStatus.DISENGAGED else 1.0
+    img_pos = rl.Vector2(center_x - self._wheel_texture.width / 2, center_y - self._wheel_texture.height / 2)
+    rl.draw_texture_v(self._wheel_texture, img_pos, rl.Color(255, 255, 255, int(255 * opacity)))

selfdrive/ui/onroad/model_renderer.py

@@ -0,0 +1,435 @@
+import colorsys
+import numpy as np
+import pyray as rl
+from cereal import messaging, car
+from dataclasses import dataclass, field
+from openpilot.common.params import Params
+from openpilot.selfdrive.ui.ui_state import ui_state
+from openpilot.system.ui.lib.application import DEFAULT_FPS
+from openpilot.system.ui.lib.shader_polygon import draw_polygon
+
+
+CLIP_MARGIN = 500
+MIN_DRAW_DISTANCE = 10.0
+MAX_DRAW_DISTANCE = 100.0
+PATH_COLOR_TRANSITION_DURATION = 0.5  # Seconds for color transition animation
+PATH_BLEND_INCREMENT = 1.0 / (PATH_COLOR_TRANSITION_DURATION * DEFAULT_FPS)
+
+MAX_POINTS = 200
+
+THROTTLE_COLORS = [
+  rl.Color(13, 248, 122, 102),   # HSLF(148/360, 0.94, 0.51, 0.4)
+  rl.Color(114, 255, 92, 89),    # HSLF(112/360, 1.0, 0.68, 0.35)
+  rl.Color(114, 255, 92, 0),     # HSLF(112/360, 1.0, 0.68, 0.0)
+]
+
+NO_THROTTLE_COLORS = [
+  rl.Color(242, 242, 242, 102), # HSLF(148/360, 0.0, 0.95, 0.4)
+  rl.Color(242, 242, 242, 89),  # HSLF(112/360, 0.0, 0.95, 0.35)
+  rl.Color(242, 242, 242, 0),   # HSLF(112/360, 0.0, 0.95, 0.0)
+]
+
+
+@dataclass
+class ModelPoints:
+  raw_points: np.ndarray = field(default_factory=lambda: np.empty((0, 3), dtype=np.float32))
+  projected_points: np.ndarray = field(default_factory=lambda: np.empty((0, 2), dtype=np.float32))
+
+@dataclass
+class LeadVehicle:
+  glow: list[float] = field(default_factory=list)
+  chevron: list[float] = field(default_factory=list)
+  fill_alpha: int = 0
+
+
+class ModelRenderer:
+  def __init__(self):
+    self._longitudinal_control = False
+    self._experimental_mode = False
+    self._blend_factor = 1.0
+    self._prev_allow_throttle = True
+    self._lane_line_probs = np.zeros(4, dtype=np.float32)
+    self._road_edge_stds = np.zeros(2, dtype=np.float32)
+    self._lead_vehicles = [LeadVehicle(), LeadVehicle()]
+    self._path_offset_z = 1.22
+
+    # Initialize ModelPoints objects
+    self._path = ModelPoints()
+    self._lane_lines = [ModelPoints() for _ in range(4)]
+    self._road_edges = [ModelPoints() for _ in range(2)]
+    self._acceleration_x = np.empty((0,), dtype=np.float32)
+
+    # Transform matrix (3x3 for car space to screen space)
+    self._car_space_transform = np.zeros((3, 3), dtype=np.float32)
+    self._transform_dirty = True
+    self._clip_region = None
+    self._rect = None
+
+    # Pre-allocated arrays for polygon conversion
+    self._temp_points_3d = np.empty((MAX_POINTS * 2, 3), dtype=np.float32)
+    self._temp_proj = np.empty((3, MAX_POINTS * 2), dtype=np.float32)
+
+    self._exp_gradient = {
+      'start': (0.0, 1.0),  # Bottom of path
+      'end': (0.0, 0.0),  # Top of path
+      'colors': [],
+      'stops': [],
+    }
+
+    # Get longitudinal control setting from car parameters
+    if car_params := Params().get("CarParams"):
+      cp = messaging.log_from_bytes(car_params, car.CarParams)
+      self._longitudinal_control = cp.openpilotLongitudinalControl
+
+  def set_transform(self, transform: np.ndarray):
+    self._car_space_transform = transform.astype(np.float32)
+    self._transform_dirty = True
+
+  def draw(self, rect: rl.Rectangle, sm: messaging.SubMaster):
+    # Check if data is up-to-date
+    if (sm.recv_frame["liveCalibration"] < ui_state.started_frame or
+        sm.recv_frame["modelV2"] < ui_state.started_frame):
+      return
+
+    # Set up clipping region
+    self._rect = rect
+    self._clip_region = rl.Rectangle(
+      rect.x - CLIP_MARGIN, rect.y - CLIP_MARGIN, rect.width + 2 * CLIP_MARGIN, rect.height + 2 * CLIP_MARGIN
+    )
+
+    # Update state
+    self._experimental_mode = sm['selfdriveState'].experimentalMode
+    self._path_offset_z = sm['liveCalibration'].height[0]
+    if sm.updated['carParams']:
+      self._longitudinal_control = sm['carParams'].openpilotLongitudinalControl
+
+    model = sm['modelV2']
+    radar_state = sm['radarState'] if sm.valid['radarState'] else None
+    lead_one = radar_state.leadOne if radar_state else None
+    render_lead_indicator = self._longitudinal_control and radar_state is not None
+
+    # Update model data when needed
+    model_updated = sm.updated['modelV2']
+    if model_updated or sm.updated['radarState'] or self._transform_dirty:
+      if model_updated:
+        self._update_raw_points(model)
+
+      path_x_array = self._path.raw_points[:, 0]
+      if path_x_array.size == 0:
+        return
+
+      self._update_model(lead_one, path_x_array)
+      if render_lead_indicator:
+        self._update_leads(radar_state, path_x_array)
+      self._transform_dirty = False
+
+
+    # Draw elements
+    self._draw_lane_lines()
+    self._draw_path(sm)
+
+    if render_lead_indicator and radar_state:
+      self._draw_lead_indicator()
+
+  def _update_raw_points(self, model):
+    """Update raw 3D points from model data"""
+    self._path.raw_points = np.array([model.position.x, model.position.y, model.position.z], dtype=np.float32).T
+
+    for i, lane_line in enumerate(model.laneLines):
+      self._lane_lines[i].raw_points = np.array([lane_line.x, lane_line.y, lane_line.z], dtype=np.float32).T
+
+    for i, road_edge in enumerate(model.roadEdges):
+      self._road_edges[i].raw_points = np.array([road_edge.x, road_edge.y, road_edge.z], dtype=np.float32).T
+
+    self._lane_line_probs = np.array(model.laneLineProbs, dtype=np.float32)
+    self._road_edge_stds = np.array(model.roadEdgeStds, dtype=np.float32)
+    self._acceleration_x = np.array(model.acceleration.x, dtype=np.float32)
+
+  def _update_leads(self, radar_state, path_x_array):
+    """Update positions of lead vehicles"""
+    self._lead_vehicles = [LeadVehicle(), LeadVehicle()]
+    leads = [radar_state.leadOne, radar_state.leadTwo]
+
+    for i, lead_data in enumerate(leads):
+      if lead_data and lead_data.status:
+        d_rel, y_rel, v_rel = lead_data.dRel, lead_data.yRel, lead_data.vRel
+        idx = self._get_path_length_idx(path_x_array, d_rel)
+
+        # Get z-coordinate from path at the lead vehicle position
+        z = self._path.raw_points[idx, 2] if idx < len(self._path.raw_points) else 0.0
+        point = self._map_to_screen(d_rel, -y_rel, z + self._path_offset_z)
+        if point:
+          self._lead_vehicles[i] = self._update_lead_vehicle(d_rel, v_rel, point, self._rect)
+
+  def _update_model(self, lead, path_x_array):
+    """Update model visualization data based on model message"""
+    max_distance = np.clip(path_x_array[-1], MIN_DRAW_DISTANCE, MAX_DRAW_DISTANCE)
+    max_idx = self._get_path_length_idx(self._lane_lines[0].raw_points[:, 0], max_distance)
+
+    # Update lane lines using raw points
+    for i, lane_line in enumerate(self._lane_lines):
+      lane_line.projected_points = self._map_line_to_polygon(
+        lane_line.raw_points, 0.025 * self._lane_line_probs[i], 0.0, max_idx
+      )
+
+    # Update road edges using raw points
+    for road_edge in self._road_edges:
+      road_edge.projected_points = self._map_line_to_polygon(road_edge.raw_points, 0.025, 0.0, max_idx)
+
+    # Update path using raw points
+    if lead and lead.status:
+      lead_d = lead.dRel * 2.0
+      max_distance = np.clip(lead_d - min(lead_d * 0.35, 10.0), 0.0, max_distance)
+
+    max_idx = self._get_path_length_idx(path_x_array, max_distance)
+    self._path.projected_points = self._map_line_to_polygon(
+      self._path.raw_points, 0.9, self._path_offset_z, max_idx, allow_invert=False
+    )
+
+    self._update_experimental_gradient(self._rect.height)
+
+  def _update_experimental_gradient(self, height):
+    """Pre-calculate experimental mode gradient colors"""
+    if not self._experimental_mode:
+      return
+
+    max_len = min(len(self._path.projected_points) // 2, len(self._acceleration_x))
+
+    segment_colors = []
+    gradient_stops = []
+
+    i = 0
+    while i < max_len:
+      track_idx = max_len - i - 1  # flip idx to start from bottom right
+      track_y = self._path.projected_points[track_idx][1]
+      if track_y < 0 or track_y > height:
+        i += 1
+        continue
+
+      # Calculate color based on acceleration
+      lin_grad_point = (height - track_y) / height
+
+      # speed up: 120, slow down: 0
+      path_hue = max(min(60 + self._acceleration_x[i] * 35, 120), 0)
+      path_hue = int(path_hue * 100 + 0.5) / 100
+
+      saturation = min(abs(self._acceleration_x[i] * 1.5), 1)
+      lightness = self._map_val(saturation, 0.0, 1.0, 0.95, 0.62)
+      alpha = self._map_val(lin_grad_point, 0.75 / 2.0, 0.75, 0.4, 0.0)
+
+      # Use HSL to RGB conversion
+      color = self._hsla_to_color(path_hue / 360.0, saturation, lightness, alpha)
+
+      gradient_stops.append(lin_grad_point)
+      segment_colors.append(color)
+
+      # Skip a point, unless next is last
+      i += 1 + (1 if (i + 2) < max_len else 0)
+
+    # Store the gradient in the path object
+    self._exp_gradient['colors'] = segment_colors
+    self._exp_gradient['stops'] = gradient_stops
+
+  def _update_lead_vehicle(self, d_rel, v_rel, point, rect):
+    speed_buff, lead_buff = 10.0, 40.0
+
+    # Calculate fill alpha
+    fill_alpha = 0
+    if d_rel < lead_buff:
+      fill_alpha = 255 * (1.0 - (d_rel / lead_buff))
+      if v_rel < 0:
+        fill_alpha += 255 * (-1 * (v_rel / speed_buff))
+      fill_alpha = min(fill_alpha, 255)
+
+    # Calculate size and position
+    sz = np.clip((25 * 30) / (d_rel / 3 + 30), 15.0, 30.0) * 2.35
+    x = np.clip(point[0], 0.0, rect.width - sz / 2)
+    y = min(point[1], rect.height - sz * 0.6)
+
+    g_xo = sz / 5
+    g_yo = sz / 10
+
+    glow = [(x + (sz * 1.35) + g_xo, y + sz + g_yo), (x, y - g_yo), (x - (sz * 1.35) - g_xo, y + sz + g_yo)]
+    chevron = [(x + (sz * 1.25), y + sz), (x, y), (x - (sz * 1.25), y + sz)]
+
+    return LeadVehicle(glow=glow,chevron=chevron, fill_alpha=int(fill_alpha))
+
+  def _draw_lane_lines(self):
+    """Draw lane lines and road edges"""
+    for i, lane_line in enumerate(self._lane_lines):
+      if lane_line.projected_points.size == 0:
+        continue
+
+      alpha = np.clip(self._lane_line_probs[i], 0.0, 0.7)
+      color = rl.Color(255, 255, 255, int(alpha * 255))
+      draw_polygon(self._rect, lane_line.projected_points, color)
+
+    for i, road_edge in enumerate(self._road_edges):
+      if road_edge.projected_points.size == 0:
+        continue
+
+      alpha = np.clip(1.0 - self._road_edge_stds[i], 0.0, 1.0)
+      color = rl.Color(255, 0, 0, int(alpha * 255))
+      draw_polygon(self._rect, road_edge.projected_points, color)
+
+  def _draw_path(self, sm):
+    """Draw path with dynamic coloring based on mode and throttle state."""
+    if not self._path.projected_points.size:
+      return
+
+    if self._experimental_mode:
+      # Draw with acceleration coloring
+      if len(self._exp_gradient['colors']) > 2:
+        draw_polygon(self._rect, self._path.projected_points, gradient=self._exp_gradient)
+      else:
+        draw_polygon(self._rect, self._path.projected_points, rl.Color(255, 255, 255, 30))
+    else:
+      # Draw with throttle/no throttle gradient
+      allow_throttle = sm['longitudinalPlan'].allowThrottle or not self._longitudinal_control
+
+      # Start transition if throttle state changes
+      if allow_throttle != self._prev_allow_throttle:
+        self._prev_allow_throttle = allow_throttle
+        self._blend_factor = max(1.0 - self._blend_factor, 0.0)
+
+      # Update blend factor
+      if self._blend_factor < 1.0:
+        self._blend_factor = min(self._blend_factor + PATH_BLEND_INCREMENT, 1.0)
+
+      begin_colors = NO_THROTTLE_COLORS if allow_throttle else THROTTLE_COLORS
+      end_colors = THROTTLE_COLORS if allow_throttle else NO_THROTTLE_COLORS
+
+      # Blend colors based on transition
+      blended_colors = self._blend_colors(begin_colors, end_colors, self._blend_factor)
+      gradient = {
+        'start': (0.0, 1.0),  # Bottom of path
+        'end': (0.0, 0.0),  # Top of path
+        'colors': blended_colors,
+        'stops': [0.0, 0.5, 1.0],
+      }
+      draw_polygon(self._rect, self._path.projected_points, gradient=gradient)
+
+  def _draw_lead_indicator(self):
+    # Draw lead vehicles if available
+    for lead in self._lead_vehicles:
+      if not lead.glow or not lead.chevron:
+        continue
+
+      rl.draw_triangle_fan(lead.glow, len(lead.glow), rl.Color(218, 202, 37, 255))
+      rl.draw_triangle_fan(lead.chevron, len(lead.chevron), rl.Color(201, 34, 49, lead.fill_alpha))
+
+  @staticmethod
+  def _get_path_length_idx(pos_x_array: np.ndarray, path_height: float) -> int:
+    """Get the index corresponding to the given path height"""
+    if len(pos_x_array) == 0:
+      return 0
+    indices = np.where(pos_x_array <= path_height)[0]
+    return indices[-1] if indices.size > 0 else 0
+
+  def _map_to_screen(self, in_x, in_y, in_z):
+    """Project a point in car space to screen space"""
+    input_pt = np.array([in_x, in_y, in_z])
+    pt = self._car_space_transform @ input_pt
+
+    if abs(pt[2]) < 1e-6:
+      return None
+
+    x, y = pt[0] / pt[2], pt[1] / pt[2]
+
+    clip = self._clip_region
+    if not (clip.x <= x <= clip.x + clip.width and clip.y <= y <= clip.y + clip.height):
+      return None
+
+    return (x, y)
+
+  def _map_line_to_polygon(self, line: np.ndarray, y_off: float, z_off: float, max_idx: int, allow_invert: bool = True) -> np.ndarray:
+    """Convert 3D line to 2D polygon for rendering."""
+    if line.shape[0] == 0:
+      return np.empty((0, 2), dtype=np.float32)
+
+    # Slice points and filter non-negative x-coordinates
+    points = line[:max_idx + 1]
+    points = points[points[:, 0] >= 0]
+    if points.shape[0] == 0:
+      return np.empty((0, 2), dtype=np.float32)
+
+    # Create left and right 3D points in one array
+    n_points = points.shape[0]
+    points_3d = self._temp_points_3d[:n_points * 2]
+    points_3d[:n_points, 0] = points_3d[n_points:, 0] = points[:, 0]
+    points_3d[:n_points, 1] = points[:, 1] - y_off
+    points_3d[n_points:, 1] = points[:, 1] + y_off
+    points_3d[:n_points, 2] = points_3d[n_points:, 2] = points[:, 2] + z_off
+
+    # Single matrix multiplication for projections
+    proj = np.ascontiguousarray(self._temp_proj[:, :n_points * 2])  # Slice the pre-allocated array
+    np.dot(self._car_space_transform, points_3d.T, out=proj)
+    valid_z = np.abs(proj[2]) > 1e-6
+    if not np.any(valid_z):
+      return np.empty((0, 2), dtype=np.float32)
+
+    # Compute screen coordinates
+    screen = proj[:2, valid_z] / proj[2, valid_z][None, :]
+    left_screen = screen[:, :n_points].T
+    right_screen = screen[:, n_points:].T
+
+    # Ensure consistent shapes by re-aligning valid points
+    valid_points = np.minimum(left_screen.shape[0], right_screen.shape[0])
+    if valid_points == 0:
+      return np.empty((0, 2), dtype=np.float32)
+    left_screen = left_screen[:valid_points]
+    right_screen = right_screen[:valid_points]
+
+    if self._clip_region:
+      clip = self._clip_region
+      bounds_mask = (
+        (left_screen[:, 0] >= clip.x) & (left_screen[:, 0] <= clip.x + clip.width) &
+        (left_screen[:, 1] >= clip.y) & (left_screen[:, 1] <= clip.y + clip.height) &
+        (right_screen[:, 0] >= clip.x) & (right_screen[:, 0] <= clip.x + clip.width) &
+        (right_screen[:, 1] >= clip.y) & (right_screen[:, 1] <= clip.y + clip.height)
+      )
+      if not np.any(bounds_mask):
+        return np.empty((0, 2), dtype=np.float32)
+      left_screen = left_screen[bounds_mask]
+      right_screen = right_screen[bounds_mask]
+
+    if not allow_invert and left_screen.shape[0] > 1:
+      keep = np.concatenate(([True], np.diff(left_screen[:, 1]) < 0))
+      left_screen = left_screen[keep]
+      right_screen = right_screen[keep]
+      if left_screen.shape[0] == 0:
+        return np.empty((0, 2), dtype=np.float32)
+
+    return np.vstack((left_screen, right_screen[::-1])).astype(np.float32)
+
+  @staticmethod
+  def _map_val(x, x0, x1, y0, y1):
+    x = np.clip(x, x0, x1)
+    ra = x1 - x0
+    rb = y1 - y0
+    return (x - x0) * rb / ra + y0 if ra != 0 else y0
+
+  @staticmethod
+  def _hsla_to_color(h, s, l, a):
+    rgb = colorsys.hls_to_rgb(h, l, s)
+    return rl.Color(
+        int(rgb[0] * 255),
+        int(rgb[1] * 255),
+        int(rgb[2] * 255),
+        int(a * 255)
+    )
+
+  @staticmethod
+  def _blend_colors(begin_colors, end_colors, t):
+    if t >= 1.0:
+      return end_colors
+    if t <= 0.0:
+      return begin_colors
+
+    inv_t = 1.0 - t
+    return [rl.Color(
+      int(inv_t * start.r + t * end.r),
+      int(inv_t * start.g + t * end.g),
+      int(inv_t * start.b + t * end.b),
+      int(inv_t * start.a + t * end.a)
+    ) for start, end in zip(begin_colors, end_colors, strict=True)]

selfdrive/ui/qt/network/networking.cc

@@ -79,7 +79,8 @@ Networking::Networking(QWidget* parent, bool show_advanced) : QFrame(parent) {
 }
 
 void Networking::setPrimeType(PrimeState::Type type) {
-  an->setGsmVisible(type == PrimeState::PRIME_TYPE_NONE || type == PrimeState::PRIME_TYPE_LITE);
+  an->setGsmVisible(type == PrimeState::PRIME_TYPE_NONE || type == PrimeState::PRIME_TYPE_UNKNOWN || \
+                    type == PrimeState::PRIME_TYPE_PURPLE ||  type == PrimeState::PRIME_TYPE_LITE);
   wifi->ipv4_forward = (type == PrimeState::PRIME_TYPE_NONE || type == PrimeState::PRIME_TYPE_LITE);
 }
 

selfdrive/ui/qt/onroad/annotated_camera.h

@@ -9,7 +9,9 @@
 #ifdef SUNNYPILOT
 #include "selfdrive/ui/sunnypilot/qt/onroad/buttons.h"
 #include "selfdrive/ui/sunnypilot/qt/onroad/hud.h"
+#include "selfdrive/ui/sunnypilot/qt/onroad/model.h"
 #define ExperimentalButton ExperimentalButtonSP
+#define ModelRenderer ModelRendererSP
 #else
 #include "selfdrive/ui/qt/onroad/buttons.h"
 #include "selfdrive/ui/qt/onroad/hud.h"

selfdrive/ui/qt/onroad/model.cc

@@ -1,18 +1,5 @@
 #include "selfdrive/ui/qt/onroad/model.h"
 
-constexpr int CLIP_MARGIN = 500;
-constexpr float MIN_DRAW_DISTANCE = 10.0;
-constexpr float MAX_DRAW_DISTANCE = 100.0;
-
-static int get_path_length_idx(const cereal::XYZTData::Reader &line, const float path_height) {
-  const auto &line_x = line.getX();
-  int max_idx = 0;
-  for (int i = 1; i < line_x.size() && line_x[i] <= path_height; ++i) {
-    max_idx = i;
-  }
-  return max_idx;
-}
-
 void ModelRenderer::draw(QPainter &painter, const QRect &surface_rect) {
   auto *s = uiState();
   auto &sm = *(s->sm);
@@ -35,7 +22,7 @@ void ModelRenderer::draw(QPainter &painter, const QRect &surface_rect) {
 
   update_model(model, lead_one);
   drawLaneLines(painter);
-  drawPath(painter, model, surface_rect.height());
+  drawPath(painter, model, surface_rect);
 
   if (longitudinal_control && sm.alive("radarState")) {
     update_leads(radar_state, model.getPosition());

selfdrive/ui/qt/onroad/model.h

@@ -9,21 +9,39 @@
 #include "selfdrive/ui/ui.h"
 #endif
 
+constexpr int CLIP_MARGIN = 500;
+constexpr float MIN_DRAW_DISTANCE = 10.0;
+constexpr float MAX_DRAW_DISTANCE = 100.0;
+
+inline int get_path_length_idx(const cereal::XYZTData::Reader &line, const float path_height) {
+  const auto &line_x = line.getX();
+  int max_idx = 0;
+  for (int i = 1; i < line_x.size() && line_x[i] <= path_height; ++i) {
+    max_idx = i;
+  }
+  return max_idx;
+}
+
 class ModelRenderer {
 public:
+  virtual ~ModelRenderer() = default;
+
   ModelRenderer() {}
   void setTransform(const Eigen::Matrix3f &transform) { car_space_transform = transform; }
   void draw(QPainter &painter, const QRect &surface_rect);
 
-private:
+protected:
   bool mapToScreen(float in_x, float in_y, float in_z, QPointF *out);
   void mapLineToPolygon(const cereal::XYZTData::Reader &line, float y_off, float z_off,
                         QPolygonF *pvd, int max_idx, bool allow_invert = true);
   void drawLead(QPainter &painter, const cereal::RadarState::LeadData::Reader &lead_data, const QPointF &vd, const QRect &surface_rect);
   void update_leads(const cereal::RadarState::Reader &radar_state, const cereal::XYZTData::Reader &line);
-  void update_model(const cereal::ModelDataV2::Reader &model, const cereal::RadarState::LeadData::Reader &lead);
+  virtual void update_model(const cereal::ModelDataV2::Reader &model, const cereal::RadarState::LeadData::Reader &lead);
   void drawLaneLines(QPainter &painter);
   void drawPath(QPainter &painter, const cereal::ModelDataV2::Reader &model, int height);
+  virtual void drawPath(QPainter &painter, const cereal::ModelDataV2::Reader &model, const QRect &surface_rect) {;
+    drawPath(painter, model, surface_rect.height());
+  }
   void updatePathGradient(QLinearGradient &bg);
   QColor blendColors(const QColor &start, const QColor &end, float t);
 

selfdrive/ui/sunnypilot/SConscript

@@ -31,6 +31,7 @@ qt_src = [
   "sunnypilot/qt/offroad/settings/sunnylink/sponsor_widget.cc",
   "sunnypilot/qt/offroad/settings/trips_panel.cc",
   "sunnypilot/qt/offroad/settings/vehicle_panel.cc",
+  "sunnypilot/qt/offroad/settings/visuals_panel.cc",
   "sunnypilot/qt/onroad/annotated_camera.cc",
   "sunnypilot/qt/onroad/buttons.cc",
   "sunnypilot/qt/onroad/hud.cc",
@@ -39,6 +40,7 @@ qt_src = [
 ]
 
 lateral_panel_qt_src = [
+  "sunnypilot/qt/offroad/settings/lateral/blinker_pause_lateral_settings.cc",
   "sunnypilot/qt/offroad/settings/lateral/lane_change_settings.cc",
   "sunnypilot/qt/offroad/settings/lateral/mads_settings.cc",
   "sunnypilot/qt/offroad/settings/lateral/neural_network_lateral_control.cc",

selfdrive/ui/sunnypilot/qt/offroad/settings/lateral/blinker_pause_lateral_settings.cc

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
+ *
+ * This file is part of sunnypilot and is licensed under the MIT License.
+ * See the LICENSE.md file in the root directory for more details.
+ */
+
+
+#include "selfdrive/ui/sunnypilot/qt/offroad/settings/lateral/blinker_pause_lateral_settings.h"
+
+BlinkerPauseLateralSettings::BlinkerPauseLateralSettings(const QString &param, const QString &title, const QString &description, const QString &icon, QWidget *parent)
+    : ExpandableToggleRow(param, title, description, icon, parent) {
+
+  pauseLateralSpeed = new OptionControlSP("BlinkerMinLateralControlSpeed", "", "", "", {0, 255}, 5);
+  connect(pauseLateralSpeed, &OptionControlSP::updateLabels, this, &BlinkerPauseLateralSettings::refresh);
+  addItem(pauseLateralSpeed);
+
+  refresh();
+}
+
+void BlinkerPauseLateralSettings::refresh() {
+  const QString option = QString::fromStdString(params.get("BlinkerMinLateralControlSpeed"));
+  const bool is_metric = params.getBool("IsMetric");
+  const QString unit = is_metric ? "km/h" : "mph";
+
+  pauseLateralSpeed->setLabel(option + " " + unit);
+}

selfdrive/ui/sunnypilot/qt/offroad/settings/lateral/blinker_pause_lateral_settings.h

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
+ *
+ * This file is part of sunnypilot and is licensed under the MIT License.
+ * See the LICENSE.md file in the root directory for more details.
+ */
+
+#pragma once
+
+#include "selfdrive/ui/sunnypilot/ui.h"
+#include "selfdrive/ui/sunnypilot/qt/offroad/settings/settings.h"
+#include "selfdrive/ui/sunnypilot/qt/widgets/controls.h"
+#include "selfdrive/ui/sunnypilot/qt/widgets/expandable_row.h"
+
+class BlinkerPauseLateralSettings : public ExpandableToggleRow {
+  Q_OBJECT
+
+public:
+  BlinkerPauseLateralSettings(const QString &param, const QString &title, const QString &description, const QString &icon, QWidget *parent = nullptr);
+  void refresh();
+
+private:
+  Params params;
+  OptionControlSP *pauseLateralSpeed;
+};

selfdrive/ui/sunnypilot/qt/offroad/settings/lateral_panel.cc

@@ -64,6 +64,17 @@ LateralPanel::LateralPanel(SettingsWindowSP *parent) : QFrame(parent) {
   list->addItem(vertical_space(0));
   list->addItem(horizontal_line());
 
+  // Blinker Pause Lateral Control
+  blinkerPauseLateralSettings = new BlinkerPauseLateralSettings(
+    "BlinkerPauseLateralControl",
+    tr("Pause Lateral Control with Blinker"),
+    tr("Pause lateral control with blinker when traveling below the desired speed selected."),
+    "",
+    this);
+  list->addItem(blinkerPauseLateralSettings);
+
+  list->addItem(horizontal_line());
+
   // Neural Network Lateral Control
   nnlcToggle = new NeuralNetworkLateralControl();
   list->addItem(nnlcToggle);
@@ -139,5 +150,7 @@ void LateralPanel::updateToggles(bool _offroad) {
 
   madsSettingsButton->setEnabled(madsToggle->isToggled());
 
+  blinkerPauseLateralSettings->refresh();
+
   offroad = _offroad;
 }

selfdrive/ui/sunnypilot/qt/offroad/settings/lateral_panel.h

@@ -11,6 +11,7 @@
 #include <string>
 
 #include "selfdrive/ui/sunnypilot/ui.h"
+#include "selfdrive/ui/sunnypilot/qt/offroad/settings/lateral/blinker_pause_lateral_settings.h"
 #include "selfdrive/ui/sunnypilot/qt/offroad/settings/lateral/mads_settings.h"
 #include "selfdrive/ui/sunnypilot/qt/offroad/settings/lateral/neural_network_lateral_control.h"
 #include "selfdrive/ui/sunnypilot/qt/offroad/settings/lateral/lane_change_settings.h"
@@ -43,6 +44,7 @@ private:
   PushButtonSP *laneChangeSettingsButton;
   LaneChangeSettings *laneChangeWidget = nullptr;
   NeuralNetworkLateralControl *nnlcToggle = nullptr;
+  BlinkerPauseLateralSettings *blinkerPauseLateralSettings = nullptr;
 
   const QString MADS_BASE_DESC = tr("Enables independent engagements of Automatic Lane Centering (ALC) and Adaptive Cruise Control (ACC).");
 

selfdrive/ui/sunnypilot/qt/offroad/settings/settings.cc

@@ -20,6 +20,7 @@
 #include "selfdrive/ui/sunnypilot/qt/offroad/settings/longitudinal_panel.h"
 #include "selfdrive/ui/sunnypilot/qt/offroad/settings/trips_panel.h"
 #include "selfdrive/ui/sunnypilot/qt/offroad/settings/vehicle_panel.h"
+#include "selfdrive/ui/sunnypilot/qt/offroad/settings/visuals_panel.h"
 
 TogglesPanelSP::TogglesPanelSP(SettingsWindowSP *parent) : TogglesPanel(parent) {
   QObject::connect(uiStateSP(), &UIStateSP::uiUpdate, this, &TogglesPanelSP::updateState);
@@ -83,6 +84,7 @@ SettingsWindowSP::SettingsWindowSP(QWidget *parent) : SettingsWindow(parent) {
     PanelInfo("   " + tr("Models"), new ModelsPanel(this), "../../sunnypilot/selfdrive/assets/offroad/icon_models.png"),
     PanelInfo("   " + tr("Steering"), new LateralPanel(this), "../../sunnypilot/selfdrive/assets/offroad/icon_lateral.png"),
     PanelInfo("   " + tr("Cruise"), new LongitudinalPanel(this), "../assets/icons/speed_limit.png"),
+    PanelInfo("   " + tr("Visuals"), new VisualsPanel(this), "../../sunnypilot/selfdrive/assets/offroad/icon_visuals.png"),
     PanelInfo("   " + tr("Trips"), new TripsPanel(this), "../../sunnypilot/selfdrive/assets/offroad/icon_trips.png"),
     PanelInfo("   " + tr("Vehicle"), new VehiclePanel(this), "../../sunnypilot/selfdrive/assets/offroad/icon_vehicle.png"),
     PanelInfo("   " + tr("Firehose"), new FirehosePanel(this), "../../sunnypilot/selfdrive/assets/offroad/icon_firehose.svg"),

selfdrive/ui/sunnypilot/qt/offroad/settings/visuals_panel.cc

@@ -0,0 +1,12 @@
+/**
+ * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
+ *
+ * This file is part of sunnypilot and is licensed under the MIT License.
+ * See the LICENSE.md file in the root directory for more details.
+ */
+
+#include "selfdrive/ui/sunnypilot/qt/offroad/settings/visuals_panel.h"
+
+VisualsPanel::VisualsPanel(QWidget *parent) : QWidget(parent) {
+
+}

selfdrive/ui/sunnypilot/qt/offroad/settings/visuals_panel.h

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
+ *
+ * This file is part of sunnypilot and is licensed under the MIT License.
+ * See the LICENSE.md file in the root directory for more details.
+ */
+
+#pragma once
+
+#include "selfdrive/ui/sunnypilot/qt/offroad/settings/settings.h"
+
+class VisualsPanel : public QWidget {
+ Q_OBJECT
+
+public:
+ explicit VisualsPanel(QWidget *parent = nullptr);
+
+};

selfdrive/ui/sunnypilot/qt/onroad/model.h

@@ -11,5 +11,5 @@
 
 class ModelRendererSP : public ModelRenderer {
 public:
-  ModelRendererSP() {}
+  ModelRendererSP() = default;
 };

selfdrive/ui/tests/test_ui/run.py

@@ -252,6 +252,12 @@ def setup_settings_driving(click, pm: PubMaster, scroll=None):
   click(278, 962)
   time.sleep(UI_DELAY)
 
+def setup_settings_visuals(click, pm: PubMaster, scroll=None):
+  setup_settings_device(click, pm)
+  scroll(-400, 278, 962)
+  click(278, 560)
+  time.sleep(UI_DELAY)
+
 def setup_settings_trips(click, pm: PubMaster, scroll=None):
   setup_settings_device(click, pm)
   scroll(-400, 278, 962)
@@ -307,6 +313,7 @@ CASES.update({
   "settings_steering_mads": setup_settings_steering_mads,
   "settings_steering_alc": setup_settings_steering_alc,
   "settings_driving": setup_settings_driving,
+  "settings_visuals": setup_settings_visuals,
   "settings_trips": setup_settings_trips,
   "settings_vehicle": setup_settings_vehicle,
 })

selfdrive/ui/translations/main_ar.ts

@@ -1397,6 +1397,10 @@ This may take up to a minute.</source>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_de.ts

@@ -1377,6 +1377,10 @@ Dies kann bis zu einer Minute dauern.</translation>
         <source>Developer</source>
         <translation type="unfinished">Entwickler</translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_es.ts

@@ -1377,6 +1377,10 @@ Esto puede tardar un minuto.</translation>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_fr.ts

@@ -1377,6 +1377,10 @@ Cela peut prendre jusqu&apos;à une minute.</translation>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_ja.ts

@@ -1372,6 +1372,10 @@ This may take up to a minute.</source>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_ko.ts

@@ -1372,6 +1372,10 @@ This may take up to a minute.</source>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_pt-BR.ts

@@ -1377,6 +1377,10 @@ Isso pode levar até um minuto.</translation>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_th.ts

@@ -1372,6 +1372,10 @@ This may take up to a minute.</source>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_tr.ts

@@ -1370,6 +1370,10 @@ This may take up to a minute.</source>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_zh-CHS.ts

@@ -1372,6 +1372,10 @@ This may take up to a minute.</source>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/translations/main_zh-CHT.ts

@@ -1372,6 +1372,10 @@ This may take up to a minute.</source>
         <source>Cruise</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Visuals</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>Setup</name>

selfdrive/ui/ui.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+import pyray as rl
+from openpilot.system.ui.lib.application import gui_app
+from openpilot.selfdrive.ui.layouts.main import MainLayout
+from openpilot.selfdrive.ui.ui_state import ui_state
+
+
+def main():
+  gui_app.init_window("UI")
+  main_layout = MainLayout()
+  for _ in gui_app.render():
+    ui_state.update()
+
+    #TODO handle brigntness and awake state here
+
+    main_layout.render(rl.Rectangle(0, 0, gui_app.width, gui_app.height))
+
+
+if __name__ == "__main__":
+  main()

selfdrive/ui/ui_state.py

@@ -0,0 +1,129 @@
+import pyray as rl
+from enum import Enum
+from cereal import messaging, log
+from openpilot.common.params import Params, UnknownKeyName
+
+
+UI_BORDER_SIZE = 30
+
+
+class UIStatus(Enum):
+  DISENGAGED = "disengaged"
+  ENGAGED = "engaged"
+  OVERRIDE = "override"
+
+
+class UIState:
+  _instance: 'UIState | None' = None
+
+  def __new__(cls):
+    if cls._instance is None:
+      cls._instance = super().__new__(cls)
+      cls._instance._initialize()
+    return cls._instance
+
+  def _initialize(self):
+    self.params = Params()
+    self.sm = messaging.SubMaster(
+      [
+        "modelV2",
+        "controlsState",
+        "liveCalibration",
+        "radarState",
+        "deviceState",
+        "pandaStates",
+        "carParams",
+        "driverMonitoringState",
+        "carState",
+        "driverStateV2",
+        "roadCameraState",
+        "wideRoadCameraState",
+        "managerState",
+        "selfdriveState",
+        "longitudinalPlan",
+      ]
+    )
+
+    # UI Status tracking
+    self.status: UIStatus = UIStatus.DISENGAGED
+    self.started_frame: int = 0
+    self._engaged_prev: bool = False
+    self._started_prev: bool = False
+
+    # Core state variables
+    self.is_metric: bool = self.params.get_bool("IsMetric")
+    self.started: bool = False
+    self.ignition: bool = False
+    self.panda_type: log.PandaState.PandaType = log.PandaState.PandaType.unknown
+    self.personality: log.LongitudinalPersonality = log.LongitudinalPersonality.standard
+    self.light_sensor: float = -1.0
+
+    self._update_params()
+
+  @property
+  def engaged(self) -> bool:
+    return self.started and self.sm["selfdriveState"].enabled
+
+  def update(self) -> None:
+    self.sm.update(0)
+    self._update_state()
+    self._update_status()
+
+  def _update_state(self) -> None:
+    # Handle panda states updates
+    if self.sm.updated["pandaStates"]:
+      panda_states = self.sm["pandaStates"]
+
+      if len(panda_states) > 0:
+        # Get panda type from first panda
+        self.panda_type = panda_states[0].pandaType
+        # Check ignition status across all pandas
+        if self.panda_type != log.PandaState.PandaType.unknown:
+          self.ignition = any(state.ignitionLine or state.ignitionCan for state in panda_states)
+    elif self.sm.frame - self.sm.recv_frame["pandaStates"] > 5 * rl.get_fps():
+      self.panda_type = log.PandaState.PandaType.unknown
+
+    # Handle wide road camera state updates
+    if self.sm.updated["wideRoadCameraState"]:
+      cam_state = self.sm["wideRoadCameraState"]
+
+      # Scale factor based on sensor type
+      scale = 6.0 if cam_state.sensor == 'ar0231' else 1.0
+      self.light_sensor = max(100.0 - scale * cam_state.exposureValPercent, 0.0)
+    elif not self.sm.alive["wideRoadCameraState"] or not self.sm.valid["wideRoadCameraState"]:
+      self.light_sensor = -1
+
+    # Update started state
+    self.started = self.sm["deviceState"].started and self.ignition
+
+  def _update_status(self) -> None:
+    if self.started and self.sm.updated["selfdriveState"]:
+      ss = self.sm["selfdriveState"]
+      state = ss.state
+
+      if state in (log.SelfdriveState.OpenpilotState.preEnabled, log.SelfdriveState.OpenpilotState.overriding):
+        self.status = UIStatus.OVERRIDE
+      else:
+        self.status = UIStatus.ENGAGED if ss.enabled else UIStatus.DISENGAGED
+
+    # Check for engagement state changes
+    if self.engaged != self._engaged_prev:
+      self._engaged_prev = self.engaged
+
+    # Handle onroad/offroad transition
+    if self.started != self._started_prev or self.sm.frame == 1:
+      if self.started:
+        self.status = UIStatus.DISENGAGED
+        self.started_frame = self.sm.frame
+
+      self._started_prev = self.started
+
+  def _update_params(self) -> None:
+    try:
+      self.is_metric = self.params.get_bool("IsMetric")
+    except UnknownKeyName:
+      self.is_metric = False
+
+
+# Global instance
+ui_state = UIState()