From ca737c0d40b35307017d7b98a44c31984a356026 Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:56:38 -0500 Subject: [PATCH] UI --- .../ui/mici/onroad/augmented_road_view.py | 25 +++--- selfdrive/ui/onroad/cameraview.py | 86 +++++++++++++++++++ selfdrive/ui/tests/test_camera_frame_order.py | 36 ++++++++ system/ui/lib/shader_polygon.py | 16 ++-- 4 files changed, 146 insertions(+), 17 deletions(-) diff --git a/selfdrive/ui/mici/onroad/augmented_road_view.py b/selfdrive/ui/mici/onroad/augmented_road_view.py index bc9f78719..190ebb6d8 100644 --- a/selfdrive/ui/mici/onroad/augmented_road_view.py +++ b/selfdrive/ui/mici/onroad/augmented_road_view.py @@ -554,9 +554,7 @@ class AugmentedRoadView(CameraView): self.view_from_calib = view_frame_from_device_frame.copy() self.view_from_wide_calib = view_frame_from_device_frame.copy() - self._last_calib_time: float = 0 - self._last_rect_dims = (0.0, 0.0) - self._last_stream_type = stream_type + self._matrix_cache_key: tuple | None = None self._cached_matrix: np.ndarray | None = None self._content_rect = rl.Rectangle() self._last_click_time = 0.0 @@ -917,10 +915,20 @@ class AugmentedRoadView(CameraView): base[1, 1] *= driver_view_ratio return base + cache_key = ( + ui_state.sm.recv_frame['liveCalibration'], + int(self._content_rect.x), + int(self._content_rect.y), + int(self._content_rect.width), + int(self._content_rect.height), + self.stream_type, + round(float(ui_state.sm['carState'].vEgo), 1), + id(self.device_camera), + ) + if cache_key == self._matrix_cache_key and self._cached_matrix is not None: + return self._cached_matrix + # Get camera configuration - # TODO: cache with vEgo? - calib_time = ui_state.sm.recv_frame['liveCalibration'] - current_dims = (self._content_rect.width, self._content_rect.height) device_camera = self.device_camera or DEFAULT_DEVICE_CAMERA is_wide_camera = self.stream_type == WIDE_CAM intrinsic = device_camera.ecam.intrinsics if is_wide_camera else device_camera.fcam.intrinsics @@ -958,10 +966,7 @@ class AugmentedRoadView(CameraView): except (ZeroDivisionError, OverflowError): x_offset, y_offset = 0, 0 - # Cache the computed transformation matrix to avoid recalculations - self._last_calib_time = calib_time - self._last_rect_dims = current_dims - self._last_stream_type = self.stream_type + self._matrix_cache_key = cache_key self._cached_matrix = np.array([ [zoom * 2 * cx / w, 0, -x_offset / w * 2], [0, zoom * 2 * cy / h, -y_offset / h * 2], diff --git a/selfdrive/ui/onroad/cameraview.py b/selfdrive/ui/onroad/cameraview.py index b1e7a871d..47c91260d 100644 --- a/selfdrive/ui/onroad/cameraview.py +++ b/selfdrive/ui/onroad/cameraview.py @@ -174,6 +174,10 @@ class CameraView(Widget): self.texture_y: rl.Texture | None = None self.texture_uv: rl.Texture | None = None + self._camera_render_texture = None + self._camera_render_texture_size = (0, 0) + self._camera_render_key = None + # EGL resources self.egl_images: dict[int, EGLImage] = {} self.egl_texture: rl.Texture | None = None @@ -218,6 +222,7 @@ class CameraView(Widget): self._regressive_frame_count = 0 self.available_streams.clear() self._texture_needs_update = True + self._camera_render_key = None self.last_connection_attempt = 0.0 self._last_stream_discovery = -float("inf") self._last_switch_request = -float("inf") @@ -297,6 +302,7 @@ class CameraView(Widget): self._last_frame_id = -1 self._regressive_frame_count = 0 self._texture_needs_update = True + self._camera_render_key = None self._reentry_stream_selected = True @property @@ -325,6 +331,7 @@ class CameraView(Widget): self.available_streams.clear() self._onroad_reentry_pending = False self._reentry_stream_selected = False + self._release_camera_render_texture() def __del__(self): self.close() @@ -369,6 +376,7 @@ class CameraView(Widget): elif not self.client.is_connected(): # ensure we clear the displayed frame when the connection is lost self.frame = None + self._camera_render_key = None if not self.frame: self._draw_placeholder(rect) @@ -393,6 +401,12 @@ class CameraView(Widget): dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y) + if TICI: + self._render_cached_camera(rect, src_rect, dst_rect, transform) + else: + self._render_current_frame(src_rect, dst_rect) + + def _render_current_frame(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None: if self._use_egl: try: rendered = self._render_egl(src_rect, dst_rect) @@ -405,6 +419,75 @@ class CameraView(Widget): if not self._use_egl: self._render_textures(src_rect, dst_rect) + def _render_cached_camera(self, rect: rl.Rectangle, src_rect: rl.Rectangle, + dst_rect: rl.Rectangle, transform: np.ndarray) -> None: + size = (max(1, int(round(rect.width))), max(1, int(round(rect.height)))) + if self._camera_render_texture is None or self._camera_render_texture_size != size: + self._release_camera_render_texture() + try: + self._camera_render_texture = rl.load_render_texture(*size) + except Exception: + cloudlog.exception("CameraView failed to create camera render texture") + self._camera_render_texture = None + self._camera_render_texture_size = size + self._camera_render_key = None + + if (self._camera_render_texture is None or + not getattr(self._camera_render_texture.texture, "id", 0)): + self._render_current_frame(src_rect, dst_rect) + return + + frame_id = int(getattr(self.frame, "frame_id", -1)) + transform_key = tuple(round(float(value), 5) for value in transform.flat) + cache_key = (frame_id, self._stream_type, size, transform_key, self._use_egl) + if cache_key != self._camera_render_key: + local_dst = rl.Rectangle( + dst_rect.x - rect.x, + dst_rect.y - rect.y, + dst_rect.width, + dst_rect.height, + ) + scissor = (int(rect.x), int(rect.y), int(rect.width), int(rect.height)) + rl.end_scissor_mode() + rendered_to_cache = False + texture_mode_started = False + try: + rl.begin_texture_mode(self._camera_render_texture) + texture_mode_started = True + try: + rl.clear_background(rl.BLACK) + self._render_current_frame(src_rect, local_dst) + finally: + if texture_mode_started: + rl.end_texture_mode() + rendered_to_cache = True + except Exception: + cloudlog.exception("CameraView failed to render cached camera frame") + self._release_camera_render_texture() + finally: + rl.begin_scissor_mode(*scissor) + if not rendered_to_cache: + self._render_current_frame(src_rect, dst_rect) + return + self._camera_render_key = cache_key + + if self._camera_render_texture is None: + return + + source = rl.Rectangle(0, 0, float(size[0]), -float(size[1])) + rl.draw_texture_pro(self._camera_render_texture.texture, source, rect, + rl.Vector2(0, 0), 0.0, rl.WHITE) + + def _release_camera_render_texture(self) -> None: + if self._camera_render_texture is not None: + try: + rl.unload_render_texture(self._camera_render_texture) + except Exception: + cloudlog.exception("CameraView failed to release camera render texture") + self._camera_render_texture = None + self._camera_render_texture_size = (0, 0) + self._camera_render_key = None + def _draw_placeholder(self, rect: rl.Rectangle): if self._placeholder_color: rl.draw_rectangle_rec(rect, self._placeholder_color) @@ -455,6 +538,7 @@ class CameraView(Widget): self._texture_needs_update = True self._onroad_reentry_pending = False self._reentry_stream_selected = False + self._camera_render_key = None return True def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> bool: @@ -488,6 +572,7 @@ class CameraView(Widget): cloudlog.error(f"CameraView switching from EGL to texture rendering: {reason}") self._use_egl = False + self._camera_render_key = None try: self._clear_textures() except Exception: @@ -618,6 +703,7 @@ class CameraView(Widget): self._texture_needs_update = True self._onroad_reentry_pending = False self._reentry_stream_selected = False + self._camera_render_key = None # Initialize textures for new stream self._initialize_textures() diff --git a/selfdrive/ui/tests/test_camera_frame_order.py b/selfdrive/ui/tests/test_camera_frame_order.py index 19dc4dc97..bde83c32a 100644 --- a/selfdrive/ui/tests/test_camera_frame_order.py +++ b/selfdrive/ui/tests/test_camera_frame_order.py @@ -1,5 +1,6 @@ from types import SimpleNamespace +import numpy as np import pytest from openpilot.selfdrive.ui.mici.onroad import cameraview as mici_cameraview @@ -167,6 +168,41 @@ def test_newer_camera_frame_is_accepted(): assert view._texture_needs_update +def test_device_camera_render_cache_reuses_processed_frame(monkeypatch): + view = _camera_view() + view._camera_render_texture = None + view._camera_render_texture_size = (0, 0) + view._camera_render_key = None + view._use_egl = True + view.frame = FakeFrame(frame_id=10, idx=0) + + render_texture = SimpleNamespace(texture=SimpleNamespace(id=1)) + processed_frames = [] + draws = [] + + monkeypatch.setattr(big_cameraview, "TICI", True) + monkeypatch.setattr(big_cameraview.rl, "load_render_texture", lambda *_size: render_texture) + monkeypatch.setattr(big_cameraview.rl, "unload_render_texture", lambda _texture: None) + monkeypatch.setattr(big_cameraview.rl, "begin_texture_mode", lambda _texture: None) + monkeypatch.setattr(big_cameraview.rl, "end_texture_mode", lambda: None) + monkeypatch.setattr(big_cameraview.rl, "clear_background", lambda _color: None) + monkeypatch.setattr(big_cameraview.rl, "end_scissor_mode", lambda: None) + monkeypatch.setattr(big_cameraview.rl, "begin_scissor_mode", lambda *_args: None) + monkeypatch.setattr(big_cameraview.rl, "draw_texture_pro", lambda *_args: draws.append(1)) + view._render_current_frame = lambda *_args: processed_frames.append(1) + + rect = big_cameraview.rl.Rectangle(10, 20, 100, 80) + src = big_cameraview.rl.Rectangle(0, 0, 1928, 1208) + dst = big_cameraview.rl.Rectangle(0, 0, 100, 80) + transform = np.eye(3) + + view._render_cached_camera(rect, src, dst, transform) + view._render_cached_camera(rect, src, dst, transform) + + assert processed_frames == [1] + assert draws == [1, 1] + + def test_shared_camera_has_upstream_shaders_and_driver_enhancement(): assert "samplerExternalOES" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL assert "pow(color.rgb, vec3(1.0/1.28))" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL diff --git a/system/ui/lib/shader_polygon.py b/system/ui/lib/shader_polygon.py index 94af35e15..413c23374 100644 --- a/system/ui/lib/shader_polygon.py +++ b/system/ui/lib/shader_polygon.py @@ -213,20 +213,22 @@ def draw_polygon(origin_rect: rl.Rectangle, points: np.ndarray, if len(points) < 3: return - # Initialize shader on-demand - state = ShaderState.get_instance() - state.initialize() - # Ensure (N,2) float32 contiguous array pts = np.ascontiguousarray(points, dtype=np.float32) assert pts.ndim == 2 and pts.shape[1] == 2, "points must be (N,2)" - # Configure gradient shader - _configure_shader_color(state, color, gradient, origin_rect) - # Triangulate via interleaving tri_strip = triangulate(pts) + if gradient is None: + rl.draw_triangle_strip(tri_strip, len(tri_strip), color or rl.WHITE) + return + + state = ShaderState.get_instance() + state.initialize() + + _configure_shader_color(state, color, gradient, origin_rect) + # Draw strip, color here doesn't matter rl.begin_shader_mode(state.shader) rl.draw_triangle_strip(tri_strip, len(tri_strip), rl.WHITE)