UI

selfdrive/ui/mici/onroad/augmented_road_view.py

@@ -554,9 +554,7 @@ class AugmentedRoadView(CameraView):
     self.view_from_calib = view_frame_from_device_frame.copy()
     self.view_from_wide_calib = view_frame_from_device_frame.copy()
 
-    self._last_calib_time: float = 0
-    self._last_rect_dims = (0.0, 0.0)
-    self._last_stream_type = stream_type
+    self._matrix_cache_key: tuple | None = None
     self._cached_matrix: np.ndarray | None = None
     self._content_rect = rl.Rectangle()
     self._last_click_time = 0.0
@@ -917,10 +915,20 @@ class AugmentedRoadView(CameraView):
       base[1, 1] *= driver_view_ratio
       return base
 
+    cache_key = (
+      ui_state.sm.recv_frame['liveCalibration'],
+      int(self._content_rect.x),
+      int(self._content_rect.y),
+      int(self._content_rect.width),
+      int(self._content_rect.height),
+      self.stream_type,
+      round(float(ui_state.sm['carState'].vEgo), 1),
+      id(self.device_camera),
+    )
+    if cache_key == self._matrix_cache_key and self._cached_matrix is not None:
+      return self._cached_matrix
+
     # Get camera configuration
-    # TODO: cache with vEgo?
-    calib_time = ui_state.sm.recv_frame['liveCalibration']
-    current_dims = (self._content_rect.width, self._content_rect.height)
     device_camera = self.device_camera or DEFAULT_DEVICE_CAMERA
     is_wide_camera = self.stream_type == WIDE_CAM
     intrinsic = device_camera.ecam.intrinsics if is_wide_camera else device_camera.fcam.intrinsics
@@ -958,10 +966,7 @@ class AugmentedRoadView(CameraView):
     except (ZeroDivisionError, OverflowError):
       x_offset, y_offset = 0, 0
 
-    # Cache the computed transformation matrix to avoid recalculations
-    self._last_calib_time = calib_time
-    self._last_rect_dims = current_dims
-    self._last_stream_type = self.stream_type
+    self._matrix_cache_key = cache_key
     self._cached_matrix = np.array([
       [zoom * 2 * cx / w, 0, -x_offset / w * 2],
       [0, zoom * 2 * cy / h, -y_offset / h * 2],

selfdrive/ui/onroad/cameraview.py

@@ -174,6 +174,10 @@ class CameraView(Widget):
     self.texture_y: rl.Texture | None = None
     self.texture_uv: rl.Texture | None = None
 
+    self._camera_render_texture = None
+    self._camera_render_texture_size = (0, 0)
+    self._camera_render_key = None
+
     # EGL resources
     self.egl_images: dict[int, EGLImage] = {}
     self.egl_texture: rl.Texture | None = None
@@ -218,6 +222,7 @@ class CameraView(Widget):
     self._regressive_frame_count = 0
     self.available_streams.clear()
     self._texture_needs_update = True
+    self._camera_render_key = None
     self.last_connection_attempt = 0.0
     self._last_stream_discovery = -float("inf")
     self._last_switch_request = -float("inf")
@@ -297,6 +302,7 @@ class CameraView(Widget):
     self._last_frame_id = -1
     self._regressive_frame_count = 0
     self._texture_needs_update = True
+    self._camera_render_key = None
     self._reentry_stream_selected = True
 
   @property
@@ -325,6 +331,7 @@ class CameraView(Widget):
     self.available_streams.clear()
     self._onroad_reentry_pending = False
     self._reentry_stream_selected = False
+    self._release_camera_render_texture()
 
   def __del__(self):
     self.close()
@@ -369,6 +376,7 @@ class CameraView(Widget):
     elif not self.client.is_connected():
       # ensure we clear the displayed frame when the connection is lost
       self.frame = None
+      self._camera_render_key = None
 
     if not self.frame:
       self._draw_placeholder(rect)
@@ -393,6 +401,12 @@ class CameraView(Widget):
 
     dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y)
 
+    if TICI:
+      self._render_cached_camera(rect, src_rect, dst_rect, transform)
+    else:
+      self._render_current_frame(src_rect, dst_rect)
+
+  def _render_current_frame(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
     if self._use_egl:
       try:
         rendered = self._render_egl(src_rect, dst_rect)
@@ -405,6 +419,75 @@ class CameraView(Widget):
     if not self._use_egl:
       self._render_textures(src_rect, dst_rect)
 
+  def _render_cached_camera(self, rect: rl.Rectangle, src_rect: rl.Rectangle,
+                            dst_rect: rl.Rectangle, transform: np.ndarray) -> None:
+    size = (max(1, int(round(rect.width))), max(1, int(round(rect.height))))
+    if self._camera_render_texture is None or self._camera_render_texture_size != size:
+      self._release_camera_render_texture()
+      try:
+        self._camera_render_texture = rl.load_render_texture(*size)
+      except Exception:
+        cloudlog.exception("CameraView failed to create camera render texture")
+        self._camera_render_texture = None
+      self._camera_render_texture_size = size
+      self._camera_render_key = None
+
+    if (self._camera_render_texture is None or
+        not getattr(self._camera_render_texture.texture, "id", 0)):
+      self._render_current_frame(src_rect, dst_rect)
+      return
+
+    frame_id = int(getattr(self.frame, "frame_id", -1))
+    transform_key = tuple(round(float(value), 5) for value in transform.flat)
+    cache_key = (frame_id, self._stream_type, size, transform_key, self._use_egl)
+    if cache_key != self._camera_render_key:
+      local_dst = rl.Rectangle(
+        dst_rect.x - rect.x,
+        dst_rect.y - rect.y,
+        dst_rect.width,
+        dst_rect.height,
+      )
+      scissor = (int(rect.x), int(rect.y), int(rect.width), int(rect.height))
+      rl.end_scissor_mode()
+      rendered_to_cache = False
+      texture_mode_started = False
+      try:
+        rl.begin_texture_mode(self._camera_render_texture)
+        texture_mode_started = True
+        try:
+          rl.clear_background(rl.BLACK)
+          self._render_current_frame(src_rect, local_dst)
+        finally:
+          if texture_mode_started:
+            rl.end_texture_mode()
+        rendered_to_cache = True
+      except Exception:
+        cloudlog.exception("CameraView failed to render cached camera frame")
+        self._release_camera_render_texture()
+      finally:
+        rl.begin_scissor_mode(*scissor)
+      if not rendered_to_cache:
+        self._render_current_frame(src_rect, dst_rect)
+        return
+      self._camera_render_key = cache_key
+
+    if self._camera_render_texture is None:
+      return
+
+    source = rl.Rectangle(0, 0, float(size[0]), -float(size[1]))
+    rl.draw_texture_pro(self._camera_render_texture.texture, source, rect,
+                        rl.Vector2(0, 0), 0.0, rl.WHITE)
+
+  def _release_camera_render_texture(self) -> None:
+    if self._camera_render_texture is not None:
+      try:
+        rl.unload_render_texture(self._camera_render_texture)
+      except Exception:
+        cloudlog.exception("CameraView failed to release camera render texture")
+    self._camera_render_texture = None
+    self._camera_render_texture_size = (0, 0)
+    self._camera_render_key = None
+
   def _draw_placeholder(self, rect: rl.Rectangle):
     if self._placeholder_color:
       rl.draw_rectangle_rec(rect, self._placeholder_color)
@@ -455,6 +538,7 @@ class CameraView(Widget):
     self._texture_needs_update = True
     self._onroad_reentry_pending = False
     self._reentry_stream_selected = False
+    self._camera_render_key = None
     return True
 
   def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> bool:
@@ -488,6 +572,7 @@ class CameraView(Widget):
 
     cloudlog.error(f"CameraView switching from EGL to texture rendering: {reason}")
     self._use_egl = False
+    self._camera_render_key = None
     try:
       self._clear_textures()
     except Exception:
@@ -618,6 +703,7 @@ class CameraView(Widget):
     self._texture_needs_update = True
     self._onroad_reentry_pending = False
     self._reentry_stream_selected = False
+    self._camera_render_key = None
 
     # Initialize textures for new stream
     self._initialize_textures()

selfdrive/ui/tests/test_camera_frame_order.py

@@ -1,5 +1,6 @@
 from types import SimpleNamespace
 
+import numpy as np
 import pytest
 
 from openpilot.selfdrive.ui.mici.onroad import cameraview as mici_cameraview
@@ -167,6 +168,41 @@ def test_newer_camera_frame_is_accepted():
   assert view._texture_needs_update
 
 
+def test_device_camera_render_cache_reuses_processed_frame(monkeypatch):
+  view = _camera_view()
+  view._camera_render_texture = None
+  view._camera_render_texture_size = (0, 0)
+  view._camera_render_key = None
+  view._use_egl = True
+  view.frame = FakeFrame(frame_id=10, idx=0)
+
+  render_texture = SimpleNamespace(texture=SimpleNamespace(id=1))
+  processed_frames = []
+  draws = []
+
+  monkeypatch.setattr(big_cameraview, "TICI", True)
+  monkeypatch.setattr(big_cameraview.rl, "load_render_texture", lambda *_size: render_texture)
+  monkeypatch.setattr(big_cameraview.rl, "unload_render_texture", lambda _texture: None)
+  monkeypatch.setattr(big_cameraview.rl, "begin_texture_mode", lambda _texture: None)
+  monkeypatch.setattr(big_cameraview.rl, "end_texture_mode", lambda: None)
+  monkeypatch.setattr(big_cameraview.rl, "clear_background", lambda _color: None)
+  monkeypatch.setattr(big_cameraview.rl, "end_scissor_mode", lambda: None)
+  monkeypatch.setattr(big_cameraview.rl, "begin_scissor_mode", lambda *_args: None)
+  monkeypatch.setattr(big_cameraview.rl, "draw_texture_pro", lambda *_args: draws.append(1))
+  view._render_current_frame = lambda *_args: processed_frames.append(1)
+
+  rect = big_cameraview.rl.Rectangle(10, 20, 100, 80)
+  src = big_cameraview.rl.Rectangle(0, 0, 1928, 1208)
+  dst = big_cameraview.rl.Rectangle(0, 0, 100, 80)
+  transform = np.eye(3)
+
+  view._render_cached_camera(rect, src, dst, transform)
+  view._render_cached_camera(rect, src, dst, transform)
+
+  assert processed_frames == [1]
+  assert draws == [1, 1]
+
+
 def test_shared_camera_has_upstream_shaders_and_driver_enhancement():
   assert "samplerExternalOES" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL
   assert "pow(color.rgb, vec3(1.0/1.28))" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL

system/ui/lib/shader_polygon.py

@@ -213,20 +213,22 @@ def draw_polygon(origin_rect: rl.Rectangle, points: np.ndarray,
   if len(points) < 3:
     return
 
-  # Initialize shader on-demand
-  state = ShaderState.get_instance()
-  state.initialize()
-
   # Ensure (N,2) float32 contiguous array
   pts = np.ascontiguousarray(points, dtype=np.float32)
   assert pts.ndim == 2 and pts.shape[1] == 2, "points must be (N,2)"
 
-  # Configure gradient shader
-  _configure_shader_color(state, color, gradient, origin_rect)
-
   # Triangulate via interleaving
   tri_strip = triangulate(pts)
 
+  if gradient is None:
+    rl.draw_triangle_strip(tri_strip, len(tri_strip), color or rl.WHITE)
+    return
+
+  state = ShaderState.get_instance()
+  state.initialize()
+
+  _configure_shader_color(state, color, gradient, origin_rect)
+
   # Draw strip, color here doesn't matter
   rl.begin_shader_mode(state.shader)
   rl.draw_triangle_strip(tri_strip, len(tri_strip), rl.WHITE)