ui: re-apply mici/AugmentedRoadView _calc_frame_matrix caching (#37948)

* ui: speed up `mici/AugmentedRoadView` by optimizing _calc_frame_matrix caching (#36669)

speed up AugmentedRoadView by optimizing _calc_frame_matrix caching

* ui: apply rect.x/y as a 2D screen offset post-projection

Removes the parent rect's screen position from the cached
video_transform passed to ModelRenderer. Instead, ModelRenderer
applies (rect.x, rect.y) as a 2D offset to projected_points at draw
time.

Why this works: the rect.x/y term in video_transform gets multiplied
by P_calib[2] before the perspective divide, then divided by the same
value, which cancels out to a simple additive shift on the final
screen coordinate. So adding x to video_transform[0,2] is equivalent
to adding x to screen_x post-projection.

Net effect: the cache key in _calc_frame_matrix no longer needs to
include rect.x/y. Cache stays hot under translation (e.g. swiping
between layouts), and the model overlay tracks the camera at 60Hz
because the offset is updated cheaply each frame.

This addresses the original revert reason for #36669 (model overlay
visually desyncs from camera during a home<->onroad swipe).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* ui: update model screen offset every frame, not just on cache miss

set_screen_offset() was called inside the cache-miss path of
_calc_frame_matrix, so the offset only updated at ~20Hz (calib publish
rate). The model overlay visibly lagged the camera during a swipe.

Move it out of the cached path so it updates each frame.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fully clean up

* shorter cmts

* make non

* clean up

* fix ty

---------

Co-authored-by: Dean Lee <deanlee3@gmail.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

selfdrive/ui/mici/onroad/augmented_road_view.py

@@ -139,9 +139,7 @@ class AugmentedRoadView(CameraView):
     self.view_from_calib = view_frame_from_device_frame.copy()
     self.view_from_wide_calib = view_frame_from_device_frame.copy()
 
-    self._last_calib_time: float = 0
-    self._last_rect_dims = (0.0, 0.0)
-    self._last_stream_type = stream_type
+    self._matrix_cache_key: tuple | None = None
     self._cached_matrix: np.ndarray | None = None
     self._content_rect = rl.Rectangle()
     self._last_click_time = 0.0
@@ -295,10 +293,18 @@ class AugmentedRoadView(CameraView):
       self.view_from_wide_calib = view_frame_from_device_frame @ wide_from_device @ device_from_calib
 
   def _calc_frame_matrix(self, rect: rl.Rectangle) -> np.ndarray:
+    cache_key = (
+      ui_state.sm.recv_frame['liveCalibration'],
+      int(self._content_rect.width),
+      int(self._content_rect.height),
+      self.stream_type,
+      round(ui_state.sm['carState'].vEgo, 1),
+    )
+
+    if cache_key == self._matrix_cache_key and self._cached_matrix is not None:
+      return self._cached_matrix
+
     # Get camera configuration
-    # TODO: cache with vEgo?
-    calib_time = ui_state.sm.recv_frame['liveCalibration']
-    current_dims = (self._content_rect.width, self._content_rect.height)
     device_camera = self.device_camera or DEFAULT_DEVICE_CAMERA
     is_wide_camera = self.stream_type == WIDE_CAM
     intrinsic = device_camera.ecam.intrinsics if is_wide_camera else device_camera.fcam.intrinsics
@@ -314,7 +320,6 @@ class AugmentedRoadView(CameraView):
     kep = calib_transform @ inf_point
 
     # Calculate center points and dimensions
-    x, y = self._content_rect.x, self._content_rect.y
     w, h = self._content_rect.width, self._content_rect.height
     cx, cy = intrinsic[0, 2], intrinsic[1, 2]
 
@@ -334,18 +339,17 @@ class AugmentedRoadView(CameraView):
       x_offset, y_offset = 0, 0
 
     # Cache the computed transformation matrix to avoid recalculations
-    self._last_calib_time = calib_time
-    self._last_rect_dims = current_dims
-    self._last_stream_type = self.stream_type
+    self._matrix_cache_key = cache_key
     self._cached_matrix = np.array([
       [zoom * 2 * cx / w, 0, -x_offset / w * 2],
       [0, zoom * 2 * cy / h, -y_offset / h * 2],
       [0, 0, 1.0]
     ])
 
+    # built without rect.x/y so cache stays hot during scroll. ModelRenderer adds offset at draw time
     video_transform = np.array([
-      [zoom, 0.0, (w / 2 + x - x_offset) - (cx * zoom)],
-      [0.0, zoom, (h / 2 + y - y_offset) - (cy * zoom)],
+      [zoom, 0.0, (w / 2 - x_offset) - (cx * zoom)],
+      [0.0, zoom, (h / 2 - y_offset) - (cy * zoom)],
       [0.0, 0.0, 1.0]
     ])
     self._model_renderer.set_transform(video_transform @ calib_transform)

selfdrive/ui/mici/onroad/model_renderer.py

@@ -72,7 +72,7 @@ class ModelRenderer(Widget):
     self._torque_filter = FirstOrderFilter(0, 0.1, 1 / gui_app.target_fps)
     self._ll_color_filter = FirstOrderFilter(0.0, 0.1, 1 / gui_app.target_fps)
 
-    # Transform matrix (3x3 for car space to screen space)
+    # 3x3 car space -> rect-origin space (draw methods add rect.x/y)
     self._car_space_transform = np.zeros((3, 3), dtype=np.float32)
     self._transform_dirty = True
     self._clip_region = None
@@ -221,7 +221,8 @@ class ModelRenderer(Widget):
     if not self._experimental_mode:
       return
 
-    max_len = min(len(self._path.projected_points) // 2, len(self._acceleration_x))
+    path_pts = self._path.projected_points + np.array([self._rect.x, self._rect.y], dtype=np.float32)
+    max_len = min(len(path_pts) // 2, len(self._acceleration_x))
 
     segment_colors = []
     gradient_stops = []
@@ -229,7 +230,7 @@ class ModelRenderer(Widget):
     i = 0
     while i < max_len:
       # Some points (screen space) are out of frame (rect space)
-      track_y = self._path.projected_points[i][1]
+      track_y = path_pts[i][1]
       if track_y < self._rect.y or track_y > (self._rect.y + self._rect.height):
         i += 1
         continue
@@ -305,14 +306,15 @@ class ModelRenderer(Widget):
     return color
 
   def _draw_lane_lines(self):
-    """Draw lane lines and road edges"""
-    """Two closest lines should be green (lane line or road edges)"""
+    """Draw lane lines and road edges. Two closest lines should be green (lane line or road edges)."""
+    offset = np.array([self._rect.x, self._rect.y], dtype=np.float32)
+
     for i, lane_line in enumerate(self._lane_lines):
       if lane_line.projected_points.size == 0:
         continue
 
       color = self._get_ll_color(float(self._lane_line_probs[i]), i in (1, 2), i in (0, 1))
-      draw_polygon(self._rect, lane_line.projected_points, color)
+      draw_polygon(self._rect, lane_line.projected_points + offset, color)
 
     for i, road_edge in enumerate(self._road_edges):
       if road_edge.projected_points.size == 0:
@@ -320,7 +322,7 @@ class ModelRenderer(Widget):
 
       # if closest lane lines are not confident, make road edges green
       color = self._get_ll_color(float(1.0 - self._road_edge_stds[i]), float(self._lane_line_probs[i + 1]) < 0.25, i == 0)
-      draw_polygon(self._rect, road_edge.projected_points, color)
+      draw_polygon(self._rect, road_edge.projected_points + offset, color)
 
   def _draw_path(self, sm):
     """Draw path with dynamic coloring based on mode and throttle state."""
@@ -330,14 +332,16 @@ class ModelRenderer(Widget):
     allow_throttle = sm['longitudinalPlan'].allowThrottle or not self._longitudinal_control
     self._blend_filter.update(int(allow_throttle))
 
+    path_pts = self._path.projected_points + np.array([self._rect.x, self._rect.y], dtype=np.float32)
+
     if self._experimental_mode:
       # Draw with acceleration coloring
       if ui_state.status == UIStatus.DISENGAGED:
-        draw_polygon(self._rect, self._path.projected_points, rl.Color(0, 0, 0, 90))
+        draw_polygon(self._rect, path_pts, rl.Color(0, 0, 0, 90))
       elif len(self._exp_gradient.colors) > 1:
-        draw_polygon(self._rect, self._path.projected_points, gradient=self._exp_gradient)
+        draw_polygon(self._rect, path_pts, gradient=self._exp_gradient)
       else:
-        draw_polygon(self._rect, self._path.projected_points, rl.Color(255, 255, 255, 30))
+        draw_polygon(self._rect, path_pts, rl.Color(255, 255, 255, 30))
     else:
       # Blend throttle/no throttle colors based on transition
       blend_factor = round(self._blend_filter.x * 100) / 100
@@ -350,9 +354,9 @@ class ModelRenderer(Widget):
       )
 
       if ui_state.status == UIStatus.DISENGAGED:
-        draw_polygon(self._rect, self._path.projected_points, rl.Color(0, 0, 0, 90))
+        draw_polygon(self._rect, path_pts, rl.Color(0, 0, 0, 90))
       else:
-        draw_polygon(self._rect, self._path.projected_points, gradient=gradient)
+        draw_polygon(self._rect, path_pts, gradient=gradient)
 
   def _draw_lead_indicator(self):
     # Draw lead vehicles if available