ui: share ModelRenderer between tizi and mici (#39198)

openpilot/selfdrive/ui/mici/onroad/model_renderer.py

@@ -1,10 +1,8 @@
-import colorsys
 import math
 import numpy as np
 from openpilot.starpilot.ui.raylib import rl
 from openpilot.cereal import log, messaging
 from opendbc.car.structs import car
-from dataclasses import dataclass, field
 from types import SimpleNamespace
 from openpilot.common.params import Params
 from openpilot.starpilot.ui.appearance_preferences import format_lead_info
@@ -22,6 +20,7 @@ from openpilot.system.ui.lib.application import FontWeight
 from openpilot.system.ui.lib.text_measure import measure_text_cached
 from openpilot.system.ui.lib.shader_polygon import draw_polygon, Gradient
 from openpilot.system.ui.widgets import Widget
+from openpilot.selfdrive.ui.onroad.model_renderer import ModelGeometry, ModelPoints
 
 CLIP_MARGIN = 500
 MIN_DRAW_DISTANCE = 10.0
@@ -48,10 +47,6 @@ LANE_LINE_COLORS = {
 }
 
 
-@dataclass
-class ModelPoints:
-  raw_points: np.ndarray = field(default_factory=lambda: np.empty((0, 3), dtype=np.float32))
-  projected_points: np.ndarray = field(default_factory=lambda: np.empty((0, 2), dtype=np.float32))
 
 
 class LeadVehicle:
@@ -65,7 +60,7 @@ class LeadVehicle:
 
 
 
-class ModelRenderer(Widget):
+class ModelRenderer(ModelGeometry, Widget):
   def __init__(self):
     super().__init__()
     self._longitudinal_control = False
@@ -117,11 +112,6 @@ class ModelRenderer(Widget):
       cp = messaging.log_from_bytes(car_params, car.CarParams)
       self._longitudinal_control = cp.openpilotLongitudinalControl
 
-  def set_transform(self, transform: np.ndarray):
-    transform = transform.astype(np.float32)
-    if not np.array_equal(transform, self._car_space_transform):
-      self._car_space_transform = transform
-      self._transform_dirty = True
 
   def render_with_lead(self, rect: rl.Rectangle, enabled: bool, info_mode: int = 0, metric: bool | None = None,
                        lateral_active: bool = False, paint: bool = True):
@@ -231,20 +221,6 @@ class ModelRenderer(Widget):
     except (AttributeError, TypeError, ValueError, OverflowError):
       return False
 
-  def _update_raw_points(self, model):
-    """Update raw 3D points from model data"""
-    self._path.raw_points = np.array([model.position.x, model.position.y, model.position.z], dtype=np.float32).T
-
-    # Older logs may append lane centers after the four boundaries.
-    for points, lane_line in zip(self._lane_lines, model.laneLines, strict=False):
-      points.raw_points = np.array([lane_line.x, lane_line.y, lane_line.z], dtype=np.float32).T
-
-    for i, road_edge in enumerate(model.roadEdges):
-      self._road_edges[i].raw_points = np.array([road_edge.x, road_edge.y, road_edge.z], dtype=np.float32).T
-
-    self._lane_line_probs = np.array(model.laneLineProbs, dtype=np.float32)
-    self._road_edge_stds = np.array(model.roadEdgeStds, dtype=np.float32)
-    self._acceleration_x = np.array(model.acceleration.x, dtype=np.float32)
 
   def _update_leads(self, sm):
     plan = sm['longitudinalPlan']
@@ -621,27 +597,4 @@ class ModelRenderer(Widget):
       start = end
     return polygons
 
-  @staticmethod
-  def _hsla_to_color(h, s, l, a):
-    rgb = colorsys.hls_to_rgb(h, l, s)
-    return rl.Color(
-      int(rgb[0] * 255),
-      int(rgb[1] * 255),
-      int(rgb[2] * 255),
-      int(a * 255)
-    )
 
-  @staticmethod
-  def _blend_colors(begin_colors, end_colors, t):
-    if t >= 1.0:
-      return end_colors
-    if t <= 0.0:
-      return begin_colors
-
-    inv_t = 1.0 - t
-    return [rl.Color(
-      int(inv_t * start.r + t * end.r),
-      int(inv_t * start.g + t * end.g),
-      int(inv_t * start.b + t * end.b),
-      int(inv_t * start.a + t * end.a)
-    ) for start, end in zip(begin_colors, end_colors, strict=True)]

openpilot/selfdrive/ui/onroad/model_renderer.py

@@ -48,7 +48,65 @@ class LeadVehicle:
   fill_alpha: int = 0
 
 
-class ModelRenderer(Widget):
+class ModelGeometry:
+  _path: ModelPoints
+  _lane_lines: list[ModelPoints]
+  _road_edges: list[ModelPoints]
+  _car_space_transform: np.ndarray
+  _transform_dirty: bool
+  _lane_line_probs: np.ndarray
+  _road_edge_stds: np.ndarray
+  _acceleration_x: np.ndarray
+
+  def set_transform(self, transform: np.ndarray):
+    transform = transform.astype(np.float32)
+    if not np.array_equal(transform, self._car_space_transform):
+      self._car_space_transform = transform
+      self._transform_dirty = True
+
+  def _update_raw_points(self, model):
+    """Update raw 3D points from model data"""
+    self._path.raw_points = np.array([model.position.x, model.position.y, model.position.z], dtype=np.float32).T
+
+    # Older logs may append lane centers after the four boundaries.
+    for points, lane_line in zip(self._lane_lines, model.laneLines, strict=False):
+      points.raw_points = np.array([lane_line.x, lane_line.y, lane_line.z], dtype=np.float32).T
+
+    for i, road_edge in enumerate(model.roadEdges):
+      self._road_edges[i].raw_points = np.array([road_edge.x, road_edge.y, road_edge.z], dtype=np.float32).T
+
+    self._lane_line_probs = np.array(model.laneLineProbs, dtype=np.float32)
+    self._road_edge_stds = np.array(model.roadEdgeStds, dtype=np.float32)
+    self._acceleration_x = np.array(model.acceleration.x, dtype=np.float32)
+
+  @staticmethod
+  def _hsla_to_color(h, s, l, a):
+    rgb = colorsys.hls_to_rgb(h, l, s)
+    return rl.Color(
+      int(rgb[0] * 255),
+      int(rgb[1] * 255),
+      int(rgb[2] * 255),
+      int(a * 255)
+    )
+
+  @staticmethod
+  def _blend_colors(begin_colors, end_colors, t):
+    if t >= 1.0:
+      return end_colors
+    if t <= 0.0:
+      return begin_colors
+
+    inv_t = 1.0 - t
+    return [rl.Color(
+      int(inv_t * start.r + t * end.r),
+      int(inv_t * start.g + t * end.g),
+      int(inv_t * start.b + t * end.b),
+      int(inv_t * start.a + t * end.a)
+    ) for start, end in zip(begin_colors, end_colors, strict=True)]
+
+
+
+class ModelRenderer(ModelGeometry, Widget):
   def __init__(self):
     super().__init__()
     self._longitudinal_control = False
@@ -92,11 +150,6 @@ class ModelRenderer(Widget):
       cp = messaging.log_from_bytes(car_params, car.CarParams)
       self._longitudinal_control = cp.openpilotLongitudinalControl
 
-  def set_transform(self, transform: np.ndarray):
-    transform = transform.astype(np.float32)
-    if not np.array_equal(transform, self._car_space_transform):
-      self._car_space_transform = transform
-      self._transform_dirty = True
 
   def _render(self, rect: rl.Rectangle):
     sm = ui_state.sm
@@ -157,20 +210,6 @@ class ModelRenderer(Widget):
         if text:
           self._draw_lead_info(self._lead_vehicles[0], text)
 
-  def _update_raw_points(self, model):
-    """Update raw 3D points from model data"""
-    self._path.raw_points = np.array([model.position.x, model.position.y, model.position.z], dtype=np.float32).T
-
-    # Older logs may append lane centers after the four boundaries.
-    for points, lane_line in zip(self._lane_lines, model.laneLines, strict=False):
-      points.raw_points = np.array([lane_line.x, lane_line.y, lane_line.z], dtype=np.float32).T
-
-    for i, road_edge in enumerate(model.roadEdges):
-      self._road_edges[i].raw_points = np.array([road_edge.x, road_edge.y, road_edge.z], dtype=np.float32).T
-
-    self._lane_line_probs = np.array(model.laneLineProbs, dtype=np.float32)
-    self._road_edge_stds = np.array(model.roadEdgeStds, dtype=np.float32)
-    self._acceleration_x = np.array(model.acceleration.x, dtype=np.float32)
 
   def _update_leads(self, radar_state, path_x_array):
     """Update positions of lead vehicles"""
@@ -485,27 +524,4 @@ class ModelRenderer(Widget):
     """Convert one 3D line to a 2D polygon for rendering."""
     return self._map_lines_to_polygons([line], [y_off], z_off, max_idx, max_distance, allow_invert)[0]
 
-  @staticmethod
-  def _hsla_to_color(h, s, l, a):
-    rgb = colorsys.hls_to_rgb(h, l, s)
-    return rl.Color(
-      int(rgb[0] * 255),
-      int(rgb[1] * 255),
-      int(rgb[2] * 255),
-      int(a * 255)
-    )
 
-  @staticmethod
-  def _blend_colors(begin_colors, end_colors, t):
-    if t >= 1.0:
-      return end_colors
-    if t <= 0.0:
-      return begin_colors
-
-    inv_t = 1.0 - t
-    return [rl.Color(
-      int(inv_t * start.r + t * end.r),
-      int(inv_t * start.g + t * end.g),
-      int(inv_t * start.b + t * end.b),
-      int(inv_t * start.a + t * end.a)
-    ) for start, end in zip(begin_colors, end_colors, strict=True)]

openpilot/starpilot/ui/tests/test_camera_resources.py

@@ -22,6 +22,7 @@ from unittest.mock import patch
 
 for path in ("openpilot.selfdrive.ui.onroad.cameraview", "openpilot.selfdrive.ui.mici.onroad.cameraview"):
   module = import_module(path)
+  owner = import_module(module.CameraView._clear_textures.__module__)
   for hardware in (False, True):
     view = object.__new__(module.CameraView)
     view.texture_y, view.texture_uv = SimpleNamespace(id=11), SimpleNamespace(id=12)
@@ -32,9 +33,10 @@ for path in ("openpilot.selfdrive.ui.onroad.cameraview", "openpilot.selfdrive.ui
     view._clients = {}
     textures, shaders, images = [], [], []
     with patch.object(module, "COMMA_HARDWARE", hardware), \\
+         patch.object(owner, "COMMA_HARDWARE", hardware), \\
          patch.object(module.rl, "unload_texture", side_effect=lambda texture: textures.append(texture.id)), \\
          patch.object(module.rl, "unload_shader", side_effect=lambda shader: shaders.append(shader.id)), \\
-         patch.object(module, "destroy_egl_image", side_effect=images.append):
+         patch.object(owner, "destroy_egl_image", side_effect=images.append):
       try:
         view.close()
         view.close()

openpilot/starpilot/ui/tests/test_shared_view_inheritance.py

@@ -0,0 +1,58 @@
+from types import SimpleNamespace as NS
+from unittest.mock import Mock
+
+import numpy as np
+import pytest
+
+from openpilot.selfdrive.ui.onroad import cameraview as large_camera
+from openpilot.selfdrive.ui.mici.onroad import cameraview as small_camera
+from openpilot.selfdrive.ui.onroad import model_renderer as large_model
+from openpilot.selfdrive.ui.mici.onroad import model_renderer as small_model
+
+
+@pytest.mark.parametrize("module", [large_camera, small_camera])
+def test_shared_camera_constructor_preserves_display_specific_clients_and_uniforms(module, monkeypatch):
+  ui = NS(add_offroad_transition_callback=Mock())
+  clients = []
+  def client(name, stream, conflate):
+    clients.append((name, stream, conflate))
+    return NS(stream=stream)
+  for target in (large_camera, small_camera):
+    monkeypatch.setattr(target, "COMMA_HARDWARE", False)
+    monkeypatch.setattr(target, "ui_state", ui)
+    monkeypatch.setattr(target, "VisionIpcClient", client)
+  monkeypatch.setattr(module.CameraView, "close", lambda self: None)
+  shader = Mock(return_value=NS(id=1))
+  monkeypatch.setattr(module.rl, "load_shader_from_memory", shader)
+  monkeypatch.setattr(module.rl, "get_shader_location", lambda *_args: 1)
+  stream = module.VisionStreamType.VISION_STREAM_CABIN
+  camera = module.CameraView("camerad", stream)
+  assert shader.call_args.args[1] == module.FRAME_FRAGMENT_SHADER
+  assert camera.stream_type == stream
+  assert camera.frame is None and camera._color_strength_val.tolist() == [0.0]
+  assert camera.texture_y is None and camera.texture_uv is None
+  ui.add_offroad_transition_callback.assert_called_once_with(camera._offroad_transition)
+  if module is small_camera:
+    assert clients == [("camerad", stream, True)]
+    assert camera._engaged_val.tolist() == [1]
+    assert camera._enhance_driver_val.tolist() == [1]
+  else:
+    assert len(clients) == len(list(module.VisionStreamType))
+    assert camera._road_camera_val.tolist() == [0]
+  camera.shader.id = 0
+
+
+def test_shared_model_constructor_keeps_distinct_lead_types_and_native_geometry():
+  assert issubclass(small_model.ModelRenderer, large_model.ModelGeometry)
+  assert issubclass(large_model.ModelRenderer, large_model.ModelGeometry)
+  for module in (large_model, small_model):
+    renderer = module.ModelRenderer()
+    assert all(isinstance(lead, module.LeadVehicle) for lead in renderer._lead_vehicles)
+    assert len(renderer._lane_lines) == 4 and len(renderer._road_edges) == 2
+    assert renderer._path.raw_points.shape == (0, 3)
+    assert renderer._path.raw_points.dtype == np.float32
+    renderer.set_transform(np.eye(3))
+    np.testing.assert_array_equal(renderer._car_space_transform, np.eye(3, dtype=np.float32))
+  assert small_model.ModelRenderer.set_transform is large_model.ModelRenderer.set_transform
+  assert small_model.ModelRenderer._update_raw_points is large_model.ModelRenderer._update_raw_points
+  assert small_model.ModelRenderer._draw_lane_lines is not large_model.ModelRenderer._draw_lane_lines