Keep replay camera frames stable during rendering

openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py

@@ -229,11 +229,12 @@ class AugmentedRoadView(CameraView):
   def render_camera_model_layer(self, rect: rl.Rectangle, *, camera_view: int = CAMERA_VIEW_AUTO,
                                 reverse_driver: bool = False, lead_indicator: bool = False,
                                 lead_info_mode: int = 0, lead_info_metric: bool | None = None,
-                                lateral_active: bool = False, road_style: dict | None = None, paint: bool = True) -> None:
+                                lateral_active: bool = False, road_style: dict | None = None, paint: bool = True,
+                                poll_camera: bool = True) -> None:
     """Draw the existing calibrated camera/model without stock HUD or alert."""
     if not ui_state.started and ui_state.replay_clock is None:
       return
-    if camera_view == CAMERA_VIEW_DRIVER or reverse_driver:
+    if poll_camera and (camera_view == CAMERA_VIEW_DRIVER or reverse_driver):
       self._ensure_connection()
     target = self._switch_stream_if_needed(ui_state.sm, camera_view, reverse_driver)
     if target is None:
@@ -241,7 +242,7 @@ class AugmentedRoadView(CameraView):
         rl.draw_rectangle_rec(rect, rl.BLACK)
       return
     driver_transition = DRIVER_CAM in (self.stream_type, target, self._target_stream_type)
-    if self._switching and driver_transition:
+    if poll_camera and self._switching and driver_transition:
       self._handle_switch()
     if (target == DRIVER_CAM and DRIVER_CAM not in self.available_streams) or \
        (driver_transition and self.stream_type != target):
@@ -254,7 +255,7 @@ class AugmentedRoadView(CameraView):
     self._update_calibration()
     self._content_rect = rect
     self._model_renderer.road_style = road_style or {}
-    gui_app.measure_frame_phase("camera", CameraView._render, self, rect, paint=paint)
+    gui_app.measure_frame_phase("camera", CameraView._render, self, rect, paint=paint, poll_camera=poll_camera)
     if target != DRIVER_CAM and self.frame is not None:
       gui_app.measure_frame_phase("model", self._model_renderer.render_with_lead, rect, lead_indicator,
                                   lead_info_mode, lead_info_metric, lateral_active=lateral_active, paint=paint)
@@ -292,9 +293,9 @@ class AugmentedRoadView(CameraView):
     if not self._bookmark_icon.interacting():
       super()._handle_mouse_release(mouse_pos)
 
-  def _render(self, rect: rl.Rectangle, /, *, paint: bool = True):
+  def _render(self, rect: rl.Rectangle, /, *, paint: bool = True, poll_camera: bool = True):
     if not paint:
-      super()._render(rect, paint=False)
+      super()._render(rect, paint=False, poll_camera=poll_camera)
       return
     # Draw text if not onroad
     if not ui_state.started:
@@ -325,7 +326,7 @@ class AugmentedRoadView(CameraView):
     )
 
     # Render the base camera view
-    super()._render(self._content_rect)
+    super()._render(self._content_rect, poll_camera=poll_camera)
 
     # Draw all UI overlays
     self._model_renderer.render(self._content_rect)

openpilot/selfdrive/ui/mici/onroad/cameraview.py

@@ -227,25 +227,24 @@ class CameraView(Widget):
       [0.0, 0.0, 1.0]
     ])
 
-  def _render(self, rect: rl.Rectangle, /, *, paint: bool = True):
+  def _poll_camera(self) -> bool:
     if self._switching:
       self._handle_switch()
 
     if not self._ensure_connection():
-      if paint:
-        self._draw_placeholder(rect)
-      return
+      return False
 
-    # Try to get a new buffer without blocking
     buffer = self.client.recv(timeout_ms=0)
     if buffer:
       self._texture_needs_update = True
       self.frame = buffer
     elif not self.client.is_connected():
-      # ensure we clear the displayed frame when the connection is lost
       self.frame = None
 
-    if not self.frame:
+    return self.frame is not None
+
+  def _render(self, rect: rl.Rectangle, /, *, paint: bool = True, poll_camera: bool = True):
+    if (poll_camera and not self._poll_camera()) or self.frame is None:
       if paint:
         self._draw_placeholder(rect)
       return

openpilot/selfdrive/ui/onroad/augmented_road_view.py

@@ -49,7 +49,7 @@ class AugmentedRoadView(CameraView):
     self.driver_state_renderer = DriverStateRenderer()
 
   def render_camera_model_layer(self, rect: rl.Rectangle, *, road_style: dict | None = None,
-                                lead_display: tuple[bool, int, bool | None] | None = None) -> None:
+                                lead_display: tuple[bool, int, bool | None] | None = None, poll_camera: bool = True) -> None:
     """Draw the existing calibrated camera/model without stock HUD or alert."""
     if not ui_state.started and ui_state.replay_clock is None:
       return
@@ -61,10 +61,10 @@ class AugmentedRoadView(CameraView):
       self.model_renderer._transform_dirty = True
     self.model_renderer.road_style = style
     self.model_renderer.lead_display = lead_display
-    gui_app.measure_frame_phase("camera", CameraView._render, self, rect)
+    gui_app.measure_frame_phase("camera", CameraView._render, self, rect, poll_camera=poll_camera)
     gui_app.measure_frame_phase("model", self.model_renderer.render, rect)
 
-  def _render(self, rect):
+  def _render(self, rect, *, poll_camera: bool = True):
     # Only render when system is started to avoid invalid data access
     if not ui_state.started:
       return
@@ -92,7 +92,7 @@ class AugmentedRoadView(CameraView):
     )
 
     # Render the base camera view
-    super()._render(self._content_rect)
+    super()._render(self._content_rect, poll_camera=poll_camera)
 
     # Draw all UI overlays
     self.model_renderer.render(self._content_rect)

openpilot/selfdrive/ui/onroad/cabin_camera_dialog.py

@@ -29,8 +29,8 @@ class CabinCameraDialog(CameraView):
   def __del__(self):
     self.close()
 
-  def _render(self, rect):
-    super()._render(rect)
+  def _render(self, rect, *, poll_camera: bool = True):
+    super()._render(rect, poll_camera=poll_camera)
 
     if not self.frame:
       gui_label(

openpilot/selfdrive/ui/onroad/cameraview.py

@@ -213,24 +213,24 @@ class CameraView(Widget):
       [0.0, 0.0, 1.0]
     ])
 
-  def _render(self, rect: rl.Rectangle):
+  def _poll_camera(self) -> bool:
     if self._switching:
       self._handle_switch()
 
     if not self._ensure_connection():
-      self._draw_placeholder(rect)
-      return
+      return False
 
-    # Try to get a new buffer without blocking
     buffer = self.client.recv(timeout_ms=0)
     if buffer:
       self._texture_needs_update = True
       self.frame = buffer
     elif not self.client.is_connected():
-      # ensure we clear the displayed frame when the connection is lost
       self.frame = None
 
-    if not self.frame:
+    return self.frame is not None
+
+  def _render(self, rect: rl.Rectangle, *, poll_camera: bool = True):
+    if (poll_camera and not self._poll_camera()) or self.frame is None:
       self._draw_placeholder(rect)
       return
 

openpilot/starpilot/replay/render.py

@@ -330,17 +330,16 @@ class Compositor:
       self.view.compact_sidebar.personality_renderer = self._personality_layer
 
   def _camera_layer(self, rect, state):
+    self.primary_ready = self._primary_matches()
     if self.profile == 'compact':
       if state.appearance.camera_view == 4:
         self.rl.draw_rectangle_rec(rect, self.rl.BLACK)
-        self._receive_primary()
         return
       if ((state.appearance.camera_view == 1 or state.reverse_driver_camera) and not self.cabin or
           state.appearance.camera_view == 3 and not self.wide):
         self.rl.draw_rectangle_rec(rect, self.rl.BLACK)
         from openpilot.system.ui.widgets.label import gui_label
         gui_label(rect, 'recorded camera unavailable', font_size=28)
-        self._receive_primary()
         return
     sm = self.ui.sm
     receipts = {name: sm.recv_frame[name] for name in
@@ -362,9 +361,10 @@ class Compositor:
                                          reverse_driver=state.reverse_driver_camera,
                                          lead_indicator=state.appearance.show_lead_indicator and state.lead_indicator_source_fresh,
                                          lead_info_mode=state.appearance.lead_info_mode, lead_info_metric=state.appearance.lead_info_metric,
-                                         lateral_active=state.lateral_active, road_style=state.customization['roadColors']['compact'])
+                                         lateral_active=state.lateral_active, road_style=state.customization['roadColors']['compact'],
+                                         poll_camera=False)
       else:
-        self.camera.render_camera_model_layer(rect, road_style=state.customization['roadColors']['large'])
+        self.camera.render_camera_model_layer(rect, road_style=state.customization['roadColors']['large'], poll_camera=False)
     finally:
       sm.updated = original
     self.render_receipts.update({name: receipts[name] for name in flags.read if name in receipts})
@@ -397,10 +397,7 @@ class Compositor:
       self.camera.client, self.camera.frame, state, sm.logMonoTime[name], sample.boot_ns))
 
   def _receive_primary(self):
-    if self.camera._ensure_connection():
-      frame = self.camera.client.recv(timeout_ms=0)
-      if frame is not None:
-        self.camera.frame = frame
+    self.camera._poll_camera()
     self.primary_ready = self._primary_matches()
 
   def _pip_layer(self, rect, state, *, submit=None):
@@ -466,13 +463,12 @@ class Compositor:
     self.gui._fallback_fonts.clear()
 
   def update(self):
+    # Select the camera before HUD messages and keep that selection through drawing.
+    self._receive_primary()
     self.ui.update()
     sample = self.ui.replay_sample
     if sample is not None and sample.valid and sample.epoch != self.epoch:
       self.reset_epoch(sample.epoch)
-    # Poll before re-priming: msgq's first receive synchronizes a subscriber
-    # created before its publisher and can discard that publisher's first frame.
-    self._receive_primary()
     # Connect the optional PiP client before the one-shot paused primer.
     if self.cabin and self.pip_saved.enabled is True:
       self.pip_stream.set_active(True)

openpilot/starpilot/replay/tests/test_render.py

@@ -111,6 +111,10 @@ def test_recorded_camera_preview_does_not_require_initial_device_state(frame_rea
 @pytest.fixture(params=(Profile.LARGE, Profile.COMPACT))
 def draw_core(request):
   from openpilot.cereal.visionipc import VisionStreamType
+  if request.param == Profile.COMPACT:
+    from openpilot.selfdrive.ui.mici.onroad.cameraview import CameraView
+  else:
+    from openpilot.selfdrive.ui.onroad.cameraview import CameraView
   from openpilot.starpilot.ui.onroad_state import OnroadState, SpeedLimitObservation
 
   class Messages(dict):
@@ -133,7 +137,9 @@ def draw_core(request):
   core.wide = core.cabin = False
   core.ui = SimpleNamespace(sm=sm, replay_sample=ClockSample(1_000_000_000, 9_050_000_000, 100, epoch=2, valid=True))
   core.camera = Mock(client=client, frame=frame, stream_type=VisionStreamType.VISION_STREAM_NARROW_ROAD)
+  core.camera._switching = False
   core.camera._ensure_connection.return_value = True
+  core.camera._poll_camera.side_effect = lambda: CameraView._poll_camera(core.camera)
   core.last_route_frame = None
   core.render_receipts = {'modelV2': 1}
   core.context, core.placement, core.adapter = Mock(), Mock(), Mock()
@@ -143,7 +149,11 @@ def draw_core(request):
   core.scale = 1
   core.view, core.converter, core.readback = Mock(), Mock(), Mock()
   core.output = None
-  core.camera.render_camera_model_layer.side_effect = lambda *args, **kwargs: core._receive_primary()
+
+  def paint(*args, **kwargs):
+    assert kwargs['poll_camera'] is False
+  core.camera.render_camera_model_layer.side_effect = paint
+
   def render_view(state):
     # The real OnroadView enters its camera layer only with this availability flag.
     if state.camera_available:
@@ -185,21 +195,82 @@ def test_invalid_or_other_epoch_clock_never_advances_draw(draw_core, sample):
   core.adapter.build.assert_not_called()
 
 
+def test_camera_poll_precedes_message_snapshot():
+  core: Any = object.__new__(Compositor)
+  order = []
+  core.ui = SimpleNamespace(update=lambda: order.append('messages'), replay_sample=ClockSample(1, 1, 1, epoch=2, valid=True))
+  core.epoch, core.cabin = 2, False
+  core._receive_primary = lambda: order.append('camera')
+  assert core.update() is core.ui.replay_sample
+  assert order == ['camera', 'messages']
+
+
+@pytest.mark.parametrize('profile', ('onroad', 'mici.onroad'))
+def test_camera_paint_does_not_receive_or_switch_selected_frame(monkeypatch, profile):
+  import importlib
+  import numpy as np
+  import pyray as rl
+  from openpilot.cereal.visionipc import VisionStreamType
+  module = importlib.import_module(f'openpilot.selfdrive.ui.{profile}.cameraview')
+  monkeypatch.setattr(module, 'COMMA_HARDWARE', True)
+  camera = object.__new__(module.CameraView)
+  camera.close = Mock()
+  camera.client = Mock()
+  camera.frame = SimpleNamespace(frame_id=5, width=100, height=50)
+  frame = camera.frame
+  camera._stream_type = VisionStreamType.VISION_STREAM_NARROW_ROAD
+  camera._switching = True
+  camera._handle_switch = Mock()
+  camera._ensure_connection = Mock()
+  camera._calc_frame_matrix = Mock(return_value=np.eye(3))
+  camera._render_egl = Mock()
+  camera._render(rl.Rectangle(0, 0, 100, 50), poll_camera=False)
+  assert camera.frame is frame
+  camera.client.recv.assert_not_called()
+  camera._handle_switch.assert_not_called()
+  camera._ensure_connection.assert_not_called()
+  camera._render_egl.assert_called_once()
+
+
+def test_compact_driver_switch_waits_for_camera_poll(monkeypatch):
+  from openpilot.selfdrive.ui.mici.onroad import augmented_road_view as module
+  camera = object.__new__(module.AugmentedRoadView)
+  camera.close = Mock()
+  camera._stream_type = module.NARROW_ROAD_CAM
+  camera._target_stream_type = module.DRIVER_CAM
+  camera._switching = True
+  camera.available_streams = [module.DRIVER_CAM]
+  camera._ensure_connection = Mock()
+  camera._handle_switch = Mock()
+  camera._switch_stream_if_needed = Mock(return_value=module.DRIVER_CAM)
+  monkeypatch.setattr(module.ui_state, 'started', True)
+  monkeypatch.setattr(module.rl, 'draw_rectangle_rec', Mock())
+  monkeypatch.setattr(module, 'gui_label', Mock())
+  camera.render_camera_model_layer(Mock(), camera_view=module.CAMERA_VIEW_DRIVER, poll_camera=False)
+  camera._handle_switch.assert_not_called()
+  camera._ensure_connection.assert_not_called()
+
+
 @pytest.mark.parametrize('replacement_id', (5, 6))
-def test_later_receive_keeps_final_identity_check(draw_core, replacement_id):
+def test_selected_camera_is_painted_before_next_poll(draw_core, replacement_id):
   core = draw_core
   def receive(**kwargs):
     core.camera.client.frame_id = replacement_id
     return SimpleNamespace(frame_id=replacement_id)
   core.camera.client.recv.side_effect = receive
-  assert core.draw(2, 0., 100) is (replacement_id == 5)
+  assert core.draw(2, 0., 100)
+  core.camera.client.recv.assert_not_called()
+  core._receive_primary()
+  core.ui.replay_sample = replace(core.ui.replay_sample, now_ns=1_050_000_000)
+  assert core.draw(2, .05, 150) is (replacement_id == 5)
   core.camera.client.recv.assert_called_once_with(timeout_ms=0)
-  assert core.readback.start.call_count == int(replacement_id == 5)
+  assert core.readback.start.call_count == 1 + int(replacement_id == 5)
 
 
-def test_optional_placeholder_still_allows_later_camera_recovery(draw_core):
+def test_optional_placeholder_still_allows_later_camera_recovery(draw_core, monkeypatch):
   from openpilot.cereal.visionipc import VisionStreamType
   core = draw_core
+  monkeypatch.setattr('openpilot.system.ui.widgets.label.gui_label', Mock())
   core.camera.stream_type = VisionStreamType.VISION_STREAM_WIDE_ROAD
   core.camera.client.frame_id = 4
   def receive(**kwargs):
@@ -207,8 +278,19 @@ def test_optional_placeholder_still_allows_later_camera_recovery(draw_core):
     return SimpleNamespace(frame_id=5)
   core.camera.client.recv.side_effect = receive
   assert core.draw(2, 0., 100)
+  core.camera.client.recv.assert_not_called()
+  core._receive_primary()
+  core.ui.replay_sample = replace(core.ui.replay_sample, now_ns=1_050_000_000)
+  assert core.draw(2, .05, 150)
   core.camera.client.recv.assert_called_once_with(timeout_ms=0)
-  core.readback.start.assert_called_once()
+  assert core.readback.start.call_count == 2
+
+
+def test_selected_buffer_reuse_still_fails_final_identity_check(draw_core):
+  core = draw_core
+  core.camera.render_camera_model_layer.side_effect = lambda *args, **kwargs: setattr(core.camera.frame, 'frame_id', 6)
+  assert not core.draw(2, 0., 100)
+  core.readback.start.assert_not_called()
 
 
 def test_eligible_retained_camera_draws_fresh_hud_and_route_time(draw_core):
@@ -235,7 +317,7 @@ def test_rejected_camera_can_match_next_update_without_losing_model_receipt(draw
   core.ui.sm['narrowRoadEncodeIdx'].frameId = 6
   def consume_model(*args, **kwargs):
     assert core.ui.sm.updated['modelV2']
-    core._receive_primary()
+    assert kwargs['poll_camera'] is False
   core.camera.client.recv.return_value = None
   core.camera.render_camera_model_layer.side_effect = consume_model
   assert core.draw(2, 0., 100)
@@ -258,11 +340,12 @@ def test_optional_missing_camera_still_publishes_existing_placeholder(draw_core,
 
 @pytest.mark.parametrize('camera_view', (1, 3, 4))
 @pytest.mark.parametrize('draw_core', (Profile.COMPACT,), indirect=True)
-def test_compact_hidden_and_unrecorded_camera_keep_acquisition(draw_core, camera_view, monkeypatch):
+def test_compact_hidden_and_unrecorded_camera_keep_selected_frame(draw_core, camera_view, monkeypatch):
   core = draw_core
   core.adapter.build.return_value.onroad = replace(core.adapter.build.return_value.onroad, appearance=replace(
     core.adapter.build.return_value.onroad.appearance, camera_view=camera_view))
   monkeypatch.setattr('openpilot.system.ui.widgets.label.gui_label', Mock())
+  core._receive_primary()
   assert core.draw(2, 0., 100)
   core.camera.client.recv.assert_called_once_with(timeout_ms=0)
   core.camera.render_camera_model_layer.assert_not_called()
@@ -483,6 +566,7 @@ def test_initial_hud_primer_required_even_when_camera_received_a_frame(tmp_path,
 
 def test_early_camera_subscriber_is_polled_before_paused_reprime():
   from openpilot.cereal.visionipc import VisionStreamType
+  from openpilot.selfdrive.ui.onroad.cameraview import CameraView
   sample = SimpleNamespace(valid=True, epoch=1, boot_ns=9_050_000_000)
   idx = SimpleNamespace(frameId=5, timestampSof=9_000_000_000, timestampEof=9_010_000_000)
   sm = Mock()
@@ -493,11 +577,13 @@ def test_early_camera_subscriber_is_polled_before_paused_reprime():
   client = Mock(frame_id=5, timestamp_sof=idx.timestampSof, timestamp_eof=idx.timestampEof)
   # The first receive synchronizes with a publisher created after this client.
   client.recv.side_effect = [None, frame]
-  core = object.__new__(Compositor)
+  core: Any = object.__new__(Compositor)
   core.ui = Mock(replay_sample=sample, sm=sm)
   core.epoch, core.cabin = 1, False
   core.camera = Mock(client=client, frame=None, stream_type=VisionStreamType.VISION_STREAM_NARROW_ROAD)
+  core.camera._switching = False
   core.camera._ensure_connection.return_value = True
+  core.camera._poll_camera.side_effect = lambda: CameraView._poll_camera(core.camera)
   core.update()
   assert not core._primary_matches()
   core.update()

openpilot/starpilot/ui/tests/test_runtime_navigation.py

@@ -104,7 +104,7 @@ class TestNativeCameraComposition(unittest.TestCase):
           module.AugmentedRoadView.render_camera_model_layer(view, rect)
         view._switch_stream_if_needed.assert_called_once()
         view._update_calibration.assert_called_once_with()
-        camera.assert_called_once_with(view, rect, **({"paint": True} if module is compact else {}))
+        camera.assert_called_once_with(view, rect, poll_camera=True, **({"paint": True} if module is compact else {}))
         if model_field == "_model_renderer":
           model.render_with_lead.assert_called_once_with(rect, False, 0, None, lateral_active=False, paint=True)
           model.render_with_lead.reset_mock()