diff --git a/openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py b/openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py index 8f1ccc7f61..364b426bb8 100644 --- a/openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py +++ b/openpilot/selfdrive/ui/mici/onroad/augmented_road_view.py @@ -229,11 +229,12 @@ class AugmentedRoadView(CameraView): def render_camera_model_layer(self, rect: rl.Rectangle, *, camera_view: int = CAMERA_VIEW_AUTO, reverse_driver: bool = False, lead_indicator: bool = False, lead_info_mode: int = 0, lead_info_metric: bool | None = None, - lateral_active: bool = False, road_style: dict | None = None, paint: bool = True) -> None: + lateral_active: bool = False, road_style: dict | None = None, paint: bool = True, + poll_camera: bool = True) -> None: """Draw the existing calibrated camera/model without stock HUD or alert.""" if not ui_state.started and ui_state.replay_clock is None: return - if camera_view == CAMERA_VIEW_DRIVER or reverse_driver: + if poll_camera and (camera_view == CAMERA_VIEW_DRIVER or reverse_driver): self._ensure_connection() target = self._switch_stream_if_needed(ui_state.sm, camera_view, reverse_driver) if target is None: @@ -241,7 +242,7 @@ class AugmentedRoadView(CameraView): rl.draw_rectangle_rec(rect, rl.BLACK) return driver_transition = DRIVER_CAM in (self.stream_type, target, self._target_stream_type) - if self._switching and driver_transition: + if poll_camera and self._switching and driver_transition: self._handle_switch() if (target == DRIVER_CAM and DRIVER_CAM not in self.available_streams) or \ (driver_transition and self.stream_type != target): @@ -254,7 +255,7 @@ class AugmentedRoadView(CameraView): self._update_calibration() self._content_rect = rect self._model_renderer.road_style = road_style or {} - gui_app.measure_frame_phase("camera", CameraView._render, self, rect, paint=paint) + gui_app.measure_frame_phase("camera", CameraView._render, self, rect, paint=paint, poll_camera=poll_camera) if target != DRIVER_CAM and self.frame is not None: gui_app.measure_frame_phase("model", self._model_renderer.render_with_lead, rect, lead_indicator, lead_info_mode, lead_info_metric, lateral_active=lateral_active, paint=paint) @@ -292,9 +293,9 @@ class AugmentedRoadView(CameraView): if not self._bookmark_icon.interacting(): super()._handle_mouse_release(mouse_pos) - def _render(self, rect: rl.Rectangle, /, *, paint: bool = True): + def _render(self, rect: rl.Rectangle, /, *, paint: bool = True, poll_camera: bool = True): if not paint: - super()._render(rect, paint=False) + super()._render(rect, paint=False, poll_camera=poll_camera) return # Draw text if not onroad if not ui_state.started: @@ -325,7 +326,7 @@ class AugmentedRoadView(CameraView): ) # Render the base camera view - super()._render(self._content_rect) + super()._render(self._content_rect, poll_camera=poll_camera) # Draw all UI overlays self._model_renderer.render(self._content_rect) diff --git a/openpilot/selfdrive/ui/mici/onroad/cameraview.py b/openpilot/selfdrive/ui/mici/onroad/cameraview.py index 1e48ba2130..2d7bdc112b 100644 --- a/openpilot/selfdrive/ui/mici/onroad/cameraview.py +++ b/openpilot/selfdrive/ui/mici/onroad/cameraview.py @@ -227,25 +227,24 @@ class CameraView(Widget): [0.0, 0.0, 1.0] ]) - def _render(self, rect: rl.Rectangle, /, *, paint: bool = True): + def _poll_camera(self) -> bool: if self._switching: self._handle_switch() if not self._ensure_connection(): - if paint: - self._draw_placeholder(rect) - return + return False - # Try to get a new buffer without blocking buffer = self.client.recv(timeout_ms=0) if buffer: self._texture_needs_update = True self.frame = buffer elif not self.client.is_connected(): - # ensure we clear the displayed frame when the connection is lost self.frame = None - if not self.frame: + return self.frame is not None + + def _render(self, rect: rl.Rectangle, /, *, paint: bool = True, poll_camera: bool = True): + if (poll_camera and not self._poll_camera()) or self.frame is None: if paint: self._draw_placeholder(rect) return diff --git a/openpilot/selfdrive/ui/onroad/augmented_road_view.py b/openpilot/selfdrive/ui/onroad/augmented_road_view.py index f85adb84ff..52894b4310 100644 --- a/openpilot/selfdrive/ui/onroad/augmented_road_view.py +++ b/openpilot/selfdrive/ui/onroad/augmented_road_view.py @@ -49,7 +49,7 @@ class AugmentedRoadView(CameraView): self.driver_state_renderer = DriverStateRenderer() def render_camera_model_layer(self, rect: rl.Rectangle, *, road_style: dict | None = None, - lead_display: tuple[bool, int, bool | None] | None = None) -> None: + lead_display: tuple[bool, int, bool | None] | None = None, poll_camera: bool = True) -> None: """Draw the existing calibrated camera/model without stock HUD or alert.""" if not ui_state.started and ui_state.replay_clock is None: return @@ -61,10 +61,10 @@ class AugmentedRoadView(CameraView): self.model_renderer._transform_dirty = True self.model_renderer.road_style = style self.model_renderer.lead_display = lead_display - gui_app.measure_frame_phase("camera", CameraView._render, self, rect) + gui_app.measure_frame_phase("camera", CameraView._render, self, rect, poll_camera=poll_camera) gui_app.measure_frame_phase("model", self.model_renderer.render, rect) - def _render(self, rect): + def _render(self, rect, *, poll_camera: bool = True): # Only render when system is started to avoid invalid data access if not ui_state.started: return @@ -92,7 +92,7 @@ class AugmentedRoadView(CameraView): ) # Render the base camera view - super()._render(self._content_rect) + super()._render(self._content_rect, poll_camera=poll_camera) # Draw all UI overlays self.model_renderer.render(self._content_rect) diff --git a/openpilot/selfdrive/ui/onroad/cabin_camera_dialog.py b/openpilot/selfdrive/ui/onroad/cabin_camera_dialog.py index 58a645381b..4843b772e9 100644 --- a/openpilot/selfdrive/ui/onroad/cabin_camera_dialog.py +++ b/openpilot/selfdrive/ui/onroad/cabin_camera_dialog.py @@ -29,8 +29,8 @@ class CabinCameraDialog(CameraView): def __del__(self): self.close() - def _render(self, rect): - super()._render(rect) + def _render(self, rect, *, poll_camera: bool = True): + super()._render(rect, poll_camera=poll_camera) if not self.frame: gui_label( diff --git a/openpilot/selfdrive/ui/onroad/cameraview.py b/openpilot/selfdrive/ui/onroad/cameraview.py index 5c4ac478e3..c05097cfe8 100644 --- a/openpilot/selfdrive/ui/onroad/cameraview.py +++ b/openpilot/selfdrive/ui/onroad/cameraview.py @@ -213,24 +213,24 @@ class CameraView(Widget): [0.0, 0.0, 1.0] ]) - def _render(self, rect: rl.Rectangle): + def _poll_camera(self) -> bool: if self._switching: self._handle_switch() if not self._ensure_connection(): - self._draw_placeholder(rect) - return + return False - # Try to get a new buffer without blocking buffer = self.client.recv(timeout_ms=0) if buffer: self._texture_needs_update = True self.frame = buffer elif not self.client.is_connected(): - # ensure we clear the displayed frame when the connection is lost self.frame = None - if not self.frame: + return self.frame is not None + + def _render(self, rect: rl.Rectangle, *, poll_camera: bool = True): + if (poll_camera and not self._poll_camera()) or self.frame is None: self._draw_placeholder(rect) return diff --git a/openpilot/starpilot/replay/render.py b/openpilot/starpilot/replay/render.py index 724cfbb2c3..f6ecd32c21 100644 --- a/openpilot/starpilot/replay/render.py +++ b/openpilot/starpilot/replay/render.py @@ -330,17 +330,16 @@ class Compositor: self.view.compact_sidebar.personality_renderer = self._personality_layer def _camera_layer(self, rect, state): + self.primary_ready = self._primary_matches() if self.profile == 'compact': if state.appearance.camera_view == 4: self.rl.draw_rectangle_rec(rect, self.rl.BLACK) - self._receive_primary() return if ((state.appearance.camera_view == 1 or state.reverse_driver_camera) and not self.cabin or state.appearance.camera_view == 3 and not self.wide): self.rl.draw_rectangle_rec(rect, self.rl.BLACK) from openpilot.system.ui.widgets.label import gui_label gui_label(rect, 'recorded camera unavailable', font_size=28) - self._receive_primary() return sm = self.ui.sm receipts = {name: sm.recv_frame[name] for name in @@ -362,9 +361,10 @@ class Compositor: reverse_driver=state.reverse_driver_camera, lead_indicator=state.appearance.show_lead_indicator and state.lead_indicator_source_fresh, lead_info_mode=state.appearance.lead_info_mode, lead_info_metric=state.appearance.lead_info_metric, - lateral_active=state.lateral_active, road_style=state.customization['roadColors']['compact']) + lateral_active=state.lateral_active, road_style=state.customization['roadColors']['compact'], + poll_camera=False) else: - self.camera.render_camera_model_layer(rect, road_style=state.customization['roadColors']['large']) + self.camera.render_camera_model_layer(rect, road_style=state.customization['roadColors']['large'], poll_camera=False) finally: sm.updated = original self.render_receipts.update({name: receipts[name] for name in flags.read if name in receipts}) @@ -397,10 +397,7 @@ class Compositor: self.camera.client, self.camera.frame, state, sm.logMonoTime[name], sample.boot_ns)) def _receive_primary(self): - if self.camera._ensure_connection(): - frame = self.camera.client.recv(timeout_ms=0) - if frame is not None: - self.camera.frame = frame + self.camera._poll_camera() self.primary_ready = self._primary_matches() def _pip_layer(self, rect, state, *, submit=None): @@ -466,13 +463,12 @@ class Compositor: self.gui._fallback_fonts.clear() def update(self): + # Select the camera before HUD messages and keep that selection through drawing. + self._receive_primary() self.ui.update() sample = self.ui.replay_sample if sample is not None and sample.valid and sample.epoch != self.epoch: self.reset_epoch(sample.epoch) - # Poll before re-priming: msgq's first receive synchronizes a subscriber - # created before its publisher and can discard that publisher's first frame. - self._receive_primary() # Connect the optional PiP client before the one-shot paused primer. if self.cabin and self.pip_saved.enabled is True: self.pip_stream.set_active(True) diff --git a/openpilot/starpilot/replay/tests/test_render.py b/openpilot/starpilot/replay/tests/test_render.py index 0426945903..f5ed6fe1f3 100644 --- a/openpilot/starpilot/replay/tests/test_render.py +++ b/openpilot/starpilot/replay/tests/test_render.py @@ -111,6 +111,10 @@ def test_recorded_camera_preview_does_not_require_initial_device_state(frame_rea @pytest.fixture(params=(Profile.LARGE, Profile.COMPACT)) def draw_core(request): from openpilot.cereal.visionipc import VisionStreamType + if request.param == Profile.COMPACT: + from openpilot.selfdrive.ui.mici.onroad.cameraview import CameraView + else: + from openpilot.selfdrive.ui.onroad.cameraview import CameraView from openpilot.starpilot.ui.onroad_state import OnroadState, SpeedLimitObservation class Messages(dict): @@ -133,7 +137,9 @@ def draw_core(request): core.wide = core.cabin = False core.ui = SimpleNamespace(sm=sm, replay_sample=ClockSample(1_000_000_000, 9_050_000_000, 100, epoch=2, valid=True)) core.camera = Mock(client=client, frame=frame, stream_type=VisionStreamType.VISION_STREAM_NARROW_ROAD) + core.camera._switching = False core.camera._ensure_connection.return_value = True + core.camera._poll_camera.side_effect = lambda: CameraView._poll_camera(core.camera) core.last_route_frame = None core.render_receipts = {'modelV2': 1} core.context, core.placement, core.adapter = Mock(), Mock(), Mock() @@ -143,7 +149,11 @@ def draw_core(request): core.scale = 1 core.view, core.converter, core.readback = Mock(), Mock(), Mock() core.output = None - core.camera.render_camera_model_layer.side_effect = lambda *args, **kwargs: core._receive_primary() + + def paint(*args, **kwargs): + assert kwargs['poll_camera'] is False + core.camera.render_camera_model_layer.side_effect = paint + def render_view(state): # The real OnroadView enters its camera layer only with this availability flag. if state.camera_available: @@ -185,21 +195,82 @@ def test_invalid_or_other_epoch_clock_never_advances_draw(draw_core, sample): core.adapter.build.assert_not_called() +def test_camera_poll_precedes_message_snapshot(): + core: Any = object.__new__(Compositor) + order = [] + core.ui = SimpleNamespace(update=lambda: order.append('messages'), replay_sample=ClockSample(1, 1, 1, epoch=2, valid=True)) + core.epoch, core.cabin = 2, False + core._receive_primary = lambda: order.append('camera') + assert core.update() is core.ui.replay_sample + assert order == ['camera', 'messages'] + + +@pytest.mark.parametrize('profile', ('onroad', 'mici.onroad')) +def test_camera_paint_does_not_receive_or_switch_selected_frame(monkeypatch, profile): + import importlib + import numpy as np + import pyray as rl + from openpilot.cereal.visionipc import VisionStreamType + module = importlib.import_module(f'openpilot.selfdrive.ui.{profile}.cameraview') + monkeypatch.setattr(module, 'COMMA_HARDWARE', True) + camera = object.__new__(module.CameraView) + camera.close = Mock() + camera.client = Mock() + camera.frame = SimpleNamespace(frame_id=5, width=100, height=50) + frame = camera.frame + camera._stream_type = VisionStreamType.VISION_STREAM_NARROW_ROAD + camera._switching = True + camera._handle_switch = Mock() + camera._ensure_connection = Mock() + camera._calc_frame_matrix = Mock(return_value=np.eye(3)) + camera._render_egl = Mock() + camera._render(rl.Rectangle(0, 0, 100, 50), poll_camera=False) + assert camera.frame is frame + camera.client.recv.assert_not_called() + camera._handle_switch.assert_not_called() + camera._ensure_connection.assert_not_called() + camera._render_egl.assert_called_once() + + +def test_compact_driver_switch_waits_for_camera_poll(monkeypatch): + from openpilot.selfdrive.ui.mici.onroad import augmented_road_view as module + camera = object.__new__(module.AugmentedRoadView) + camera.close = Mock() + camera._stream_type = module.NARROW_ROAD_CAM + camera._target_stream_type = module.DRIVER_CAM + camera._switching = True + camera.available_streams = [module.DRIVER_CAM] + camera._ensure_connection = Mock() + camera._handle_switch = Mock() + camera._switch_stream_if_needed = Mock(return_value=module.DRIVER_CAM) + monkeypatch.setattr(module.ui_state, 'started', True) + monkeypatch.setattr(module.rl, 'draw_rectangle_rec', Mock()) + monkeypatch.setattr(module, 'gui_label', Mock()) + camera.render_camera_model_layer(Mock(), camera_view=module.CAMERA_VIEW_DRIVER, poll_camera=False) + camera._handle_switch.assert_not_called() + camera._ensure_connection.assert_not_called() + + @pytest.mark.parametrize('replacement_id', (5, 6)) -def test_later_receive_keeps_final_identity_check(draw_core, replacement_id): +def test_selected_camera_is_painted_before_next_poll(draw_core, replacement_id): core = draw_core def receive(**kwargs): core.camera.client.frame_id = replacement_id return SimpleNamespace(frame_id=replacement_id) core.camera.client.recv.side_effect = receive - assert core.draw(2, 0., 100) is (replacement_id == 5) + assert core.draw(2, 0., 100) + core.camera.client.recv.assert_not_called() + core._receive_primary() + core.ui.replay_sample = replace(core.ui.replay_sample, now_ns=1_050_000_000) + assert core.draw(2, .05, 150) is (replacement_id == 5) core.camera.client.recv.assert_called_once_with(timeout_ms=0) - assert core.readback.start.call_count == int(replacement_id == 5) + assert core.readback.start.call_count == 1 + int(replacement_id == 5) -def test_optional_placeholder_still_allows_later_camera_recovery(draw_core): +def test_optional_placeholder_still_allows_later_camera_recovery(draw_core, monkeypatch): from openpilot.cereal.visionipc import VisionStreamType core = draw_core + monkeypatch.setattr('openpilot.system.ui.widgets.label.gui_label', Mock()) core.camera.stream_type = VisionStreamType.VISION_STREAM_WIDE_ROAD core.camera.client.frame_id = 4 def receive(**kwargs): @@ -207,8 +278,19 @@ def test_optional_placeholder_still_allows_later_camera_recovery(draw_core): return SimpleNamespace(frame_id=5) core.camera.client.recv.side_effect = receive assert core.draw(2, 0., 100) + core.camera.client.recv.assert_not_called() + core._receive_primary() + core.ui.replay_sample = replace(core.ui.replay_sample, now_ns=1_050_000_000) + assert core.draw(2, .05, 150) core.camera.client.recv.assert_called_once_with(timeout_ms=0) - core.readback.start.assert_called_once() + assert core.readback.start.call_count == 2 + + +def test_selected_buffer_reuse_still_fails_final_identity_check(draw_core): + core = draw_core + core.camera.render_camera_model_layer.side_effect = lambda *args, **kwargs: setattr(core.camera.frame, 'frame_id', 6) + assert not core.draw(2, 0., 100) + core.readback.start.assert_not_called() def test_eligible_retained_camera_draws_fresh_hud_and_route_time(draw_core): @@ -235,7 +317,7 @@ def test_rejected_camera_can_match_next_update_without_losing_model_receipt(draw core.ui.sm['narrowRoadEncodeIdx'].frameId = 6 def consume_model(*args, **kwargs): assert core.ui.sm.updated['modelV2'] - core._receive_primary() + assert kwargs['poll_camera'] is False core.camera.client.recv.return_value = None core.camera.render_camera_model_layer.side_effect = consume_model assert core.draw(2, 0., 100) @@ -258,11 +340,12 @@ def test_optional_missing_camera_still_publishes_existing_placeholder(draw_core, @pytest.mark.parametrize('camera_view', (1, 3, 4)) @pytest.mark.parametrize('draw_core', (Profile.COMPACT,), indirect=True) -def test_compact_hidden_and_unrecorded_camera_keep_acquisition(draw_core, camera_view, monkeypatch): +def test_compact_hidden_and_unrecorded_camera_keep_selected_frame(draw_core, camera_view, monkeypatch): core = draw_core core.adapter.build.return_value.onroad = replace(core.adapter.build.return_value.onroad, appearance=replace( core.adapter.build.return_value.onroad.appearance, camera_view=camera_view)) monkeypatch.setattr('openpilot.system.ui.widgets.label.gui_label', Mock()) + core._receive_primary() assert core.draw(2, 0., 100) core.camera.client.recv.assert_called_once_with(timeout_ms=0) core.camera.render_camera_model_layer.assert_not_called() @@ -483,6 +566,7 @@ def test_initial_hud_primer_required_even_when_camera_received_a_frame(tmp_path, def test_early_camera_subscriber_is_polled_before_paused_reprime(): from openpilot.cereal.visionipc import VisionStreamType + from openpilot.selfdrive.ui.onroad.cameraview import CameraView sample = SimpleNamespace(valid=True, epoch=1, boot_ns=9_050_000_000) idx = SimpleNamespace(frameId=5, timestampSof=9_000_000_000, timestampEof=9_010_000_000) sm = Mock() @@ -493,11 +577,13 @@ def test_early_camera_subscriber_is_polled_before_paused_reprime(): client = Mock(frame_id=5, timestamp_sof=idx.timestampSof, timestamp_eof=idx.timestampEof) # The first receive synchronizes with a publisher created after this client. client.recv.side_effect = [None, frame] - core = object.__new__(Compositor) + core: Any = object.__new__(Compositor) core.ui = Mock(replay_sample=sample, sm=sm) core.epoch, core.cabin = 1, False core.camera = Mock(client=client, frame=None, stream_type=VisionStreamType.VISION_STREAM_NARROW_ROAD) + core.camera._switching = False core.camera._ensure_connection.return_value = True + core.camera._poll_camera.side_effect = lambda: CameraView._poll_camera(core.camera) core.update() assert not core._primary_matches() core.update() diff --git a/openpilot/starpilot/ui/tests/test_runtime_navigation.py b/openpilot/starpilot/ui/tests/test_runtime_navigation.py index 5e186d36cf..6e2eeade66 100644 --- a/openpilot/starpilot/ui/tests/test_runtime_navigation.py +++ b/openpilot/starpilot/ui/tests/test_runtime_navigation.py @@ -104,7 +104,7 @@ class TestNativeCameraComposition(unittest.TestCase): module.AugmentedRoadView.render_camera_model_layer(view, rect) view._switch_stream_if_needed.assert_called_once() view._update_calibration.assert_called_once_with() - camera.assert_called_once_with(view, rect, **({"paint": True} if module is compact else {})) + camera.assert_called_once_with(view, rect, poll_camera=True, **({"paint": True} if module is compact else {})) if model_field == "_model_renderer": model.render_with_lead.assert_called_once_with(rect, False, 0, None, lateral_active=False, paint=True) model.render_with_lead.reset_mock()