diff --git a/selfdrive/ui/mici/onroad/augmented_road_view.py b/selfdrive/ui/mici/onroad/augmented_road_view.py index 4bf5d4f3c..e4cded2d3 100644 --- a/selfdrive/ui/mici/onroad/augmented_road_view.py +++ b/selfdrive/ui/mici/onroad/augmented_road_view.py @@ -19,7 +19,7 @@ from openpilot.selfdrive.ui.mici.onroad.starpilot_status import ( TRAFFIC_COLOR, get_border_color, ) -from openpilot.selfdrive.ui.mici.onroad.cameraview import CameraView +from openpilot.selfdrive.ui.onroad.cameraview import CameraView from openpilot.selfdrive.ui.lib.starpilot_visuals import get_border_width from openpilot.starpilot.common.favorite_slots import is_favorite_action_key, load_favorite_slots, toggle_favorite_slot from openpilot.system.ui.lib.application import FontWeight, gui_app, MousePos, MouseEvent @@ -830,14 +830,16 @@ class AugmentedRoadView(CameraView): def _switch_stream_if_needed(self, sm, camera_view: int): if camera_view == CAMERA_VIEW_NONE: + self._cancel_pending_switch() self._reverse_driver_camera_frames = 0 self._reverse_driver_camera_active = False return + if getattr(self, "_onroad_reentry_pending", False): + self._refresh_available_streams() + if self._update_reverse_driver_camera_state(): - target = DRIVER_CAM - if self.stream_type != target: - self.switch_stream(target) + self.switch_stream(DRIVER_CAM) return if camera_view == CAMERA_VIEW_DRIVER: @@ -858,7 +860,8 @@ class AugmentedRoadView(CameraView): else: target = ROAD_CAM - if self.stream_type != target: + if (getattr(self, "_onroad_reentry_pending", False) or + self.stream_type != target or (self._switching and self._target_stream_type != target)): self.switch_stream(target) def _update_calibration(self): diff --git a/selfdrive/ui/mici/onroad/cameraview.py b/selfdrive/ui/mici/onroad/cameraview.py index d5f7a9d98..1aa0694ff 100644 --- a/selfdrive/ui/mici/onroad/cameraview.py +++ b/selfdrive/ui/mici/onroad/cameraview.py @@ -1,476 +1,5 @@ -import os -import platform -import weakref -import numpy as np -import pyray as rl +"""Compatibility import for the shared CameraView implementation.""" -from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf -from openpilot.common.swaglog import cloudlog -from openpilot.system.hardware import TICI -from openpilot.system.ui.lib.application import gui_app -from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage -from openpilot.system.ui.widgets import Widget -from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus +from openpilot.selfdrive.ui.onroad.cameraview import CameraView -CONNECTION_RETRY_INTERVAL = 0.2 # seconds between connection attempts -MICI_FORCE_TEXTURE_CAMERA = os.getenv("MICI_FORCE_TEXTURE_CAMERA", "0") == "1" - -VERSION = """ -#version 300 es -precision mediump float; -""" -if platform.system() == "Darwin": - VERSION = """ - #version 330 core - """ - - -VERTEX_SHADER = VERSION + """ -in vec3 vertexPosition; -in vec2 vertexTexCoord; -in vec3 vertexNormal; -in vec4 vertexColor; -uniform mat4 mvp; -out vec2 fragTexCoord; -out vec4 fragColor; -void main() { - fragTexCoord = vertexTexCoord; - fragColor = vertexColor; - gl_Position = mvp * vec4(vertexPosition, 1.0); -} -""" - -FRAME_FRAGMENT_SHADER_EXTERNAL = """ - #version 300 es - #extension GL_OES_EGL_image_external_essl3 : enable - precision mediump float; - in vec2 fragTexCoord; - uniform samplerExternalOES texture0; - out vec4 fragColor; - uniform int engaged; - uniform int enhance_driver; - - void main() { - vec4 color = texture(texture0, fragTexCoord); - // Keep the onroad camera feed full-color in every driving state. - if (engaged == 1) { - color.rgb = color.rgb; - } - if (enhance_driver == 1) { - float brightness = 1.1; - color.rgb = color.rgb + 0.15; - color.rgb = clamp((color.rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0); - color.rgb = color.rgb * color.rgb * (3.0 - 2.0 * color.rgb); - color.rgb = pow(color.rgb, vec3(0.8)); - } - fragColor = vec4(color.rgb, color.a); - } - """ - -FRAME_FRAGMENT_SHADER_YUV = VERSION + """ - in vec2 fragTexCoord; - uniform sampler2D texture0; - uniform sampler2D texture1; - out vec4 fragColor; - uniform int engaged; - uniform int enhance_driver; - - void main() { - float y = texture(texture0, fragTexCoord).r; - vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; - vec3 rgb = vec3(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x); - // Keep the onroad camera feed full-color in every driving state. - if (engaged == 1) { - rgb = rgb; - } - // TODO: the images out of camerad need some more correction and - // the ui should apply a gamma curve for the device display - if (enhance_driver == 1) { - float brightness = 1.1; - rgb = rgb + 0.15; - rgb = clamp((rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0); - rgb = rgb * rgb * (3.0 - 2.0 * rgb); - rgb = pow(rgb, vec3(0.8)); - } - fragColor = vec4(rgb, 1.0); - } - """ - - -class CameraView(Widget): - def __init__(self, name: str, stream_type: VisionStreamType): - super().__init__() - self._name = name - # Primary stream - self.client = VisionIpcClient(name, stream_type, conflate=True) - self._stream_type = stream_type - self.available_streams: list[VisionStreamType] = [] - - # Target stream for switching - self._target_client: VisionIpcClient | None = None - self._target_stream_type: VisionStreamType | None = None - self._switching: bool = False - - self._texture_needs_update = True - self.last_connection_attempt: float = 0.0 - self._use_egl = TICI and not MICI_FORCE_TEXTURE_CAMERA and init_egl() - if TICI and MICI_FORCE_TEXTURE_CAMERA: - cloudlog.warning("CameraView EGL disabled by MICI_FORCE_TEXTURE_CAMERA, using texture rendering") - elif TICI and not self._use_egl: - cloudlog.error("CameraView EGL init failed, falling back to texture rendering") - - frame_shader = FRAME_FRAGMENT_SHADER_EXTERNAL if self._use_egl else FRAME_FRAGMENT_SHADER_YUV - self.shader = rl.load_shader_from_memory(VERTEX_SHADER, frame_shader) - self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not self._use_egl else -1 - self._engaged_loc = rl.get_shader_location(self.shader, "engaged") - self._engaged_val = rl.ffi.new("int[1]", [1]) - self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver") - self._enhance_driver_val = rl.ffi.new("int[1]", [1 if stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0]) - - self.frame: VisionBuf | None = None - self._last_frame_id = -1 - self._regressive_frame_count = 0 - self.texture_y: rl.Texture | None = None - self.texture_uv: rl.Texture | None = None - - # EGL resources - self.egl_images: dict[int, EGLImage] = {} - self.egl_texture: rl.Texture | None = None - - self._placeholder_color: rl.Color | None = None - self._closed = False - - # Initialize EGL for zero-copy rendering when available. - if self._use_egl: - self._create_egl_texture() - - self_ref = weakref.ref(self) - - def offroad_transition_callback(): - if (view := self_ref()) is not None: - view._offroad_transition() - - self._offroad_transition_callback = offroad_transition_callback - ui_state.add_offroad_transition_callback(self._offroad_transition_callback) - - def _offroad_transition(self): - self._reset_camera_connection() - - def _reset_camera_connection(self): - # EGL images and VisionBuf objects both retain the imported camera buffer. - # Release them on every road-state transition instead of pinning the old - # camerad allocation until this view happens to render again. - self._clear_textures() - self.frame = None - self._last_frame_id = -1 - self.available_streams.clear() - self.client = VisionIpcClient(self._name, self._stream_type, conflate=True) - self._target_client = None - self._target_stream_type = None - self._switching = False - self._texture_needs_update = True - self.last_connection_attempt = 0.0 - - def _set_placeholder_color(self, color: rl.Color): - """Set a placeholder color to be drawn when no frame is available.""" - self._placeholder_color = color - - def switch_stream(self, stream_type: VisionStreamType) -> None: - if self._stream_type == stream_type: - return - - if self._switching and self._target_stream_type == stream_type: - return - - cloudlog.debug(f'Preparing switch from {self._stream_type} to {stream_type}') - - if self._target_client: - del self._target_client - - self._target_stream_type = stream_type - self._target_client = VisionIpcClient(self._name, stream_type, conflate=True) - self._switching = True - - @property - def stream_type(self) -> VisionStreamType: - return self._stream_type - - def close(self) -> None: - if self._closed: - return - self._closed = True - - callback = getattr(self, "_offroad_transition_callback", None) - if callback is not None: - ui_state.remove_offroad_transition_callback(callback) - self._offroad_transition_callback = None - self._clear_textures() - - # Clean up shader - if self.shader and self.shader.id: - rl.unload_shader(self.shader) - self.shader.id = 0 - - self.frame = None - self._last_frame_id = -1 - self.available_streams.clear() - self.client = None - self._target_client = None - - def __del__(self): - self.close() - - def _calc_frame_matrix(self, rect: rl.Rectangle) -> np.ndarray: - if not self.frame: - return np.eye(3) - - # Calculate aspect ratios - widget_aspect_ratio = rect.width / rect.height - frame_aspect_ratio = self.frame.width / self.frame.height - - # Calculate scaling factors to maintain aspect ratio - zx = min(frame_aspect_ratio / widget_aspect_ratio, 1.0) - zy = min(widget_aspect_ratio / frame_aspect_ratio, 1.0) - - return np.array([ - [zx, 0.0, 0.0], - [0.0, zy, 0.0], - [0.0, 0.0, 1.0] - ]) - - def _render(self, rect: rl.Rectangle): - if self._switching: - self._handle_switch() - - if not self._ensure_connection(): - self._draw_placeholder(rect) - return - - if self._use_egl: - self._observe_displayed_frame() - - # Try to get a new buffer without blocking - buffer = self.client.recv(timeout_ms=0) - if buffer: - self._accept_frame(buffer, self.client.frame_id) - elif not self.client.is_connected(): - # ensure we clear the displayed frame when the connection is lost - self.frame = None - - if not self.frame: - self._draw_placeholder(rect) - return - - transform = self._calc_frame_matrix(rect) - src_rect = rl.Rectangle(0, 0, float(self.frame.width), float(self.frame.height)) - # Flip driver camera horizontally - if self._stream_type == VisionStreamType.VISION_STREAM_DRIVER: - src_rect.width = -src_rect.width - - # Calculate scale - scale_x = rect.width * transform[0, 0] # zx - scale_y = rect.height * transform[1, 1] # zy - - # Calculate base position (centered) - x_offset = rect.x + (rect.width - scale_x) / 2 - y_offset = rect.y + (rect.height - scale_y) / 2 - - x_offset += transform[0, 2] * rect.width / 2 - y_offset += transform[1, 2] * rect.height / 2 - - dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y) - - # Render with appropriate method - if self._use_egl: - self._render_egl(src_rect, dst_rect) - else: - self._render_textures(src_rect, dst_rect) - - def _draw_placeholder(self, rect: rl.Rectangle): - if self._placeholder_color: - rl.draw_rectangle_rec(rect, self._placeholder_color) - - def _observe_displayed_frame(self) -> None: - if self.frame is not None: - client_frame_id = getattr(self.client, "frame_id", -1) if hasattr(self, "client") and self.client is not None else -1 - frame_id = getattr(self.frame, "frame_id", client_frame_id) - self._last_frame_id = max(self._last_frame_id, int(frame_id)) - - def _accept_frame(self, frame: VisionBuf, packet_frame_id: int) -> bool: - content_frame_id = int(getattr(frame, "frame_id", packet_frame_id)) - if content_frame_id < self._last_frame_id: - self._regressive_frame_count += 1 - if self._regressive_frame_count == 1 or self._regressive_frame_count % 100 == 0: - message = f"Dropping regressive {self._name} frame: content={content_frame_id}, packet={packet_frame_id}, " - message += f"displayed={self._last_frame_id}, idx={frame.idx}, count={self._regressive_frame_count}" - cloudlog.warning(message) - return False - - self.frame = frame - self._last_frame_id = content_frame_id - self._texture_needs_update = True - return True - - def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None: - """Render using EGL for direct buffer access""" - if self.frame is None or self.egl_texture is None: - return - - idx = self.frame.idx - egl_image = self.egl_images.get(idx) - - # Create EGL image if needed - if egl_image is None: - egl_image = create_egl_image(self.frame.width, self.frame.height, self.frame.stride, self.frame.fd, self.frame.uv_offset) - if egl_image: - self.egl_images[idx] = egl_image - else: - return - - # Update texture dimensions to match current frame - self.egl_texture.width = self.frame.width - self.egl_texture.height = self.frame.height - - # Bind the EGL image to our texture - bind_egl_image_to_texture(self.egl_texture.id, egl_image) - - # Render with shader - rl.begin_shader_mode(self.shader) - self._update_texture_color_filtering() - rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) - rl.end_shader_mode() - - def _render_textures(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None: - """Render using texture copies""" - if not self.texture_y or not self.texture_uv or self.frame is None: - return - - # Update textures with new frame data - if self._texture_needs_update: - y_data = self.frame.data[: self.frame.uv_offset] - uv_data = self.frame.data[self.frame.uv_offset:] - - rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data)) - rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data)) - self._texture_needs_update = False - - # Render with shader - rl.begin_shader_mode(self.shader) - self._update_texture_color_filtering() - rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv) - rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) - rl.end_shader_mode() - - def _update_texture_color_filtering(self): - self._engaged_val[0] = 1 if ui_state.status != UIStatus.DISENGAGED else 0 - if self._engaged_loc >= 0: - rl.set_shader_value(self.shader, self._engaged_loc, self._engaged_val, rl.ShaderUniformDataType.SHADER_UNIFORM_INT) - if self._enhance_driver_loc >= 0: - rl.set_shader_value(self.shader, self._enhance_driver_loc, self._enhance_driver_val, rl.ShaderUniformDataType.SHADER_UNIFORM_INT) - - def _ensure_connection(self) -> bool: - if not self.client.is_connected(): - self.frame = None - self._last_frame_id = -1 - self.available_streams.clear() - - # Throttle connection attempts - current_time = rl.get_time() - if current_time - self.last_connection_attempt < CONNECTION_RETRY_INTERVAL: - return False - self.last_connection_attempt = current_time - - # A GL texture can retain the last EGL image after camerad exits. Release - # it before connect() frees and replaces the client's imported buffers. - self._clear_textures() - if not self.client.connect(False) or not self.client.num_buffers: - return False - - cloudlog.debug(f"Connected to {self._name} stream: {self._stream_type}, buffers: {self.client.num_buffers}") - self._initialize_textures() - self.available_streams = self.client.available_streams(self._name, block=False) - - return True - - def _handle_switch(self) -> None: - """Check if target stream is ready and switch immediately.""" - if not self._target_client or not self._switching: - return - - # Try to connect target if needed - if not self._target_client.is_connected(): - if not self._target_client.connect(False) or not self._target_client.num_buffers: - return - - cloudlog.debug(f"Target stream connected: {self._target_stream_type}") - - # Check if target has frames ready - target_frame = self._target_client.recv(timeout_ms=0) - if target_frame: - self.frame = target_frame # Update current frame to target frame - self._complete_switch() - - def _complete_switch(self) -> None: - """Instantly switch to target stream.""" - cloudlog.debug(f"Switching to {self._target_stream_type}") - # Delete the GL texture before releasing the old client. Merely destroying - # the EGLImage handle leaves its storage alive while a texture sibling exists. - self._clear_textures() - - # Switch to target - self.client = self._target_client - self._stream_type = self._target_stream_type - client_frame_id = getattr(self.client, "frame_id", -1) if hasattr(self, "client") and self.client is not None else -1 - self._last_frame_id = int(getattr(self.frame, "frame_id", client_frame_id)) if self.frame is not None else -1 - self._texture_needs_update = True - - # Reset state - self._target_client = None - self._target_stream_type = None - self._switching = False - - # Initialize textures for new stream - self._initialize_textures() - - def _initialize_textures(self): - self._clear_textures() - if self._use_egl: - self._create_egl_texture() - else: - self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride), - int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE)) - self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2), - int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA)) - - def _create_egl_texture(self): - # A fresh texture has no EGL image sibling from a previous camera client. - temp_image = rl.gen_image_color(1, 1, rl.BLACK) - self.egl_texture = rl.load_texture_from_image(temp_image) - rl.unload_image(temp_image) - - def _clear_textures(self): - if self.texture_y and self.texture_y.id: - rl.unload_texture(self.texture_y) - self.texture_y = None - - if self.texture_uv and self.texture_uv.id: - rl.unload_texture(self.texture_uv) - self.texture_uv = None - - # Delete the texture first. eglDestroyImageKHR only destroys the EGLImage - # handle; the image storage stays alive while a GL texture sibling exists. - if self._use_egl: - if self.egl_texture and self.egl_texture.id: - rl.unload_texture(self.egl_texture) - self.egl_texture = None - - for data in self.egl_images.values(): - destroy_egl_image(data) - self.egl_images = {} - - -if __name__ == "__main__": - gui_app.init_window("camera view") - road = CameraView("camerad", VisionStreamType.VISION_STREAM_ROAD) - for _ in gui_app.render(): - road.render(rl.Rectangle(0, 0, gui_app.width, gui_app.height)) +__all__ = ["CameraView"] diff --git a/selfdrive/ui/mici/onroad/driver_camera_dialog.py b/selfdrive/ui/mici/onroad/driver_camera_dialog.py index 220542592..9eac9309f 100644 --- a/selfdrive/ui/mici/onroad/driver_camera_dialog.py +++ b/selfdrive/ui/mici/onroad/driver_camera_dialog.py @@ -1,7 +1,7 @@ import pyray as rl from cereal import car, log, messaging from msgq.visionipc import VisionStreamType -from openpilot.selfdrive.ui.mici.onroad.cameraview import CameraView +from openpilot.selfdrive.ui.onroad.cameraview import CameraView from openpilot.selfdrive.ui.mici.onroad.driver_state import DriverStateRenderer from openpilot.selfdrive.ui.ui_state import ui_state, device from openpilot.system.ui.lib.application import gui_app, FontWeight diff --git a/selfdrive/ui/mici/tests/test_camera_cleanup.py b/selfdrive/ui/mici/tests/test_camera_cleanup.py index 2164d456a..94e4655d9 100644 --- a/selfdrive/ui/mici/tests/test_camera_cleanup.py +++ b/selfdrive/ui/mici/tests/test_camera_cleanup.py @@ -2,14 +2,13 @@ import gc from types import SimpleNamespace import weakref -import pytest - -from openpilot.selfdrive.ui.mici.onroad import cameraview as mici_cameraview from openpilot.selfdrive.ui.onroad import cameraview as big_cameraview +from openpilot.selfdrive.ui.mici.onroad import augmented_road_view as mici_augmented_road_view -@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview)) -def test_road_transition_releases_camera_buffers(monkeypatch, module): +def test_road_transition_releases_camera_buffers(monkeypatch): + module = big_cameraview + class FakeClient: pass @@ -25,6 +24,7 @@ def test_road_transition_releases_camera_buffers(monkeypatch, module): view._target_stream_type = object() view._switching = True view._texture_needs_update = False + view._regressive_frame_count = 2 view.last_connection_attempt = 123.0 view._closed = True cleared = [] @@ -44,11 +44,13 @@ def test_road_transition_releases_camera_buffers(monkeypatch, module): assert view._target_stream_type is None assert view._switching is False assert view._texture_needs_update + assert view._regressive_frame_count == 0 assert view.last_connection_attempt == 0.0 -@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview)) -def test_transition_callback_does_not_retain_camera_view(monkeypatch, module): +def test_transition_callback_does_not_retain_camera_view(monkeypatch): + module = big_cameraview + class FakeClient: pass @@ -72,8 +74,9 @@ def test_transition_callback_does_not_retain_camera_view(monkeypatch, module): assert callbacks == [] -@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview)) -def test_stream_switch_releases_graphics_before_old_client(module): +def test_stream_switch_releases_graphics_before_old_client(): + module = big_cameraview + events = [] class FakeClient: @@ -86,6 +89,8 @@ def test_stream_switch_releases_graphics_before_old_client(module): view._target_stream_type = object() view._stream_type = object() view._switching = True + view.frame = SimpleNamespace(frame_id=10) + view._regressive_frame_count = 2 view._texture_needs_update = False view._closed = True view._clear_textures = lambda: events.append("graphics") @@ -96,10 +101,12 @@ def test_stream_switch_releases_graphics_before_old_client(module): assert old_client_finalizer.alive is False assert events == ["graphics", "client", "initialize"] + assert view._regressive_frame_count == 0 -@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview)) -def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module): +def test_egl_cleanup_deletes_texture_before_images(monkeypatch): + module = big_cameraview + events = [] view = module.CameraView.__new__(module.CameraView) view.texture_y = None @@ -108,10 +115,7 @@ def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module): view.egl_images = {0: object(), 1: object()} view._closed = True - if module is mici_cameraview: - view._use_egl = True - else: - monkeypatch.setattr(module, "TICI", True) + view._use_egl = True monkeypatch.setattr(module.rl, "unload_texture", lambda _texture: events.append("texture")) monkeypatch.setattr(module, "destroy_egl_image", lambda _image: events.append("image")) @@ -121,3 +125,73 @@ def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module): assert events == ["texture", "image", "image"] assert view.egl_texture is None assert view.egl_images == {} + + +def test_egl_cleanup_synchronizes_after_backend_switch(monkeypatch): + module = big_cameraview + + events = [] + view = module.CameraView.__new__(module.CameraView) + view.texture_y = None + view.texture_uv = None + view.egl_texture = SimpleNamespace(id=7) + view.egl_images = {0: object()} + view._use_egl = False + view._closed = True + + monkeypatch.setattr(module, "is_egl_initialized", lambda: True) + monkeypatch.setattr(module.rl, "rl_draw_render_batch_active", lambda: events.append("flush")) + monkeypatch.setattr(module, "finish_gl", lambda: events.append("finish")) + monkeypatch.setattr(module.rl, "unload_texture", lambda _texture: events.append("texture")) + monkeypatch.setattr(module, "destroy_egl_image", lambda _image: events.append("image")) + + view._clear_textures() + + assert events == ["flush", "finish", "texture", "image"] + + +def test_reverse_activation_cancels_mismatched_pending_switch(): + view = mici_augmented_road_view.AugmentedRoadView.__new__(mici_augmented_road_view.AugmentedRoadView) + view._stream_type = mici_augmented_road_view.DRIVER_CAM + view._target_stream_type = mici_augmented_road_view.WIDE_CAM + view._target_client = object() + view._switching = True + view._closed = True + view._update_reverse_driver_camera_state = lambda: True + + view._switch_stream_if_needed(None, mici_augmented_road_view.CAMERA_VIEW_AUTO) + + assert view._target_client is None + assert view._target_stream_type is None + assert not view._switching + + +def test_onroad_transition_marks_camera_reentry(monkeypatch): + module = big_cameraview + + class FakeClient: + pass + + view = module.CameraView.__new__(module.CameraView) + view._name = "camerad" + view._stream_type = object() + view.client = FakeClient() + view.frame = None + view.available_streams = [] + view._target_client = None + view._target_stream_type = None + view._switching = False + view._texture_needs_update = False + view._regressive_frame_count = 1 + view._closed = True + view._onroad_reentry_pending = False + view._reentry_stream_selected = False + view._clear_textures = lambda: None + + monkeypatch.setattr(module, "VisionIpcClient", lambda *_args, **_kwargs: FakeClient()) + monkeypatch.setattr(module.ui_state, "is_onroad", lambda: True) + + view._offroad_transition() + + assert view._onroad_reentry_pending + assert not view._reentry_stream_selected diff --git a/selfdrive/ui/onroad/augmented_road_view.py b/selfdrive/ui/onroad/augmented_road_view.py index 36c475f8c..a07f4419b 100644 --- a/selfdrive/ui/onroad/augmented_road_view.py +++ b/selfdrive/ui/onroad/augmented_road_view.py @@ -200,10 +200,14 @@ class AugmentedRoadView(CameraView): def _switch_stream_if_needed(self, sm, camera_view: int): if camera_view == CAMERA_VIEW_NONE: + self._cancel_pending_switch() self._reverse_driver_camera_frames = 0 self._reverse_driver_camera_active = False return + if getattr(self, "_onroad_reentry_pending", False): + self._refresh_available_streams() + if self._update_reverse_driver_camera_state(): target = DRIVER_CAM elif camera_view == CAMERA_VIEW_DRIVER: @@ -224,7 +228,8 @@ class AugmentedRoadView(CameraView): else: target = ROAD_CAM - if self.stream_type != target: + if (getattr(self, "_onroad_reentry_pending", False) or + self.stream_type != target or (self._switching and self._target_stream_type != target)): self.switch_stream(target) def _update_calibration(self): diff --git a/selfdrive/ui/onroad/cameraview.py b/selfdrive/ui/onroad/cameraview.py index 8be38a389..22e6f0690 100644 --- a/selfdrive/ui/onroad/cameraview.py +++ b/selfdrive/ui/onroad/cameraview.py @@ -1,3 +1,4 @@ +import os import platform import weakref import numpy as np @@ -7,11 +8,17 @@ from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf from openpilot.common.swaglog import cloudlog from openpilot.system.hardware import TICI from openpilot.system.ui.lib.application import gui_app -from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage +from openpilot.system.ui.lib.egl import ( + init_egl, is_egl_initialized, finish_gl, create_egl_image, destroy_egl_image, + bind_egl_image_to_texture, EGLImage, +) from openpilot.system.ui.widgets import Widget from openpilot.selfdrive.ui.ui_state import ui_state CONNECTION_RETRY_INTERVAL = 0.2 # seconds between connection attempts +MICI_FORCE_TEXTURE_CAMERA = os.getenv("MICI_FORCE_TEXTURE_CAMERA", "0") == "1" +# One stale frame can be normal ring-buffer reuse; repeated consecutive regressions demote EGL. +EGL_REGRESSIVE_FRAME_FALLBACK_THRESHOLD = 3 VERSION = """ #version 300 es @@ -38,32 +45,48 @@ void main() { } """ -# Choose fragment shader based on platform capabilities -if TICI: - FRAME_FRAGMENT_SHADER = """ - #version 300 es - #extension GL_OES_EGL_image_external_essl3 : enable - precision mediump float; - in vec2 fragTexCoord; - uniform samplerExternalOES texture0; - out vec4 fragColor; - void main() { - vec4 color = texture(texture0, fragTexCoord); - fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a); +FRAME_FRAGMENT_SHADER_EXTERNAL = """ + #version 300 es + #extension GL_OES_EGL_image_external_essl3 : enable + precision mediump float; + in vec2 fragTexCoord; + uniform samplerExternalOES texture0; + uniform int enhance_driver; + out vec4 fragColor; + void main() { + vec4 color = texture(texture0, fragTexCoord); + color.rgb = pow(color.rgb, vec3(1.0/1.28)); + if (enhance_driver == 1) { + float brightness = 1.1; + color.rgb = color.rgb + 0.15; + color.rgb = clamp((color.rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0); + color.rgb = color.rgb * color.rgb * (3.0 - 2.0 * color.rgb); + color.rgb = pow(color.rgb, vec3(0.8)); } - """ -else: - FRAME_FRAGMENT_SHADER = VERSION + """ - in vec2 fragTexCoord; - uniform sampler2D texture0; - uniform sampler2D texture1; - out vec4 fragColor; - void main() { - float y = texture(texture0, fragTexCoord).r; - vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; - fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0); + fragColor = vec4(color.rgb, color.a); + } + """ + +FRAME_FRAGMENT_SHADER_YUV = VERSION + """ + in vec2 fragTexCoord; + uniform sampler2D texture0; + uniform sampler2D texture1; + uniform int enhance_driver; + out vec4 fragColor; + void main() { + float y = texture(texture0, fragTexCoord).r; + vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; + vec3 rgb = vec3(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x); + if (enhance_driver == 1) { + float brightness = 1.1; + rgb = rgb + 0.15; + rgb = clamp((rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0); + rgb = rgb * rgb * (3.0 - 2.0 * rgb); + rgb = pow(rgb, vec3(0.8)); } - """ + fragColor = vec4(rgb, 1.0); + } + """ class CameraView(Widget): @@ -82,8 +105,18 @@ class CameraView(Widget): self._texture_needs_update = True self.last_connection_attempt: float = 0.0 - self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER) - self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1 + self._use_egl = TICI and not MICI_FORCE_TEXTURE_CAMERA and init_egl() + if TICI and MICI_FORCE_TEXTURE_CAMERA: + cloudlog.warning("CameraView EGL disabled by MICI_FORCE_TEXTURE_CAMERA, using texture rendering") + elif TICI and not self._use_egl: + cloudlog.error("CameraView EGL init failed, falling back to texture rendering") + + self._enhance_driver_val = rl.ffi.new("int[1]", [0]) + self._load_frame_shader() + if self._use_egl and not self.shader.id: + cloudlog.error("CameraView EGL shader failed, falling back to texture rendering") + self._use_egl = False + self._load_frame_shader() self.frame: VisionBuf | None = None self._last_frame_id = -1 @@ -97,12 +130,17 @@ class CameraView(Widget): self._placeholder_color: rl.Color | None = None self._closed = False + self._onroad_reentry_pending = False + self._reentry_stream_selected = False - # Initialize EGL for zero-copy rendering on TICI - if TICI: - if not init_egl(): - raise RuntimeError("Failed to initialize EGL") - self._create_egl_texture() + if self._use_egl and not self._create_egl_texture(): + cloudlog.error("CameraView EGL texture creation failed, falling back to texture rendering") + self._use_egl = False + if self.shader and self.shader.id: + rl.unload_shader(self.shader) + self.shader.id = 0 + self._load_frame_shader() + cloudlog.info(f"CameraView using {'EGL zero-copy' if self._use_egl else 'texture-copy'} rendering for {stream_type}") self_ref = weakref.ref(self) @@ -117,12 +155,10 @@ class CameraView(Widget): self._reset_camera_connection() def _reset_camera_connection(self): - # EGL images and VisionBuf objects both retain the imported camera buffer. - # Release them on every road-state transition instead of pinning the old - # camerad allocation until this view happens to render again. self._clear_textures() self.frame = None self._last_frame_id = -1 + self._regressive_frame_count = 0 self.available_streams.clear() self.client = VisionIpcClient(self._name, self._stream_type, conflate=True) self._target_client = None @@ -130,16 +166,29 @@ class CameraView(Widget): self._switching = False self._texture_needs_update = True self.last_connection_attempt = 0.0 + self._onroad_reentry_pending = ui_state.is_onroad() + self._reentry_stream_selected = False def _set_placeholder_color(self, color: rl.Color): """Set a placeholder color to be drawn when no frame is available.""" self._placeholder_color = color + def _refresh_available_streams(self) -> None: + streams = VisionIpcClient.available_streams(self._name, block=False) + if streams: + self.available_streams = list(streams) + def switch_stream(self, stream_type: VisionStreamType) -> None: - if self._stream_type == stream_type: + if getattr(self, "_onroad_reentry_pending", False): + self._select_reentry_stream(stream_type) return - if self._switching and self._target_stream_type == stream_type: + if self._switching: + if self._target_stream_type == stream_type: + return + self._cancel_pending_switch() + + if self._stream_type == stream_type: return cloudlog.debug(f'Preparing switch from {self._stream_type} to {stream_type}') @@ -151,6 +200,30 @@ class CameraView(Widget): self._target_client = VisionIpcClient(self._name, stream_type, conflate=True) self._switching = True + def _cancel_pending_switch(self) -> None: + if self._target_client is not None: + cloudlog.debug(f"Cancelling pending camera switch to {self._target_stream_type}") + self._target_client = None + self._target_stream_type = None + self._switching = False + + def _select_reentry_stream(self, stream_type: VisionStreamType) -> None: + """Select the desired stream before displaying any post-transition frame.""" + self._cancel_pending_switch() + + if self._stream_type != stream_type: + old_client = self.client + self.client = None + del old_client + self.client = VisionIpcClient(self._name, stream_type, conflate=True) + self._stream_type = stream_type + + self.frame = None + self._last_frame_id = -1 + self._regressive_frame_count = 0 + self._texture_needs_update = True + self._reentry_stream_selected = True + @property def stream_type(self) -> VisionStreamType: return self._stream_type @@ -169,12 +242,15 @@ class CameraView(Widget): # Clean up shader if self.shader and self.shader.id: rl.unload_shader(self.shader) + self.shader.id = 0 self.frame = None self._last_frame_id = -1 self.available_streams.clear() self.client = None self._target_client = None + self._onroad_reentry_pending = False + self._reentry_stream_selected = False def __del__(self): self.close() @@ -201,13 +277,15 @@ class CameraView(Widget): if self._switching: self._handle_switch() + if self._onroad_reentry_pending and not self._reentry_stream_selected: + # Standalone CameraView users have no higher-level stream selector. + self._select_reentry_stream(self._stream_type) + if not self._ensure_connection(): self._draw_placeholder(rect) return - # An EGL image references camerad's reusable ring-buffer slot. Account for - # that slot advancing before accepting another (possibly older) slot. - if TICI: + if self._use_egl: self._observe_displayed_frame() # Try to get a new buffer without blocking @@ -241,16 +319,34 @@ class CameraView(Widget): dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y) - # Render with appropriate method - if TICI: - self._render_egl(src_rect, dst_rect) - else: + if self._use_egl: + try: + rendered = self._render_egl(src_rect, dst_rect) + except Exception: + cloudlog.exception("CameraView EGL rendering failed") + rendered = False + if not rendered: + self._fallback_to_textures("EGL frame rendering failed") + + if not self._use_egl: self._render_textures(src_rect, dst_rect) def _draw_placeholder(self, rect: rl.Rectangle): if self._placeholder_color: rl.draw_rectangle_rec(rect, self._placeholder_color) + def _load_frame_shader(self) -> None: + frame_shader = FRAME_FRAGMENT_SHADER_EXTERNAL if self._use_egl else FRAME_FRAGMENT_SHADER_YUV + self.shader = rl.load_shader_from_memory(VERTEX_SHADER, frame_shader) + self._texture1_loc = -1 if self._use_egl else rl.get_shader_location(self.shader, "texture1") + self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver") + + def _update_shader_state(self) -> None: + self._enhance_driver_val[0] = 1 if self._stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0 + if self._enhance_driver_loc >= 0: + rl.set_shader_value(self.shader, self._enhance_driver_loc, self._enhance_driver_val, + rl.ShaderUniformDataType.SHADER_UNIFORM_INT) + def _observe_displayed_frame(self) -> None: if self.frame is not None: client_frame_id = getattr(self.client, "frame_id", -1) if hasattr(self, "client") and self.client is not None else -1 @@ -265,44 +361,77 @@ class CameraView(Widget): message = f"Dropping regressive {self._name} frame: content={content_frame_id}, packet={packet_frame_id}, " message += f"displayed={self._last_frame_id}, idx={frame.idx}, count={self._regressive_frame_count}" cloudlog.warning(message) + if getattr(self, "_use_egl", False) and self._regressive_frame_count >= EGL_REGRESSIVE_FRAME_FALLBACK_THRESHOLD: + self._fallback_to_textures("repeated regressive frames") return False self.frame = frame self._last_frame_id = content_frame_id + self._regressive_frame_count = 0 self._texture_needs_update = True + self._onroad_reentry_pending = False + self._reentry_stream_selected = False return True - def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None: - """Render using EGL for direct buffer access""" - if self.frame is None or self.egl_texture is None: - return + def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> bool: + """Render using EGL for direct buffer access.""" + if self.frame is None or self.egl_texture is None or not self.egl_texture.id: + return False idx = self.frame.idx egl_image = self.egl_images.get(idx) - - # Create EGL image if needed if egl_image is None: egl_image = create_egl_image(self.frame.width, self.frame.height, self.frame.stride, self.frame.fd, self.frame.uv_offset) - if egl_image: - self.egl_images[idx] = egl_image - else: - return + if egl_image is None: + return False + self.egl_images[idx] = egl_image - # Update texture dimensions to match current frame self.egl_texture.width = self.frame.width self.egl_texture.height = self.frame.height - - # Bind the EGL image to our texture bind_egl_image_to_texture(self.egl_texture.id, egl_image) - # Render with shader rl.begin_shader_mode(self.shader) - rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) - rl.end_shader_mode() + try: + self._update_shader_state() + rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) + finally: + rl.end_shader_mode() + return True + + def _fallback_to_textures(self, reason: str) -> None: + if not self._use_egl: + return + + cloudlog.error(f"CameraView switching from EGL to texture rendering: {reason}") + self._use_egl = False + try: + self._clear_textures() + except Exception: + cloudlog.exception("CameraView EGL cleanup failed during texture fallback") + + if self.shader and self.shader.id: + try: + rl.unload_shader(self.shader) + except Exception: + cloudlog.exception("CameraView EGL shader cleanup failed during texture fallback") + self.shader.id = 0 + try: + self._load_frame_shader() + self._initialize_textures() + self._texture_needs_update = True + except Exception: + cloudlog.exception("CameraView texture fallback initialization failed") def _render_textures(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None: - """Render using texture copies""" - if not self.texture_y or not self.texture_uv or self.frame is None: + """Copy camera data into ordinary Raylib textures before drawing. + + Raylib batches camera draws as GL_TEXTURE_2D. Imported EGL images are + GL_TEXTURE_EXTERNAL_OES objects and cannot safely pass through that path; + copying also prevents the GPU from sampling camerad's reusable buffers + after they have been handed back to the producer. + """ + if (self.texture_y is None or not self.texture_y.id or + self.texture_uv is None or not self.texture_uv.id or self.frame is None): return # Update textures with new frame data @@ -316,14 +445,18 @@ class CameraView(Widget): # Render with shader rl.begin_shader_mode(self.shader) - rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv) - rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) - rl.end_shader_mode() + try: + self._update_shader_state() + rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv) + rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) + finally: + rl.end_shader_mode() def _ensure_connection(self) -> bool: if not self.client.is_connected(): self.frame = None self._last_frame_id = -1 + self._regressive_frame_count = 0 self.available_streams.clear() # Throttle connection attempts @@ -332,8 +465,6 @@ class CameraView(Widget): return False self.last_connection_attempt = current_time - # A GL texture can retain the last EGL image after camerad exits. Release - # it before connect() frees and replaces the client's imported buffers. self._clear_textures() if not self.client.connect(False) or not self.client.num_buffers: return False @@ -365,15 +496,14 @@ class CameraView(Widget): def _complete_switch(self) -> None: """Instantly switch to target stream.""" cloudlog.debug(f"Switching to {self._target_stream_type}") - # Delete the GL texture before releasing the old client. Merely destroying - # the EGLImage handle leaves its storage alive while a texture sibling exists. self._clear_textures() # Switch to target self.client = self._target_client self._stream_type = self._target_stream_type - client_frame_id = getattr(self.client, "frame_id", -1) if hasattr(self, "client") and self.client is not None else -1 + client_frame_id = getattr(self.client, "frame_id", -1) if self.client is not None else -1 self._last_frame_id = int(getattr(self.frame, "frame_id", client_frame_id)) if self.frame is not None else -1 + self._regressive_frame_count = 0 self._texture_needs_update = True # Reset state @@ -383,42 +513,72 @@ class CameraView(Widget): # Initialize textures for new stream self._initialize_textures() + available_streams = getattr(self.client, "available_streams", None) + if available_streams is not None: + self.available_streams = available_streams(self._name, block=False) def _initialize_textures(self): self._clear_textures() - if TICI: - self._create_egl_texture() + if self._use_egl: + if not self._create_egl_texture(): + self._fallback_to_textures("EGL texture creation failed") else: self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride), int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE)) self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2), int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA)) + if not self.texture_y.id or not self.texture_uv.id: + cloudlog.error("CameraView texture-copy texture creation failed") + self._clear_textures() - def _create_egl_texture(self): - # A fresh texture has no EGL image sibling from a previous camera client. - temp_image = rl.gen_image_color(1, 1, rl.BLACK) - self.egl_texture = rl.load_texture_from_image(temp_image) - rl.unload_image(temp_image) + def _create_egl_texture(self) -> bool: + temp_image = None + try: + temp_image = rl.gen_image_color(1, 1, rl.BLACK) + texture = rl.load_texture_from_image(temp_image) + if texture is None or not texture.id: + self.egl_texture = None + return False + self.egl_texture = texture + return True + except Exception: + self.egl_texture = None + cloudlog.exception("CameraView failed to create EGL texture") + return False + finally: + if temp_image is not None: + try: + rl.unload_image(temp_image) + except Exception: + cloudlog.exception("CameraView failed to unload temporary EGL image") def _clear_textures(self): - if self.texture_y and self.texture_y.id: - rl.unload_texture(self.texture_y) + if ((self.egl_texture is not None or self.egl_images) and is_egl_initialized()): + try: + # Raylib queues draw calls. Submit them before waiting for the GPU so + # no pending batch can still reference an EGL-backed texture. + rl.rl_draw_render_batch_active() + finish_gl() + except Exception: + cloudlog.exception("CameraView failed to synchronize EGL resources") + + if self.texture_y is not None: + if self.texture_y.id: + rl.unload_texture(self.texture_y) self.texture_y = None - if self.texture_uv and self.texture_uv.id: - rl.unload_texture(self.texture_uv) + if self.texture_uv is not None: + if self.texture_uv.id: + rl.unload_texture(self.texture_uv) self.texture_uv = None - # Delete the texture first. eglDestroyImageKHR only destroys the EGLImage - # handle; the image storage stays alive while a GL texture sibling exists. - if TICI: - if self.egl_texture and self.egl_texture.id: - rl.unload_texture(self.egl_texture) - self.egl_texture = None + if self.egl_texture and self.egl_texture.id: + rl.unload_texture(self.egl_texture) + self.egl_texture = None - for data in self.egl_images.values(): - destroy_egl_image(data) - self.egl_images = {} + for data in self.egl_images.values(): + destroy_egl_image(data) + self.egl_images = {} if __name__ == "__main__": diff --git a/selfdrive/ui/tests/test_camera_frame_order.py b/selfdrive/ui/tests/test_camera_frame_order.py index 002f37695..d698b0452 100644 --- a/selfdrive/ui/tests/test_camera_frame_order.py +++ b/selfdrive/ui/tests/test_camera_frame_order.py @@ -1,3 +1,5 @@ +from types import SimpleNamespace + import pytest from openpilot.selfdrive.ui.mici.onroad import cameraview as mici_cameraview @@ -10,9 +12,10 @@ class FakeFrame: self.idx = idx -def _camera_view(cameraview): - view = cameraview.CameraView.__new__(cameraview.CameraView) +def _camera_view(): + view = big_cameraview.CameraView.__new__(big_cameraview.CameraView) view._name = "camerad" + view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_ROAD view.frame = None view._last_frame_id = -1 view._regressive_frame_count = 0 @@ -21,10 +24,88 @@ def _camera_view(cameraview): return view -@pytest.mark.parametrize("cameraview", [big_cameraview, mici_cameraview]) -def test_reused_egl_slot_cannot_move_camera_backwards(monkeypatch, cameraview): - monkeypatch.setattr(cameraview.cloudlog, "warning", lambda *_args, **_kwargs: None) - view = _camera_view(cameraview) +def test_mici_uses_shared_camera_view(): + assert mici_cameraview.CameraView is big_cameraview.CameraView + + +def test_pending_switch_is_cancelled_when_requested_stream_is_current(): + view = _camera_view() + view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_ROAD + view._target_stream_type = big_cameraview.VisionStreamType.VISION_STREAM_DRIVER + view._target_client = object() + view._switching = True + + view.switch_stream(big_cameraview.VisionStreamType.VISION_STREAM_ROAD) + + assert view._target_client is None + assert view._target_stream_type is None + assert not view._switching + + +def test_onroad_reentry_selects_requested_stream_before_rendering(monkeypatch): + view = _camera_view() + view._name = "camerad" + view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_WIDE_ROAD + view.client = object() + view._target_client = object() + view._target_stream_type = big_cameraview.VisionStreamType.VISION_STREAM_ROAD + view._switching = True + view._onroad_reentry_pending = True + view._reentry_stream_selected = False + clients = [] + + class FakeClient: + def __init__(self, name, stream_type, conflate): + self.name = name + self.stream_type = stream_type + self.conflate = conflate + clients.append(self) + + monkeypatch.setattr(big_cameraview, "VisionIpcClient", FakeClient) + view.switch_stream(big_cameraview.VisionStreamType.VISION_STREAM_ROAD) + + assert len(clients) == 1 + assert view.client is clients[0] + assert view.client.stream_type == big_cameraview.VisionStreamType.VISION_STREAM_ROAD + assert view.stream_type == big_cameraview.VisionStreamType.VISION_STREAM_ROAD + assert view._target_client is None + assert view._target_stream_type is None + assert not view._switching + assert view._reentry_stream_selected + + +def test_onroad_reentry_guard_clears_on_first_fresh_frame(): + view = _camera_view() + view._onroad_reentry_pending = True + view._reentry_stream_selected = True + + assert view._accept_frame(FakeFrame(frame_id=1, idx=0), packet_frame_id=1) + assert not view._onroad_reentry_pending + assert not view._reentry_stream_selected + + +def test_standalone_camera_reentry_selects_configured_stream(): + view = _camera_view() + view._switching = False + view._onroad_reentry_pending = True + view._reentry_stream_selected = False + selected = [] + placeholders = [] + view._select_reentry_stream = lambda stream_type: ( + selected.append(stream_type), setattr(view, "_reentry_stream_selected", True) + ) + view._draw_placeholder = lambda rect: placeholders.append(rect) + view._ensure_connection = lambda: False + + view._render(object()) + + assert selected == [view._stream_type] + assert len(placeholders) == 1 + + +def test_reused_egl_slot_cannot_move_camera_backwards(monkeypatch): + monkeypatch.setattr(big_cameraview.cloudlog, "warning", lambda *_args, **_kwargs: None) + view = _camera_view() displayed = FakeFrame(frame_id=10, idx=0) assert view._accept_frame(displayed, packet_frame_id=10) @@ -40,13 +121,145 @@ def test_reused_egl_slot_cannot_move_camera_backwards(monkeypatch, cameraview): assert view._regressive_frame_count == 1 -@pytest.mark.parametrize("cameraview", [big_cameraview, mici_cameraview]) -def test_newer_camera_frame_is_accepted(cameraview): - view = _camera_view(cameraview) +def test_newer_camera_frame_is_accepted(): + view = _camera_view() view._last_frame_id = 30 + view._regressive_frame_count = 2 newer = FakeFrame(frame_id=31, idx=2) assert view._accept_frame(newer, packet_frame_id=31) assert view.frame is newer assert view._last_frame_id == 31 + assert view._regressive_frame_count == 0 assert view._texture_needs_update + + +def test_shared_camera_has_upstream_shaders_and_driver_enhancement(): + assert "samplerExternalOES" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL + assert "pow(color.rgb, vec3(1.0/1.28))" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL + assert "uniform sampler2D texture0" in big_cameraview.FRAME_FRAGMENT_SHADER_YUV + assert "uniform sampler2D texture1" in big_cameraview.FRAME_FRAGMENT_SHADER_YUV + assert "uniform int enhance_driver" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL + assert "uniform int enhance_driver" in big_cameraview.FRAME_FRAGMENT_SHADER_YUV + assert "uniform int engaged" not in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL + assert "uniform int engaged" not in big_cameraview.FRAME_FRAGMENT_SHADER_YUV + assert hasattr(big_cameraview.CameraView, "_render_egl") + assert hasattr(big_cameraview.CameraView, "_fallback_to_textures") + + +def test_shared_camera_falls_back_after_repeated_regressive_frames(monkeypatch): + monkeypatch.setattr(big_cameraview.cloudlog, "warning", lambda *_args, **_kwargs: None) + view = _camera_view() + view._use_egl = True + view.frame = FakeFrame(frame_id=30, idx=0) + view._last_frame_id = 30 + fallback_reasons = [] + view._fallback_to_textures = fallback_reasons.append + + for frame_id in (20, 19, 18): + assert not view._accept_frame(FakeFrame(frame_id=frame_id, idx=1), packet_frame_id=frame_id) + + assert fallback_reasons == ["repeated regressive frames"] + assert view.frame.frame_id == 30 + + +def test_shared_camera_fallback_reloads_texture_backend(monkeypatch): + view = _camera_view() + view._use_egl = True + view.shader = SimpleNamespace(id=1) + events = [] + view._clear_textures = lambda: events.append("clear") + view._load_frame_shader = lambda: events.append(("shader", view._use_egl)) + view._initialize_textures = lambda: events.append("textures") + monkeypatch.setattr(big_cameraview.cloudlog, "error", lambda *_args, **_kwargs: None) + monkeypatch.setattr(big_cameraview.rl, "unload_shader", lambda _shader: events.append("unload_shader")) + + view._fallback_to_textures("test") + + assert events == ["clear", "unload_shader", ("shader", False), "textures"] + assert not view._use_egl + + +def test_egl_image_creation_failure_is_reported(monkeypatch): + view = _camera_view() + view.frame = SimpleNamespace(idx=0, width=1928, height=1208, stride=2048, fd=7, uv_offset=2473984) + view.egl_texture = SimpleNamespace(id=1) + view.egl_images = {} + monkeypatch.setattr(big_cameraview, "create_egl_image", lambda *_args: None) + + assert not view._render_egl(None, None) + assert view.egl_images == {} + + +def test_invalid_egl_texture_is_reported_without_binding(monkeypatch): + view = _camera_view() + view.frame = SimpleNamespace(idx=0) + view.egl_texture = SimpleNamespace(id=0) + view.egl_images = {0: object()} + monkeypatch.setattr(big_cameraview, "bind_egl_image_to_texture", + lambda *_args: pytest.fail("invalid EGL texture was bound")) + + assert not view._render_egl(None, None) + + +def test_egl_render_always_ends_shader_mode(monkeypatch): + view = _camera_view() + view.frame = SimpleNamespace(idx=0, width=1928, height=1208) + view.egl_texture = SimpleNamespace(id=1, width=0, height=0) + view.egl_images = {0: object()} + view.shader = SimpleNamespace(id=1) + view._update_shader_state = lambda: None + events = [] + monkeypatch.setattr(big_cameraview, "bind_egl_image_to_texture", lambda *_args: None) + monkeypatch.setattr(big_cameraview.rl, "begin_shader_mode", lambda *_args: events.append("begin")) + def fail_draw(*_args): + raise RuntimeError("draw failed") + + monkeypatch.setattr(big_cameraview.rl, "draw_texture_pro", fail_draw) + monkeypatch.setattr(big_cameraview.rl, "end_shader_mode", lambda: events.append("end")) + + with pytest.raises(RuntimeError, match="draw failed"): + view._render_egl(None, None) + + assert events == ["begin", "end"] + + +def test_driver_enhancement_tracks_active_stream(monkeypatch): + view = _camera_view() + view.shader = SimpleNamespace(id=1) + view._enhance_driver_loc = 2 + view._enhance_driver_val = [0] + values = [] + monkeypatch.setattr(big_cameraview.rl, "set_shader_value", + lambda _shader, _loc, value, _type: values.append(value[0])) + + view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_ROAD + view._update_shader_state() + view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_DRIVER + view._update_shader_state() + view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_WIDE_ROAD + view._update_shader_state() + + assert values == [0, 1, 0] + + +def test_texture_fallback_survives_egl_cleanup_failure(monkeypatch): + view = _camera_view() + view._use_egl = True + view.shader = SimpleNamespace(id=1) + events = [] + + def fail_cleanup(): + raise RuntimeError("cleanup failed") + + view._clear_textures = fail_cleanup + view._load_frame_shader = lambda: events.append(("shader", view._use_egl)) + view._initialize_textures = lambda: events.append("textures") + monkeypatch.setattr(big_cameraview.cloudlog, "error", lambda *_args, **_kwargs: None) + monkeypatch.setattr(big_cameraview.cloudlog, "exception", lambda *_args, **_kwargs: None) + monkeypatch.setattr(big_cameraview.rl, "unload_shader", lambda _shader: events.append("unload_shader")) + + view._fallback_to_textures("test") + + assert not view._use_egl + assert events == ["unload_shader", ("shader", False), "textures"] diff --git a/system/ui/lib/egl.py b/system/ui/lib/egl.py index 69236482b..f81999f14 100644 --- a/system/ui/lib/egl.py +++ b/system/ui/lib/egl.py @@ -56,6 +56,7 @@ class EGLState: get_error: Any = None bind_texture: Any = None active_texture: Any = None + gl_finish: Any = None # Create a single instance of the state @@ -92,6 +93,7 @@ def init_egl() -> bool: void glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image); void glBindTexture(GLenum target, unsigned int texture); void glActiveTexture(GLenum texture); + void glFinish(void); """) # Load libraries @@ -111,6 +113,7 @@ def init_egl() -> bool: _egl.get_error = _egl.egl_lib.eglGetError _egl.bind_texture = _egl.gles_lib.glBindTexture _egl.active_texture = _egl.gles_lib.glActiveTexture + _egl.gl_finish = _egl.gles_lib.glFinish # Initialize EGL display once here _egl.display = _egl.get_current_display() @@ -125,6 +128,15 @@ def init_egl() -> bool: return False +def is_egl_initialized() -> bool: + return _egl.initialized + + +def finish_gl() -> None: + if _egl.initialized: + _egl.gl_finish() + + def create_egl_image(width: int, height: int, stride: int, fd: int, uv_offset: int) -> EGLImage | None: assert _egl.initialized, "EGL not initialized" @@ -160,10 +172,12 @@ def create_egl_image(width: int, height: int, stride: int, fd: int, uv_offset: i return EGLImage(egl_image=egl_image, fd=dup_fd) -def destroy_egl_image(egl_image: EGLImage) -> None: +def destroy_egl_image(egl_image: EGLImage) -> bool: assert _egl.initialized, "EGL not initialized" - _egl.destroy_image_khr(_egl.display, egl_image.egl_image) + destroyed = bool(_egl.destroy_image_khr(_egl.display, egl_image.egl_image)) + if not destroyed: + cloudlog.error(f"Failed to destroy EGL image: {_egl.get_error()}") # Close the duplicated fd we created in create_egl_image() # We need to handle OSError since the fd might already be closed @@ -172,6 +186,8 @@ def destroy_egl_image(egl_image: EGLImage) -> None: except OSError: pass + return destroyed + def bind_egl_image_to_texture(texture_id: int, egl_image: EGLImage) -> None: assert _egl.initialized, "EGL not initialized"