mawei/StarPilot
1
0
Fork 0
mirror of https://github.com/firestar5683/StarPilot.git synced 2026-08-03 08:41:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Metapod

Browse Source
This commit is contained in:
firestarsdog
2026-07-29 15:31:08 -04:00
parent 6277ecee4e
commit 8c17630899
8 changed files with 860 additions and 638 deletions
Download Patch File Download Diff File Expand all files Collapse all files
selfdrive/ui/mici/onroad/augmented_road_view.py
+8 -5
View File
@@ -19,7 +19,7 @@ from openpilot.selfdrive.ui.mici.onroad.starpilot_status import (
TRAFFIC_COLOR,
get_border_color,
)
from openpilot.selfdrive.ui.mici.onroad.cameraview import CameraView
from openpilot.selfdrive.ui.onroad.cameraview import CameraView
from openpilot.selfdrive.ui.lib.starpilot_visuals import get_border_width
from openpilot.starpilot.common.favorite_slots import is_favorite_action_key, load_favorite_slots, toggle_favorite_slot
from openpilot.system.ui.lib.application import FontWeight, gui_app, MousePos, MouseEvent
@@ -830,14 +830,16 @@ class AugmentedRoadView(CameraView):
def _switch_stream_if_needed(self, sm, camera_view: int):
if camera_view == CAMERA_VIEW_NONE:
self._cancel_pending_switch()
self._reverse_driver_camera_frames = 0
self._reverse_driver_camera_active = False
return
if getattr(self, "_onroad_reentry_pending", False):
self._refresh_available_streams()
if self._update_reverse_driver_camera_state():
target = DRIVER_CAM
if self.stream_type != target:
self.switch_stream(target)
self.switch_stream(DRIVER_CAM)
return
if camera_view == CAMERA_VIEW_DRIVER:
@@ -858,7 +860,8 @@ class AugmentedRoadView(CameraView):
else:
target = ROAD_CAM
if self.stream_type != target:
if (getattr(self, "_onroad_reentry_pending", False) or
self.stream_type != target or (self._switching and self._target_stream_type != target)):
self.switch_stream(target)
def _update_calibration(self):
selfdrive/ui/mici/onroad/cameraview.py
+3 -474
View File
@@ -1,476 +1,5 @@
import os
import platform
import weakref
import numpy as np
import pyray as rl
"""Compatibility import for the shared CameraView implementation."""
from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
from openpilot.common.swaglog import cloudlog
from openpilot.system.hardware import TICI
from openpilot.system.ui.lib.application import gui_app
from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage
from openpilot.system.ui.widgets import Widget
from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus
from openpilot.selfdrive.ui.onroad.cameraview import CameraView
CONNECTION_RETRY_INTERVAL = 0.2 # seconds between connection attempts
MICI_FORCE_TEXTURE_CAMERA = os.getenv("MICI_FORCE_TEXTURE_CAMERA", "0") == "1"
VERSION = """
#version 300 es
precision mediump float;
"""
if platform.system() == "Darwin":
VERSION = """
#version 330 core
"""
VERTEX_SHADER = VERSION + """
in vec3 vertexPosition;
in vec2 vertexTexCoord;
in vec3 vertexNormal;
in vec4 vertexColor;
uniform mat4 mvp;
out vec2 fragTexCoord;
out vec4 fragColor;
void main() {
fragTexCoord = vertexTexCoord;
fragColor = vertexColor;
gl_Position = mvp * vec4(vertexPosition, 1.0);
}
"""
FRAME_FRAGMENT_SHADER_EXTERNAL = """
#version 300 es
#extension GL_OES_EGL_image_external_essl3 : enable
precision mediump float;
in vec2 fragTexCoord;
uniform samplerExternalOES texture0;
out vec4 fragColor;
uniform int engaged;
uniform int enhance_driver;
void main() {
vec4 color = texture(texture0, fragTexCoord);
// Keep the onroad camera feed full-color in every driving state.
if (engaged == 1) {
color.rgb = color.rgb;
}
if (enhance_driver == 1) {
float brightness = 1.1;
color.rgb = color.rgb + 0.15;
color.rgb = clamp((color.rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0);
color.rgb = color.rgb * color.rgb * (3.0 - 2.0 * color.rgb);
color.rgb = pow(color.rgb, vec3(0.8));
}
fragColor = vec4(color.rgb, color.a);
}
"""
FRAME_FRAGMENT_SHADER_YUV = VERSION + """
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
out vec4 fragColor;
uniform int engaged;
uniform int enhance_driver;
void main() {
float y = texture(texture0, fragTexCoord).r;
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
vec3 rgb = vec3(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x);
// Keep the onroad camera feed full-color in every driving state.
if (engaged == 1) {
rgb = rgb;
}
// TODO: the images out of camerad need some more correction and
// the ui should apply a gamma curve for the device display
if (enhance_driver == 1) {
float brightness = 1.1;
rgb = rgb + 0.15;
rgb = clamp((rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0);
rgb = rgb * rgb * (3.0 - 2.0 * rgb);
rgb = pow(rgb, vec3(0.8));
}
fragColor = vec4(rgb, 1.0);
}
"""
class CameraView(Widget):
def __init__(self, name: str, stream_type: VisionStreamType):
super().__init__()
self._name = name
# Primary stream
self.client = VisionIpcClient(name, stream_type, conflate=True)
self._stream_type = stream_type
self.available_streams: list[VisionStreamType] = []
# Target stream for switching
self._target_client: VisionIpcClient | None = None
self._target_stream_type: VisionStreamType | None = None
self._switching: bool = False
self._texture_needs_update = True
self.last_connection_attempt: float = 0.0
self._use_egl = TICI and not MICI_FORCE_TEXTURE_CAMERA and init_egl()
if TICI and MICI_FORCE_TEXTURE_CAMERA:
cloudlog.warning("CameraView EGL disabled by MICI_FORCE_TEXTURE_CAMERA, using texture rendering")
elif TICI and not self._use_egl:
cloudlog.error("CameraView EGL init failed, falling back to texture rendering")
frame_shader = FRAME_FRAGMENT_SHADER_EXTERNAL if self._use_egl else FRAME_FRAGMENT_SHADER_YUV
self.shader = rl.load_shader_from_memory(VERTEX_SHADER, frame_shader)
self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not self._use_egl else -1
self._engaged_loc = rl.get_shader_location(self.shader, "engaged")
self._engaged_val = rl.ffi.new("int[1]", [1])
self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver")
self._enhance_driver_val = rl.ffi.new("int[1]", [1 if stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0])
self.frame: VisionBuf | None = None
self._last_frame_id = -1
self._regressive_frame_count = 0
self.texture_y: rl.Texture | None = None
self.texture_uv: rl.Texture | None = None
# EGL resources
self.egl_images: dict[int, EGLImage] = {}
self.egl_texture: rl.Texture | None = None
self._placeholder_color: rl.Color | None = None
self._closed = False
# Initialize EGL for zero-copy rendering when available.
if self._use_egl:
self._create_egl_texture()
self_ref = weakref.ref(self)
def offroad_transition_callback():
if (view := self_ref()) is not None:
view._offroad_transition()
self._offroad_transition_callback = offroad_transition_callback
ui_state.add_offroad_transition_callback(self._offroad_transition_callback)
def _offroad_transition(self):
self._reset_camera_connection()
def _reset_camera_connection(self):
# EGL images and VisionBuf objects both retain the imported camera buffer.
# Release them on every road-state transition instead of pinning the old
# camerad allocation until this view happens to render again.
self._clear_textures()
self.frame = None
self._last_frame_id = -1
self.available_streams.clear()
self.client = VisionIpcClient(self._name, self._stream_type, conflate=True)
self._target_client = None
self._target_stream_type = None
self._switching = False
self._texture_needs_update = True
self.last_connection_attempt = 0.0
def _set_placeholder_color(self, color: rl.Color):
"""Set a placeholder color to be drawn when no frame is available."""
self._placeholder_color = color
def switch_stream(self, stream_type: VisionStreamType) -> None:
if self._stream_type == stream_type:
return
if self._switching and self._target_stream_type == stream_type:
return
cloudlog.debug(f'Preparing switch from {self._stream_type} to {stream_type}')
if self._target_client:
del self._target_client
self._target_stream_type = stream_type
self._target_client = VisionIpcClient(self._name, stream_type, conflate=True)
self._switching = True
@property
def stream_type(self) -> VisionStreamType:
return self._stream_type
def close(self) -> None:
if self._closed:
return
self._closed = True
callback = getattr(self, "_offroad_transition_callback", None)
if callback is not None:
ui_state.remove_offroad_transition_callback(callback)
self._offroad_transition_callback = None
self._clear_textures()
# Clean up shader
if self.shader and self.shader.id:
rl.unload_shader(self.shader)
self.shader.id = 0
self.frame = None
self._last_frame_id = -1
self.available_streams.clear()
self.client = None
self._target_client = None
def __del__(self):
self.close()
def _calc_frame_matrix(self, rect: rl.Rectangle) -> np.ndarray:
if not self.frame:
return np.eye(3)
# Calculate aspect ratios
widget_aspect_ratio = rect.width / rect.height
frame_aspect_ratio = self.frame.width / self.frame.height
# Calculate scaling factors to maintain aspect ratio
zx = min(frame_aspect_ratio / widget_aspect_ratio, 1.0)
zy = min(widget_aspect_ratio / frame_aspect_ratio, 1.0)
return np.array([
[zx, 0.0, 0.0],
[0.0, zy, 0.0],
[0.0, 0.0, 1.0]
])
def _render(self, rect: rl.Rectangle):
if self._switching:
self._handle_switch()
if not self._ensure_connection():
self._draw_placeholder(rect)
return
if self._use_egl:
self._observe_displayed_frame()
# Try to get a new buffer without blocking
buffer = self.client.recv(timeout_ms=0)
if buffer:
self._accept_frame(buffer, self.client.frame_id)
elif not self.client.is_connected():
# ensure we clear the displayed frame when the connection is lost
self.frame = None
if not self.frame:
self._draw_placeholder(rect)
return
transform = self._calc_frame_matrix(rect)
src_rect = rl.Rectangle(0, 0, float(self.frame.width), float(self.frame.height))
# Flip driver camera horizontally
if self._stream_type == VisionStreamType.VISION_STREAM_DRIVER:
src_rect.width = -src_rect.width
# Calculate scale
scale_x = rect.width * transform[0, 0] # zx
scale_y = rect.height * transform[1, 1] # zy
# Calculate base position (centered)
x_offset = rect.x + (rect.width - scale_x) / 2
y_offset = rect.y + (rect.height - scale_y) / 2
x_offset += transform[0, 2] * rect.width / 2
y_offset += transform[1, 2] * rect.height / 2
dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y)
# Render with appropriate method
if self._use_egl:
self._render_egl(src_rect, dst_rect)
else:
self._render_textures(src_rect, dst_rect)
def _draw_placeholder(self, rect: rl.Rectangle):
if self._placeholder_color:
rl.draw_rectangle_rec(rect, self._placeholder_color)
def _observe_displayed_frame(self) -> None:
if self.frame is not None:
client_frame_id = getattr(self.client, "frame_id", -1) if hasattr(self, "client") and self.client is not None else -1
frame_id = getattr(self.frame, "frame_id", client_frame_id)
self._last_frame_id = max(self._last_frame_id, int(frame_id))
def _accept_frame(self, frame: VisionBuf, packet_frame_id: int) -> bool:
content_frame_id = int(getattr(frame, "frame_id", packet_frame_id))
if content_frame_id < self._last_frame_id:
self._regressive_frame_count += 1
if self._regressive_frame_count == 1 or self._regressive_frame_count % 100 == 0:
message = f"Dropping regressive {self._name} frame: content={content_frame_id}, packet={packet_frame_id}, "
message += f"displayed={self._last_frame_id}, idx={frame.idx}, count={self._regressive_frame_count}"
cloudlog.warning(message)
return False
self.frame = frame
self._last_frame_id = content_frame_id
self._texture_needs_update = True
return True
def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
"""Render using EGL for direct buffer access"""
if self.frame is None or self.egl_texture is None:
return
idx = self.frame.idx
egl_image = self.egl_images.get(idx)
# Create EGL image if needed
if egl_image is None:
egl_image = create_egl_image(self.frame.width, self.frame.height, self.frame.stride, self.frame.fd, self.frame.uv_offset)
if egl_image:
self.egl_images[idx] = egl_image
else:
return
# Update texture dimensions to match current frame
self.egl_texture.width = self.frame.width
self.egl_texture.height = self.frame.height
# Bind the EGL image to our texture
bind_egl_image_to_texture(self.egl_texture.id, egl_image)
# Render with shader
rl.begin_shader_mode(self.shader)
self._update_texture_color_filtering()
rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
def _render_textures(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
"""Render using texture copies"""
if not self.texture_y or not self.texture_uv or self.frame is None:
return
# Update textures with new frame data
if self._texture_needs_update:
y_data = self.frame.data[: self.frame.uv_offset]
uv_data = self.frame.data[self.frame.uv_offset:]
rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data))
rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data))
self._texture_needs_update = False
# Render with shader
rl.begin_shader_mode(self.shader)
self._update_texture_color_filtering()
rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv)
rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
def _update_texture_color_filtering(self):
self._engaged_val[0] = 1 if ui_state.status != UIStatus.DISENGAGED else 0
if self._engaged_loc >= 0:
rl.set_shader_value(self.shader, self._engaged_loc, self._engaged_val, rl.ShaderUniformDataType.SHADER_UNIFORM_INT)
if self._enhance_driver_loc >= 0:
rl.set_shader_value(self.shader, self._enhance_driver_loc, self._enhance_driver_val, rl.ShaderUniformDataType.SHADER_UNIFORM_INT)
def _ensure_connection(self) -> bool:
if not self.client.is_connected():
self.frame = None
self._last_frame_id = -1
self.available_streams.clear()
# Throttle connection attempts
current_time = rl.get_time()
if current_time - self.last_connection_attempt < CONNECTION_RETRY_INTERVAL:
return False
self.last_connection_attempt = current_time
# A GL texture can retain the last EGL image after camerad exits. Release
# it before connect() frees and replaces the client's imported buffers.
self._clear_textures()
if not self.client.connect(False) or not self.client.num_buffers:
return False
cloudlog.debug(f"Connected to {self._name} stream: {self._stream_type}, buffers: {self.client.num_buffers}")
self._initialize_textures()
self.available_streams = self.client.available_streams(self._name, block=False)
return True
def _handle_switch(self) -> None:
"""Check if target stream is ready and switch immediately."""
if not self._target_client or not self._switching:
return
# Try to connect target if needed
if not self._target_client.is_connected():
if not self._target_client.connect(False) or not self._target_client.num_buffers:
return
cloudlog.debug(f"Target stream connected: {self._target_stream_type}")
# Check if target has frames ready
target_frame = self._target_client.recv(timeout_ms=0)
if target_frame:
self.frame = target_frame # Update current frame to target frame
self._complete_switch()
def _complete_switch(self) -> None:
"""Instantly switch to target stream."""
cloudlog.debug(f"Switching to {self._target_stream_type}")
# Delete the GL texture before releasing the old client. Merely destroying
# the EGLImage handle leaves its storage alive while a texture sibling exists.
self._clear_textures()
# Switch to target
self.client = self._target_client
self._stream_type = self._target_stream_type
client_frame_id = getattr(self.client, "frame_id", -1) if hasattr(self, "client") and self.client is not None else -1
self._last_frame_id = int(getattr(self.frame, "frame_id", client_frame_id)) if self.frame is not None else -1
self._texture_needs_update = True
# Reset state
self._target_client = None
self._target_stream_type = None
self._switching = False
# Initialize textures for new stream
self._initialize_textures()
def _initialize_textures(self):
self._clear_textures()
if self._use_egl:
self._create_egl_texture()
else:
self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
def _create_egl_texture(self):
# A fresh texture has no EGL image sibling from a previous camera client.
temp_image = rl.gen_image_color(1, 1, rl.BLACK)
self.egl_texture = rl.load_texture_from_image(temp_image)
rl.unload_image(temp_image)
def _clear_textures(self):
if self.texture_y and self.texture_y.id:
rl.unload_texture(self.texture_y)
self.texture_y = None
if self.texture_uv and self.texture_uv.id:
rl.unload_texture(self.texture_uv)
self.texture_uv = None
# Delete the texture first. eglDestroyImageKHR only destroys the EGLImage
# handle; the image storage stays alive while a GL texture sibling exists.
if self._use_egl:
if self.egl_texture and self.egl_texture.id:
rl.unload_texture(self.egl_texture)
self.egl_texture = None
for data in self.egl_images.values():
destroy_egl_image(data)
self.egl_images = {}
if __name__ == "__main__":
gui_app.init_window("camera view")
road = CameraView("camerad", VisionStreamType.VISION_STREAM_ROAD)
for _ in gui_app.render():
road.render(rl.Rectangle(0, 0, gui_app.width, gui_app.height))
__all__ = ["CameraView"]
selfdrive/ui/mici/onroad/driver_camera_dialog.py
+1 -1
View File
@@ -1,7 +1,7 @@
import pyray as rl
from cereal import car, log, messaging
from msgq.visionipc import VisionStreamType
from openpilot.selfdrive.ui.mici.onroad.cameraview import CameraView
from openpilot.selfdrive.ui.onroad.cameraview import CameraView
from openpilot.selfdrive.ui.mici.onroad.driver_state import DriverStateRenderer
from openpilot.selfdrive.ui.ui_state import ui_state, device
from openpilot.system.ui.lib.application import gui_app, FontWeight
selfdrive/ui/mici/tests/test_camera_cleanup.py
+103 -20
View File
@@ -2,14 +2,13 @@ import gc
from types import SimpleNamespace
import weakref
import pytest
from openpilot.selfdrive.ui.mici.onroad import cameraview as mici_cameraview
from openpilot.selfdrive.ui.onroad import cameraview as big_cameraview
from openpilot.selfdrive.ui.mici.onroad import augmented_road_view as mici_augmented_road_view
@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview))
def test_road_transition_releases_camera_buffers(monkeypatch, module):
def test_road_transition_releases_camera_buffers(monkeypatch):
module = big_cameraview
class FakeClient:
pass
@@ -25,12 +24,12 @@ def test_road_transition_releases_camera_buffers(monkeypatch, module):
view._target_stream_type = object()
view._switching = True
view._texture_needs_update = False
view._regressive_frame_count = 2
view.last_connection_attempt = 123.0
view._closed = True
cleared = []
view._clear_textures = lambda: cleared.append(True)
monkeypatch.setattr(module, "VisionIpcClient", lambda *_args, **_kwargs: FakeClient())
del old_client
view._offroad_transition()
@@ -44,11 +43,13 @@ def test_road_transition_releases_camera_buffers(monkeypatch, module):
assert view._target_stream_type is None
assert view._switching is False
assert view._texture_needs_update
assert view._regressive_frame_count == 0
assert view.last_connection_attempt == 0.0
@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview))
def test_transition_callback_does_not_retain_camera_view(monkeypatch, module):
def test_transition_callback_does_not_retain_camera_view(monkeypatch):
module = big_cameraview
class FakeClient:
pass
@@ -72,34 +73,50 @@ def test_transition_callback_does_not_retain_camera_view(monkeypatch, module):
assert callbacks == []
@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview))
def test_stream_switch_releases_graphics_before_old_client(module):
def test_stream_switch_releases_graphics_before_old_client():
module = big_cameraview
events = []
class FakeClient:
pass
class FakeFrame:
pass
view = module.CameraView.__new__(module.CameraView)
view.client = FakeClient()
old_client_finalizer = weakref.finalize(view.client, events.append, "client")
old_client = FakeClient()
old_client_finalizer = weakref.finalize(old_client, events.append, "client")
old_frame = FakeFrame()
old_frame.frame_id = 10
old_frame.owner = old_client
old_frame_finalizer = weakref.finalize(old_frame, events.append, "frame")
view.client = old_client
view._target_client = FakeClient()
view._target_stream_type = object()
view._stream_type = object()
view._switching = True
view.frame = old_frame
view._regressive_frame_count = 2
view._texture_needs_update = False
view._closed = True
view._clear_textures = lambda: events.append("graphics")
view._initialize_textures = lambda: events.append("initialize")
del old_frame
del old_client
view._complete_switch()
view._complete_switch(SimpleNamespace(frame_id=11))
gc.collect()
assert old_client_finalizer.alive is False
assert events == ["graphics", "client", "initialize"]
assert old_frame_finalizer.alive is False
assert events == ["graphics", "frame", "client", "initialize"]
assert view._regressive_frame_count == 0
@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview))
def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module):
def test_egl_cleanup_deletes_texture_before_images(monkeypatch):
module = big_cameraview
events = []
view = module.CameraView.__new__(module.CameraView)
view.texture_y = None
@@ -108,10 +125,7 @@ def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module):
view.egl_images = {0: object(), 1: object()}
view._closed = True
if module is mici_cameraview:
view._use_egl = True
else:
monkeypatch.setattr(module, "TICI", True)
view._use_egl = True
monkeypatch.setattr(module.rl, "unload_texture", lambda _texture: events.append("texture"))
monkeypatch.setattr(module, "destroy_egl_image", lambda _image: events.append("image"))
@@ -121,3 +135,72 @@ def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module):
assert events == ["texture", "image", "image"]
assert view.egl_texture is None
assert view.egl_images == {}
def test_egl_cleanup_synchronizes_after_backend_switch(monkeypatch):
module = big_cameraview
events = []
view = module.CameraView.__new__(module.CameraView)
view.texture_y = None
view.texture_uv = None
view.egl_texture = SimpleNamespace(id=7)
view.egl_images = {0: object()}
view._use_egl = False
view._closed = True
monkeypatch.setattr(module, "is_egl_initialized", lambda: True)
monkeypatch.setattr(module.rl, "rl_draw_render_batch_active", lambda: events.append("flush"))
monkeypatch.setattr(module, "finish_gl", lambda: events.append("finish"))
monkeypatch.setattr(module.rl, "unload_texture", lambda _texture: events.append("texture"))
monkeypatch.setattr(module, "destroy_egl_image", lambda _image: events.append("image"))
view._clear_textures()
assert events == ["flush", "finish", "texture", "image"]
def test_reverse_activation_cancels_mismatched_pending_switch():
view = mici_augmented_road_view.AugmentedRoadView.__new__(mici_augmented_road_view.AugmentedRoadView)
view._stream_type = mici_augmented_road_view.DRIVER_CAM
view._target_stream_type = mici_augmented_road_view.WIDE_CAM
view._target_client = object()
view._switching = True
view._closed = True
view._update_reverse_driver_camera_state = lambda: True
view._switch_stream_if_needed(None, mici_augmented_road_view.CAMERA_VIEW_AUTO)
assert view._target_client is None
assert view._target_stream_type is None
assert not view._switching
def test_onroad_transition_marks_camera_reentry(monkeypatch):
module = big_cameraview
class FakeClient:
pass
view = module.CameraView.__new__(module.CameraView)
view._name = "camerad"
view._stream_type = object()
view.client = FakeClient()
view.frame = None
view.available_streams = []
view._target_client = None
view._target_stream_type = None
view._switching = False
view._texture_needs_update = False
view._regressive_frame_count = 1
view._closed = True
view._onroad_reentry_pending = False
view._reentry_stream_selected = False
view._clear_textures = lambda: None
monkeypatch.setattr(module.ui_state, "is_onroad", lambda: True)
view._offroad_transition()
assert view._onroad_reentry_pending
assert not view._reentry_stream_selected
selfdrive/ui/onroad/augmented_road_view.py
+6 -1
View File
@@ -200,10 +200,14 @@ class AugmentedRoadView(CameraView):
def _switch_stream_if_needed(self, sm, camera_view: int):
if camera_view == CAMERA_VIEW_NONE:
self._cancel_pending_switch()
self._reverse_driver_camera_frames = 0
self._reverse_driver_camera_active = False
return
if getattr(self, "_onroad_reentry_pending", False):
self._refresh_available_streams()
if self._update_reverse_driver_camera_state():
target = DRIVER_CAM
elif camera_view == CAMERA_VIEW_DRIVER:
@@ -224,7 +228,8 @@ class AugmentedRoadView(CameraView):
else:
target = ROAD_CAM
if self.stream_type != target:
if (getattr(self, "_onroad_reentry_pending", False) or
self.stream_type != target or (self._switching and self._target_stream_type != target)):
self.switch_stream(target)
def _update_calibration(self):
selfdrive/ui/onroad/cameraview.py
+323 -126
View File
@@ -1,3 +1,4 @@
import os
import platform
import weakref
import numpy as np
@@ -7,11 +8,17 @@ from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
from openpilot.common.swaglog import cloudlog
from openpilot.system.hardware import TICI
from openpilot.system.ui.lib.application import gui_app
from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage
from openpilot.system.ui.lib.egl import (
init_egl, is_egl_initialized, finish_gl, create_egl_image, destroy_egl_image,
bind_egl_image_to_texture, EGLImage,
)
from openpilot.system.ui.widgets import Widget
from openpilot.selfdrive.ui.ui_state import ui_state
CONNECTION_RETRY_INTERVAL = 0.2 # seconds between connection attempts
MICI_FORCE_TEXTURE_CAMERA = os.getenv("MICI_FORCE_TEXTURE_CAMERA", "0") == "1"
# One stale frame can be normal ring-buffer reuse; repeated consecutive regressions demote EGL.
EGL_REGRESSIVE_FRAME_FALLBACK_THRESHOLD = 3
VERSION = """
#version 300 es
@@ -38,32 +45,48 @@ void main() {
}
"""
# Choose fragment shader based on platform capabilities
if TICI:
FRAME_FRAGMENT_SHADER = """
#version 300 es
#extension GL_OES_EGL_image_external_essl3 : enable
precision mediump float;
in vec2 fragTexCoord;
uniform samplerExternalOES texture0;
out vec4 fragColor;
void main() {
vec4 color = texture(texture0, fragTexCoord);
fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a);
FRAME_FRAGMENT_SHADER_EXTERNAL = """
#version 300 es
#extension GL_OES_EGL_image_external_essl3 : enable
precision mediump float;
in vec2 fragTexCoord;
uniform samplerExternalOES texture0;
uniform int enhance_driver;
out vec4 fragColor;
void main() {
vec4 color = texture(texture0, fragTexCoord);
color.rgb = pow(color.rgb, vec3(1.0/1.28));
if (enhance_driver == 1) {
float brightness = 1.1;
color.rgb = color.rgb + 0.15;
color.rgb = clamp((color.rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0);
color.rgb = color.rgb * color.rgb * (3.0 - 2.0 * color.rgb);
color.rgb = pow(color.rgb, vec3(0.8));
}
"""
else:
FRAME_FRAGMENT_SHADER = VERSION + """
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
out vec4 fragColor;
void main() {
float y = texture(texture0, fragTexCoord).r;
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0);
fragColor = vec4(color.rgb, color.a);
}
"""
FRAME_FRAGMENT_SHADER_YUV = VERSION + """
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
uniform int enhance_driver;
out vec4 fragColor;
void main() {
float y = texture(texture0, fragTexCoord).r;
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
vec3 rgb = vec3(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x);
if (enhance_driver == 1) {
float brightness = 1.1;
rgb = rgb + 0.15;
rgb = clamp((rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0);
rgb = rgb * rgb * (3.0 - 2.0 * rgb);
rgb = pow(rgb, vec3(0.8));
}
"""
fragColor = vec4(rgb, 1.0);
}
"""
class CameraView(Widget):
@@ -71,7 +94,7 @@ class CameraView(Widget):
super().__init__()
self._name = name
# Primary stream
self.client = VisionIpcClient(name, stream_type, conflate=True)
self.client: VisionIpcClient | None = None
self._stream_type = stream_type
self.available_streams: list[VisionStreamType] = []
@@ -82,8 +105,18 @@ class CameraView(Widget):
self._texture_needs_update = True
self.last_connection_attempt: float = 0.0
self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER)
self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1
self._use_egl = TICI and not MICI_FORCE_TEXTURE_CAMERA and init_egl()
if TICI and MICI_FORCE_TEXTURE_CAMERA:
cloudlog.warning("CameraView EGL disabled by MICI_FORCE_TEXTURE_CAMERA, using texture rendering")
elif TICI and not self._use_egl:
cloudlog.error("CameraView EGL init failed, falling back to texture rendering")
self._enhance_driver_val = rl.ffi.new("int[1]", [0])
self._load_frame_shader()
if self._use_egl and not self.shader.id:
cloudlog.error("CameraView EGL shader failed, falling back to texture rendering")
self._use_egl = False
self._load_frame_shader()
self.frame: VisionBuf | None = None
self._last_frame_id = -1
@@ -97,12 +130,17 @@ class CameraView(Widget):
self._placeholder_color: rl.Color | None = None
self._closed = False
self._onroad_reentry_pending = False
self._reentry_stream_selected = False
# Initialize EGL for zero-copy rendering on TICI
if TICI:
if not init_egl():
raise RuntimeError("Failed to initialize EGL")
self._create_egl_texture()
if self._use_egl and not self._create_egl_texture():
cloudlog.error("CameraView EGL texture creation failed, falling back to texture rendering")
self._use_egl = False
if self.shader and self.shader.id:
rl.unload_shader(self.shader)
self.shader.id = 0
self._load_frame_shader()
cloudlog.info(f"CameraView using {'EGL zero-copy' if self._use_egl else 'texture-copy'} rendering for {stream_type}")
self_ref = weakref.ref(self)
@@ -116,30 +154,43 @@ class CameraView(Widget):
def _offroad_transition(self):
self._reset_camera_connection()
def _reset_camera_connection(self):
# EGL images and VisionBuf objects both retain the imported camera buffer.
# Release them on every road-state transition instead of pinning the old
# camerad allocation until this view happens to render again.
def _retire_active_client(self) -> None:
"""Release graphics, frame, and client as one camera generation."""
self._clear_textures()
self.frame = None
self.client = None
def _reset_camera_connection(self):
self._cancel_pending_switch()
self._retire_active_client()
self._last_frame_id = -1
self._regressive_frame_count = 0
self.available_streams.clear()
self.client = VisionIpcClient(self._name, self._stream_type, conflate=True)
self._target_client = None
self._target_stream_type = None
self._switching = False
self._texture_needs_update = True
self.last_connection_attempt = 0.0
self._onroad_reentry_pending = ui_state.is_onroad()
self._reentry_stream_selected = False
def _set_placeholder_color(self, color: rl.Color):
"""Set a placeholder color to be drawn when no frame is available."""
self._placeholder_color = color
def _refresh_available_streams(self) -> None:
streams = VisionIpcClient.available_streams(self._name, block=False)
if streams:
self.available_streams = list(streams)
def switch_stream(self, stream_type: VisionStreamType) -> None:
if self._stream_type == stream_type:
if getattr(self, "_onroad_reentry_pending", False):
self._select_reentry_stream(stream_type)
return
if self._switching and self._target_stream_type == stream_type:
if self._switching:
if self._target_stream_type == stream_type:
return
self._cancel_pending_switch()
if self._stream_type == stream_type:
return
cloudlog.debug(f'Preparing switch from {self._stream_type} to {stream_type}')
@@ -151,6 +202,32 @@ class CameraView(Widget):
self._target_client = VisionIpcClient(self._name, stream_type, conflate=True)
self._switching = True
def _cancel_pending_switch(self) -> None:
if self._target_client is not None:
cloudlog.debug(f"Cancelling pending camera switch to {self._target_stream_type}")
self._target_client = None
self._target_stream_type = None
self._switching = False
def _discard_pending_client(self) -> None:
"""Discard a failed candidate while retaining the requested stream."""
self._target_client = None
self._switching = False
def _select_reentry_stream(self, stream_type: VisionStreamType) -> None:
"""Select the desired stream before displaying any post-transition frame."""
self._cancel_pending_switch()
if self._stream_type != stream_type:
self._retire_active_client()
self._stream_type = stream_type
self.frame = None
self._last_frame_id = -1
self._regressive_frame_count = 0
self._texture_needs_update = True
self._reentry_stream_selected = True
@property
def stream_type(self) -> VisionStreamType:
return self._stream_type
@@ -164,17 +241,19 @@ class CameraView(Widget):
if callback is not None:
ui_state.remove_offroad_transition_callback(callback)
self._offroad_transition_callback = None
self._clear_textures()
self._cancel_pending_switch()
self._retire_active_client()
# Clean up shader
if self.shader and self.shader.id:
rl.unload_shader(self.shader)
self.shader.id = 0
self.frame = None
self._last_frame_id = -1
self.available_streams.clear()
self.client = None
self._target_client = None
self._onroad_reentry_pending = False
self._reentry_stream_selected = False
def __del__(self):
self.close()
@@ -201,13 +280,15 @@ class CameraView(Widget):
if self._switching:
self._handle_switch()
if self._onroad_reentry_pending and not self._reentry_stream_selected:
# Standalone CameraView users have no higher-level stream selector.
self._select_reentry_stream(self._stream_type)
if not self._ensure_connection():
self._draw_placeholder(rect)
return
# An EGL image references camerad's reusable ring-buffer slot. Account for
# that slot advancing before accepting another (possibly older) slot.
if TICI:
if self._use_egl:
self._observe_displayed_frame()
# Try to get a new buffer without blocking
@@ -241,16 +322,34 @@ class CameraView(Widget):
dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y)
# Render with appropriate method
if TICI:
self._render_egl(src_rect, dst_rect)
else:
if self._use_egl:
try:
rendered = self._render_egl(src_rect, dst_rect)
except Exception:
cloudlog.exception("CameraView EGL rendering failed")
rendered = False
if not rendered:
self._fallback_to_textures("EGL frame rendering failed")
if not self._use_egl:
self._render_textures(src_rect, dst_rect)
def _draw_placeholder(self, rect: rl.Rectangle):
if self._placeholder_color:
rl.draw_rectangle_rec(rect, self._placeholder_color)
def _load_frame_shader(self) -> None:
frame_shader = FRAME_FRAGMENT_SHADER_EXTERNAL if self._use_egl else FRAME_FRAGMENT_SHADER_YUV
self.shader = rl.load_shader_from_memory(VERTEX_SHADER, frame_shader)
self._texture1_loc = -1 if self._use_egl else rl.get_shader_location(self.shader, "texture1")
self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver")
def _update_shader_state(self) -> None:
self._enhance_driver_val[0] = 1 if self._stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0
if self._enhance_driver_loc >= 0:
rl.set_shader_value(self.shader, self._enhance_driver_loc, self._enhance_driver_val,
rl.ShaderUniformDataType.SHADER_UNIFORM_INT)
def _observe_displayed_frame(self) -> None:
if self.frame is not None:
client_frame_id = getattr(self.client, "frame_id", -1) if hasattr(self, "client") and self.client is not None else -1
@@ -259,50 +358,88 @@ class CameraView(Widget):
def _accept_frame(self, frame: VisionBuf, packet_frame_id: int) -> bool:
content_frame_id = int(getattr(frame, "frame_id", packet_frame_id))
if content_frame_id != packet_frame_id:
cloudlog.debug(
f"Dropping inconsistent {self._name} frame: content={content_frame_id}, packet={packet_frame_id}"
)
return False
if content_frame_id < self._last_frame_id:
self._regressive_frame_count += 1
if self._regressive_frame_count == 1 or self._regressive_frame_count % 100 == 0:
message = f"Dropping regressive {self._name} frame: content={content_frame_id}, packet={packet_frame_id}, "
message += f"displayed={self._last_frame_id}, idx={frame.idx}, count={self._regressive_frame_count}"
cloudlog.warning(message)
if getattr(self, "_use_egl", False) and self._regressive_frame_count >= EGL_REGRESSIVE_FRAME_FALLBACK_THRESHOLD:
self._fallback_to_textures("repeated regressive frames")
return False
self.frame = frame
self._last_frame_id = content_frame_id
self._regressive_frame_count = 0
self._texture_needs_update = True
self._onroad_reentry_pending = False
self._reentry_stream_selected = False
return True
def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
"""Render using EGL for direct buffer access"""
if self.frame is None or self.egl_texture is None:
return
def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> bool:
"""Render using EGL for direct buffer access."""
if self.frame is None or self.egl_texture is None or not self.egl_texture.id:
return False
idx = self.frame.idx
egl_image = self.egl_images.get(idx)
# Create EGL image if needed
if egl_image is None:
egl_image = create_egl_image(self.frame.width, self.frame.height, self.frame.stride, self.frame.fd, self.frame.uv_offset)
if egl_image:
self.egl_images[idx] = egl_image
else:
return
if egl_image is None:
return False
self.egl_images[idx] = egl_image
# Update texture dimensions to match current frame
self.egl_texture.width = self.frame.width
self.egl_texture.height = self.frame.height
# Bind the EGL image to our texture
bind_egl_image_to_texture(self.egl_texture.id, egl_image)
# Render with shader
rl.begin_shader_mode(self.shader)
rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
try:
self._update_shader_state()
rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
finally:
rl.end_shader_mode()
return True
def _fallback_to_textures(self, reason: str) -> None:
if not self._use_egl:
return
cloudlog.error(f"CameraView switching from EGL to texture rendering: {reason}")
self._use_egl = False
try:
self._clear_textures()
except Exception:
cloudlog.exception("CameraView EGL cleanup failed during texture fallback")
if self.shader and self.shader.id:
try:
rl.unload_shader(self.shader)
except Exception:
cloudlog.exception("CameraView EGL shader cleanup failed during texture fallback")
self.shader.id = 0
try:
self._load_frame_shader()
self._initialize_textures()
self._texture_needs_update = True
except Exception:
cloudlog.exception("CameraView texture fallback initialization failed")
def _render_textures(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
"""Render using texture copies"""
if not self.texture_y or not self.texture_uv or self.frame is None:
"""Copy camera data into ordinary Raylib textures before drawing.
Raylib batches camera draws as GL_TEXTURE_2D. Imported EGL images are
GL_TEXTURE_EXTERNAL_OES objects and cannot safely pass through that path;
copying also prevents the GPU from sampling camerad's reusable buffers
after they have been handed back to the producer.
"""
if (self.texture_y is None or not self.texture_y.id or
self.texture_uv is None or not self.texture_uv.id or self.frame is None):
return
# Update textures with new frame data
@@ -316,33 +453,45 @@ class CameraView(Widget):
# Render with shader
rl.begin_shader_mode(self.shader)
rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv)
rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
try:
self._update_shader_state()
rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv)
rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
finally:
rl.end_shader_mode()
def _ensure_connection(self) -> bool:
if not self.client.is_connected():
self.frame = None
self._last_frame_id = -1
self.available_streams.clear()
if self.client is not None and self.client.is_connected():
return True
# Throttle connection attempts
current_time = rl.get_time()
if current_time - self.last_connection_attempt < CONNECTION_RETRY_INTERVAL:
return False
self.last_connection_attempt = current_time
# A pending candidate owns the connection attempt. Poll it until its first
# frame arrives instead of reconnecting the same client in place.
if self._switching:
self._handle_switch()
return self.client is not None and self.client.is_connected()
# A GL texture can retain the last EGL image after camerad exits. Release
# it before connect() frees and replaces the client's imported buffers.
self._clear_textures()
if not self.client.connect(False) or not self.client.num_buffers:
return False
if self.client is not None:
self._retire_active_client()
self._last_frame_id = -1
self._regressive_frame_count = 0
self.available_streams.clear()
cloudlog.debug(f"Connected to {self._name} stream: {self._stream_type}, buffers: {self.client.num_buffers}")
self._initialize_textures()
self.available_streams = self.client.available_streams(self._name, block=False)
# Throttle connection attempts
current_time = rl.get_time()
if current_time - self.last_connection_attempt < CONNECTION_RETRY_INTERVAL:
return False
self.last_connection_attempt = current_time
return True
# Do not create a client until camerad advertises the requested stream.
stream_type = self._target_stream_type or self._stream_type
if stream_type not in VisionIpcClient.available_streams(self._name, block=False):
return False
self._target_stream_type = stream_type
self._target_client = VisionIpcClient(self._name, stream_type, conflate=True)
self._switching = True
self._handle_switch()
return self.client is not None and self.client.is_connected()
def _handle_switch(self) -> None:
"""Check if target stream is ready and switch immediately."""
@@ -352,6 +501,7 @@ class CameraView(Widget):
# Try to connect target if needed
if not self._target_client.is_connected():
if not self._target_client.connect(False) or not self._target_client.num_buffers:
self._discard_pending_client()
return
cloudlog.debug(f"Target stream connected: {self._target_stream_type}")
@@ -359,66 +509,113 @@ class CameraView(Widget):
# Check if target has frames ready
target_frame = self._target_client.recv(timeout_ms=0)
if target_frame:
self.frame = target_frame # Update current frame to target frame
self._complete_switch()
packet_frame_id = int(getattr(self._target_client, "frame_id", -1))
content_frame_id = int(getattr(target_frame, "frame_id", packet_frame_id))
if content_frame_id != packet_frame_id:
cloudlog.warning(
f"Discarding inconsistent {self._name} target frame: "
f"content={content_frame_id}, packet={packet_frame_id}, stream={self._target_stream_type}"
)
self._discard_pending_client()
return
self._complete_switch(target_frame)
elif not self._target_client.is_connected():
# A failed recv can invalidate the server/buffer generation. Never
# reconnect this client; the next attempt must use a fresh candidate.
self._discard_pending_client()
def _complete_switch(self) -> None:
def _complete_switch(self, target_frame: VisionBuf) -> None:
"""Instantly switch to target stream."""
cloudlog.debug(f"Switching to {self._target_stream_type}")
# Delete the GL texture before releasing the old client. Merely destroying
# the EGLImage handle leaves its storage alive while a texture sibling exists.
self._clear_textures()
# Switch to target
self.client = self._target_client
self._stream_type = self._target_stream_type
client_frame_id = getattr(self.client, "frame_id", -1) if hasattr(self, "client") and self.client is not None else -1
self._last_frame_id = int(getattr(self.frame, "frame_id", client_frame_id)) if self.frame is not None else -1
self._texture_needs_update = True
# Reset state
target_client = self._target_client
target_stream_type = self._target_stream_type
self._target_client = None
self._target_stream_type = None
self._switching = False
# Retire the old generation before exposing the new client and frame.
self._retire_active_client()
# Switch to target
self.client = target_client
self._stream_type = target_stream_type
self.frame = target_frame
client_frame_id = getattr(self.client, "frame_id", -1) if self.client is not None else -1
self._last_frame_id = int(getattr(self.frame, "frame_id", client_frame_id)) if self.frame is not None else -1
self._regressive_frame_count = 0
self._texture_needs_update = True
self._onroad_reentry_pending = False
self._reentry_stream_selected = False
# Initialize textures for new stream
self._initialize_textures()
available_streams = getattr(self.client, "available_streams", None)
if available_streams is not None:
self.available_streams = available_streams(self._name, block=False)
def _initialize_textures(self):
self._clear_textures()
if TICI:
self._create_egl_texture()
if self._use_egl:
if not self._create_egl_texture():
self._fallback_to_textures("EGL texture creation failed")
else:
self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
if not self.texture_y.id or not self.texture_uv.id:
cloudlog.error("CameraView texture-copy texture creation failed")
self._clear_textures()
def _create_egl_texture(self):
# A fresh texture has no EGL image sibling from a previous camera client.
temp_image = rl.gen_image_color(1, 1, rl.BLACK)
self.egl_texture = rl.load_texture_from_image(temp_image)
rl.unload_image(temp_image)
def _create_egl_texture(self) -> bool:
temp_image = None
try:
temp_image = rl.gen_image_color(1, 1, rl.BLACK)
texture = rl.load_texture_from_image(temp_image)
if texture is None or not texture.id:
self.egl_texture = None
return False
self.egl_texture = texture
return True
except Exception:
self.egl_texture = None
cloudlog.exception("CameraView failed to create EGL texture")
return False
finally:
if temp_image is not None:
try:
rl.unload_image(temp_image)
except Exception:
cloudlog.exception("CameraView failed to unload temporary EGL image")
def _clear_textures(self):
if self.texture_y and self.texture_y.id:
rl.unload_texture(self.texture_y)
if ((self.egl_texture is not None or self.egl_images) and is_egl_initialized()):
try:
# Raylib queues draw calls. Submit them before waiting for the GPU so
# no pending batch can still reference an EGL-backed texture.
rl.rl_draw_render_batch_active()
finish_gl()
except Exception:
cloudlog.exception("CameraView failed to synchronize EGL resources")
if self.texture_y is not None:
if self.texture_y.id:
rl.unload_texture(self.texture_y)
self.texture_y = None
if self.texture_uv and self.texture_uv.id:
rl.unload_texture(self.texture_uv)
if self.texture_uv is not None:
if self.texture_uv.id:
rl.unload_texture(self.texture_uv)
self.texture_uv = None
# Delete the texture first. eglDestroyImageKHR only destroys the EGLImage
# handle; the image storage stays alive while a GL texture sibling exists.
if TICI:
if self.egl_texture and self.egl_texture.id:
rl.unload_texture(self.egl_texture)
self.egl_texture = None
if self.egl_texture and self.egl_texture.id:
rl.unload_texture(self.egl_texture)
self.egl_texture = None
for data in self.egl_images.values():
destroy_egl_image(data)
self.egl_images = {}
for data in self.egl_images.values():
destroy_egl_image(data)
self.egl_images = {}
if __name__ == "__main__":
selfdrive/ui/tests/test_camera_frame_order.py
+398 -9
View File
@@ -1,3 +1,5 @@
from types import SimpleNamespace
import pytest
from openpilot.selfdrive.ui.mici.onroad import cameraview as mici_cameraview
@@ -10,9 +12,10 @@ class FakeFrame:
self.idx = idx
def _camera_view(cameraview):
view = cameraview.CameraView.__new__(cameraview.CameraView)
def _camera_view():
view = big_cameraview.CameraView.__new__(big_cameraview.CameraView)
view._name = "camerad"
view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_ROAD
view.frame = None
view._last_frame_id = -1
view._regressive_frame_count = 0
@@ -21,10 +24,88 @@ def _camera_view(cameraview):
return view
@pytest.mark.parametrize("cameraview", [big_cameraview, mici_cameraview])
def test_reused_egl_slot_cannot_move_camera_backwards(monkeypatch, cameraview):
monkeypatch.setattr(cameraview.cloudlog, "warning", lambda *_args, **_kwargs: None)
view = _camera_view(cameraview)
def test_mici_uses_shared_camera_view():
assert mici_cameraview.CameraView is big_cameraview.CameraView
def test_pending_switch_is_cancelled_when_requested_stream_is_current():
view = _camera_view()
view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_ROAD
view._target_stream_type = big_cameraview.VisionStreamType.VISION_STREAM_DRIVER
view._target_client = object()
view._switching = True
view.switch_stream(big_cameraview.VisionStreamType.VISION_STREAM_ROAD)
assert view._target_client is None
assert view._target_stream_type is None
assert not view._switching
def test_onroad_reentry_selects_requested_stream_before_rendering(monkeypatch):
view = _camera_view()
view._name = "camerad"
view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_WIDE_ROAD
view.client = object()
view._target_client = object()
view._target_stream_type = big_cameraview.VisionStreamType.VISION_STREAM_ROAD
view._switching = True
view._onroad_reentry_pending = True
view._reentry_stream_selected = False
view._clear_textures = lambda: None
clients = []
class FakeClient:
def __init__(self, name, stream_type, conflate):
self.name = name
self.stream_type = stream_type
self.conflate = conflate
clients.append(self)
monkeypatch.setattr(big_cameraview, "VisionIpcClient", FakeClient)
view.switch_stream(big_cameraview.VisionStreamType.VISION_STREAM_ROAD)
assert clients == []
assert view.client is None
assert view.stream_type == big_cameraview.VisionStreamType.VISION_STREAM_ROAD
assert view._target_client is None
assert view._target_stream_type is None
assert not view._switching
assert view._reentry_stream_selected
def test_onroad_reentry_guard_clears_on_first_fresh_frame():
view = _camera_view()
view._onroad_reentry_pending = True
view._reentry_stream_selected = True
assert view._accept_frame(FakeFrame(frame_id=1, idx=0), packet_frame_id=1)
assert not view._onroad_reentry_pending
assert not view._reentry_stream_selected
def test_standalone_camera_reentry_selects_configured_stream():
view = _camera_view()
view._switching = False
view._onroad_reentry_pending = True
view._reentry_stream_selected = False
selected = []
placeholders = []
view._select_reentry_stream = lambda stream_type: (
selected.append(stream_type), setattr(view, "_reentry_stream_selected", True)
)
view._draw_placeholder = lambda rect: placeholders.append(rect)
view._ensure_connection = lambda: False
view._render(object())
assert selected == [view._stream_type]
assert len(placeholders) == 1
def test_reused_egl_slot_cannot_move_camera_backwards(monkeypatch):
monkeypatch.setattr(big_cameraview.cloudlog, "warning", lambda *_args, **_kwargs: None)
view = _camera_view()
displayed = FakeFrame(frame_id=10, idx=0)
assert view._accept_frame(displayed, packet_frame_id=10)
@@ -40,13 +121,321 @@ def test_reused_egl_slot_cannot_move_camera_backwards(monkeypatch, cameraview):
assert view._regressive_frame_count == 1
@pytest.mark.parametrize("cameraview", [big_cameraview, mici_cameraview])
def test_newer_camera_frame_is_accepted(cameraview):
view = _camera_view(cameraview)
def test_newer_camera_frame_is_accepted():
view = _camera_view()
view._last_frame_id = 30
view._regressive_frame_count = 2
newer = FakeFrame(frame_id=31, idx=2)
assert view._accept_frame(newer, packet_frame_id=31)
assert view.frame is newer
assert view._last_frame_id == 31
assert view._regressive_frame_count == 0
assert view._texture_needs_update
def test_shared_camera_has_upstream_shaders_and_driver_enhancement():
assert "samplerExternalOES" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL
assert "pow(color.rgb, vec3(1.0/1.28))" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL
assert "uniform sampler2D texture0" in big_cameraview.FRAME_FRAGMENT_SHADER_YUV
assert "uniform sampler2D texture1" in big_cameraview.FRAME_FRAGMENT_SHADER_YUV
assert "uniform int enhance_driver" in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL
assert "uniform int enhance_driver" in big_cameraview.FRAME_FRAGMENT_SHADER_YUV
assert "uniform int engaged" not in big_cameraview.FRAME_FRAGMENT_SHADER_EXTERNAL
assert "uniform int engaged" not in big_cameraview.FRAME_FRAGMENT_SHADER_YUV
assert hasattr(big_cameraview.CameraView, "_render_egl")
assert hasattr(big_cameraview.CameraView, "_fallback_to_textures")
def test_shared_camera_falls_back_after_repeated_regressive_frames(monkeypatch):
monkeypatch.setattr(big_cameraview.cloudlog, "warning", lambda *_args, **_kwargs: None)
view = _camera_view()
view._use_egl = True
view.frame = FakeFrame(frame_id=30, idx=0)
view._last_frame_id = 30
fallback_reasons = []
view._fallback_to_textures = fallback_reasons.append
for frame_id in (20, 19, 18):
assert not view._accept_frame(FakeFrame(frame_id=frame_id, idx=1), packet_frame_id=frame_id)
assert fallback_reasons == ["repeated regressive frames"]
assert view.frame.frame_id == 30
def test_shared_camera_fallback_reloads_texture_backend(monkeypatch):
view = _camera_view()
view._use_egl = True
view.shader = SimpleNamespace(id=1)
events = []
view._clear_textures = lambda: events.append("clear")
view._load_frame_shader = lambda: events.append(("shader", view._use_egl))
view._initialize_textures = lambda: events.append("textures")
monkeypatch.setattr(big_cameraview.cloudlog, "error", lambda *_args, **_kwargs: None)
monkeypatch.setattr(big_cameraview.rl, "unload_shader", lambda _shader: events.append("unload_shader"))
view._fallback_to_textures("test")
assert events == ["clear", "unload_shader", ("shader", False), "textures"]
assert not view._use_egl
def test_connection_retry_discards_failed_client_and_uses_fresh_candidate(monkeypatch):
view = _camera_view()
view._name = "camerad"
view._clear_textures = lambda: None
view.client = SimpleNamespace(is_connected=lambda: False)
view._target_client = None
view._target_stream_type = None
view._switching = False
view.available_streams = []
view.last_connection_attempt = 0.0
candidates = []
class FakeClient:
@staticmethod
def available_streams(_name, block=False):
return [view._stream_type]
def __init__(self, *_args, **_kwargs):
candidates.append(self)
self.connected = False
self.num_buffers = 0
def is_connected(self):
return self.connected
def connect(self, _block):
return False
monkeypatch.setattr(big_cameraview, "VisionIpcClient", FakeClient)
monkeypatch.setattr(big_cameraview.rl, "get_time", lambda: 1.0)
assert not view._ensure_connection()
assert view.client is None
assert len(candidates) == 1
monkeypatch.setattr(big_cameraview.rl, "get_time", lambda: 1.3)
assert not view._ensure_connection()
assert len(candidates) == 2
assert candidates[0] is not candidates[1]
def test_candidate_is_not_active_until_first_consistent_frame(monkeypatch):
view = _camera_view()
view._name = "camerad"
view._clear_textures = lambda: None
view._initialize_textures = lambda: None
view.client = None
view._target_client = None
view._target_stream_type = None
view._switching = False
view.available_streams = []
view.last_connection_attempt = 0.0
class FakeClient:
@staticmethod
def available_streams(_name, block=False):
return [view._stream_type]
def __init__(self, *_args, **_kwargs):
self.connected = False
self.num_buffers = 1
self.frame_id = -1
self.frames = [None, FakeFrame(frame_id=42, idx=0)]
def is_connected(self):
return self.connected
def connect(self, _block):
self.connected = True
return True
def recv(self, timeout_ms=0):
frame = self.frames.pop(0)
if frame is not None:
self.frame_id = frame.frame_id
return frame
monkeypatch.setattr(big_cameraview, "VisionIpcClient", FakeClient)
monkeypatch.setattr(big_cameraview.rl, "get_time", lambda: 1.0)
assert not view._ensure_connection()
assert view.client is None
candidate = view._target_client
assert candidate is not None
assert view._ensure_connection()
assert view.client is candidate
assert view.frame.frame_id == 42
assert view._target_client is None
assert not view._switching
def test_inconsistent_candidate_frame_is_discarded(monkeypatch):
view = _camera_view()
view._name = "camerad"
view._clear_textures = lambda: None
view.client = None
view._target_client = None
view._target_stream_type = None
view._switching = False
view.available_streams = []
view.last_connection_attempt = 0.0
class FakeClient:
@staticmethod
def available_streams(_name, block=False):
return [view._stream_type]
def __init__(self, *_args, **_kwargs):
self.connected = False
self.num_buffers = 1
self.frame_id = 10
def is_connected(self):
return self.connected
def connect(self, _block):
self.connected = True
return True
def recv(self, timeout_ms=0):
return FakeFrame(frame_id=9, idx=0)
monkeypatch.setattr(big_cameraview, "VisionIpcClient", FakeClient)
monkeypatch.setattr(big_cameraview.rl, "get_time", lambda: 1.0)
assert not view._ensure_connection()
assert view.client is None
assert view._target_client is None
assert view._target_stream_type == view._stream_type
assert not view._switching
def test_disconnected_candidate_is_discarded_without_reconnect():
view = _camera_view()
class Candidate:
num_buffers = 1
def __init__(self):
self.connected = True
def is_connected(self):
return self.connected
def connect(self, _block):
pytest.fail("discarded candidate was reconnected")
def recv(self, timeout_ms=0):
self.connected = False
return None
candidate = Candidate()
view._target_client = candidate
view._target_stream_type = view._stream_type
view._switching = True
view._handle_switch()
assert view._target_client is None
assert not view._switching
assert view._target_stream_type == view._stream_type
def test_steady_state_packet_content_mismatch_is_rejected():
view = _camera_view()
displayed = FakeFrame(frame_id=10, idx=0)
assert view._accept_frame(displayed, packet_frame_id=10)
delayed = FakeFrame(frame_id=12, idx=1)
assert not view._accept_frame(delayed, packet_frame_id=11)
assert view.frame is displayed
assert view._last_frame_id == 10
def test_egl_image_creation_failure_is_reported(monkeypatch):
view = _camera_view()
view.frame = SimpleNamespace(idx=0, width=1928, height=1208, stride=2048, fd=7, uv_offset=2473984)
view.egl_texture = SimpleNamespace(id=1)
view.egl_images = {}
monkeypatch.setattr(big_cameraview, "create_egl_image", lambda *_args: None)
assert not view._render_egl(None, None)
assert view.egl_images == {}
def test_invalid_egl_texture_is_reported_without_binding(monkeypatch):
view = _camera_view()
view.frame = SimpleNamespace(idx=0)
view.egl_texture = SimpleNamespace(id=0)
view.egl_images = {0: object()}
monkeypatch.setattr(big_cameraview, "bind_egl_image_to_texture",
lambda *_args: pytest.fail("invalid EGL texture was bound"))
assert not view._render_egl(None, None)
def test_egl_render_always_ends_shader_mode(monkeypatch):
view = _camera_view()
view.frame = SimpleNamespace(idx=0, width=1928, height=1208)
view.egl_texture = SimpleNamespace(id=1, width=0, height=0)
view.egl_images = {0: object()}
view.shader = SimpleNamespace(id=1)
view._update_shader_state = lambda: None
events = []
monkeypatch.setattr(big_cameraview, "bind_egl_image_to_texture", lambda *_args: None)
monkeypatch.setattr(big_cameraview.rl, "begin_shader_mode", lambda *_args: events.append("begin"))
def fail_draw(*_args):
raise RuntimeError("draw failed")
monkeypatch.setattr(big_cameraview.rl, "draw_texture_pro", fail_draw)
monkeypatch.setattr(big_cameraview.rl, "end_shader_mode", lambda: events.append("end"))
with pytest.raises(RuntimeError, match="draw failed"):
view._render_egl(None, None)
assert events == ["begin", "end"]
def test_driver_enhancement_tracks_active_stream(monkeypatch):
view = _camera_view()
view.shader = SimpleNamespace(id=1)
view._enhance_driver_loc = 2
view._enhance_driver_val = [0]
values = []
monkeypatch.setattr(big_cameraview.rl, "set_shader_value",
lambda _shader, _loc, value, _type: values.append(value[0]))
view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_ROAD
view._update_shader_state()
view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_DRIVER
view._update_shader_state()
view._stream_type = big_cameraview.VisionStreamType.VISION_STREAM_WIDE_ROAD
view._update_shader_state()
assert values == [0, 1, 0]
def test_texture_fallback_survives_egl_cleanup_failure(monkeypatch):
view = _camera_view()
view._use_egl = True
view.shader = SimpleNamespace(id=1)
events = []
def fail_cleanup():
raise RuntimeError("cleanup failed")
view._clear_textures = fail_cleanup
view._load_frame_shader = lambda: events.append(("shader", view._use_egl))
view._initialize_textures = lambda: events.append("textures")
monkeypatch.setattr(big_cameraview.cloudlog, "error", lambda *_args, **_kwargs: None)
monkeypatch.setattr(big_cameraview.cloudlog, "exception", lambda *_args, **_kwargs: None)
monkeypatch.setattr(big_cameraview.rl, "unload_shader", lambda _shader: events.append("unload_shader"))
view._fallback_to_textures("test")
assert not view._use_egl
assert events == ["unload_shader", ("shader", False), "textures"]
system/ui/lib/egl.py
+18 -2
View File
@@ -56,6 +56,7 @@ class EGLState:
get_error: Any = None
bind_texture: Any = None
active_texture: Any = None
gl_finish: Any = None
# Create a single instance of the state
@@ -92,6 +93,7 @@ def init_egl() -> bool:
void glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image);
void glBindTexture(GLenum target, unsigned int texture);
void glActiveTexture(GLenum texture);
void glFinish(void);
""")
# Load libraries
@@ -111,6 +113,7 @@ def init_egl() -> bool:
_egl.get_error = _egl.egl_lib.eglGetError
_egl.bind_texture = _egl.gles_lib.glBindTexture
_egl.active_texture = _egl.gles_lib.glActiveTexture
_egl.gl_finish = _egl.gles_lib.glFinish
# Initialize EGL display once here
_egl.display = _egl.get_current_display()
@@ -125,6 +128,15 @@ def init_egl() -> bool:
return False
def is_egl_initialized() -> bool:
return _egl.initialized
def finish_gl() -> None:
if _egl.initialized:
_egl.gl_finish()
def create_egl_image(width: int, height: int, stride: int, fd: int, uv_offset: int) -> EGLImage | None:
assert _egl.initialized, "EGL not initialized"
@@ -160,10 +172,12 @@ def create_egl_image(width: int, height: int, stride: int, fd: int, uv_offset: i
return EGLImage(egl_image=egl_image, fd=dup_fd)
def destroy_egl_image(egl_image: EGLImage) -> None:
def destroy_egl_image(egl_image: EGLImage) -> bool:
assert _egl.initialized, "EGL not initialized"
_egl.destroy_image_khr(_egl.display, egl_image.egl_image)
destroyed = bool(_egl.destroy_image_khr(_egl.display, egl_image.egl_image))
if not destroyed:
cloudlog.error(f"Failed to destroy EGL image: {_egl.get_error()}")
# Close the duplicated fd we created in create_egl_image()
# We need to handle OSError since the fd might already be closed
@@ -172,6 +186,8 @@ def destroy_egl_image(egl_image: EGLImage) -> None:
except OSError:
pass
return destroyed
def bind_egl_image_to_texture(texture_id: int, egl_image: EGLImage) -> None:
assert _egl.initialized, "EGL not initialized"