From d304d14342544829d61a35722e4d5c1143b0cdf3 Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Sat, 1 Aug 2026 17:28:44 -0500 Subject: [PATCH] Time Warp --- selfdrive/ui/mici/onroad/cameraview.py | 18 +++++--- .../ui/mici/tests/test_camera_cleanup.py | 32 +++++++++++++- selfdrive/ui/onroad/cameraview.py | 18 +++++--- system/ui/lib/egl.py | 44 +++++++++++++++++++ system/ui/lib/tests/test_egl.py | 34 ++++++++++++++ 5 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 system/ui/lib/tests/test_egl.py diff --git a/selfdrive/ui/mici/onroad/cameraview.py b/selfdrive/ui/mici/onroad/cameraview.py index 8cd0addf6..1cfc11a12 100644 --- a/selfdrive/ui/mici/onroad/cameraview.py +++ b/selfdrive/ui/mici/onroad/cameraview.py @@ -8,7 +8,8 @@ from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf from openpilot.common.swaglog import cloudlog from openpilot.system.hardware import TICI from openpilot.system.ui.lib.application import gui_app -from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage +from openpilot.system.ui.lib.egl import (init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, + create_external_texture, destroy_external_texture, EGLImage) from openpilot.system.ui.widgets import Widget from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus @@ -136,6 +137,7 @@ class CameraView(Widget): # EGL resources self.egl_images: dict[int, EGLImage] = {} self.egl_texture: rl.Texture | None = None + self._external_texture_id = 0 self._placeholder_color: rl.Color | None = None self._closed = False @@ -313,7 +315,7 @@ class CameraView(Widget): def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None: """Render using EGL for direct buffer access""" - if self.frame is None or self.egl_texture is None: + if self.frame is None or self.egl_texture is None or not self._external_texture_id: return idx = self.frame.idx @@ -332,7 +334,7 @@ class CameraView(Widget): self.egl_texture.height = self.frame.height # Bind the EGL image to our texture - bind_egl_image_to_texture(self.egl_texture.id, egl_image) + bind_egl_image_to_texture(self._external_texture_id, egl_image) # Render with shader rl.begin_shader_mode(self.shader) @@ -447,10 +449,12 @@ class CameraView(Widget): int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA)) def _create_egl_texture(self): - # A fresh texture has no EGL image sibling from a previous camera client. temp_image = rl.gen_image_color(1, 1, rl.BLACK) self.egl_texture = rl.load_texture_from_image(temp_image) rl.unload_image(temp_image) + self._external_texture_id = create_external_texture() + if not self._external_texture_id: + raise RuntimeError("Failed to create external camera texture") def _clear_textures(self): if self.texture_y and self.texture_y.id: @@ -461,9 +465,11 @@ class CameraView(Widget): rl.unload_texture(self.texture_uv) self.texture_uv = None - # Delete the texture first. eglDestroyImageKHR only destroys the EGLImage - # handle; the image storage stays alive while a GL texture sibling exists. if self._use_egl: + if self._external_texture_id: + destroy_external_texture(self._external_texture_id) + self._external_texture_id = 0 + if self.egl_texture and self.egl_texture.id: rl.unload_texture(self.egl_texture) self.egl_texture = None diff --git a/selfdrive/ui/mici/tests/test_camera_cleanup.py b/selfdrive/ui/mici/tests/test_camera_cleanup.py index a53d5ef7a..081806d5f 100644 --- a/selfdrive/ui/mici/tests/test_camera_cleanup.py +++ b/selfdrive/ui/mici/tests/test_camera_cleanup.py @@ -132,6 +132,7 @@ def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module): view.texture_y = None view.texture_uv = None view.egl_texture = SimpleNamespace(id=7) + view._external_texture_id = 11 view.egl_images = {0: object(), 1: object()} view._closed = True @@ -141,10 +142,39 @@ def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module): monkeypatch.setattr(module, "TICI", True) monkeypatch.setattr(module.rl, "unload_texture", lambda _texture: events.append("texture")) + monkeypatch.setattr(module, "destroy_external_texture", lambda _texture: events.append("external")) monkeypatch.setattr(module, "destroy_egl_image", lambda _image: events.append("image")) view._clear_textures() - assert events == ["texture", "image", "image"] + assert events == ["external", "texture", "image", "image"] assert view.egl_texture is None + assert view._external_texture_id == 0 assert view.egl_images == {} + + +@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview)) +def test_egl_render_keeps_external_and_raylib_texture_targets_separate(monkeypatch, module): + frame = SimpleNamespace(idx=3, width=1928, height=1208, stride=2048, fd=9, uv_offset=2473984) + image = object() + view = module.CameraView.__new__(module.CameraView) + view.frame = frame + view.egl_texture = SimpleNamespace(id=7, width=1, height=1) + view._external_texture_id = 11 + view.egl_images = {frame.idx: image} + view.shader = object() + view._closed = True + view._update_texture_color_filtering = lambda: None + + bound = [] + drawn = [] + monkeypatch.setattr(module, "bind_egl_image_to_texture", lambda texture_id, egl_image: bound.append((texture_id, egl_image))) + monkeypatch.setattr(module.rl, "begin_shader_mode", lambda _shader: None) + monkeypatch.setattr(module.rl, "end_shader_mode", lambda: None) + monkeypatch.setattr(module.rl, "draw_texture_pro", lambda texture, *_args: drawn.append(texture.id)) + + rect = SimpleNamespace() + view._render_egl(rect, rect) + + assert bound == [(11, image)] + assert drawn == [7] diff --git a/selfdrive/ui/onroad/cameraview.py b/selfdrive/ui/onroad/cameraview.py index 1e1f53e5c..e3cdeafbe 100644 --- a/selfdrive/ui/onroad/cameraview.py +++ b/selfdrive/ui/onroad/cameraview.py @@ -7,7 +7,8 @@ from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf from openpilot.common.swaglog import cloudlog from openpilot.system.hardware import TICI from openpilot.system.ui.lib.application import gui_app -from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage +from openpilot.system.ui.lib.egl import (init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, + create_external_texture, destroy_external_texture, EGLImage) from openpilot.system.ui.widgets import Widget from openpilot.selfdrive.ui.ui_state import ui_state @@ -94,6 +95,7 @@ class CameraView(Widget): # EGL resources self.egl_images: dict[int, EGLImage] = {} self.egl_texture: rl.Texture | None = None + self._external_texture_id = 0 self._placeholder_color: rl.Color | None = None self._closed = False @@ -274,7 +276,7 @@ class CameraView(Widget): def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None: """Render using EGL for direct buffer access""" - if self.frame is None or self.egl_texture is None: + if self.frame is None or self.egl_texture is None or not self._external_texture_id: return idx = self.frame.idx @@ -293,7 +295,7 @@ class CameraView(Widget): self.egl_texture.height = self.frame.height # Bind the EGL image to our texture - bind_egl_image_to_texture(self.egl_texture.id, egl_image) + bind_egl_image_to_texture(self._external_texture_id, egl_image) # Render with shader rl.begin_shader_mode(self.shader) @@ -396,10 +398,12 @@ class CameraView(Widget): int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA)) def _create_egl_texture(self): - # A fresh texture has no EGL image sibling from a previous camera client. temp_image = rl.gen_image_color(1, 1, rl.BLACK) self.egl_texture = rl.load_texture_from_image(temp_image) rl.unload_image(temp_image) + self._external_texture_id = create_external_texture() + if not self._external_texture_id: + raise RuntimeError("Failed to create external camera texture") def _clear_textures(self): if self.texture_y and self.texture_y.id: @@ -410,9 +414,11 @@ class CameraView(Widget): rl.unload_texture(self.texture_uv) self.texture_uv = None - # Delete the texture first. eglDestroyImageKHR only destroys the EGLImage - # handle; the image storage stays alive while a GL texture sibling exists. if TICI: + if self._external_texture_id: + destroy_external_texture(self._external_texture_id) + self._external_texture_id = 0 + if self.egl_texture and self.egl_texture.id: rl.unload_texture(self.egl_texture) self.egl_texture = None diff --git a/system/ui/lib/egl.py b/system/ui/lib/egl.py index 69236482b..6676ca314 100644 --- a/system/ui/lib/egl.py +++ b/system/ui/lib/egl.py @@ -18,6 +18,7 @@ EGL_DMA_BUF_PLANE1_PITCH_EXT = 0x3277 EGL_NONE = 0x3038 GL_TEXTURE0 = 0x84C0 GL_TEXTURE_EXTERNAL_OES = 0x8D65 +GL_NO_ERROR = 0 # DRM Format for NV12 DRM_FORMAT_NV12 = 842094158 @@ -54,8 +55,11 @@ class EGLState: destroy_image_khr: Any = None image_target_texture: Any = None get_error: Any = None + get_gl_error: Any = None bind_texture: Any = None active_texture: Any = None + gen_textures: Any = None + delete_textures: Any = None # Create a single instance of the state @@ -92,6 +96,9 @@ def init_egl() -> bool: void glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image); void glBindTexture(GLenum target, unsigned int texture); void glActiveTexture(GLenum texture); + void glGenTextures(int n, unsigned int *textures); + void glDeleteTextures(int n, const unsigned int *textures); + GLenum glGetError(void); """) # Load libraries @@ -109,8 +116,11 @@ def init_egl() -> bool: _egl.destroy_image_khr = _egl.egl_lib.eglDestroyImageKHR _egl.image_target_texture = _egl.gles_lib.glEGLImageTargetTexture2DOES _egl.get_error = _egl.egl_lib.eglGetError + _egl.get_gl_error = _egl.gles_lib.glGetError _egl.bind_texture = _egl.gles_lib.glBindTexture _egl.active_texture = _egl.gles_lib.glActiveTexture + _egl.gen_textures = _egl.gles_lib.glGenTextures + _egl.delete_textures = _egl.gles_lib.glDeleteTextures # Initialize EGL display once here _egl.display = _egl.get_current_display() @@ -173,6 +183,40 @@ def destroy_egl_image(egl_image: EGLImage) -> None: pass +def create_external_texture() -> int: + """Create a texture name whose target is exclusively GL_TEXTURE_EXTERNAL_OES.""" + assert _egl.initialized, "EGL not initialized" + + while _egl.get_gl_error() != GL_NO_ERROR: + pass + + texture = _egl.ffi.new("unsigned int[1]") + _egl.gen_textures(1, texture) + texture_id = int(texture[0]) + if texture_id == 0: + cloudlog.error("Failed to generate external camera texture") + return 0 + + _egl.active_texture(GL_TEXTURE0) + _egl.bind_texture(GL_TEXTURE_EXTERNAL_OES, texture_id) + error = _egl.get_gl_error() + if error != GL_NO_ERROR: + cloudlog.error(f"Failed to bind external camera texture: GL error {error:#x}") + _egl.delete_textures(1, texture) + return 0 + + return texture_id + + +def destroy_external_texture(texture_id: int) -> None: + if not texture_id: + return + + assert _egl.initialized, "EGL not initialized" + texture = _egl.ffi.new("unsigned int[1]", [texture_id]) + _egl.delete_textures(1, texture) + + def bind_egl_image_to_texture(texture_id: int, egl_image: EGLImage) -> None: assert _egl.initialized, "EGL not initialized" diff --git a/system/ui/lib/tests/test_egl.py b/system/ui/lib/tests/test_egl.py new file mode 100644 index 000000000..1d3143a41 --- /dev/null +++ b/system/ui/lib/tests/test_egl.py @@ -0,0 +1,34 @@ +import cffi + +from openpilot.system.ui.lib import egl + + +def test_external_texture_has_its_own_gl_target(monkeypatch): + calls = [] + ffi = cffi.FFI() + + def gen_textures(count, textures): + calls.append(("generate", count)) + textures[0] = 23 + + state = egl.EGLState( + initialized=True, + ffi=ffi, + get_gl_error=lambda: egl.GL_NO_ERROR, + active_texture=lambda texture: calls.append(("active", texture)), + bind_texture=lambda target, texture: calls.append(("bind", target, texture)), + gen_textures=gen_textures, + delete_textures=lambda count, textures: calls.append(("delete", count, int(textures[0]))), + ) + monkeypatch.setattr(egl, "_egl", state) + + texture_id = egl.create_external_texture() + egl.destroy_external_texture(texture_id) + + assert texture_id == 23 + assert calls == [ + ("generate", 1), + ("active", egl.GL_TEXTURE0), + ("bind", egl.GL_TEXTURE_EXTERNAL_OES, 23), + ("delete", 1, 23), + ]