Time Warp

selfdrive/ui/mici/onroad/cameraview.py

@@ -8,7 +8,8 @@ from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.hardware import TICI
 from openpilot.system.ui.lib.application import gui_app
-from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage
+from openpilot.system.ui.lib.egl import (init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture,
+                                        create_external_texture, destroy_external_texture, EGLImage)
 from openpilot.system.ui.widgets import Widget
 from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus
 
@@ -136,6 +137,7 @@ class CameraView(Widget):
     # EGL resources
     self.egl_images: dict[int, EGLImage] = {}
     self.egl_texture: rl.Texture | None = None
+    self._external_texture_id = 0
 
     self._placeholder_color: rl.Color | None = None
     self._closed = False
@@ -313,7 +315,7 @@ class CameraView(Widget):
 
   def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
     """Render using EGL for direct buffer access"""
-    if self.frame is None or self.egl_texture is None:
+    if self.frame is None or self.egl_texture is None or not self._external_texture_id:
       return
 
     idx = self.frame.idx
@@ -332,7 +334,7 @@ class CameraView(Widget):
     self.egl_texture.height = self.frame.height
 
     # Bind the EGL image to our texture
-    bind_egl_image_to_texture(self.egl_texture.id, egl_image)
+    bind_egl_image_to_texture(self._external_texture_id, egl_image)
 
     # Render with shader
     rl.begin_shader_mode(self.shader)
@@ -447,10 +449,12 @@ class CameraView(Widget):
         int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
 
   def _create_egl_texture(self):
-    # A fresh texture has no EGL image sibling from a previous camera client.
     temp_image = rl.gen_image_color(1, 1, rl.BLACK)
     self.egl_texture = rl.load_texture_from_image(temp_image)
     rl.unload_image(temp_image)
+    self._external_texture_id = create_external_texture()
+    if not self._external_texture_id:
+      raise RuntimeError("Failed to create external camera texture")
 
   def _clear_textures(self):
     if self.texture_y and self.texture_y.id:
@@ -461,9 +465,11 @@ class CameraView(Widget):
       rl.unload_texture(self.texture_uv)
       self.texture_uv = None
 
-    # Delete the texture first. eglDestroyImageKHR only destroys the EGLImage
-    # handle; the image storage stays alive while a GL texture sibling exists.
     if self._use_egl:
+      if self._external_texture_id:
+        destroy_external_texture(self._external_texture_id)
+      self._external_texture_id = 0
+
       if self.egl_texture and self.egl_texture.id:
         rl.unload_texture(self.egl_texture)
       self.egl_texture = None

selfdrive/ui/mici/tests/test_camera_cleanup.py

@@ -132,6 +132,7 @@ def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module):
   view.texture_y = None
   view.texture_uv = None
   view.egl_texture = SimpleNamespace(id=7)
+  view._external_texture_id = 11
   view.egl_images = {0: object(), 1: object()}
   view._closed = True
 
@@ -141,10 +142,39 @@ def test_egl_cleanup_deletes_texture_before_images(monkeypatch, module):
     monkeypatch.setattr(module, "TICI", True)
 
   monkeypatch.setattr(module.rl, "unload_texture", lambda _texture: events.append("texture"))
+  monkeypatch.setattr(module, "destroy_external_texture", lambda _texture: events.append("external"))
   monkeypatch.setattr(module, "destroy_egl_image", lambda _image: events.append("image"))
 
   view._clear_textures()
 
-  assert events == ["texture", "image", "image"]
+  assert events == ["external", "texture", "image", "image"]
   assert view.egl_texture is None
+  assert view._external_texture_id == 0
   assert view.egl_images == {}
+
+
+@pytest.mark.parametrize("module", (mici_cameraview, big_cameraview))
+def test_egl_render_keeps_external_and_raylib_texture_targets_separate(monkeypatch, module):
+  frame = SimpleNamespace(idx=3, width=1928, height=1208, stride=2048, fd=9, uv_offset=2473984)
+  image = object()
+  view = module.CameraView.__new__(module.CameraView)
+  view.frame = frame
+  view.egl_texture = SimpleNamespace(id=7, width=1, height=1)
+  view._external_texture_id = 11
+  view.egl_images = {frame.idx: image}
+  view.shader = object()
+  view._closed = True
+  view._update_texture_color_filtering = lambda: None
+
+  bound = []
+  drawn = []
+  monkeypatch.setattr(module, "bind_egl_image_to_texture", lambda texture_id, egl_image: bound.append((texture_id, egl_image)))
+  monkeypatch.setattr(module.rl, "begin_shader_mode", lambda _shader: None)
+  monkeypatch.setattr(module.rl, "end_shader_mode", lambda: None)
+  monkeypatch.setattr(module.rl, "draw_texture_pro", lambda texture, *_args: drawn.append(texture.id))
+
+  rect = SimpleNamespace()
+  view._render_egl(rect, rect)
+
+  assert bound == [(11, image)]
+  assert drawn == [7]

selfdrive/ui/onroad/cameraview.py

@@ -7,7 +7,8 @@ from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.hardware import TICI
 from openpilot.system.ui.lib.application import gui_app
-from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage
+from openpilot.system.ui.lib.egl import (init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture,
+                                        create_external_texture, destroy_external_texture, EGLImage)
 from openpilot.system.ui.widgets import Widget
 from openpilot.selfdrive.ui.ui_state import ui_state
 
@@ -94,6 +95,7 @@ class CameraView(Widget):
     # EGL resources
     self.egl_images: dict[int, EGLImage] = {}
     self.egl_texture: rl.Texture | None = None
+    self._external_texture_id = 0
 
     self._placeholder_color: rl.Color | None = None
     self._closed = False
@@ -274,7 +276,7 @@ class CameraView(Widget):
 
   def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
     """Render using EGL for direct buffer access"""
-    if self.frame is None or self.egl_texture is None:
+    if self.frame is None or self.egl_texture is None or not self._external_texture_id:
       return
 
     idx = self.frame.idx
@@ -293,7 +295,7 @@ class CameraView(Widget):
     self.egl_texture.height = self.frame.height
 
     # Bind the EGL image to our texture
-    bind_egl_image_to_texture(self.egl_texture.id, egl_image)
+    bind_egl_image_to_texture(self._external_texture_id, egl_image)
 
     # Render with shader
     rl.begin_shader_mode(self.shader)
@@ -396,10 +398,12 @@ class CameraView(Widget):
         int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
 
   def _create_egl_texture(self):
-    # A fresh texture has no EGL image sibling from a previous camera client.
     temp_image = rl.gen_image_color(1, 1, rl.BLACK)
     self.egl_texture = rl.load_texture_from_image(temp_image)
     rl.unload_image(temp_image)
+    self._external_texture_id = create_external_texture()
+    if not self._external_texture_id:
+      raise RuntimeError("Failed to create external camera texture")
 
   def _clear_textures(self):
     if self.texture_y and self.texture_y.id:
@@ -410,9 +414,11 @@ class CameraView(Widget):
       rl.unload_texture(self.texture_uv)
       self.texture_uv = None
 
-    # Delete the texture first. eglDestroyImageKHR only destroys the EGLImage
-    # handle; the image storage stays alive while a GL texture sibling exists.
     if TICI:
+      if self._external_texture_id:
+        destroy_external_texture(self._external_texture_id)
+      self._external_texture_id = 0
+
       if self.egl_texture and self.egl_texture.id:
         rl.unload_texture(self.egl_texture)
       self.egl_texture = None

system/ui/lib/egl.py

@@ -18,6 +18,7 @@ EGL_DMA_BUF_PLANE1_PITCH_EXT = 0x3277
 EGL_NONE = 0x3038
 GL_TEXTURE0 = 0x84C0
 GL_TEXTURE_EXTERNAL_OES = 0x8D65
+GL_NO_ERROR = 0
 
 # DRM Format for NV12
 DRM_FORMAT_NV12 = 842094158
@@ -54,8 +55,11 @@ class EGLState:
   destroy_image_khr: Any = None
   image_target_texture: Any = None
   get_error: Any = None
+  get_gl_error: Any = None
   bind_texture: Any = None
   active_texture: Any = None
+  gen_textures: Any = None
+  delete_textures: Any = None
 
 
 # Create a single instance of the state
@@ -92,6 +96,9 @@ def init_egl() -> bool:
       void glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image);
       void glBindTexture(GLenum target, unsigned int texture);
       void glActiveTexture(GLenum texture);
+      void glGenTextures(int n, unsigned int *textures);
+      void glDeleteTextures(int n, const unsigned int *textures);
+      GLenum glGetError(void);
     """)
 
     # Load libraries
@@ -109,8 +116,11 @@ def init_egl() -> bool:
     _egl.destroy_image_khr = _egl.egl_lib.eglDestroyImageKHR
     _egl.image_target_texture = _egl.gles_lib.glEGLImageTargetTexture2DOES
     _egl.get_error = _egl.egl_lib.eglGetError
+    _egl.get_gl_error = _egl.gles_lib.glGetError
     _egl.bind_texture = _egl.gles_lib.glBindTexture
     _egl.active_texture = _egl.gles_lib.glActiveTexture
+    _egl.gen_textures = _egl.gles_lib.glGenTextures
+    _egl.delete_textures = _egl.gles_lib.glDeleteTextures
 
     # Initialize EGL display once here
     _egl.display = _egl.get_current_display()
@@ -173,6 +183,40 @@ def destroy_egl_image(egl_image: EGLImage) -> None:
     pass
 
 
+def create_external_texture() -> int:
+  """Create a texture name whose target is exclusively GL_TEXTURE_EXTERNAL_OES."""
+  assert _egl.initialized, "EGL not initialized"
+
+  while _egl.get_gl_error() != GL_NO_ERROR:
+    pass
+
+  texture = _egl.ffi.new("unsigned int[1]")
+  _egl.gen_textures(1, texture)
+  texture_id = int(texture[0])
+  if texture_id == 0:
+    cloudlog.error("Failed to generate external camera texture")
+    return 0
+
+  _egl.active_texture(GL_TEXTURE0)
+  _egl.bind_texture(GL_TEXTURE_EXTERNAL_OES, texture_id)
+  error = _egl.get_gl_error()
+  if error != GL_NO_ERROR:
+    cloudlog.error(f"Failed to bind external camera texture: GL error {error:#x}")
+    _egl.delete_textures(1, texture)
+    return 0
+
+  return texture_id
+
+
+def destroy_external_texture(texture_id: int) -> None:
+  if not texture_id:
+    return
+
+  assert _egl.initialized, "EGL not initialized"
+  texture = _egl.ffi.new("unsigned int[1]", [texture_id])
+  _egl.delete_textures(1, texture)
+
+
 def bind_egl_image_to_texture(texture_id: int, egl_image: EGLImage) -> None:
   assert _egl.initialized, "EGL not initialized"
 

system/ui/lib/tests/test_egl.py

@@ -0,0 +1,34 @@
+import cffi
+
+from openpilot.system.ui.lib import egl
+
+
+def test_external_texture_has_its_own_gl_target(monkeypatch):
+  calls = []
+  ffi = cffi.FFI()
+
+  def gen_textures(count, textures):
+    calls.append(("generate", count))
+    textures[0] = 23
+
+  state = egl.EGLState(
+    initialized=True,
+    ffi=ffi,
+    get_gl_error=lambda: egl.GL_NO_ERROR,
+    active_texture=lambda texture: calls.append(("active", texture)),
+    bind_texture=lambda target, texture: calls.append(("bind", target, texture)),
+    gen_textures=gen_textures,
+    delete_textures=lambda count, textures: calls.append(("delete", count, int(textures[0]))),
+  )
+  monkeypatch.setattr(egl, "_egl", state)
+
+  texture_id = egl.create_external_texture()
+  egl.destroy_external_texture(texture_id)
+
+  assert texture_id == 23
+  assert calls == [
+    ("generate", 1),
+    ("active", egl.GL_TEXTURE0),
+    ("bind", egl.GL_TEXTURE_EXTERNAL_OES, 23),
+    ("delete", 1, 23),
+  ]