diff --git a/selfdrive/ui/mici/onroad/cameraview.py b/selfdrive/ui/mici/onroad/cameraview.py index e5d45501..776bc8b6 100644 --- a/selfdrive/ui/mici/onroad/cameraview.py +++ b/selfdrive/ui/mici/onroad/cameraview.py @@ -8,7 +8,7 @@ from openpilot.system.hardware import TICI from openpilot.system.ui.lib.application import gui_app from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage from openpilot.system.ui.widgets import Widget -from openpilot.selfdrive.ui.ui_state import ui_state +from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus CONNECTION_RETRY_INTERVAL = 0.2 # seconds between connection attempts @@ -37,52 +37,70 @@ void main() { } """ -FRAME_FRAGMENT_SHADER_EGL = """ - #version 300 es - #extension GL_OES_EGL_image_external_essl3 : enable - precision mediump float; - in vec2 fragTexCoord; - uniform samplerExternalOES texture0; - out vec4 fragColor; - uniform int enhance_driver; +# Choose fragment shader based on platform capabilities +if TICI: + FRAME_FRAGMENT_SHADER = """ + #version 300 es + #extension GL_OES_EGL_image_external_essl3 : enable + precision mediump float; + in vec2 fragTexCoord; + uniform samplerExternalOES texture0; + out vec4 fragColor; + uniform int engaged; + uniform int enhance_driver; - void main() { - vec4 color = texture(texture0, fragTexCoord); - color.rgb = pow(color.rgb, vec3(1.0/1.28)); - if (enhance_driver == 1) { - float brightness = 1.1; - color.rgb = color.rgb + 0.15; - color.rgb = clamp((color.rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0); - color.rgb = color.rgb * color.rgb * (3.0 - 2.0 * color.rgb); - color.rgb = pow(color.rgb, vec3(0.8)); + void main() { + vec4 color = texture(texture0, fragTexCoord); + if (engaged == 1) { + float gray = dot(color.rgb, vec3(0.299, 0.587, 0.114)); // Luma + color.rgb = mix(vec3(gray), color.rgb, 0.2); // 20% saturation + color.rgb = clamp((color.rgb - 0.5) * 1.2 + 0.5, 0.0, 1.0); // +20% contrast + color.rgb = pow(color.rgb, vec3(1.0/1.28)); + } else { + color.rgb *= 0.85; // 85% opacity + } + if (enhance_driver == 1) { + float brightness = 1.1; + color.rgb = color.rgb + 0.15; + color.rgb = clamp((color.rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0); + color.rgb = color.rgb * color.rgb * (3.0 - 2.0 * color.rgb); + color.rgb = pow(color.rgb, vec3(0.8)); + } + fragColor = vec4(color.rgb, color.a); } - fragColor = vec4(color.rgb, color.a); - } - """ + """ +else: + FRAME_FRAGMENT_SHADER = VERSION + """ + in vec2 fragTexCoord; + uniform sampler2D texture0; + uniform sampler2D texture1; + out vec4 fragColor; + uniform int engaged; + uniform int enhance_driver; -FRAME_FRAGMENT_SHADER_TEXTURES = VERSION + """ - in vec2 fragTexCoord; - uniform sampler2D texture0; - uniform sampler2D texture1; - out vec4 fragColor; - uniform int enhance_driver; - - void main() { - float y = texture(texture0, fragTexCoord).r; - vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; - vec3 rgb = vec3(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x); - // TODO: the images out of camerad need some more correction and - // the ui should apply a gamma curve for the device display - if (enhance_driver == 1) { - float brightness = 1.1; - rgb = rgb + 0.15; - rgb = clamp((rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0); - rgb = rgb * rgb * (3.0 - 2.0 * rgb); - rgb = pow(rgb, vec3(0.8)); + void main() { + float y = texture(texture0, fragTexCoord).r; + vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; + vec3 rgb = vec3(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x); + if (engaged == 1) { + float gray = dot(rgb, vec3(0.299, 0.587, 0.114)); + rgb = mix(vec3(gray), rgb, 0.2); // 20% saturation + rgb = clamp((rgb - 0.5) * 1.2 + 0.5, 0.0, 1.0); // +20% contrast + } else { + rgb *= 0.85; // 85% opacity + } + // TODO: the images out of camerad need some more correction and + // the ui should apply a gamma curve for the device display + if (enhance_driver == 1) { + float brightness = 1.1; + rgb = rgb + 0.15; + rgb = clamp((rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0); + rgb = rgb * rgb * (3.0 - 2.0 * rgb); + rgb = pow(rgb, vec3(0.8)); + } + fragColor = vec4(rgb, 1.0); } - fragColor = vec4(rgb, 1.0); - } - """ + """ class CameraView(Widget): @@ -101,6 +119,12 @@ class CameraView(Widget): self._texture_needs_update = True self.last_connection_attempt: float = 0.0 + self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER) + self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1 + self._engaged_loc = rl.get_shader_location(self.shader, "engaged") + self._engaged_val = rl.ffi.new("int[1]", [1]) + self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver") + self._enhance_driver_val = rl.ffi.new("int[1]", [1 if stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0]) self.frame: VisionBuf | None = None self.texture_y: rl.Texture | None = None @@ -111,18 +135,12 @@ class CameraView(Widget): self.egl_texture: rl.Texture | None = None self._placeholder_color: rl.Color | None = None - self._use_egl = TICI and init_egl() - fragment_shader = FRAME_FRAGMENT_SHADER_EGL if self._use_egl else FRAME_FRAGMENT_SHADER_TEXTURES - self.shader = rl.load_shader_from_memory(VERTEX_SHADER, fragment_shader) - self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not self._use_egl else -1 - self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver") - self._enhance_driver_val = rl.ffi.new("int[1]", [1 if stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0]) + # Initialize EGL for zero-copy rendering on TICI + if TICI: + if not init_egl(): + raise RuntimeError("Failed to initialize EGL") - # Keep the UI alive if EGL cannot be used on device startup. - if TICI and not self._use_egl: - cloudlog.warning(f"Failed to initialize EGL for {self._name}; falling back to texture rendering") - elif self._use_egl: # Create a 1x1 pixel placeholder texture for EGL image binding temp_image = rl.gen_image_color(1, 1, rl.BLACK) self.egl_texture = rl.load_texture_from_image(temp_image) @@ -136,11 +154,11 @@ class CameraView(Widget): # Prevent old frames from showing when going onroad. Qt has a separate thread # which drains the VisionIpcClient SubSocket for us. Re-connecting is not enough # and only clears internal buffers, not the message queue. - self.frame = None self.available_streams.clear() if self.client: del self.client self.client = VisionIpcClient(self._name, self._stream_type, conflate=True) + self.frame = None def _set_placeholder_color(self, color: rl.Color): """Set a placeholder color to be drawn when no frame is available.""" @@ -170,7 +188,7 @@ class CameraView(Widget): self._clear_textures() # Clean up EGL texture - if self.egl_texture: + if TICI and self.egl_texture: rl.unload_texture(self.egl_texture) self.egl_texture = None @@ -245,7 +263,7 @@ class CameraView(Widget): dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y) # Render with appropriate method - if self._use_egl: + if TICI: self._render_egl(src_rect, dst_rect) else: self._render_textures(src_rect, dst_rect) @@ -279,7 +297,7 @@ class CameraView(Widget): # Render with shader rl.begin_shader_mode(self.shader) - self._update_shader_uniforms() + self._update_texture_color_filtering() rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) rl.end_shader_mode() @@ -299,12 +317,14 @@ class CameraView(Widget): # Render with shader rl.begin_shader_mode(self.shader) - self._update_shader_uniforms() + self._update_texture_color_filtering() rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv) rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) rl.end_shader_mode() - def _update_shader_uniforms(self): + def _update_texture_color_filtering(self): + self._engaged_val[0] = 1 if ui_state.status != UIStatus.DISENGAGED else 0 + rl.set_shader_value(self.shader, self._engaged_loc, self._engaged_val, rl.ShaderUniformDataType.SHADER_UNIFORM_INT) rl.set_shader_value(self.shader, self._enhance_driver_loc, self._enhance_driver_val, rl.ShaderUniformDataType.SHADER_UNIFORM_INT) def _ensure_connection(self) -> bool: @@ -356,7 +376,6 @@ class CameraView(Widget): self.client = self._target_client self._stream_type = self._target_stream_type self._texture_needs_update = True - self._enhance_driver_val[0] = 1 if self._stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0 # Reset state self._target_client = None @@ -368,7 +387,7 @@ class CameraView(Widget): def _initialize_textures(self): self._clear_textures() - if not self._use_egl: + if not TICI: self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride), int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE)) self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2), @@ -384,7 +403,7 @@ class CameraView(Widget): self.texture_uv = None # Clean up EGL resources - if self._use_egl: + if TICI: for data in self.egl_images.values(): destroy_egl_image(data) self.egl_images = {} diff --git a/selfdrive/ui/onroad/cameraview.py b/selfdrive/ui/onroad/cameraview.py index e71578af..54439484 100644 --- a/selfdrive/ui/onroad/cameraview.py +++ b/selfdrive/ui/onroad/cameraview.py @@ -37,30 +37,32 @@ void main() { } """ -FRAME_FRAGMENT_SHADER_EGL = """ - #version 300 es - #extension GL_OES_EGL_image_external_essl3 : enable - precision mediump float; - in vec2 fragTexCoord; - uniform samplerExternalOES texture0; - out vec4 fragColor; - void main() { - vec4 color = texture(texture0, fragTexCoord); - fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a); - } - """ - -FRAME_FRAGMENT_SHADER_TEXTURES = VERSION + """ - in vec2 fragTexCoord; - uniform sampler2D texture0; - uniform sampler2D texture1; - out vec4 fragColor; - void main() { - float y = texture(texture0, fragTexCoord).r; - vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; - fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0); - } - """ +# Choose fragment shader based on platform capabilities +if TICI: + FRAME_FRAGMENT_SHADER = """ + #version 300 es + #extension GL_OES_EGL_image_external_essl3 : enable + precision mediump float; + in vec2 fragTexCoord; + uniform samplerExternalOES texture0; + out vec4 fragColor; + void main() { + vec4 color = texture(texture0, fragTexCoord); + fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a); + } + """ +else: + FRAME_FRAGMENT_SHADER = VERSION + """ + in vec2 fragTexCoord; + uniform sampler2D texture0; + uniform sampler2D texture1; + out vec4 fragColor; + void main() { + float y = texture(texture0, fragTexCoord).r; + vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; + fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0); + } + """ class CameraView(Widget): @@ -79,6 +81,8 @@ class CameraView(Widget): self._texture_needs_update = True self.last_connection_attempt: float = 0.0 + self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER) + self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1 self.frame: VisionBuf | None = None self.texture_y: rl.Texture | None = None @@ -89,16 +93,12 @@ class CameraView(Widget): self.egl_texture: rl.Texture | None = None self._placeholder_color: rl.Color | None = None - self._use_egl = TICI and init_egl() - fragment_shader = FRAME_FRAGMENT_SHADER_EGL if self._use_egl else FRAME_FRAGMENT_SHADER_TEXTURES - self.shader = rl.load_shader_from_memory(VERTEX_SHADER, fragment_shader) - self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not self._use_egl else -1 + # Initialize EGL for zero-copy rendering on TICI + if TICI: + if not init_egl(): + raise RuntimeError("Failed to initialize EGL") - # Keep the UI alive if EGL cannot be used on device startup. - if TICI and not self._use_egl: - cloudlog.warning(f"Failed to initialize EGL for {self._name}; falling back to texture rendering") - elif self._use_egl: # Create a 1x1 pixel placeholder texture for EGL image binding temp_image = rl.gen_image_color(1, 1, rl.BLACK) self.egl_texture = rl.load_texture_from_image(temp_image) @@ -146,7 +146,7 @@ class CameraView(Widget): self._clear_textures() # Clean up EGL texture - if self.egl_texture: + if TICI and self.egl_texture: rl.unload_texture(self.egl_texture) self.egl_texture = None @@ -220,7 +220,7 @@ class CameraView(Widget): dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y) # Render with appropriate method - if self._use_egl: + if TICI: self._render_egl(src_rect, dst_rect) else: self._render_textures(src_rect, dst_rect) @@ -337,7 +337,7 @@ class CameraView(Widget): def _initialize_textures(self): self._clear_textures() - if not self._use_egl: + if not TICI: self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride), int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE)) self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2), @@ -353,7 +353,7 @@ class CameraView(Widget): self.texture_uv = None # Clean up EGL resources - if self._use_egl: + if TICI: for data in self.egl_images.values(): destroy_egl_image(data) self.egl_images = {}