mawei/onepilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

calibration

Browse Source
This commit is contained in:
firestar5683
2026-04-11 19:06:07 -05:00
parent 8c93f6b896
commit d2e5f06395
2 changed files with 117 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
selfdrive/ui/mici/onroad/cameraview.py
+81 -62
View File
@@ -8,7 +8,7 @@ from openpilot.system.hardware import TICI
from openpilot.system.ui.lib.application import gui_app
from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage
from openpilot.system.ui.widgets import Widget
from openpilot.selfdrive.ui.ui_state import ui_state
from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus
CONNECTION_RETRY_INTERVAL = 0.2 # seconds between connection attempts
@@ -37,52 +37,70 @@ void main() {
}
"""
FRAME_FRAGMENT_SHADER_EGL = """
#version 300 es
#extension GL_OES_EGL_image_external_essl3 : enable
precision mediump float;
in vec2 fragTexCoord;
uniform samplerExternalOES texture0;
out vec4 fragColor;
uniform int enhance_driver;
# Choose fragment shader based on platform capabilities
if TICI:
FRAME_FRAGMENT_SHADER = """
#version 300 es
#extension GL_OES_EGL_image_external_essl3 : enable
precision mediump float;
in vec2 fragTexCoord;
uniform samplerExternalOES texture0;
out vec4 fragColor;
uniform int engaged;
uniform int enhance_driver;
void main() {
vec4 color = texture(texture0, fragTexCoord);
color.rgb = pow(color.rgb, vec3(1.0/1.28));
if (enhance_driver == 1) {
float brightness = 1.1;
color.rgb = color.rgb + 0.15;
color.rgb = clamp((color.rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0);
color.rgb = color.rgb * color.rgb * (3.0 - 2.0 * color.rgb);
color.rgb = pow(color.rgb, vec3(0.8));
void main() {
vec4 color = texture(texture0, fragTexCoord);
if (engaged == 1) {
float gray = dot(color.rgb, vec3(0.299, 0.587, 0.114)); // Luma
color.rgb = mix(vec3(gray), color.rgb, 0.2); // 20% saturation
color.rgb = clamp((color.rgb - 0.5) * 1.2 + 0.5, 0.0, 1.0); // +20% contrast
color.rgb = pow(color.rgb, vec3(1.0/1.28));
} else {
color.rgb *= 0.85; // 85% opacity
}
if (enhance_driver == 1) {
float brightness = 1.1;
color.rgb = color.rgb + 0.15;
color.rgb = clamp((color.rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0);
color.rgb = color.rgb * color.rgb * (3.0 - 2.0 * color.rgb);
color.rgb = pow(color.rgb, vec3(0.8));
}
fragColor = vec4(color.rgb, color.a);
}
fragColor = vec4(color.rgb, color.a);
}
"""
"""
else:
FRAME_FRAGMENT_SHADER = VERSION + """
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
out vec4 fragColor;
uniform int engaged;
uniform int enhance_driver;
FRAME_FRAGMENT_SHADER_TEXTURES = VERSION + """
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
out vec4 fragColor;
uniform int enhance_driver;
void main() {
float y = texture(texture0, fragTexCoord).r;
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
vec3 rgb = vec3(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x);
// TODO: the images out of camerad need some more correction and
// the ui should apply a gamma curve for the device display
if (enhance_driver == 1) {
float brightness = 1.1;
rgb = rgb + 0.15;
rgb = clamp((rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0);
rgb = rgb * rgb * (3.0 - 2.0 * rgb);
rgb = pow(rgb, vec3(0.8));
void main() {
float y = texture(texture0, fragTexCoord).r;
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
vec3 rgb = vec3(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x);
if (engaged == 1) {
float gray = dot(rgb, vec3(0.299, 0.587, 0.114));
rgb = mix(vec3(gray), rgb, 0.2); // 20% saturation
rgb = clamp((rgb - 0.5) * 1.2 + 0.5, 0.0, 1.0); // +20% contrast
} else {
rgb *= 0.85; // 85% opacity
}
// TODO: the images out of camerad need some more correction and
// the ui should apply a gamma curve for the device display
if (enhance_driver == 1) {
float brightness = 1.1;
rgb = rgb + 0.15;
rgb = clamp((rgb - 0.5) * (brightness * 0.8) + 0.5, 0.0, 1.0);
rgb = rgb * rgb * (3.0 - 2.0 * rgb);
rgb = pow(rgb, vec3(0.8));
}
fragColor = vec4(rgb, 1.0);
}
fragColor = vec4(rgb, 1.0);
}
"""
"""
class CameraView(Widget):
@@ -101,6 +119,12 @@ class CameraView(Widget):
self._texture_needs_update = True
self.last_connection_attempt: float = 0.0
self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER)
self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1
self._engaged_loc = rl.get_shader_location(self.shader, "engaged")
self._engaged_val = rl.ffi.new("int[1]", [1])
self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver")
self._enhance_driver_val = rl.ffi.new("int[1]", [1 if stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0])
self.frame: VisionBuf | None = None
self.texture_y: rl.Texture | None = None
@@ -111,18 +135,12 @@ class CameraView(Widget):
self.egl_texture: rl.Texture | None = None
self._placeholder_color: rl.Color | None = None
self._use_egl = TICI and init_egl()
fragment_shader = FRAME_FRAGMENT_SHADER_EGL if self._use_egl else FRAME_FRAGMENT_SHADER_TEXTURES
self.shader = rl.load_shader_from_memory(VERTEX_SHADER, fragment_shader)
self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not self._use_egl else -1
self._enhance_driver_loc = rl.get_shader_location(self.shader, "enhance_driver")
self._enhance_driver_val = rl.ffi.new("int[1]", [1 if stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0])
# Initialize EGL for zero-copy rendering on TICI
if TICI:
if not init_egl():
raise RuntimeError("Failed to initialize EGL")
# Keep the UI alive if EGL cannot be used on device startup.
if TICI and not self._use_egl:
cloudlog.warning(f"Failed to initialize EGL for {self._name}; falling back to texture rendering")
elif self._use_egl:
# Create a 1x1 pixel placeholder texture for EGL image binding
temp_image = rl.gen_image_color(1, 1, rl.BLACK)
self.egl_texture = rl.load_texture_from_image(temp_image)
@@ -136,11 +154,11 @@ class CameraView(Widget):
# Prevent old frames from showing when going onroad. Qt has a separate thread
# which drains the VisionIpcClient SubSocket for us. Re-connecting is not enough
# and only clears internal buffers, not the message queue.
self.frame = None
self.available_streams.clear()
if self.client:
del self.client
self.client = VisionIpcClient(self._name, self._stream_type, conflate=True)
self.frame = None
def _set_placeholder_color(self, color: rl.Color):
"""Set a placeholder color to be drawn when no frame is available."""
@@ -170,7 +188,7 @@ class CameraView(Widget):
self._clear_textures()
# Clean up EGL texture
if self.egl_texture:
if TICI and self.egl_texture:
rl.unload_texture(self.egl_texture)
self.egl_texture = None
@@ -245,7 +263,7 @@ class CameraView(Widget):
dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y)
# Render with appropriate method
if self._use_egl:
if TICI:
self._render_egl(src_rect, dst_rect)
else:
self._render_textures(src_rect, dst_rect)
@@ -279,7 +297,7 @@ class CameraView(Widget):
# Render with shader
rl.begin_shader_mode(self.shader)
self._update_shader_uniforms()
self._update_texture_color_filtering()
rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
@@ -299,12 +317,14 @@ class CameraView(Widget):
# Render with shader
rl.begin_shader_mode(self.shader)
self._update_shader_uniforms()
self._update_texture_color_filtering()
rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv)
rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
def _update_shader_uniforms(self):
def _update_texture_color_filtering(self):
self._engaged_val[0] = 1 if ui_state.status != UIStatus.DISENGAGED else 0
rl.set_shader_value(self.shader, self._engaged_loc, self._engaged_val, rl.ShaderUniformDataType.SHADER_UNIFORM_INT)
rl.set_shader_value(self.shader, self._enhance_driver_loc, self._enhance_driver_val, rl.ShaderUniformDataType.SHADER_UNIFORM_INT)
def _ensure_connection(self) -> bool:
@@ -356,7 +376,6 @@ class CameraView(Widget):
self.client = self._target_client
self._stream_type = self._target_stream_type
self._texture_needs_update = True
self._enhance_driver_val[0] = 1 if self._stream_type == VisionStreamType.VISION_STREAM_DRIVER else 0
# Reset state
self._target_client = None
@@ -368,7 +387,7 @@ class CameraView(Widget):
def _initialize_textures(self):
self._clear_textures()
if not self._use_egl:
if not TICI:
self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
@@ -384,7 +403,7 @@ class CameraView(Widget):
self.texture_uv = None
# Clean up EGL resources
if self._use_egl:
if TICI:
for data in self.egl_images.values():
destroy_egl_image(data)
self.egl_images = {}
selfdrive/ui/onroad/cameraview.py
+36 -36
View File
@@ -37,30 +37,32 @@ void main() {
}
"""
FRAME_FRAGMENT_SHADER_EGL = """
#version 300 es
#extension GL_OES_EGL_image_external_essl3 : enable
precision mediump float;
in vec2 fragTexCoord;
uniform samplerExternalOES texture0;
out vec4 fragColor;
void main() {
vec4 color = texture(texture0, fragTexCoord);
fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a);
}
"""
FRAME_FRAGMENT_SHADER_TEXTURES = VERSION + """
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
out vec4 fragColor;
void main() {
float y = texture(texture0, fragTexCoord).r;
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0);
}
"""
# Choose fragment shader based on platform capabilities
if TICI:
FRAME_FRAGMENT_SHADER = """
#version 300 es
#extension GL_OES_EGL_image_external_essl3 : enable
precision mediump float;
in vec2 fragTexCoord;
uniform samplerExternalOES texture0;
out vec4 fragColor;
void main() {
vec4 color = texture(texture0, fragTexCoord);
fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a);
}
"""
else:
FRAME_FRAGMENT_SHADER = VERSION + """
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
out vec4 fragColor;
void main() {
float y = texture(texture0, fragTexCoord).r;
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0);
}
"""
class CameraView(Widget):
@@ -79,6 +81,8 @@ class CameraView(Widget):
self._texture_needs_update = True
self.last_connection_attempt: float = 0.0
self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER)
self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1
self.frame: VisionBuf | None = None
self.texture_y: rl.Texture | None = None
@@ -89,16 +93,12 @@ class CameraView(Widget):
self.egl_texture: rl.Texture | None = None
self._placeholder_color: rl.Color | None = None
self._use_egl = TICI and init_egl()
fragment_shader = FRAME_FRAGMENT_SHADER_EGL if self._use_egl else FRAME_FRAGMENT_SHADER_TEXTURES
self.shader = rl.load_shader_from_memory(VERTEX_SHADER, fragment_shader)
self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not self._use_egl else -1
# Initialize EGL for zero-copy rendering on TICI
if TICI:
if not init_egl():
raise RuntimeError("Failed to initialize EGL")
# Keep the UI alive if EGL cannot be used on device startup.
if TICI and not self._use_egl:
cloudlog.warning(f"Failed to initialize EGL for {self._name}; falling back to texture rendering")
elif self._use_egl:
# Create a 1x1 pixel placeholder texture for EGL image binding
temp_image = rl.gen_image_color(1, 1, rl.BLACK)
self.egl_texture = rl.load_texture_from_image(temp_image)
@@ -146,7 +146,7 @@ class CameraView(Widget):
self._clear_textures()
# Clean up EGL texture
if self.egl_texture:
if TICI and self.egl_texture:
rl.unload_texture(self.egl_texture)
self.egl_texture = None
@@ -220,7 +220,7 @@ class CameraView(Widget):
dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y)
# Render with appropriate method
if self._use_egl:
if TICI:
self._render_egl(src_rect, dst_rect)
else:
self._render_textures(src_rect, dst_rect)
@@ -337,7 +337,7 @@ class CameraView(Widget):
def _initialize_textures(self):
self._clear_textures()
if not self._use_egl:
if not TICI:
self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
@@ -353,7 +353,7 @@ class CameraView(Widget):
self.texture_uv = None
# Clean up EGL resources
if self._use_egl:
if TICI:
for data in self.egl_images.values():
destroy_egl_image(data)
self.egl_images = {}