mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-10-01 03:43:46 +08:00
move onroad/driving stuff from system/ui to selfdrive/ui (#35425)
* mv system/ui/onroad->selfdrive/ui/onroad * mv ui_state * fix import path * fix imports * mv cameraview * remove from application
This commit is contained in:
Dean Lee
@@ -1,267 +0,0 @@
|
||||
import numpy as np
|
||||
import pyray as rl
|
||||
|
||||
from openpilot.system.hardware import TICI
|
||||
from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.system.ui.lib.application import gui_app
|
||||
from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage
|
||||
|
||||
|
||||
CONNECTION_RETRY_INTERVAL = 0.2 # seconds between connection attempts
|
||||
|
||||
VERTEX_SHADER = """
|
||||
#version 300 es
|
||||
in vec3 vertexPosition;
|
||||
in vec2 vertexTexCoord;
|
||||
in vec3 vertexNormal;
|
||||
in vec4 vertexColor;
|
||||
uniform mat4 mvp;
|
||||
out vec2 fragTexCoord;
|
||||
out vec4 fragColor;
|
||||
void main() {
|
||||
fragTexCoord = vertexTexCoord;
|
||||
fragColor = vertexColor;
|
||||
gl_Position = mvp * vec4(vertexPosition, 1.0);
|
||||
}
|
||||
"""
|
||||
|
||||
# Choose fragment shader based on platform capabilities
|
||||
if TICI:
|
||||
FRAME_FRAGMENT_SHADER = """
|
||||
#version 300 es
|
||||
#extension GL_OES_EGL_image_external_essl3 : enable
|
||||
precision mediump float;
|
||||
in vec2 fragTexCoord;
|
||||
uniform samplerExternalOES texture0;
|
||||
out vec4 fragColor;
|
||||
void main() {
|
||||
vec4 color = texture(texture0, fragTexCoord);
|
||||
fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a);
|
||||
}
|
||||
"""
|
||||
else:
|
||||
FRAME_FRAGMENT_SHADER = """
|
||||
#version 300 es
|
||||
precision mediump float;
|
||||
in vec2 fragTexCoord;
|
||||
uniform sampler2D texture0;
|
||||
uniform sampler2D texture1;
|
||||
out vec4 fragColor;
|
||||
void main() {
|
||||
float y = texture(texture0, fragTexCoord).r;
|
||||
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
|
||||
fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0);
|
||||
}
|
||||
"""
|
||||
|
||||
class CameraView:
|
||||
def __init__(self, name: str, stream_type: VisionStreamType):
|
||||
self.client = VisionIpcClient(name, stream_type, False)
|
||||
self._name = name
|
||||
self._stream_type = stream_type
|
||||
|
||||
self._texture_needs_update = True
|
||||
self.last_connection_attempt: float = 0.0
|
||||
self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER)
|
||||
self._texture1_loc: int = rl.get_shader_location(self.shader, "texture1") if not TICI else -1
|
||||
|
||||
self.frame: VisionBuf | None = None
|
||||
self.texture_y: rl.Texture | None = None
|
||||
self.texture_uv: rl.Texture | None = None
|
||||
|
||||
# EGL resources
|
||||
self.egl_images: dict[int, EGLImage] = {}
|
||||
self.egl_texture: rl.Texture | None = None
|
||||
|
||||
# Initialize EGL for zero-copy rendering on TICI
|
||||
if TICI:
|
||||
if not init_egl():
|
||||
raise RuntimeError("Failed to initialize EGL")
|
||||
|
||||
# Create a 1x1 pixel placeholder texture for EGL image binding
|
||||
temp_image = rl.gen_image_color(1, 1, rl.BLACK)
|
||||
self.egl_texture = rl.load_texture_from_image(temp_image)
|
||||
rl.unload_image(temp_image)
|
||||
|
||||
def switch_stream(self, stream_type: VisionStreamType) -> None:
|
||||
if self._stream_type != stream_type:
|
||||
cloudlog.debug(f'switching stream from {self._stream_type} to {stream_type}')
|
||||
self._clear_textures()
|
||||
self.frame = None
|
||||
self._stream_type = stream_type
|
||||
self.client = VisionIpcClient(self._name, stream_type, False)
|
||||
|
||||
@property
|
||||
def stream_type(self) -> VisionStreamType:
|
||||
return self._stream_type
|
||||
|
||||
def close(self) -> None:
|
||||
self._clear_textures()
|
||||
|
||||
# Clean up EGL texture
|
||||
if TICI and self.egl_texture:
|
||||
rl.unload_texture(self.egl_texture)
|
||||
self.egl_texture = None
|
||||
|
||||
# Clean up shader
|
||||
if self.shader and self.shader.id:
|
||||
rl.unload_shader(self.shader)
|
||||
|
||||
self.client = None
|
||||
|
||||
def _calc_frame_matrix(self, rect: rl.Rectangle) -> np.ndarray:
|
||||
if not self.frame:
|
||||
return np.eye(3)
|
||||
|
||||
# Calculate aspect ratios
|
||||
widget_aspect_ratio = rect.width / rect.height
|
||||
frame_aspect_ratio = self.frame.width / self.frame.height
|
||||
|
||||
# Calculate scaling factors to maintain aspect ratio
|
||||
zx = min(frame_aspect_ratio / widget_aspect_ratio, 1.0)
|
||||
zy = min(widget_aspect_ratio / frame_aspect_ratio, 1.0)
|
||||
|
||||
return np.array([
|
||||
[zx, 0.0, 0.0],
|
||||
[0.0, zy, 0.0],
|
||||
[0.0, 0.0, 1.0]
|
||||
])
|
||||
|
||||
def render(self, rect: rl.Rectangle):
|
||||
if not self._ensure_connection():
|
||||
return
|
||||
|
||||
# Try to get a new buffer without blocking
|
||||
buffer = self.client.recv(timeout_ms=0)
|
||||
if buffer:
|
||||
self._texture_needs_update = True
|
||||
self.frame = buffer
|
||||
|
||||
if not self.frame:
|
||||
return
|
||||
|
||||
transform = self._calc_frame_matrix(rect)
|
||||
src_rect = rl.Rectangle(0, 0, float(self.frame.width), float(self.frame.height))
|
||||
|
||||
# Calculate scale
|
||||
scale_x = rect.width * transform[0, 0] # zx
|
||||
scale_y = rect.height * transform[1, 1] # zy
|
||||
|
||||
# Calculate base position (centered)
|
||||
x_offset = rect.x + (rect.width - scale_x) / 2
|
||||
y_offset = rect.y + (rect.height - scale_y) / 2
|
||||
|
||||
x_offset += transform[0, 2] * rect.width / 2
|
||||
y_offset += transform[1, 2] * rect.height / 2
|
||||
|
||||
dst_rect = rl.Rectangle(x_offset, y_offset, scale_x, scale_y)
|
||||
|
||||
# Render with appropriate method
|
||||
if TICI:
|
||||
self._render_egl(src_rect, dst_rect)
|
||||
else:
|
||||
self._render_textures(src_rect, dst_rect)
|
||||
|
||||
def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
|
||||
"""Render using EGL for direct buffer access"""
|
||||
if self.frame is None or self.egl_texture is None:
|
||||
return
|
||||
|
||||
idx = self.frame.idx
|
||||
egl_image = self.egl_images.get(idx)
|
||||
|
||||
# Create EGL image if needed
|
||||
if egl_image is None:
|
||||
egl_image = create_egl_image(self.frame.width, self.frame.height, self.frame.stride, self.frame.fd, self.frame.uv_offset)
|
||||
if egl_image:
|
||||
self.egl_images[idx] = egl_image
|
||||
else:
|
||||
return
|
||||
|
||||
# Update texture dimensions to match current frame
|
||||
self.egl_texture.width = self.frame.width
|
||||
self.egl_texture.height = self.frame.height
|
||||
|
||||
# Bind the EGL image to our texture
|
||||
bind_egl_image_to_texture(self.egl_texture.id, egl_image)
|
||||
|
||||
# Render with shader
|
||||
rl.begin_shader_mode(self.shader)
|
||||
rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
|
||||
rl.end_shader_mode()
|
||||
|
||||
def _render_textures(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
|
||||
"""Render using texture copies"""
|
||||
if not self.texture_y or not self.texture_uv or self.frame is None:
|
||||
return
|
||||
|
||||
# Update textures with new frame data
|
||||
if self._texture_needs_update:
|
||||
y_data = self.frame.data[: self.frame.uv_offset]
|
||||
uv_data = self.frame.data[self.frame.uv_offset :]
|
||||
|
||||
rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data))
|
||||
rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data))
|
||||
self._texture_needs_update = False
|
||||
|
||||
# Render with shader
|
||||
rl.begin_shader_mode(self.shader)
|
||||
rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv)
|
||||
rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
|
||||
rl.end_shader_mode()
|
||||
|
||||
def _ensure_connection(self) -> bool:
|
||||
if not self.client.is_connected():
|
||||
self.frame = None
|
||||
|
||||
# Throttle connection attempts
|
||||
current_time = rl.get_time()
|
||||
if current_time - self.last_connection_attempt < CONNECTION_RETRY_INTERVAL:
|
||||
return False
|
||||
self.last_connection_attempt = current_time
|
||||
|
||||
if not self.client.connect(False) or not self.client.num_buffers:
|
||||
return False
|
||||
|
||||
cloudlog.debug(f"Connected to {self._name} stream: {self._stream_type}, buffers: {self.client.num_buffers}")
|
||||
self._clear_textures()
|
||||
|
||||
if not TICI:
|
||||
self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
|
||||
int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
|
||||
self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
|
||||
int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
|
||||
|
||||
return True
|
||||
|
||||
def _clear_textures(self):
|
||||
if self.texture_y and self.texture_y.id:
|
||||
rl.unload_texture(self.texture_y)
|
||||
self.texture_y = None
|
||||
|
||||
if self.texture_uv and self.texture_uv.id:
|
||||
rl.unload_texture(self.texture_uv)
|
||||
self.texture_uv = None
|
||||
|
||||
# Clean up EGL resources
|
||||
if TICI:
|
||||
for data in self.egl_images.values():
|
||||
destroy_egl_image(data)
|
||||
self.egl_images = {}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
gui_app.init_window("watch3")
|
||||
road_camera_view = CameraView("camerad", VisionStreamType.VISION_STREAM_ROAD)
|
||||
driver_camera_view = CameraView("camerad", VisionStreamType.VISION_STREAM_DRIVER)
|
||||
wide_road_camera_view = CameraView("camerad", VisionStreamType.VISION_STREAM_WIDE_ROAD)
|
||||
try:
|
||||
for _ in gui_app.render():
|
||||
road_camera_view.render(rl.Rectangle(gui_app.width // 4, 0, gui_app.width // 2, gui_app.height // 2))
|
||||
driver_camera_view.render(rl.Rectangle(0, gui_app.height // 2, gui_app.width // 2, gui_app.height // 2))
|
||||
wide_road_camera_view.render(rl.Rectangle(gui_app.width // 2, gui_app.height // 2, gui_app.width // 2, gui_app.height // 2))
|
||||
finally:
|
||||
road_camera_view.close()
|
||||
driver_camera_view.close()
|
||||
wide_road_camera_view.close()
|
||||
@@ -1,94 +0,0 @@
|
||||
import numpy as np
|
||||
import pyray as rl
|
||||
from cereal import messaging
|
||||
from openpilot.system.ui.widgets.cameraview import CameraView
|
||||
from msgq.visionipc import VisionStreamType
|
||||
from openpilot.system.ui.lib.application import gui_app, FontWeight
|
||||
from openpilot.system.ui.lib.label import gui_label
|
||||
from openpilot.system.ui.onroad.driver_state import DriverStateRenderer
|
||||
|
||||
|
||||
class DriverCameraView(CameraView):
|
||||
def __init__(self, stream_type: VisionStreamType):
|
||||
super().__init__("camerad", stream_type)
|
||||
self.driver_state_renderer = DriverStateRenderer()
|
||||
|
||||
def render(self, rect, sm):
|
||||
super().render(rect)
|
||||
|
||||
if not self.frame:
|
||||
gui_label(
|
||||
rect,
|
||||
"camera starting",
|
||||
font_size=100,
|
||||
font_weight=FontWeight.BOLD,
|
||||
alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER,
|
||||
)
|
||||
return
|
||||
|
||||
self._draw_face_detection(rect, sm)
|
||||
self.driver_state_renderer.draw(rect, sm)
|
||||
|
||||
def _draw_face_detection(self, rect: rl.Rectangle, sm) -> None:
|
||||
driver_state = sm["driverStateV2"]
|
||||
is_rhd = driver_state.wheelOnRightProb > 0.5
|
||||
driver_data = driver_state.rightDriverData if is_rhd else driver_state.leftDriverData
|
||||
face_detect = driver_data.faceProb > 0.7
|
||||
if not face_detect:
|
||||
return
|
||||
|
||||
# Get face position and orientation
|
||||
face_x, face_y = driver_data.facePosition
|
||||
face_std = max(driver_data.faceOrientationStd[0], driver_data.faceOrientationStd[1])
|
||||
alpha = 0.7
|
||||
if face_std > 0.15:
|
||||
alpha = max(0.7 - (face_std - 0.15) * 3.5, 0.0)
|
||||
|
||||
# use approx instead of distort_points
|
||||
# TODO: replace with distort_points
|
||||
fbox_x = int(1080.0 - 1714.0 * face_x)
|
||||
fbox_y = int(-135.0 + (504.0 + abs(face_x) * 112.0) + (1205.0 - abs(face_x) * 724.0) * face_y)
|
||||
box_size = 220
|
||||
|
||||
line_color = rl.Color(255, 255, 255, int(alpha * 255))
|
||||
rl.draw_rectangle_rounded_lines_ex(
|
||||
rl.Rectangle(fbox_x - box_size / 2, fbox_y - box_size / 2, box_size, box_size),
|
||||
35.0 / box_size / 2,
|
||||
10,
|
||||
10,
|
||||
line_color,
|
||||
)
|
||||
|
||||
def _calc_frame_matrix(self, rect: rl.Rectangle) -> np.ndarray:
|
||||
driver_view_ratio = 2.0
|
||||
|
||||
# Get stream dimensions
|
||||
if self.frame:
|
||||
stream_width = self.frame.width
|
||||
stream_height = self.frame.height
|
||||
else:
|
||||
# Default values if frame not available
|
||||
stream_width = 1928
|
||||
stream_height = 1208
|
||||
|
||||
yscale = stream_height * driver_view_ratio / stream_width
|
||||
xscale = yscale * rect.height / rect.width * stream_width / stream_height
|
||||
|
||||
return np.array([
|
||||
[xscale, 0.0, 0.0],
|
||||
[0.0, yscale, 0.0],
|
||||
[0.0, 0.0, 1.0]
|
||||
])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
gui_app.init_window("Driver Camera View")
|
||||
sm = messaging.SubMaster(["selfdriveState", "driverStateV2", "driverMonitoringState"])
|
||||
|
||||
driver_camera_view = DriverCameraView(VisionStreamType.VISION_STREAM_DRIVER)
|
||||
try:
|
||||
for _ in gui_app.render():
|
||||
sm.update()
|
||||
driver_camera_view.render(rl.Rectangle(0, 0, gui_app.width, gui_app.height), sm)
|
||||
finally:
|
||||
driver_camera_view.close()
|
||||
Reference in New Issue
Block a user