diff --git a/selfdrive/ui/mici/onroad/augmented_road_view.py b/selfdrive/ui/mici/onroad/augmented_road_view.py index 190ebb6d8..8afe43aed 100644 --- a/selfdrive/ui/mici/onroad/augmented_road_view.py +++ b/selfdrive/ui/mici/onroad/augmented_road_view.py @@ -20,6 +20,7 @@ from openpilot.selfdrive.ui.mici.onroad.starpilot_status import ( get_border_color, ) from openpilot.selfdrive.ui.mici.onroad.cameraview import CameraView +from openpilot.selfdrive.ui.onroad.starpilot.pip_sidecam import PipSideCamera from openpilot.selfdrive.ui.lib.starpilot_visuals import get_border_width from openpilot.starpilot.common.favorite_slots import is_favorite_action_key, load_favorite_slots, toggle_favorite_slot from openpilot.system.ui.lib.application import FontWeight, gui_app, MousePos, MouseEvent @@ -583,6 +584,10 @@ class AugmentedRoadView(CameraView): # debug self._pm = messaging.PubMaster(['uiDebug']) + # C4 sidecam: fills road preview as a curved rectangle. Only shown + # on the road camera screen, gated via widget visibility + self._pip_sidecam = self._child(PipSideCamera(shape="curved")) + self._pip_sidecam.set_visible(lambda: self.stream_type == ROAD_CAM) @staticmethod def _controls_ready() -> bool: @@ -787,6 +792,17 @@ class AugmentedRoadView(CameraView): rl.draw_rectangle(int(self.rect.x), int(self.rect.y), int(self.rect.width), int(self.rect.height), rl.Color(0, 0, 0, 175)) self._offroad_label.render(self._content_rect) + # C4 sidecam renders last (on top) and only when showing the road camera + # Inset by the border so the pill never covers the green/orange status border. + border = self._get_border_width() + preview_rect = rl.Rectangle( + self._content_rect.x + border, + self._content_rect.y + border, + max(1, self._content_rect.width - 2 * border), + max(1, self._content_rect.height - 2 * border), + ) + self._pip_sidecam.render(preview_rect) + # publish uiDebug msg = messaging.new_message('uiDebug') msg.uiDebug.drawTimeMillis = (time.monotonic() - start_draw) * 1000 diff --git a/selfdrive/ui/onroad/starpilot/pip_sidecam.py b/selfdrive/ui/onroad/starpilot/pip_sidecam.py index 23441a26c..1bc47a125 100644 --- a/selfdrive/ui/onroad/starpilot/pip_sidecam.py +++ b/selfdrive/ui/onroad/starpilot/pip_sidecam.py @@ -10,6 +10,7 @@ import pyray as rl from msgq.visionipc import VisionIpcClient, VisionStreamType from openpilot.selfdrive.ui.ui_state import ui_state from openpilot.starpilot.common.vision_bsm import get_fresh_vasm_state +from openpilot.system.ui.widgets import Widget PIP_SHADER_VERSION = """ #version 300 es @@ -93,6 +94,70 @@ void main() { } """ +# curved-rectangle variant for the mici display. +PIP_CURVED_FRAGMENT_SHADER = PIP_SHADER_VERSION + """ +in vec2 fragTexCoord; +uniform sampler2D texture0; +uniform sampler2D texture1; +uniform vec2 uCropMin; +uniform vec2 uCropSize; +uniform int uFlipX; +uniform vec2 uRectSize; +out vec4 fragColor; + +const float CORNER_RADIUS_FRACTION = 0.22; +const float CURVE_AMOUNT = 0.07; +const float EDGE_DARKEN = 0.14; +const float RIM_BLEND = 0.06; + +void main() { + vec2 p = fragTexCoord * 2.0 - 1.0; + float halfW = uRectSize.x * 0.5; + float halfH = uRectSize.y * 0.5; + float radius = CORNER_RADIUS_FRACTION * min(uRectSize.x, uRectSize.y); + + // Rounded-rectangle SDF in pixel space; mask before sampling. + vec2 q = abs(vec2(p.x * halfW, p.y * halfH)) - (vec2(halfW, halfH) - radius); + float dist = length(max(q, 0.0)) + min(max(q.x, q.y), 0.0) - radius; + float aa = max(fwidth(dist), 0.00001); + float alpha = 1.0 - smoothstep(-aa, aa, dist); + if (dist > aa) { + discard; + } + + // Gentle convex curvature along both axes to mimic the curved OLED panel. + float curve = CURVE_AMOUNT * (1.0 - p.x * p.x) * (1.0 - p.y * p.y); + vec2 sampleCoord = clamp(fragTexCoord + curve * vec2(0.0, 0.5), 0.001, 0.999); + + // The saved mask is a square crop, but the curved panel is wider than tall. + // Sample an aspect-matched horizontal band of that square (centered) instead + // of stretching it, so the image is never distorted. The circle's diameter + // (the square crop) becomes the panel's length; the height follows the aspect. + float aspect = uRectSize.x / max(uRectSize.y, 0.0001); + vec2 cropCoord = sampleCoord; + if (aspect >= 1.0) { + cropCoord.y = 0.5 + (sampleCoord.y - 0.5) / aspect; + } else { + cropCoord.x = 0.5 + (sampleCoord.x - 0.5) * aspect; + } + if (uFlipX == 1) { + cropCoord.x = 1.0 - cropCoord.x; + } + vec2 uv = uCropMin + cropCoord * uCropSize; + float y = texture(texture0, uv).r; + vec2 c = texture(texture1, uv).ra - 0.5; + vec3 rgb = vec3(y + 1.402 * c.y, y - 0.344 * c.x - 0.714 * c.y, y + 1.772 * c.x); + + // Let the rim blend into the camera image and the UI underneath it. + float edgeShade = smoothstep(-radius, 0.0, dist); + rgb *= mix(1.0, 1.0 - EDGE_DARKEN, edgeShade); + float rim = smoothstep(radius * 0.55, radius, radius - dist); + rgb = mix(rgb, vec3(0.48, 0.70, 1.0), rim * RIM_BLEND); + + fragColor = vec4(rgb, alpha); +} +""" + UNIFORM_VEC2 = rl.ShaderUniformDataType.SHADER_UNIFORM_VEC2 UNIFORM_INT = rl.ShaderUniformDataType.SHADER_UNIFORM_INT @@ -110,10 +175,18 @@ BUBBLE_RADIUS_MIN = 180 BUBBLE_RADIUS_MAX = 420 BUBBLE_MARGIN = 24 +class PipSideCamera(Widget): + """Overlays the adjacent side window from the dcamera. + + Drawn as a circular bubble on the big screen (shape="bubble") or as a + curved rectangle filling the whole road preview on the C4 (shape="curved"). + """ + def __init__(self, shape: str = "bubble"): + super().__init__() + if shape not in ("bubble", "curved"): + raise ValueError(f"Unknown PipSideCamera shape: {shape!r}") + self._shape = shape -class PipSideCamera: - """Renders a circular pip bubble of the adjacent side window from the dcamera.""" - def __init__(self): self._params = ui_state.params self._params_memory = ui_state.params_memory @@ -132,6 +205,8 @@ class PipSideCamera: self._show_on_bsm = False self._mask = {} self._last_param_refresh = 0.0 + self._side_activation_time: dict[str, float] = {} + self._active_sides: set[str] = set() self.shader = rl.load_shader_from_memory(PIP_VERTEX_SHADER, PIP_FRAGMENT_SHADER) self._texture1_loc = rl.get_shader_location(self.shader, "texture1") @@ -140,6 +215,13 @@ class PipSideCamera: self._flip_x_loc = rl.get_shader_location(self.shader, "uFlipX") self._flip_x_value = rl.ffi.new("int[1]", [1]) + self.curved_shader = rl.load_shader_from_memory(PIP_VERTEX_SHADER, PIP_CURVED_FRAGMENT_SHADER) + self._curved_texture1_loc = rl.get_shader_location(self.curved_shader, "texture1") + self._curved_crop_min_loc = rl.get_shader_location(self.curved_shader, "uCropMin") + self._curved_crop_size_loc = rl.get_shader_location(self.curved_shader, "uCropSize") + self._curved_flip_x_loc = rl.get_shader_location(self.curved_shader, "uFlipX") + self._curved_rect_size_loc = rl.get_shader_location(self.curved_shader, "uRectSize") + self_ref = weakref.ref(self) def offroad_transition_callback(): @@ -157,16 +239,19 @@ class PipSideCamera: self.client = VisionIpcClient("camerad", self._stream_type, conflate=True) def close(self): - if self._closed: + if getattr(self, "_closed", False): return self._closed = True if getattr(self, "_offroad_transition_callback", None) is not None: ui_state.remove_offroad_transition_callback(self._offroad_transition_callback) self._offroad_transition_callback = None self._clear_textures() - if self.shader and self.shader.id: - rl.unload_shader(self.shader) - self.shader.id = 0 + if (shader := getattr(self, "shader", None)) is not None and shader.id: + rl.unload_shader(shader) + shader.id = 0 + if (curved := getattr(self, "curved_shader", None)) is not None and curved.id: + rl.unload_shader(curved) + curved.id = 0 self.frame = None self.client = None @@ -247,16 +332,10 @@ class PipSideCamera: cy = content_rect.y + content_rect.height - margin - radius return rl.Rectangle(cx - radius, cy - radius, radius * 2, radius * 2) - def render(self, content_rect: rl.Rectangle): - if not ui_state.started: - return - - sides = self.active_sides() - if not sides: - return - + def _acquire_frame(self) -> bool: + """Ensure connection and refresh the Y/UV textures from the latest driver frame.""" if not self._ensure_connection(): - return + return False buffer = self.client.recv(timeout_ms=0) if buffer: @@ -264,10 +343,10 @@ class PipSideCamera: self._last_frame_id = int(getattr(buffer, "frame_id", -1)) self._texture_needs_update = True if self.frame is None: - return + return False if not self.texture_y or not self.texture_uv: - return + return False if self._texture_needs_update: y_data = self.frame.data[: self.frame.uv_offset] @@ -275,13 +354,47 @@ class PipSideCamera: rl.update_texture(self.texture_y, rl.ffi.cast("void *", rl.ffi.from_buffer(y_data))) rl.update_texture(self.texture_uv, rl.ffi.cast("void *", rl.ffi.from_buffer(uv_data))) self._texture_needs_update = False + return True - for side in sides: - crop = self._crop_rect(side) - if crop is None: - continue - bubble = self._bubble_rect(content_rect, side) - self._draw_bubble(bubble, crop) + def _render(self, content_rect: rl.Rectangle): + """Fetch the current driver frame, then draw it for the configured shape.""" + if not ui_state.started: + return None + + sides = self.active_sides() + if not sides or not self._acquire_frame(): + return None + + if self._shape == "curved": + # C4: one crop fills the whole road preview as a curved rectangle. + side = self._pick_side(sides) + if side is not None: + crop = self._crop_rect(side) + if crop is not None: + self._draw_curved(content_rect, crop) + else: + # Raybig: one circular bubble per active side. + for side in sides: + crop = self._crop_rect(side) + if crop is None: + continue + bubble = self._bubble_rect(content_rect, side) + self._draw_bubble(bubble, crop) + return None + + def _pick_side(self, sides: list[str]) -> str | None: + """Return the active side whose blinker/BSM most recently turned on. + + Only a rising edge (inactive -> active) refreshes the timestamp + """ + now = time.monotonic() + active = set(sides) + for side in active - self._active_sides: + self._side_activation_time[side] = now + self._active_sides = active + if not sides: + return None + return max(sides, key=lambda side: self._side_activation_time.get(side, 0.0)) def _draw_bubble(self, bubble: rl.Rectangle, crop: rl.Rectangle): tex_w = float(self.texture_y.width) @@ -300,6 +413,25 @@ class PipSideCamera: rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) rl.end_shader_mode() + def _draw_curved(self, content_rect: rl.Rectangle, crop: rl.Rectangle): + tex_w = float(self.texture_y.width) + tex_h = float(self.texture_y.height) + crop_min = rl.Vector2(crop.x / tex_w, crop.y / tex_h) + crop_size = rl.Vector2(crop.width / tex_w, crop.height / tex_h) + rect_size = rl.Vector2(content_rect.width, content_rect.height) + + src_rect = rl.Rectangle(0, 0, tex_w, tex_h) + dst_rect = rl.Rectangle(content_rect.x, content_rect.y, content_rect.width, content_rect.height) + + rl.begin_shader_mode(self.curved_shader) + rl.set_shader_value(self.curved_shader, self._curved_crop_min_loc, crop_min, UNIFORM_VEC2) + rl.set_shader_value(self.curved_shader, self._curved_crop_size_loc, crop_size, UNIFORM_VEC2) + rl.set_shader_value(self.curved_shader, self._curved_flip_x_loc, self._flip_x_value, UNIFORM_INT) + rl.set_shader_value(self.curved_shader, self._curved_rect_size_loc, rect_size, UNIFORM_VEC2) + rl.set_shader_value_texture(self.curved_shader, self._curved_texture1_loc, self.texture_uv) + rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) + rl.end_shader_mode() + def _ensure_connection(self) -> bool: if not self.client.is_connected(): self.frame = None @@ -324,9 +456,9 @@ class PipSideCamera: int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA)) def _clear_textures(self): - if self.texture_y and self.texture_y.id: - rl.unload_texture(self.texture_y) + if (texture_y := getattr(self, "texture_y", None)) is not None and texture_y.id: + rl.unload_texture(texture_y) self.texture_y = None - if self.texture_uv and self.texture_uv.id: - rl.unload_texture(self.texture_uv) + if (texture_uv := getattr(self, "texture_uv", None)) is not None and texture_uv.id: + rl.unload_texture(texture_uv) self.texture_uv = None diff --git a/selfdrive/ui/onroad/starpilot/starpilot_onroad_view.py b/selfdrive/ui/onroad/starpilot/starpilot_onroad_view.py index ef3325e24..f19b0e85e 100644 --- a/selfdrive/ui/onroad/starpilot/starpilot_onroad_view.py +++ b/selfdrive/ui/onroad/starpilot/starpilot_onroad_view.py @@ -39,7 +39,7 @@ class StarPilotOnroadView(AugmentedRoadView): self._max_fps = 0.0 self._avg_fps = 0.0 - self._pip_sidecam = PipSideCamera() + self._pip_sidecam = self._child(PipSideCamera()) self.layout_manager = WidgetLayoutManager(self._content_rect) diff --git a/selfdrive/ui/tests/test_pip_sidecam.py b/selfdrive/ui/tests/test_pip_sidecam.py index 9a86dbf36..dcdd1855a 100644 --- a/selfdrive/ui/tests/test_pip_sidecam.py +++ b/selfdrive/ui/tests/test_pip_sidecam.py @@ -1,6 +1,7 @@ from openpilot.selfdrive.ui.onroad.starpilot.pip_sidecam import ( IMAGE_TO_VEHICLE_SIDE, PIP_FRAGMENT_SHADER, + PIP_CURVED_FRAGMENT_SHADER, PipSideCamera, ) @@ -47,3 +48,39 @@ def test_pip_driver_camera_shader_uses_analytic_bubble_shading_without_new_unifo assert "rgb = mix(rgb, vec3(1.0)" not in PIP_FRAGMENT_SHADER assert "uniform sampler2D texture2" not in PIP_FRAGMENT_SHADER assert "uRefraction" not in PIP_FRAGMENT_SHADER + + +def test_pip_c4_curved_shader_masks_a_rounded_rectangle_before_sampling(): + assert "uRectSize" in PIP_CURVED_FRAGMENT_SHADER + assert "CORNER_RADIUS_FRACTION" in PIP_CURVED_FRAGMENT_SHADER + assert "CURVE_AMOUNT" in PIP_CURVED_FRAGMENT_SHADER + assert "length(max(q, 0.0))" in PIP_CURVED_FRAGMENT_SHADER + assert PIP_CURVED_FRAGMENT_SHADER.index("if (dist > aa)") < PIP_CURVED_FRAGMENT_SHADER.index("texture(texture0") + assert PIP_CURVED_FRAGMENT_SHADER.count("texture(texture0") == 1 + assert PIP_CURVED_FRAGMENT_SHADER.count("texture(texture1") == 1 + assert "cropCoord.x = 1.0 - cropCoord.x" in PIP_CURVED_FRAGMENT_SHADER + assert "y + 1.402 * c.y" in PIP_CURVED_FRAGMENT_SHADER + assert "uRefraction" not in PIP_CURVED_FRAGMENT_SHADER + + +def test_pip_sidecam_is_a_widget_with_curved_and_bubble_shapes(): + bubble = PipSideCamera.__new__(PipSideCamera) + bubble._closed = True + assert isinstance(bubble, PipSideCamera) + assert hasattr(bubble, "render") + assert hasattr(bubble, "_draw_bubble") + assert hasattr(bubble, "_draw_curved") + curved = PipSideCamera.__new__(PipSideCamera) + curved._closed = True + curved._shape = "curved" + assert curved._shape == "curved" + + +def test_pip_sidecam_rejects_unknown_shapes(): + camera = object.__new__(PipSideCamera) + try: + PipSideCamera.__init__(camera, shape="hexagon") + except ValueError: + pass + else: + raise AssertionError("expected ValueError for unknown shape") diff --git a/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.js b/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.js index 14e00a4ba..d365024b0 100644 --- a/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.js +++ b/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.js @@ -35,6 +35,97 @@ const state = reactive({ let _loadedImage = null; let _lastCanvas = null; let loadedConfig = null; +let deviceType = null; + +const C4_ROAD_ASPECT = 476 / 240; + +function isC4() { + return (deviceType || "").toLowerCase() === "mici"; +} + +function drawCurvedRect(ctx, x, y, w, h) { + const r = Math.min(w, h) * 0.22; + ctx.beginPath(); + ctx.moveTo(x + r, y); + ctx.arcTo(x + w, y, x + w, y + h, r); + ctx.arcTo(x + w, y + h, x, y + h, r); + ctx.arcTo(x, y + h, x, y, r); + ctx.arcTo(x, y, x + w, y, r); + ctx.closePath(); +} + +function cropSource(ctx, img, center, zoom) { + const cw = ctx.canvas.width; + const ch = ctx.canvas.height; + const nativeW = img.naturalWidth; + const nativeH = img.naturalHeight; + const [cx, cy] = center; + const nativeCx = (cw - cx) * nativeW / cw; + const nativeCy = cy * nativeH / ch; + const nativeZoom = zoom * nativeW / cw; + return { cw, ch, cx, cy, nativeCx, nativeCy, nativeZoom }; +} + +function drawC4Preview(ctx, img, center, zoom, color) { + const { cw, ch, cx, cy, nativeCx, nativeCy, nativeZoom } = cropSource(ctx, img, center, zoom); + + let w = Math.min(zoom * 1.1, cw * 0.55); + w = Math.max(60, w); + let h = w / C4_ROAD_ASPECT; + if (h > ch * 0.5) { + h = ch * 0.5; + w = h * C4_ROAD_ASPECT; + } + const x = cx - w / 2; + const y = cy - h / 2; + const aspect = w / h; + const sx = nativeCx - nativeZoom / 2; + const sh = nativeZoom / aspect; + const sy = nativeCy - sh / 2; + + ctx.save(); + drawCurvedRect(ctx, x, y, w, h); + ctx.fillStyle = "#000"; + ctx.fill(); + ctx.clip(); + ctx.translate(x + w, 0); + ctx.scale(-1, 1); + ctx.drawImage(img, sx, sy, nativeZoom, sh, 0, 0, w, h); + ctx.restore(); + + ctx.save(); + drawCurvedRect(ctx, x, y, w, h); + ctx.strokeStyle = color; + ctx.lineWidth = 2.5; + ctx.stroke(); + ctx.restore(); +} + +function drawC3Preview(ctx, img, center, zoom, color) { + const { cx, cy, nativeCx, nativeCy, nativeZoom } = cropSource(ctx, img, center, zoom); + const half = zoom / 2; + const sx = nativeCx - nativeZoom / 2; + const sy = nativeCy - nativeZoom / 2; + + ctx.save(); + ctx.beginPath(); + ctx.arc(cx, cy, half, 0, Math.PI * 2); + ctx.fillStyle = "#000"; + ctx.fill(); + ctx.clip(); + ctx.translate(cx + half, 0); + ctx.scale(-1, 1); + ctx.drawImage(img, sx, sy, nativeZoom, nativeZoom, 0, 0, zoom, zoom); + ctx.restore(); + + ctx.save(); + ctx.beginPath(); + ctx.arc(cx, cy, half, 0, Math.PI * 2); + ctx.strokeStyle = color; + ctx.lineWidth = 2; + ctx.stroke(); + ctx.restore(); +} function getCanvas() { return document.getElementById("pip-sidecam-canvas"); @@ -107,19 +198,11 @@ function redraw() { const [cx, cy] = side.center; - // Crop square (what gets sampled) + circular bubble overlay. - ctx.strokeStyle = side.color; - ctx.lineWidth = 2; - ctx.setLineDash([]); - ctx.strokeRect(cx - half, cy - half, state.zoom, state.zoom); - - ctx.beginPath(); - ctx.arc(cx, cy, half, 0, Math.PI * 2); - ctx.fillStyle = side.color + "40"; - ctx.fill(); - ctx.lineWidth = 2; - ctx.strokeStyle = side.color; - ctx.stroke(); + if (isC4()) { + drawC4Preview(ctx, img, side.center, state.zoom, side.color); + } else { + drawC3Preview(ctx, img, side.center, state.zoom, side.color); + } // Center dot ctx.beginPath(); @@ -323,8 +406,9 @@ async function loadExistingConfig() { try { const resp = await fetch("/api/pip_preview/config"); if (!resp.ok) return; - const config = await resp.json(); - loadedConfig = config; + const data = await resp.json(); + deviceType = data.device_type || deviceType || null; + loadedConfig = data.mask || null; applyConfigToCanvas(); } catch (e) { console.error("PiP Preview config load failed", e); diff --git a/starpilot/system/the_galaxy/the_galaxy.py b/starpilot/system/the_galaxy/the_galaxy.py index f2ca383b0..c981a1a9b 100644 --- a/starpilot/system/the_galaxy/the_galaxy.py +++ b/starpilot/system/the_galaxy/the_galaxy.py @@ -8454,7 +8454,8 @@ def setup(app): def pip_preview_get_config(): if not params.get_bool("GalaxyDeveloperMode"): return jsonify({"error": "PiP Side Camera is available only with Galaxy Developer Mode enabled."}), 403 - return jsonify(_decode_json_object(params.get("PIPPreviewMask"))) + mask = _decode_json_object(params.get("PIPPreviewMask")) + return jsonify({"device_type": HARDWARE.get_device_type(), "mask": mask}) @app.route("/api/pip_preview/config", methods=["POST"]) def pip_preview_save_config():