Add C4 side camera support with curved rectangle rendering and update API response

selfdrive/ui/mici/onroad/augmented_road_view.py

@@ -20,6 +20,7 @@ from openpilot.selfdrive.ui.mici.onroad.starpilot_status import (
   get_border_color,
 )
 from openpilot.selfdrive.ui.mici.onroad.cameraview import CameraView
+from openpilot.selfdrive.ui.onroad.starpilot.pip_sidecam import PipSideCamera
 from openpilot.selfdrive.ui.lib.starpilot_visuals import get_border_width
 from openpilot.starpilot.common.favorite_slots import is_favorite_action_key, load_favorite_slots, toggle_favorite_slot
 from openpilot.system.ui.lib.application import FontWeight, gui_app, MousePos, MouseEvent
@@ -583,6 +584,10 @@ class AugmentedRoadView(CameraView):
 
     # debug
     self._pm = messaging.PubMaster(['uiDebug'])
+    # C4 sidecam: fills road preview as a curved rectangle. Only shown
+    # on the road camera screen, gated via widget visibility
+    self._pip_sidecam = self._child(PipSideCamera(shape="curved"))
+    self._pip_sidecam.set_visible(lambda: self.stream_type == ROAD_CAM)
 
   @staticmethod
   def _controls_ready() -> bool:
@@ -787,6 +792,17 @@ class AugmentedRoadView(CameraView):
       rl.draw_rectangle(int(self.rect.x), int(self.rect.y), int(self.rect.width), int(self.rect.height), rl.Color(0, 0, 0, 175))
       self._offroad_label.render(self._content_rect)
 
+    # C4 sidecam renders last (on top) and only when showing the road camera
+    # Inset by the border so the pill never covers the green/orange status border.
+    border = self._get_border_width()
+    preview_rect = rl.Rectangle(
+      self._content_rect.x + border,
+      self._content_rect.y + border,
+      max(1, self._content_rect.width - 2 * border),
+      max(1, self._content_rect.height - 2 * border),
+    )
+    self._pip_sidecam.render(preview_rect)
+
     # publish uiDebug
     msg = messaging.new_message('uiDebug')
     msg.uiDebug.drawTimeMillis = (time.monotonic() - start_draw) * 1000

selfdrive/ui/onroad/starpilot/pip_sidecam.py

@@ -10,6 +10,7 @@ import pyray as rl
 from msgq.visionipc import VisionIpcClient, VisionStreamType
 from openpilot.selfdrive.ui.ui_state import ui_state
 from openpilot.starpilot.common.vision_bsm import get_fresh_vasm_state
+from openpilot.system.ui.widgets import Widget
 
 PIP_SHADER_VERSION = """
 #version 300 es
@@ -93,6 +94,70 @@ void main() {
 }
 """
 
+# curved-rectangle variant for the mici display.
+PIP_CURVED_FRAGMENT_SHADER = PIP_SHADER_VERSION + """
+in vec2 fragTexCoord;
+uniform sampler2D texture0;
+uniform sampler2D texture1;
+uniform vec2 uCropMin;
+uniform vec2 uCropSize;
+uniform int uFlipX;
+uniform vec2 uRectSize;
+out vec4 fragColor;
+
+const float CORNER_RADIUS_FRACTION = 0.22;
+const float CURVE_AMOUNT = 0.07;
+const float EDGE_DARKEN = 0.14;
+const float RIM_BLEND = 0.06;
+
+void main() {
+  vec2 p = fragTexCoord * 2.0 - 1.0;
+  float halfW = uRectSize.x * 0.5;
+  float halfH = uRectSize.y * 0.5;
+  float radius = CORNER_RADIUS_FRACTION * min(uRectSize.x, uRectSize.y);
+
+  // Rounded-rectangle SDF in pixel space; mask before sampling.
+  vec2 q = abs(vec2(p.x * halfW, p.y * halfH)) - (vec2(halfW, halfH) - radius);
+  float dist = length(max(q, 0.0)) + min(max(q.x, q.y), 0.0) - radius;
+  float aa = max(fwidth(dist), 0.00001);
+  float alpha = 1.0 - smoothstep(-aa, aa, dist);
+  if (dist > aa) {
+    discard;
+  }
+
+  // Gentle convex curvature along both axes to mimic the curved OLED panel.
+  float curve = CURVE_AMOUNT * (1.0 - p.x * p.x) * (1.0 - p.y * p.y);
+  vec2 sampleCoord = clamp(fragTexCoord + curve * vec2(0.0, 0.5), 0.001, 0.999);
+
+  // The saved mask is a square crop, but the curved panel is wider than tall.
+  // Sample an aspect-matched horizontal band of that square (centered) instead
+  // of stretching it, so the image is never distorted. The circle's diameter
+  // (the square crop) becomes the panel's length; the height follows the aspect.
+  float aspect = uRectSize.x / max(uRectSize.y, 0.0001);
+  vec2 cropCoord = sampleCoord;
+  if (aspect >= 1.0) {
+    cropCoord.y = 0.5 + (sampleCoord.y - 0.5) / aspect;
+  } else {
+    cropCoord.x = 0.5 + (sampleCoord.x - 0.5) * aspect;
+  }
+  if (uFlipX == 1) {
+    cropCoord.x = 1.0 - cropCoord.x;
+  }
+  vec2 uv = uCropMin + cropCoord * uCropSize;
+  float y = texture(texture0, uv).r;
+  vec2 c = texture(texture1, uv).ra - 0.5;
+  vec3 rgb = vec3(y + 1.402 * c.y, y - 0.344 * c.x - 0.714 * c.y, y + 1.772 * c.x);
+
+  // Let the rim blend into the camera image and the UI underneath it.
+  float edgeShade = smoothstep(-radius, 0.0, dist);
+  rgb *= mix(1.0, 1.0 - EDGE_DARKEN, edgeShade);
+  float rim = smoothstep(radius * 0.55, radius, radius - dist);
+  rgb = mix(rgb, vec3(0.48, 0.70, 1.0), rim * RIM_BLEND);
+
+  fragColor = vec4(rgb, alpha);
+}
+"""
+
 UNIFORM_VEC2 = rl.ShaderUniformDataType.SHADER_UNIFORM_VEC2
 UNIFORM_INT = rl.ShaderUniformDataType.SHADER_UNIFORM_INT
 
@@ -110,10 +175,18 @@ BUBBLE_RADIUS_MIN = 180
 BUBBLE_RADIUS_MAX = 420
 BUBBLE_MARGIN = 24
 
+class PipSideCamera(Widget):
+  """Overlays the adjacent side window from the dcamera.
+
+  Drawn as a circular bubble on the big screen (shape="bubble") or as a
+  curved rectangle filling the whole road preview on the C4 (shape="curved").
+  """
+  def __init__(self, shape: str = "bubble"):
+    super().__init__()
+    if shape not in ("bubble", "curved"):
+      raise ValueError(f"Unknown PipSideCamera shape: {shape!r}")
+    self._shape = shape
 
-class PipSideCamera:
-  """Renders a circular pip bubble of the adjacent side window from the dcamera."""
-  def __init__(self):
     self._params = ui_state.params
     self._params_memory = ui_state.params_memory
 
@@ -132,6 +205,8 @@ class PipSideCamera:
     self._show_on_bsm = False
     self._mask = {}
     self._last_param_refresh = 0.0
+    self._side_activation_time: dict[str, float] = {}
+    self._active_sides: set[str] = set()
 
     self.shader = rl.load_shader_from_memory(PIP_VERTEX_SHADER, PIP_FRAGMENT_SHADER)
     self._texture1_loc = rl.get_shader_location(self.shader, "texture1")
@@ -140,6 +215,13 @@ class PipSideCamera:
     self._flip_x_loc = rl.get_shader_location(self.shader, "uFlipX")
     self._flip_x_value = rl.ffi.new("int[1]", [1])
 
+    self.curved_shader = rl.load_shader_from_memory(PIP_VERTEX_SHADER, PIP_CURVED_FRAGMENT_SHADER)
+    self._curved_texture1_loc = rl.get_shader_location(self.curved_shader, "texture1")
+    self._curved_crop_min_loc = rl.get_shader_location(self.curved_shader, "uCropMin")
+    self._curved_crop_size_loc = rl.get_shader_location(self.curved_shader, "uCropSize")
+    self._curved_flip_x_loc = rl.get_shader_location(self.curved_shader, "uFlipX")
+    self._curved_rect_size_loc = rl.get_shader_location(self.curved_shader, "uRectSize")
+
     self_ref = weakref.ref(self)
 
     def offroad_transition_callback():
@@ -157,16 +239,19 @@ class PipSideCamera:
     self.client = VisionIpcClient("camerad", self._stream_type, conflate=True)
 
   def close(self):
-    if self._closed:
+    if getattr(self, "_closed", False):
       return
     self._closed = True
     if getattr(self, "_offroad_transition_callback", None) is not None:
       ui_state.remove_offroad_transition_callback(self._offroad_transition_callback)
       self._offroad_transition_callback = None
     self._clear_textures()
-    if self.shader and self.shader.id:
-      rl.unload_shader(self.shader)
-      self.shader.id = 0
+    if (shader := getattr(self, "shader", None)) is not None and shader.id:
+      rl.unload_shader(shader)
+      shader.id = 0
+    if (curved := getattr(self, "curved_shader", None)) is not None and curved.id:
+      rl.unload_shader(curved)
+      curved.id = 0
     self.frame = None
     self.client = None
 
@@ -247,16 +332,10 @@ class PipSideCamera:
     cy = content_rect.y + content_rect.height - margin - radius
     return rl.Rectangle(cx - radius, cy - radius, radius * 2, radius * 2)
 
-  def render(self, content_rect: rl.Rectangle):
-    if not ui_state.started:
-      return
-
-    sides = self.active_sides()
-    if not sides:
-      return
-
+  def _acquire_frame(self) -> bool:
+    """Ensure connection and refresh the Y/UV textures from the latest driver frame."""
     if not self._ensure_connection():
-      return
+      return False
 
     buffer = self.client.recv(timeout_ms=0)
     if buffer:
@@ -264,10 +343,10 @@ class PipSideCamera:
       self._last_frame_id = int(getattr(buffer, "frame_id", -1))
       self._texture_needs_update = True
     if self.frame is None:
-      return
+      return False
 
     if not self.texture_y or not self.texture_uv:
-      return
+      return False
 
     if self._texture_needs_update:
       y_data = self.frame.data[: self.frame.uv_offset]
@@ -275,13 +354,47 @@ class PipSideCamera:
       rl.update_texture(self.texture_y, rl.ffi.cast("void *", rl.ffi.from_buffer(y_data)))
       rl.update_texture(self.texture_uv, rl.ffi.cast("void *", rl.ffi.from_buffer(uv_data)))
       self._texture_needs_update = False
+    return True
 
-    for side in sides:
-      crop = self._crop_rect(side)
-      if crop is None:
-        continue
-      bubble = self._bubble_rect(content_rect, side)
-      self._draw_bubble(bubble, crop)
+  def _render(self, content_rect: rl.Rectangle):
+    """Fetch the current driver frame, then draw it for the configured shape."""
+    if not ui_state.started:
+      return None
+
+    sides = self.active_sides()
+    if not sides or not self._acquire_frame():
+      return None
+
+    if self._shape == "curved":
+      # C4: one crop fills the whole road preview as a curved rectangle.
+      side = self._pick_side(sides)
+      if side is not None:
+        crop = self._crop_rect(side)
+        if crop is not None:
+          self._draw_curved(content_rect, crop)
+    else:
+      # Raybig: one circular bubble per active side.
+      for side in sides:
+        crop = self._crop_rect(side)
+        if crop is None:
+          continue
+        bubble = self._bubble_rect(content_rect, side)
+        self._draw_bubble(bubble, crop)
+    return None
+
+  def _pick_side(self, sides: list[str]) -> str | None:
+    """Return the active side whose blinker/BSM most recently turned on.
+
+    Only a rising edge (inactive -> active) refreshes the timestamp
+    """
+    now = time.monotonic()
+    active = set(sides)
+    for side in active - self._active_sides:
+      self._side_activation_time[side] = now
+    self._active_sides = active
+    if not sides:
+      return None
+    return max(sides, key=lambda side: self._side_activation_time.get(side, 0.0))
 
   def _draw_bubble(self, bubble: rl.Rectangle, crop: rl.Rectangle):
     tex_w = float(self.texture_y.width)
@@ -300,6 +413,25 @@ class PipSideCamera:
     rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
     rl.end_shader_mode()
 
+  def _draw_curved(self, content_rect: rl.Rectangle, crop: rl.Rectangle):
+    tex_w = float(self.texture_y.width)
+    tex_h = float(self.texture_y.height)
+    crop_min = rl.Vector2(crop.x / tex_w, crop.y / tex_h)
+    crop_size = rl.Vector2(crop.width / tex_w, crop.height / tex_h)
+    rect_size = rl.Vector2(content_rect.width, content_rect.height)
+
+    src_rect = rl.Rectangle(0, 0, tex_w, tex_h)
+    dst_rect = rl.Rectangle(content_rect.x, content_rect.y, content_rect.width, content_rect.height)
+
+    rl.begin_shader_mode(self.curved_shader)
+    rl.set_shader_value(self.curved_shader, self._curved_crop_min_loc, crop_min, UNIFORM_VEC2)
+    rl.set_shader_value(self.curved_shader, self._curved_crop_size_loc, crop_size, UNIFORM_VEC2)
+    rl.set_shader_value(self.curved_shader, self._curved_flip_x_loc, self._flip_x_value, UNIFORM_INT)
+    rl.set_shader_value(self.curved_shader, self._curved_rect_size_loc, rect_size, UNIFORM_VEC2)
+    rl.set_shader_value_texture(self.curved_shader, self._curved_texture1_loc, self.texture_uv)
+    rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
+    rl.end_shader_mode()
+
   def _ensure_connection(self) -> bool:
     if not self.client.is_connected():
       self.frame = None
@@ -324,9 +456,9 @@ class PipSideCamera:
       int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
 
   def _clear_textures(self):
-    if self.texture_y and self.texture_y.id:
-      rl.unload_texture(self.texture_y)
+    if (texture_y := getattr(self, "texture_y", None)) is not None and texture_y.id:
+      rl.unload_texture(texture_y)
       self.texture_y = None
-    if self.texture_uv and self.texture_uv.id:
-      rl.unload_texture(self.texture_uv)
+    if (texture_uv := getattr(self, "texture_uv", None)) is not None and texture_uv.id:
+      rl.unload_texture(texture_uv)
       self.texture_uv = None

selfdrive/ui/onroad/starpilot/starpilot_onroad_view.py

@@ -39,7 +39,7 @@ class StarPilotOnroadView(AugmentedRoadView):
     self._max_fps = 0.0
     self._avg_fps = 0.0
 
-    self._pip_sidecam = PipSideCamera()
+    self._pip_sidecam = self._child(PipSideCamera())
 
     self.layout_manager = WidgetLayoutManager(self._content_rect)
 

selfdrive/ui/tests/test_pip_sidecam.py

@@ -1,6 +1,7 @@
 from openpilot.selfdrive.ui.onroad.starpilot.pip_sidecam import (
   IMAGE_TO_VEHICLE_SIDE,
   PIP_FRAGMENT_SHADER,
+  PIP_CURVED_FRAGMENT_SHADER,
   PipSideCamera,
 )
 
@@ -47,3 +48,39 @@ def test_pip_driver_camera_shader_uses_analytic_bubble_shading_without_new_unifo
   assert "rgb = mix(rgb, vec3(1.0)" not in PIP_FRAGMENT_SHADER
   assert "uniform sampler2D texture2" not in PIP_FRAGMENT_SHADER
   assert "uRefraction" not in PIP_FRAGMENT_SHADER
+
+
+def test_pip_c4_curved_shader_masks_a_rounded_rectangle_before_sampling():
+  assert "uRectSize" in PIP_CURVED_FRAGMENT_SHADER
+  assert "CORNER_RADIUS_FRACTION" in PIP_CURVED_FRAGMENT_SHADER
+  assert "CURVE_AMOUNT" in PIP_CURVED_FRAGMENT_SHADER
+  assert "length(max(q, 0.0))" in PIP_CURVED_FRAGMENT_SHADER
+  assert PIP_CURVED_FRAGMENT_SHADER.index("if (dist > aa)") < PIP_CURVED_FRAGMENT_SHADER.index("texture(texture0")
+  assert PIP_CURVED_FRAGMENT_SHADER.count("texture(texture0") == 1
+  assert PIP_CURVED_FRAGMENT_SHADER.count("texture(texture1") == 1
+  assert "cropCoord.x = 1.0 - cropCoord.x" in PIP_CURVED_FRAGMENT_SHADER
+  assert "y + 1.402 * c.y" in PIP_CURVED_FRAGMENT_SHADER
+  assert "uRefraction" not in PIP_CURVED_FRAGMENT_SHADER
+
+
+def test_pip_sidecam_is_a_widget_with_curved_and_bubble_shapes():
+  bubble = PipSideCamera.__new__(PipSideCamera)
+  bubble._closed = True
+  assert isinstance(bubble, PipSideCamera)
+  assert hasattr(bubble, "render")
+  assert hasattr(bubble, "_draw_bubble")
+  assert hasattr(bubble, "_draw_curved")
+  curved = PipSideCamera.__new__(PipSideCamera)
+  curved._closed = True
+  curved._shape = "curved"
+  assert curved._shape == "curved"
+
+
+def test_pip_sidecam_rejects_unknown_shapes():
+  camera = object.__new__(PipSideCamera)
+  try:
+    PipSideCamera.__init__(camera, shape="hexagon")
+  except ValueError:
+    pass
+  else:
+    raise AssertionError("expected ValueError for unknown shape")

starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.js

@@ -35,6 +35,97 @@ const state = reactive({
 let _loadedImage = null;
 let _lastCanvas = null;
 let loadedConfig = null;
+let deviceType = null;
+
+const C4_ROAD_ASPECT = 476 / 240;
+
+function isC4() {
+  return (deviceType || "").toLowerCase() === "mici";
+}
+
+function drawCurvedRect(ctx, x, y, w, h) {
+  const r = Math.min(w, h) * 0.22;
+  ctx.beginPath();
+  ctx.moveTo(x + r, y);
+  ctx.arcTo(x + w, y, x + w, y + h, r);
+  ctx.arcTo(x + w, y + h, x, y + h, r);
+  ctx.arcTo(x, y + h, x, y, r);
+  ctx.arcTo(x, y, x + w, y, r);
+  ctx.closePath();
+}
+
+function cropSource(ctx, img, center, zoom) {
+  const cw = ctx.canvas.width;
+  const ch = ctx.canvas.height;
+  const nativeW = img.naturalWidth;
+  const nativeH = img.naturalHeight;
+  const [cx, cy] = center;
+  const nativeCx = (cw - cx) * nativeW / cw;
+  const nativeCy = cy * nativeH / ch;
+  const nativeZoom = zoom * nativeW / cw;
+  return { cw, ch, cx, cy, nativeCx, nativeCy, nativeZoom };
+}
+
+function drawC4Preview(ctx, img, center, zoom, color) {
+  const { cw, ch, cx, cy, nativeCx, nativeCy, nativeZoom } = cropSource(ctx, img, center, zoom);
+
+  let w = Math.min(zoom * 1.1, cw * 0.55);
+  w = Math.max(60, w);
+  let h = w / C4_ROAD_ASPECT;
+  if (h > ch * 0.5) {
+    h = ch * 0.5;
+    w = h * C4_ROAD_ASPECT;
+  }
+  const x = cx - w / 2;
+  const y = cy - h / 2;
+  const aspect = w / h;
+  const sx = nativeCx - nativeZoom / 2;
+  const sh = nativeZoom / aspect;
+  const sy = nativeCy - sh / 2;
+
+  ctx.save();
+  drawCurvedRect(ctx, x, y, w, h);
+  ctx.fillStyle = "#000";
+  ctx.fill();
+  ctx.clip();
+  ctx.translate(x + w, 0);
+  ctx.scale(-1, 1);
+  ctx.drawImage(img, sx, sy, nativeZoom, sh, 0, y, w, h);
+  ctx.restore();
+
+  ctx.save();
+  drawCurvedRect(ctx, x, y, w, h);
+  ctx.strokeStyle = color;
+  ctx.lineWidth = 2.5;
+  ctx.stroke();
+  ctx.restore();
+}
+
+function drawC3Preview(ctx, img, center, zoom, color) {
+  const { cx, cy, nativeCx, nativeCy, nativeZoom } = cropSource(ctx, img, center, zoom);
+  const half = zoom / 2;
+  const sx = nativeCx - nativeZoom / 2;
+  const sy = nativeCy - nativeZoom / 2;
+
+  ctx.save();
+  ctx.beginPath();
+  ctx.arc(cx, cy, half, 0, Math.PI * 2);
+  ctx.fillStyle = "#000";
+  ctx.fill();
+  ctx.clip();
+  ctx.translate(cx + half, 0);
+  ctx.scale(-1, 1);
+  ctx.drawImage(img, sx, sy, nativeZoom, nativeZoom, 0, cy - half, zoom, zoom);
+  ctx.restore();
+
+  ctx.save();
+  ctx.beginPath();
+  ctx.arc(cx, cy, half, 0, Math.PI * 2);
+  ctx.strokeStyle = color;
+  ctx.lineWidth = 2;
+  ctx.stroke();
+  ctx.restore();
+}
 
 function getCanvas() {
   return document.getElementById("pip-sidecam-canvas");
@@ -107,19 +198,11 @@ function redraw() {
 
     const [cx, cy] = side.center;
 
-    // Crop square (what gets sampled) + circular bubble overlay.
-    ctx.strokeStyle = side.color;
-    ctx.lineWidth = 2;
-    ctx.setLineDash([]);
-    ctx.strokeRect(cx - half, cy - half, state.zoom, state.zoom);
-
-    ctx.beginPath();
-    ctx.arc(cx, cy, half, 0, Math.PI * 2);
-    ctx.fillStyle = side.color + "40";
-    ctx.fill();
-    ctx.lineWidth = 2;
-    ctx.strokeStyle = side.color;
-    ctx.stroke();
+    if (isC4()) {
+      drawC4Preview(ctx, img, side.center, state.zoom, side.color);
+    } else {
+      drawC3Preview(ctx, img, side.center, state.zoom, side.color);
+    }
 
     // Center dot
     ctx.beginPath();
@@ -323,8 +406,9 @@ async function loadExistingConfig() {
   try {
     const resp = await fetch("/api/pip_preview/config");
     if (!resp.ok) return;
-    const config = await resp.json();
-    loadedConfig = config;
+    const data = await resp.json();
+    deviceType = data.device_type || deviceType || null;
+    loadedConfig = data.mask || null;
     applyConfigToCanvas();
   } catch (e) {
     console.error("PiP Preview config load failed", e);

starpilot/system/the_galaxy/the_galaxy.py

@@ -8454,7 +8454,8 @@ def setup(app):
   def pip_preview_get_config():
     if not params.get_bool("GalaxyDeveloperMode"):
       return jsonify({"error": "PiP Side Camera is available only with Galaxy Developer Mode enabled."}), 403
-    return jsonify(_decode_json_object(params.get("PIPPreviewMask")))
+    mask = _decode_json_object(params.get("PIPPreviewMask"))
+    return jsonify({"device_type": HARDWARE.get_device_type(), "mask": mask})
 
   @app.route("/api/pip_preview/config", methods=["POST"])
   def pip_preview_save_config():