add PiP Side Camera

This commit is contained in:
Prabhaav Pillai
2026-08-05 21:15:52 -04:00
committed by firestar5683
parent 1f33a35c5b
commit b5223b6e23
11 changed files with 945 additions and 2 deletions
+4
View File
@@ -493,6 +493,10 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"PauseLateralSpeed", {PERSISTENT, FLOAT, "0.0", "0.0", 1, SETTINGS_SIMPLE}},
{"LateralResumeDelay", {PERSISTENT, FLOAT, "0.0", "0.0", 1, SETTINGS_SIMPLE}},
{"PedalsOnUI", {PERSISTENT, BOOL, "0", "0", 1, SETTINGS_SIMPLE}},
{"PIPPreviewEnabled", {PERSISTENT, BOOL, "0", "0", 1}},
{"PIPPreviewMask", {PERSISTENT, JSON, "{\"width\":1928,\"height\":1208,\"center_left\":[315,548],\"center_right\":[1571,539],\"crop_size\":580}", "{\"width\":1928,\"height\":1208,\"center_left\":[315,548],\"center_right\":[1571,539],\"crop_size\":580}", 2}},
{"PIPPreviewShowOnBlinker", {PERSISTENT, BOOL, "0", "0", 1}},
{"PIPPreviewShowOnBSM", {PERSISTENT, BOOL, "0", "0", 1}},
{"GalaxyPaired", {PERSISTENT, BOOL, "0", "0", 0}},
{"GalaxyUploadPending", {PERSISTENT, BOOL, "0", "0", 0}},
{"PreferredSchedule", {PERSISTENT, INT, "2", "0", 0}},
@@ -0,0 +1,284 @@
from __future__ import annotations
import json
import platform
import time
import weakref
import pyray as rl
from msgq.visionipc import VisionIpcClient, VisionStreamType
from openpilot.selfdrive.ui.ui_state import ui_state
from openpilot.starpilot.common.vision_bsm import get_fresh_vasm_state
PIP_SHADER_VERSION = """
#version 300 es
precision mediump float;
"""
if platform.system() == "Darwin":
PIP_SHADER_VERSION = """
#version 330 core
"""
PIP_VERTEX_SHADER = PIP_SHADER_VERSION + """
in vec3 vertexPosition;
in vec2 vertexTexCoord;
in vec3 vertexNormal;
in vec4 vertexColor;
uniform mat4 mvp;
out vec2 fragTexCoord;
out vec4 fragColor;
void main() {
fragTexCoord = vertexTexCoord;
fragColor = vertexColor;
gl_Position = mvp * vec4(vertexPosition, 1.0);
}
"""
PIP_FRAGMENT_SHADER = PIP_SHADER_VERSION + """
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
uniform vec2 uCropMin;
uniform vec2 uCropSize;
out vec4 fragColor;
void main() {
vec2 uv = uCropMin + fragTexCoord * uCropSize;
float y = texture(texture0, uv).r;
vec2 c = texture(texture1, uv).ra - 0.5;
vec3 rgb = vec3(y + 1.402 * c.y, y - 0.344 * c.x - 0.714 * c.y, y + 1.772 * c.x);
vec2 p = fragTexCoord - 0.5;
float dist = length(p);
float edge = 0.02;
float alpha = 1.0 - smoothstep(0.5 - edge, 0.5, dist);
fragColor = vec4(rgb, alpha);
}
"""
UNIFORM_VEC2 = rl.ShaderUniformDataType.SHADER_UNIFORM_VEC2
CONNECTION_RETRY_INTERVAL = 0.2
PARAM_REFRESH_INTERVAL = 2.0
# Bubble geometry
BUBBLE_RADIUS_FRACTION = 0.3
BUBBLE_RADIUS_MIN = 180
BUBBLE_RADIUS_MAX = 420
BUBBLE_MARGIN = 24
class PipSideCamera:
"""Renders a circular pip bubble of the adjacent side window from the dcamera."""
def __init__(self):
self._params = ui_state.params
self._params_memory = ui_state.params_memory
self.client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, conflate=True)
self._stream_type = VisionStreamType.VISION_STREAM_DRIVER
self._last_connection_attempt = 0.0
self.frame = None
self._last_frame_id = -1
self._texture_needs_update = True
self.texture_y: rl.Texture | None = None
self.texture_uv: rl.Texture | None = None
self._closed = False
self._enabled = False
self._show_on_blinker = False
self._show_on_bsm = False
self._mask = {}
self._last_param_refresh = 0.0
self.shader = rl.load_shader_from_memory(PIP_VERTEX_SHADER, PIP_FRAGMENT_SHADER)
self._texture1_loc = rl.get_shader_location(self.shader, "texture1")
self._crop_min_loc = rl.get_shader_location(self.shader, "uCropMin")
self._crop_size_loc = rl.get_shader_location(self.shader, "uCropSize")
self_ref = weakref.ref(self)
def offroad_transition_callback():
if (ref := self_ref()) is not None:
ref._offroad_transition()
self._offroad_transition_callback = offroad_transition_callback
ui_state.add_offroad_transition_callback(self._offroad_transition_callback)
def _offroad_transition(self):
self._clear_textures()
self.frame = None
self._last_frame_id = -1
self._last_connection_attempt = 0.0
self.client = VisionIpcClient("camerad", self._stream_type, conflate=True)
def close(self):
if self._closed:
return
self._closed = True
if getattr(self, "_offroad_transition_callback", None) is not None:
ui_state.remove_offroad_transition_callback(self._offroad_transition_callback)
self._offroad_transition_callback = None
self._clear_textures()
if self.shader and self.shader.id:
rl.unload_shader(self.shader)
self.shader.id = 0
self.frame = None
self.client = None
def __del__(self):
self.close()
def _refresh_config(self, force: bool = False):
now = time.monotonic()
if not force and now - self._last_param_refresh < PARAM_REFRESH_INTERVAL:
return
self._last_param_refresh = now
self._enabled = self._params.get_bool("PIPPreviewEnabled")
self._show_on_blinker = self._params.get_bool("PIPPreviewShowOnBlinker")
self._show_on_bsm = self._params.get_bool("PIPPreviewShowOnBSM")
try:
raw = self._params.get("PIPPreviewMask")
if isinstance(raw, (bytes, str)):
raw = json.loads(raw)
self._mask = raw if isinstance(raw, dict) else {}
except (TypeError, ValueError, json.JSONDecodeError):
self._mask = {}
def active_sides(self) -> list[str]:
"""Return the car-side keys ('left'/'right') whose preview bubble should show."""
if not ui_state.started:
return []
self._refresh_config()
if not self._enabled or not self._mask:
return []
car_state = ui_state.sm["carState"] if ui_state.sm.valid.get("carState", False) else None
if car_state is None:
return []
vasm_left, vasm_right = get_fresh_vasm_state(self._params_memory)
left_blinker = bool(car_state.leftBlinker)
right_blinker = bool(car_state.rightBlinker)
left_bsm = bool(car_state.leftBlindspot) or vasm_left
right_bsm = bool(car_state.rightBlindspot) or vasm_right
sides = []
if self._mask.get("center_left") and ((self._show_on_blinker and left_blinker) or (self._show_on_bsm and left_bsm)):
sides.append("left")
if self._mask.get("center_right") and ((self._show_on_blinker and right_blinker) or (self._show_on_bsm and right_bsm)):
sides.append("right")
return sides
def _crop_rect(self, side: str) -> rl.Rectangle | None:
center = self._mask.get(f"center_{side}")
size = self._mask.get("crop_size")
if not center or len(center) < 2 or not size:
return None
try:
cx, cy = float(center[0]), float(center[1])
half = float(size) / 2.0
except (TypeError, ValueError):
return None
if half <= 0:
return None
return rl.Rectangle(cx - half, cy - half, size, size)
def _bubble_rect(self, content_rect: rl.Rectangle, side: str) -> rl.Rectangle:
radius = int(min(content_rect.width, content_rect.height) * BUBBLE_RADIUS_FRACTION)
radius = max(BUBBLE_RADIUS_MIN, min(radius, BUBBLE_RADIUS_MAX))
margin = BUBBLE_MARGIN
cx = content_rect.x + margin + radius if side == "left" else content_rect.x + content_rect.width - margin - radius
cy = content_rect.y + content_rect.height - margin - radius
return rl.Rectangle(cx - radius, cy - radius, radius * 2, radius * 2)
def render(self, content_rect: rl.Rectangle):
if not ui_state.started:
return
sides = self.active_sides()
if not sides:
return
if not self._ensure_connection():
return
buffer = self.client.recv(timeout_ms=0)
if buffer:
self.frame = buffer
self._last_frame_id = int(getattr(buffer, "frame_id", -1))
self._texture_needs_update = True
if self.frame is None:
return
if not self.texture_y or not self.texture_uv:
return
if self._texture_needs_update:
y_data = self.frame.data[: self.frame.uv_offset]
uv_data = self.frame.data[self.frame.uv_offset:]
rl.update_texture(self.texture_y, rl.ffi.cast("void *", rl.ffi.from_buffer(y_data)))
rl.update_texture(self.texture_uv, rl.ffi.cast("void *", rl.ffi.from_buffer(uv_data)))
self._texture_needs_update = False
for side in sides:
crop = self._crop_rect(side)
if crop is None:
continue
bubble = self._bubble_rect(content_rect, side)
self._draw_bubble(bubble, crop)
def _draw_bubble(self, bubble: rl.Rectangle, crop: rl.Rectangle):
cx = bubble.x + bubble.width / 2
cy = bubble.y + bubble.height / 2
radius = bubble.width / 2
rl.draw_circle(int(round(cx)), int(round(cy)), radius, rl.BLACK)
rl.draw_circle_lines(int(round(cx)), int(round(cy)), radius, rl.Color(255, 255, 255, 120))
tex_w = float(self.texture_y.width)
tex_h = float(self.texture_y.height)
crop_min = rl.Vector2(crop.x / tex_w, crop.y / tex_h)
crop_size = rl.Vector2(crop.width / tex_w, crop.height / tex_h)
src_rect = rl.Rectangle(0, 0, tex_w, tex_h)
dst_rect = rl.Rectangle(bubble.x, bubble.y, bubble.width, bubble.height)
rl.begin_shader_mode(self.shader)
rl.set_shader_value(self.shader, self._crop_min_loc, crop_min, UNIFORM_VEC2)
rl.set_shader_value(self.shader, self._crop_size_loc, crop_size, UNIFORM_VEC2)
rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv)
rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
def _ensure_connection(self) -> bool:
if not self.client.is_connected():
self.frame = None
self._last_frame_id = -1
now = rl.get_time()
if now - self._last_connection_attempt < CONNECTION_RETRY_INTERVAL:
return False
self._last_connection_attempt = now
self._clear_textures()
if not self.client.connect(False) or not self.client.num_buffers:
return False
self._initialize_textures()
return True
def _initialize_textures(self):
self._clear_textures()
self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
def _clear_textures(self):
if self.texture_y and self.texture_y.id:
rl.unload_texture(self.texture_y)
self.texture_y = None
if self.texture_uv and self.texture_uv.id:
rl.unload_texture(self.texture_uv)
self.texture_uv = None
@@ -13,6 +13,7 @@ from openpilot.selfdrive.ui.onroad.starpilot.widgets import (
)
from openpilot.selfdrive.ui.onroad.starpilot.stopping_point import render_stopping_point
from openpilot.selfdrive.ui.onroad.starpilot.pause_indicators import render_lateral_paused, render_longitudinal_paused
from openpilot.selfdrive.ui.onroad.starpilot.pip_sidecam import PipSideCamera
from openpilot.selfdrive.ui.onroad.starpilot.weather_icon import render_weather_icon
from openpilot.selfdrive.ui.lib.starpilot_status import (
get_screen_edge_color,
@@ -37,6 +38,8 @@ class StarPilotOnroadView(AugmentedRoadView):
self._max_fps = 0.0
self._avg_fps = 0.0
self._pip_sidecam = PipSideCamera()
self.layout_manager = WidgetLayoutManager(self._content_rect)
# Disable parent rendering calls — layout manager draws at computed bounds
@@ -94,6 +97,9 @@ class StarPilotOnroadView(AugmentedRoadView):
self._render_overlays()
self._render_road_name()
# PiP renders last so it always sits on top of every other on-road overlay.
self._pip_sidecam.render(self._content_rect)
def _draw_border(self, rect: rl.Rectangle):
border_width = self._get_border_width()
rl.draw_rectangle_rounded_lines_ex(rect, 0.12, 10, border_width, rl.BLACK)
@@ -131,8 +137,8 @@ class StarPilotOnroadView(AugmentedRoadView):
if not self._params.get_bool("EnableTorqueBarWidget", default=True):
return
rl.begin_scissor_mode(
int(self._content_rect.x), int(self._content_rect.y),
int(self._content_rect.width), int(self._content_rect.height),
int(round(self._content_rect.x)), int(round(self._content_rect.y)),
int(round(self._content_rect.width)), int(round(self._content_rect.height)),
)
self._torque_bar.render(self._content_rect)
rl.end_scissor_mode()
@@ -145,6 +151,11 @@ class StarPilotOnroadView(AugmentedRoadView):
if not mr._path.projected_points.size:
return
rl.begin_scissor_mode(
int(round(rect.x)), int(round(rect.y)),
int(round(rect.width)), int(round(rect.height)),
)
# Path edges (always rendered if track_edge_vertices exist)
if mr._track_edge_vertices.size >= 4:
render_path_edges(mr)
@@ -24,6 +24,7 @@ import { Troubleshoot } from "/assets/components/tools/troubleshoot.js"
import { TmuxLog } from "/assets/components/tools/tmux.js"
import { ToggleControl } from "/assets/components/tools/toggles.js"
import { VASMAnnotations } from "/assets/components/tools/v_asm.js"
import { PipSideCamera } from "/assets/components/tools/pip_sidecam.js"
import { UpdateManager } from "/assets/components/tools/update_manager.js"
let router, routerState
@@ -86,6 +87,7 @@ function Root() {
createRoute("updates", "/manage_updates", UpdateManager),
createRoute("vehicle_features", "/vehicle_features", VehicleFeatures),
createRoute("v_asm", "/manage_v_asm", VASMAnnotations),
createRoute("pip_sidecam", "/manage_pip_sidecam", PipSideCamera),
]
router = createRouter({
@@ -24,6 +24,7 @@ const MENU_ITEMS = {
{ name: "Testing Ground", link: "/testing_ground", icon: "bi-bezier2" },
{ name: "Troubleshoot", link: "/troubleshoot", icon: "bi-tools" },
{ name: "V-Adj Spot Monitor", link: "/manage_v_asm", icon: "bi-eye" },
{ name: "PiP Side Camera", link: "/manage_pip_sidecam", icon: "bi-badge-hd" },
{ name: "Theme Maker", link: "/theme_maker", icon: "bi-palette-fill" },
{ name: "Tmux Log", link: "/manage_tmux", icon: "bi-terminal" },
{ name: "Backup and Restore", link: "/manage_toggles", icon: "bi-arrow-repeat" },
@@ -2137,6 +2137,36 @@
"parent_key": "CustomUI",
"settings_tier": "simple"
},
{
"key": "PIPPreviewEnabled",
"label": "Enable PiP Side Preview",
"description": "Show a temporary Picture-in-Picture bubble of the side window while the turn signal is on or a blind spot is detected.",
"data_type": "bool",
"ui_type": "toggle",
"parent_key": "CustomUI",
"is_parent_toggle": true,
"requires_nonempty_key": "PIPPreviewMask",
"disabled_reason": "Configure the PiP window mask first in Galaxy > PiP Side Camera",
"settings_tier": "simple"
},
{
"key": "PIPPreviewShowOnBlinker",
"label": "Show on Turn Signal",
"description": "Display the side preview bubble while the corresponding turn signal is engaged.",
"data_type": "bool",
"ui_type": "toggle",
"parent_key": "PIPPreviewEnabled",
"settings_tier": "simple"
},
{
"key": "PIPPreviewShowOnBSM",
"label": "Show on Blind Spot Detection",
"description": "Display the side preview bubble when a vehicle is detected in the blind spot, whether from factory BSM, V-ASM, or both.",
"data_type": "bool",
"ui_type": "toggle",
"parent_key": "PIPPreviewEnabled",
"settings_tier": "simple"
},
{
"key": "Compass",
"label": "Compass",
@@ -0,0 +1,44 @@
/* PiP Side Camera reuses the V-ASM tool styling. */
@import url("./v_asm.css");
.pip-zoom-control {
margin-top: 18px;
padding: 14px 18px;
background: var(--card-bg, #16181d);
border: 1px solid var(--card-border, #2a2e37);
border-radius: 10px;
}
.pip-zoom-header {
display: flex;
justify-content: space-between;
align-items: center;
font-size: 14px;
font-weight: 600;
color: #e6e6e6;
}
.pip-zoom-value {
font-family: monospace;
color: #9ecbff;
}
.pip-zoom-slider {
width: 100%;
margin-top: 12px;
accent-color: #0d6efd;
}
.pip-zoom-labels {
display: flex;
justify-content: space-between;
font-size: 12px;
color: #8b8f98;
margin-top: 2px;
}
.pip-zoom-hint {
margin-top: 8px;
font-size: 12px;
color: #8b8f98;
}
@@ -0,0 +1,467 @@
import { html, reactive } from "/assets/vendor/arrow-core.js";
const CANVAS_W = 640;
const CANVAS_H = 480;
const ZOOM_MIN = 60;
const ZOOM_MAX = 640;
const ZOOM_STEP = 5;
let initialLoadTriggered = false;
const state = reactive({
loading: false,
error: "",
success: "",
armSide: null,
leftCenter: null,
rightCenter: null,
zoom: 240,
image: false,
configSaved: false,
configExists: false,
});
let _loadedImage = null;
let _lastCanvas = null;
let loadedConfig = null;
function getCanvas() {
return document.getElementById("pip-sidecam-canvas");
}
function canvasScale() {
const canvas = getCanvas();
const img = _loadedImage;
const nativeW = img ? img.naturalWidth : 1920;
const nativeH = img ? img.naturalHeight : 1080;
const cw = canvas ? canvas.width || CANVAS_W : CANVAS_W;
const ch = canvas ? canvas.height || CANVAS_H : CANVAS_H;
return { cw, ch, nativeW, nativeH };
}
function redraw() {
const canvas = getCanvas();
if (!canvas) return;
if (canvas !== _lastCanvas) {
_lastCanvas = canvas;
if (_loadedImage) {
canvas._img = _loadedImage;
canvas.width = Math.min(_loadedImage.naturalWidth, 1280);
canvas.height = Math.round(canvas.width * (_loadedImage.naturalHeight / _loadedImage.naturalWidth));
} else {
canvas.width = CANVAS_W;
canvas.height = CANVAS_H;
}
}
const ctx = canvas.getContext("2d");
ctx.clearRect(0, 0, canvas.width, canvas.height);
const img = _loadedImage;
if (img) {
canvas._img = img;
ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
} else {
ctx.fillStyle = "#222";
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.fillStyle = "#888";
ctx.font = "16px monospace";
ctx.textAlign = "center";
ctx.fillText("Loading camera snapshot...", canvas.width / 2, canvas.height / 2);
return;
}
const half = state.zoom / 2;
const sides = [
{ key: "left", center: state.leftCenter, color: "#0d6efd", label: "LEFT WINDOW" },
{ key: "right", center: state.rightCenter, color: "#fd7e14", label: "RIGHT WINDOW" },
];
for (const side of sides) {
if (!side.center) continue;
const [cx, cy] = side.center;
// Crop square (what gets sampled) + circular bubble overlay.
ctx.strokeStyle = side.color;
ctx.lineWidth = 2;
ctx.setLineDash([]);
ctx.strokeRect(cx - half, cy - half, state.zoom, state.zoom);
ctx.beginPath();
ctx.arc(cx, cy, half, 0, Math.PI * 2);
ctx.fillStyle = side.color + "40";
ctx.fill();
ctx.lineWidth = 2;
ctx.strokeStyle = side.color;
ctx.stroke();
// Center dot
ctx.beginPath();
ctx.arc(cx, cy, 4, 0, Math.PI * 2);
ctx.fillStyle = "#fff";
ctx.fill();
ctx.strokeStyle = side.color;
ctx.lineWidth = 1.5;
ctx.stroke();
// Label
ctx.fillStyle = side.color;
ctx.font = "bold 13px monospace";
ctx.textAlign = "center";
ctx.fillText(side.label, cx, cy - half - 8);
}
if (state.armSide) {
ctx.fillStyle = "#fff";
ctx.font = "bold 15px monospace";
ctx.textAlign = "center";
ctx.fillText(
state.armSide === "left" ? "Click to set LEFT window center" : "Click to set RIGHT window center",
canvas.width / 2,
canvas.height - 18,
);
}
}
async function loadSnapshot() {
state.error = "";
state.success = "";
state.image = false;
let blobUrl = null;
try {
const resp = await fetch("/api/pip_preview/snapshot");
if (!resp.ok) {
const payload = await resp.json().catch(() => ({}));
throw new Error(payload.error || resp.statusText || "Failed to load snapshot");
}
let src;
const contentType = resp.headers.get("content-type") || "";
if (contentType.includes("application/json")) {
const data = await resp.json();
if (!data.jpeg) throw new Error("Snapshot missing image data");
src = `data:image/jpeg;base64,${data.jpeg}`;
} else {
const blob = await resp.blob();
blobUrl = URL.createObjectURL(blob);
src = blobUrl;
}
const img = new Image();
img.onload = () => {
_loadedImage = img;
const canvas = getCanvas();
if (canvas) {
canvas._img = img;
canvas.width = Math.min(img.naturalWidth, 1280);
canvas.height = Math.round(canvas.width * (img.naturalHeight / img.naturalWidth));
}
state.image = true;
state.success = "Camera snapshot loaded. Place a center point on each window, then adjust the zoom.";
applyConfigToCanvas();
requestAnimationFrame(redraw);
if (blobUrl) {
URL.revokeObjectURL(blobUrl);
blobUrl = null;
}
};
img.onerror = () => {
state.image = false;
state.error = "Failed to decode image";
requestAnimationFrame(redraw);
if (blobUrl) {
URL.revokeObjectURL(blobUrl);
blobUrl = null;
}
};
img.src = src;
} catch (e) {
state.image = false;
state.error = e.message;
requestAnimationFrame(redraw);
if (blobUrl) {
URL.revokeObjectURL(blobUrl);
blobUrl = null;
}
}
}
function canvasClick(e) {
if (!state.image || !state.armSide || !e) return;
const canvas = getCanvas();
if (!canvas) return;
const rect = canvas.getBoundingClientRect();
const x = Math.round((e.clientX - rect.left) * (canvas.width / rect.width));
const y = Math.round((e.clientY - rect.top) * (canvas.height / rect.height));
if (x < 0 || y < 0) return;
if (state.armSide === "left") {
state.leftCenter = [x, y];
} else {
state.rightCenter = [x, y];
}
state.armSide = null;
state.error = "";
requestAnimationFrame(redraw);
}
function setArm(e) {
const side = e?.currentTarget?.value || e?.target?.value;
if (!side) return;
state.armSide = side;
state.error = "";
state.success = "";
requestAnimationFrame(redraw);
}
function updateZoom(e) {
const value = Number(e?.currentTarget?.value ?? e?.target?.value ?? 0);
if (!Number.isFinite(value)) return;
state.zoom = value;
requestAnimationFrame(redraw);
}
function clearCenter(side) {
if (side === "left") {
state.leftCenter = null;
} else {
state.rightCenter = null;
}
if (state.armSide === side) state.armSide = null;
requestAnimationFrame(redraw);
}
function clearAll() {
state.leftCenter = null;
state.rightCenter = null;
state.armSide = null;
state.configSaved = false;
requestAnimationFrame(redraw);
}
async function saveConfig() {
if (!state.leftCenter && !state.rightCenter) {
state.error = "Place at least one center point.";
return;
}
const { cw, ch, nativeW, nativeH } = canvasScale();
function toNative(center) {
if (!center) return [];
return [Math.round(center[0] * nativeW / cw), Math.round(center[1] * nativeH / ch)];
}
const config = {
width: nativeW,
height: nativeH,
center_left: toNative(state.leftCenter),
center_right: toNative(state.rightCenter),
crop_size: Math.round(state.zoom * nativeW / cw),
};
state.loading = true;
state.error = "";
state.success = "";
try {
const resp = await fetch("/api/pip_preview/config", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(config),
});
const data = await resp.json();
if (!resp.ok) throw new Error(data.error || "Failed to save");
state.configSaved = true;
state.configExists = true;
state.success = "PiP Preview mask saved!";
await loadExistingConfig();
} catch (e) {
state.error = e.message;
}
state.loading = false;
}
async function loadExistingConfig() {
try {
const resp = await fetch("/api/pip_preview/config");
if (!resp.ok) return;
const config = await resp.json();
loadedConfig = config;
applyConfigToCanvas();
} catch (e) {
console.error("PiP Preview config load failed", e);
}
}
function applyConfigToCanvas() {
const config = loadedConfig;
if (!config) {
state.leftCenter = null;
state.rightCenter = null;
state.configExists = false;
redraw();
return;
}
const { cw, ch, nativeW, nativeH } = canvasScale();
function toCanvas(center) {
if (!Array.isArray(center) || center.length < 2) return null;
return [Math.round(center[0] * cw / nativeW), Math.round(center[1] * ch / nativeH)];
}
state.leftCenter = toCanvas(config.center_left);
state.rightCenter = toCanvas(config.center_right);
if (Number.isFinite(Number(config.crop_size))) {
state.zoom = Math.round(Number(config.crop_size) * cw / nativeW);
}
state.configExists = Boolean(state.leftCenter || state.rightCenter);
redraw();
}
async function deleteConfig() {
state.loading = true;
state.error = "";
state.success = "";
try {
const resp = await fetch("/api/pip_preview/config", { method: "DELETE" });
const data = await resp.json();
if (!resp.ok) throw new Error(data.error || "Failed to delete");
state.leftCenter = null;
state.rightCenter = null;
state.armSide = null;
state.configSaved = false;
state.configExists = false;
loadedConfig = null;
state.success = "PiP Preview mask cleared.";
requestAnimationFrame(redraw);
} catch (e) {
state.error = e.message;
}
state.loading = false;
}
function scheduleInitialLoad() {
if (!initialLoadTriggered) {
initialLoadTriggered = true;
loadExistingConfig();
loadSnapshot();
}
const attempt = () => {
if (getCanvas()) {
redraw();
return;
}
requestAnimationFrame(attempt);
};
requestAnimationFrame(attempt);
}
function retrySnapshot() {
state.error = "";
state.success = "";
loadSnapshot();
}
export function PipSideCamera() {
const el = html`
<div class="v-asm-wrapper">
<div class="v-asm-section">
<div class="v-asm-header">
<h2>PiP Side Camera Preview</h2>
<div class="v-asm-card v-asm-card-info">
<div class="v-asm-card-title">About PiP Preview</div>
<ul class="v-asm-card-list">
<li>Shows a temporary Picture-in-Picture bubble of the adjacent side window while the turn signal is on or a blind spot is detected</li>
<li>Place a single center point on each window, then pick a shared zoom level</li>
<li>This mask is separate from the V-ASM detection mask, so you can tune the visual crop independently</li>
<li>Works alongside factory blind spot monitoring and/or V-ASM, and with turn signals alone</li>
</ul>
</div>
<div class="v-asm-card v-asm-card-danger">
<div class="v-asm-card-title">Setup</div>
<ul class="v-asm-card-list">
<li>Click "Set Left Center", then click the driver's side window; repeat for the right window</li>
<li>From the driver camera, the car's LEFT window appears on the right side of the image and vice versa</li>
<li>The zoom slider applies to BOTH windows so the preview stays consistent</li>
<li>At least one window center is required to enable the preview</li>
</ul>
</div>
</div>
<div class="v-asm-note">
Use the preview to enhance lateral awareness. Always check manually before merging and be aware the driver camera view is from the cabin and does not reflect your blind spot.
</div>
${state.error ? html`<div class="v-asm-error-banner">${state.error}</div>` : ""}
${state.success ? html`<div class="v-asm-success-banner">${state.success}</div>` : ""}
${state.configSaved ? html`
<div class="v-asm-success-banner">
PiP Preview mask saved! Configure the preview in <a href="/device_settings/visual-display-ui">Toggles</a>.
</div>
` : ""}
<div class="v-asm-toolbar">
<div class="v-asm-btn-group">
<button class="${state.armSide === "left" ? "v-asm-btn v-asm-btn-left-active" : "v-asm-btn v-asm-btn-outline-left"}"
@click="${setArm}" value="left">
${state.leftCenter ? "Move Left Center" : "Set Left Center"}
</button>
<button class="${state.armSide === "right" ? "v-asm-btn v-asm-btn-right-active" : "v-asm-btn v-asm-btn-outline-right"}"
@click="${setArm}" value="right">
${state.rightCenter ? "Move Right Center" : "Set Right Center"}
</button>
<button class="v-asm-btn v-asm-btn-primary" @click="${saveConfig}" .disabled="${state.loading || (!state.leftCenter && !state.rightCenter)}">
${state.loading ? "Saving..." : "Save Mask"}
</button>
${state.configExists ? html`<button class="v-asm-btn v-asm-btn-danger" @click="${deleteConfig}" .disabled="${state.loading}">Delete Mask</button>` : ""}
<button class="v-asm-btn v-asm-btn-secondary" @click="${clearAll}">Clear All</button>
<button class="v-asm-btn v-asm-btn-secondary" @click="${retrySnapshot}" .disabled="${state.loading}">
${state.loading ? "Loading..." : "Get a new Snapshot"}
</button>
</div>
</div>
${state.armSide ? html`<div class="v-asm-mode-banner ${state.armSide === "left" ? "v-asm-mode-left" : "v-asm-mode-right"}"><span>${state.armSide === "left" ? "⬅ Placing Left Center" : "➡ Placing Right Center"}</span><span>Click on the window to place its center point</span></div>` : ""}
<div class="v-asm-canvas-wrapper">
<canvas id="pip-sidecam-canvas" @click="${canvasClick}"></canvas>
<div class="v-asm-instructions">
${state.armSide ? "Click the window to place its center point." : "Place a center point on each window, then use the zoom slider below."}
</div>
</div>
<div class="pip-zoom-control">
<div class="pip-zoom-header">
<span>Zoom</span>
<span class="pip-zoom-value">${() => state.zoom} px</span>
</div>
<input type="range" class="pip-zoom-slider" min="${ZOOM_MIN}" max="${ZOOM_MAX}" step="${ZOOM_STEP}"
value="${() => state.zoom}" @input="${updateZoom}" />
<div class="pip-zoom-labels">
<span>Wide</span>
<span>Close</span>
</div>
<div class="pip-zoom-hint">Applied to both windows so the preview stays consistent.</div>
</div>
</div>
<div class="v-asm-card v-asm-card-warning">
<div class="v-asm-card-title">Preview Settings</div>
<ul class="v-asm-card-list">
<li>Enable "PiP Side Preview" in <a href="/device_settings/visual-display-ui">Toggles -> Visual (Display & UI) -> Driving Screen Widgets</a></li>
<li>Choose to show the preview on the turn signal, on blind spot detection, or both</li>
</ul>
</div>
</div>
`;
scheduleInitialLoad();
return el;
}
@@ -43,6 +43,7 @@
<link rel="stylesheet" href="/assets/components/tools/galaxy.css">
<link rel="stylesheet" href="/assets/components/tools/longitudinal_maneuvers.css">
<link rel="stylesheet" href="/assets/components/tools/v_asm.css">
<link rel="stylesheet" href="/assets/components/tools/pip_sidecam.css">
<link rel="stylesheet" href="/assets/components/tools/tsk_manager.css">
<script type="module">
@@ -151,3 +151,26 @@ def test_vasm_is_default_off_and_configured_only_in_galaxy():
REPO_ROOT / "selfdrive/ui/layouts/settings/starpilot/lateral.py",
)
assert all("VASM" not in path.read_text(encoding="utf-8") for path in physical_settings)
def test_pip_preview_is_under_driving_screen_widgets_and_configured_only_in_galaxy():
sections = _params_by_section(_layout())
visual = sections["Visual (Display & UI)"]
assert {"PIPPreviewEnabled", "PIPPreviewShowOnBlinker", "PIPPreviewShowOnBSM"} <= visual.keys()
assert visual["PIPPreviewEnabled"]["parent_key"] == "CustomUI"
assert visual["PIPPreviewShowOnBlinker"]["parent_key"] == "PIPPreviewEnabled"
assert visual["PIPPreviewShowOnBSM"]["parent_key"] == "PIPPreviewEnabled"
assert visual["PIPPreviewEnabled"]["settings_tier"] == "simple"
assert _declared_default("PIPPreviewEnabled") == "0"
assert _declared_default("PIPPreviewShowOnBlinker") == "0"
assert _declared_default("PIPPreviewShowOnBSM") == "0"
assert '"{\\"width\\":1928,\\"height\\":1208,\\"center_left\\":[315,548],\\"center_right\\":[1571,539],\\"crop_size\\":580}"' in PARAM_KEYS_PATH.read_text(encoding="utf-8")
physical_settings = (
REPO_ROOT / "selfdrive/ui/layouts/settings/starpilot/aethergrid.py",
REPO_ROOT / "selfdrive/ui/layouts/settings/starpilot/lateral.py",
REPO_ROOT / "selfdrive/ui/layouts/settings/starpilot/appearance.py",
)
assert all("PIPPreview" not in path.read_text(encoding="utf-8") for path in physical_settings)
+76
View File
@@ -2605,6 +2605,51 @@ def _normalize_vasm_config(data):
return config
def _normalize_pip_preview_config(data):
if not isinstance(data, dict):
raise ValueError("Configuration must be a JSON object.")
try:
width = int(data.get("width", 0))
height = int(data.get("height", 0))
except (TypeError, ValueError) as exc:
raise ValueError("Invalid camera dimensions.") from exc
if not (1 <= width <= 8192 and 1 <= height <= 8192):
raise ValueError("Camera dimensions are out of range.")
try:
crop_size = int(data.get("crop_size", 0))
except (TypeError, ValueError) as exc:
raise ValueError("Invalid crop size.") from exc
if not (10 <= crop_size <= 8192):
raise ValueError("Crop size is out of range.")
def normalize_center(key):
point = data.get(key)
if not point:
return []
if not isinstance(point, (list, tuple)) or len(point) != 2:
raise ValueError(f"{key} requires an (x, y) center point.")
try:
x, y = float(point[0]), float(point[1])
except (TypeError, ValueError) as exc:
raise ValueError(f"{key} contains a non-numeric point.") from exc
if not (math.isfinite(x) and math.isfinite(y) and 0 <= x <= width and 0 <= y <= height):
raise ValueError(f"{key} center is outside the camera frame.")
return [round(x), round(y)]
config = {
"width": width,
"height": height,
"center_left": normalize_center("center_left"),
"center_right": normalize_center("center_right"),
"crop_size": crop_size,
}
if not config["center_left"] and not config["center_right"]:
raise ValueError("At least one window center is required.")
return config
def _decode_json_object(value):
if isinstance(value, bytes):
value = value.decode("utf-8", errors="replace")
@@ -3992,6 +4037,8 @@ def setup(app):
"/assets/components/tools/device_settings_layout.json",
"/assets/components/tools/v_asm.js",
"/assets/components/tools/v_asm.css",
"/assets/components/tools/pip_sidecam.js",
"/assets/components/tools/pip_sidecam.css",
"/assets/components/tools/toggles.js",
}:
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0"
@@ -7667,6 +7714,35 @@ def setup(app):
update_starpilot_toggles()
return jsonify({"success": True, "message": "Annotation config cleared. V-ASM disabled."})
@app.route("/api/pip_preview/snapshot", methods=["GET"])
def pip_preview_snapshot():
jpeg = _get_live_driver_jpeg()
if jpeg is not None:
return Response(jpeg, mimetype="image/jpeg")
return jsonify({"error": "Unable to capture live frame from driver camera."}), 503
@app.route("/api/pip_preview/config", methods=["GET"])
def pip_preview_get_config():
return jsonify(_decode_json_object(params.get("PIPPreviewMask")))
@app.route("/api/pip_preview/config", methods=["POST"])
def pip_preview_save_config():
try:
config = _normalize_pip_preview_config(request.get_json(silent=True))
except ValueError as exc:
return jsonify({"error": str(exc)}), 400
params.put("PIPPreviewMask", config)
update_starpilot_toggles()
return jsonify({"success": True, "message": "PiP Preview mask saved."})
@app.route("/api/pip_preview/config", methods=["DELETE"])
def pip_preview_delete_config():
params.put("PIPPreviewMask", {})
params.put_bool("PIPPreviewEnabled", False)
update_starpilot_toggles()
return jsonify({"success": True, "message": "PiP Preview mask cleared."})
@app.route("/mapbox-help/<path:filename>", methods=["GET"])
def serve_mapbox_help(filename):
return send_from_directory("/data/openpilot/starpilot/navigation/navigation_training", filename)