From b5223b6e23a0fdde5ca445f3d765c2e56d37e529 Mon Sep 17 00:00:00 2001 From: Prabhaav Pillai Date: Wed, 5 Aug 2026 21:15:52 -0400 Subject: [PATCH] add PiP Side Camera --- common/params_keys.h | 4 + selfdrive/ui/onroad/starpilot/pip_sidecam.py | 284 +++++++++++ .../onroad/starpilot/starpilot_onroad_view.py | 15 +- .../the_galaxy/assets/components/router.js | 2 + .../the_galaxy/assets/components/sidebar.js | 1 + .../tools/device_settings_layout.json | 30 ++ .../assets/components/tools/pip_sidecam.css | 44 ++ .../assets/components/tools/pip_sidecam.js | 467 ++++++++++++++++++ .../system/the_galaxy/templates/index.html | 1 + .../tests/test_device_settings_layout.py | 23 + starpilot/system/the_galaxy/the_galaxy.py | 76 +++ 11 files changed, 945 insertions(+), 2 deletions(-) create mode 100644 selfdrive/ui/onroad/starpilot/pip_sidecam.py create mode 100644 starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.css create mode 100644 starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.js diff --git a/common/params_keys.h b/common/params_keys.h index 4548b0abc..48d198286 100644 --- a/common/params_keys.h +++ b/common/params_keys.h @@ -493,6 +493,10 @@ inline static std::unordered_map keys = { {"PauseLateralSpeed", {PERSISTENT, FLOAT, "0.0", "0.0", 1, SETTINGS_SIMPLE}}, {"LateralResumeDelay", {PERSISTENT, FLOAT, "0.0", "0.0", 1, SETTINGS_SIMPLE}}, {"PedalsOnUI", {PERSISTENT, BOOL, "0", "0", 1, SETTINGS_SIMPLE}}, + {"PIPPreviewEnabled", {PERSISTENT, BOOL, "0", "0", 1}}, + {"PIPPreviewMask", {PERSISTENT, JSON, "{\"width\":1928,\"height\":1208,\"center_left\":[315,548],\"center_right\":[1571,539],\"crop_size\":580}", "{\"width\":1928,\"height\":1208,\"center_left\":[315,548],\"center_right\":[1571,539],\"crop_size\":580}", 2}}, + {"PIPPreviewShowOnBlinker", {PERSISTENT, BOOL, "0", "0", 1}}, + {"PIPPreviewShowOnBSM", {PERSISTENT, BOOL, "0", "0", 1}}, {"GalaxyPaired", {PERSISTENT, BOOL, "0", "0", 0}}, {"GalaxyUploadPending", {PERSISTENT, BOOL, "0", "0", 0}}, {"PreferredSchedule", {PERSISTENT, INT, "2", "0", 0}}, diff --git a/selfdrive/ui/onroad/starpilot/pip_sidecam.py b/selfdrive/ui/onroad/starpilot/pip_sidecam.py new file mode 100644 index 000000000..d4f03a3da --- /dev/null +++ b/selfdrive/ui/onroad/starpilot/pip_sidecam.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +import json +import platform +import time +import weakref + +import pyray as rl + +from msgq.visionipc import VisionIpcClient, VisionStreamType +from openpilot.selfdrive.ui.ui_state import ui_state +from openpilot.starpilot.common.vision_bsm import get_fresh_vasm_state + +PIP_SHADER_VERSION = """ +#version 300 es +precision mediump float; +""" +if platform.system() == "Darwin": + PIP_SHADER_VERSION = """ + #version 330 core + """ + +PIP_VERTEX_SHADER = PIP_SHADER_VERSION + """ +in vec3 vertexPosition; +in vec2 vertexTexCoord; +in vec3 vertexNormal; +in vec4 vertexColor; +uniform mat4 mvp; +out vec2 fragTexCoord; +out vec4 fragColor; +void main() { + fragTexCoord = vertexTexCoord; + fragColor = vertexColor; + gl_Position = mvp * vec4(vertexPosition, 1.0); +} +""" + +PIP_FRAGMENT_SHADER = PIP_SHADER_VERSION + """ +in vec2 fragTexCoord; +uniform sampler2D texture0; +uniform sampler2D texture1; +uniform vec2 uCropMin; +uniform vec2 uCropSize; +out vec4 fragColor; +void main() { + vec2 uv = uCropMin + fragTexCoord * uCropSize; + float y = texture(texture0, uv).r; + vec2 c = texture(texture1, uv).ra - 0.5; + vec3 rgb = vec3(y + 1.402 * c.y, y - 0.344 * c.x - 0.714 * c.y, y + 1.772 * c.x); + + vec2 p = fragTexCoord - 0.5; + float dist = length(p); + float edge = 0.02; + float alpha = 1.0 - smoothstep(0.5 - edge, 0.5, dist); + + fragColor = vec4(rgb, alpha); +} +""" + +UNIFORM_VEC2 = rl.ShaderUniformDataType.SHADER_UNIFORM_VEC2 + +CONNECTION_RETRY_INTERVAL = 0.2 +PARAM_REFRESH_INTERVAL = 2.0 + +# Bubble geometry +BUBBLE_RADIUS_FRACTION = 0.3 +BUBBLE_RADIUS_MIN = 180 +BUBBLE_RADIUS_MAX = 420 +BUBBLE_MARGIN = 24 + + +class PipSideCamera: + """Renders a circular pip bubble of the adjacent side window from the dcamera.""" + def __init__(self): + self._params = ui_state.params + self._params_memory = ui_state.params_memory + + self.client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, conflate=True) + self._stream_type = VisionStreamType.VISION_STREAM_DRIVER + self._last_connection_attempt = 0.0 + self.frame = None + self._last_frame_id = -1 + self._texture_needs_update = True + self.texture_y: rl.Texture | None = None + self.texture_uv: rl.Texture | None = None + self._closed = False + + self._enabled = False + self._show_on_blinker = False + self._show_on_bsm = False + self._mask = {} + self._last_param_refresh = 0.0 + + self.shader = rl.load_shader_from_memory(PIP_VERTEX_SHADER, PIP_FRAGMENT_SHADER) + self._texture1_loc = rl.get_shader_location(self.shader, "texture1") + self._crop_min_loc = rl.get_shader_location(self.shader, "uCropMin") + self._crop_size_loc = rl.get_shader_location(self.shader, "uCropSize") + + self_ref = weakref.ref(self) + + def offroad_transition_callback(): + if (ref := self_ref()) is not None: + ref._offroad_transition() + + self._offroad_transition_callback = offroad_transition_callback + ui_state.add_offroad_transition_callback(self._offroad_transition_callback) + + def _offroad_transition(self): + self._clear_textures() + self.frame = None + self._last_frame_id = -1 + self._last_connection_attempt = 0.0 + self.client = VisionIpcClient("camerad", self._stream_type, conflate=True) + + def close(self): + if self._closed: + return + self._closed = True + if getattr(self, "_offroad_transition_callback", None) is not None: + ui_state.remove_offroad_transition_callback(self._offroad_transition_callback) + self._offroad_transition_callback = None + self._clear_textures() + if self.shader and self.shader.id: + rl.unload_shader(self.shader) + self.shader.id = 0 + self.frame = None + self.client = None + + def __del__(self): + self.close() + + def _refresh_config(self, force: bool = False): + now = time.monotonic() + if not force and now - self._last_param_refresh < PARAM_REFRESH_INTERVAL: + return + self._last_param_refresh = now + self._enabled = self._params.get_bool("PIPPreviewEnabled") + self._show_on_blinker = self._params.get_bool("PIPPreviewShowOnBlinker") + self._show_on_bsm = self._params.get_bool("PIPPreviewShowOnBSM") + try: + raw = self._params.get("PIPPreviewMask") + if isinstance(raw, (bytes, str)): + raw = json.loads(raw) + self._mask = raw if isinstance(raw, dict) else {} + except (TypeError, ValueError, json.JSONDecodeError): + self._mask = {} + + def active_sides(self) -> list[str]: + """Return the car-side keys ('left'/'right') whose preview bubble should show.""" + if not ui_state.started: + return [] + self._refresh_config() + if not self._enabled or not self._mask: + return [] + + car_state = ui_state.sm["carState"] if ui_state.sm.valid.get("carState", False) else None + if car_state is None: + return [] + + vasm_left, vasm_right = get_fresh_vasm_state(self._params_memory) + + left_blinker = bool(car_state.leftBlinker) + right_blinker = bool(car_state.rightBlinker) + left_bsm = bool(car_state.leftBlindspot) or vasm_left + right_bsm = bool(car_state.rightBlindspot) or vasm_right + + sides = [] + if self._mask.get("center_left") and ((self._show_on_blinker and left_blinker) or (self._show_on_bsm and left_bsm)): + sides.append("left") + if self._mask.get("center_right") and ((self._show_on_blinker and right_blinker) or (self._show_on_bsm and right_bsm)): + sides.append("right") + return sides + + def _crop_rect(self, side: str) -> rl.Rectangle | None: + center = self._mask.get(f"center_{side}") + size = self._mask.get("crop_size") + if not center or len(center) < 2 or not size: + return None + try: + cx, cy = float(center[0]), float(center[1]) + half = float(size) / 2.0 + except (TypeError, ValueError): + return None + if half <= 0: + return None + return rl.Rectangle(cx - half, cy - half, size, size) + + def _bubble_rect(self, content_rect: rl.Rectangle, side: str) -> rl.Rectangle: + radius = int(min(content_rect.width, content_rect.height) * BUBBLE_RADIUS_FRACTION) + radius = max(BUBBLE_RADIUS_MIN, min(radius, BUBBLE_RADIUS_MAX)) + margin = BUBBLE_MARGIN + cx = content_rect.x + margin + radius if side == "left" else content_rect.x + content_rect.width - margin - radius + cy = content_rect.y + content_rect.height - margin - radius + return rl.Rectangle(cx - radius, cy - radius, radius * 2, radius * 2) + + def render(self, content_rect: rl.Rectangle): + if not ui_state.started: + return + + sides = self.active_sides() + if not sides: + return + + if not self._ensure_connection(): + return + + buffer = self.client.recv(timeout_ms=0) + if buffer: + self.frame = buffer + self._last_frame_id = int(getattr(buffer, "frame_id", -1)) + self._texture_needs_update = True + if self.frame is None: + return + + if not self.texture_y or not self.texture_uv: + return + + if self._texture_needs_update: + y_data = self.frame.data[: self.frame.uv_offset] + uv_data = self.frame.data[self.frame.uv_offset:] + rl.update_texture(self.texture_y, rl.ffi.cast("void *", rl.ffi.from_buffer(y_data))) + rl.update_texture(self.texture_uv, rl.ffi.cast("void *", rl.ffi.from_buffer(uv_data))) + self._texture_needs_update = False + + for side in sides: + crop = self._crop_rect(side) + if crop is None: + continue + bubble = self._bubble_rect(content_rect, side) + self._draw_bubble(bubble, crop) + + def _draw_bubble(self, bubble: rl.Rectangle, crop: rl.Rectangle): + cx = bubble.x + bubble.width / 2 + cy = bubble.y + bubble.height / 2 + radius = bubble.width / 2 + + rl.draw_circle(int(round(cx)), int(round(cy)), radius, rl.BLACK) + rl.draw_circle_lines(int(round(cx)), int(round(cy)), radius, rl.Color(255, 255, 255, 120)) + + tex_w = float(self.texture_y.width) + tex_h = float(self.texture_y.height) + crop_min = rl.Vector2(crop.x / tex_w, crop.y / tex_h) + crop_size = rl.Vector2(crop.width / tex_w, crop.height / tex_h) + + src_rect = rl.Rectangle(0, 0, tex_w, tex_h) + dst_rect = rl.Rectangle(bubble.x, bubble.y, bubble.width, bubble.height) + + rl.begin_shader_mode(self.shader) + rl.set_shader_value(self.shader, self._crop_min_loc, crop_min, UNIFORM_VEC2) + rl.set_shader_value(self.shader, self._crop_size_loc, crop_size, UNIFORM_VEC2) + rl.set_shader_value_texture(self.shader, self._texture1_loc, self.texture_uv) + rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) + rl.end_shader_mode() + + def _ensure_connection(self) -> bool: + if not self.client.is_connected(): + self.frame = None + self._last_frame_id = -1 + + now = rl.get_time() + if now - self._last_connection_attempt < CONNECTION_RETRY_INTERVAL: + return False + self._last_connection_attempt = now + + self._clear_textures() + if not self.client.connect(False) or not self.client.num_buffers: + return False + self._initialize_textures() + return True + + def _initialize_textures(self): + self._clear_textures() + self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride), + int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE)) + self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2), + int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA)) + + def _clear_textures(self): + if self.texture_y and self.texture_y.id: + rl.unload_texture(self.texture_y) + self.texture_y = None + if self.texture_uv and self.texture_uv.id: + rl.unload_texture(self.texture_uv) + self.texture_uv = None \ No newline at end of file diff --git a/selfdrive/ui/onroad/starpilot/starpilot_onroad_view.py b/selfdrive/ui/onroad/starpilot/starpilot_onroad_view.py index 73c1b1ff6..378f4884a 100644 --- a/selfdrive/ui/onroad/starpilot/starpilot_onroad_view.py +++ b/selfdrive/ui/onroad/starpilot/starpilot_onroad_view.py @@ -13,6 +13,7 @@ from openpilot.selfdrive.ui.onroad.starpilot.widgets import ( ) from openpilot.selfdrive.ui.onroad.starpilot.stopping_point import render_stopping_point from openpilot.selfdrive.ui.onroad.starpilot.pause_indicators import render_lateral_paused, render_longitudinal_paused +from openpilot.selfdrive.ui.onroad.starpilot.pip_sidecam import PipSideCamera from openpilot.selfdrive.ui.onroad.starpilot.weather_icon import render_weather_icon from openpilot.selfdrive.ui.lib.starpilot_status import ( get_screen_edge_color, @@ -37,6 +38,8 @@ class StarPilotOnroadView(AugmentedRoadView): self._max_fps = 0.0 self._avg_fps = 0.0 + self._pip_sidecam = PipSideCamera() + self.layout_manager = WidgetLayoutManager(self._content_rect) # Disable parent rendering calls — layout manager draws at computed bounds @@ -94,6 +97,9 @@ class StarPilotOnroadView(AugmentedRoadView): self._render_overlays() self._render_road_name() + # PiP renders last so it always sits on top of every other on-road overlay. + self._pip_sidecam.render(self._content_rect) + def _draw_border(self, rect: rl.Rectangle): border_width = self._get_border_width() rl.draw_rectangle_rounded_lines_ex(rect, 0.12, 10, border_width, rl.BLACK) @@ -131,8 +137,8 @@ class StarPilotOnroadView(AugmentedRoadView): if not self._params.get_bool("EnableTorqueBarWidget", default=True): return rl.begin_scissor_mode( - int(self._content_rect.x), int(self._content_rect.y), - int(self._content_rect.width), int(self._content_rect.height), + int(round(self._content_rect.x)), int(round(self._content_rect.y)), + int(round(self._content_rect.width)), int(round(self._content_rect.height)), ) self._torque_bar.render(self._content_rect) rl.end_scissor_mode() @@ -145,6 +151,11 @@ class StarPilotOnroadView(AugmentedRoadView): if not mr._path.projected_points.size: return + rl.begin_scissor_mode( + int(round(rect.x)), int(round(rect.y)), + int(round(rect.width)), int(round(rect.height)), + ) + # Path edges (always rendered if track_edge_vertices exist) if mr._track_edge_vertices.size >= 4: render_path_edges(mr) diff --git a/starpilot/system/the_galaxy/assets/components/router.js b/starpilot/system/the_galaxy/assets/components/router.js index 5a89e33e6..a876c7ba0 100644 --- a/starpilot/system/the_galaxy/assets/components/router.js +++ b/starpilot/system/the_galaxy/assets/components/router.js @@ -24,6 +24,7 @@ import { Troubleshoot } from "/assets/components/tools/troubleshoot.js" import { TmuxLog } from "/assets/components/tools/tmux.js" import { ToggleControl } from "/assets/components/tools/toggles.js" import { VASMAnnotations } from "/assets/components/tools/v_asm.js" +import { PipSideCamera } from "/assets/components/tools/pip_sidecam.js" import { UpdateManager } from "/assets/components/tools/update_manager.js" let router, routerState @@ -86,6 +87,7 @@ function Root() { createRoute("updates", "/manage_updates", UpdateManager), createRoute("vehicle_features", "/vehicle_features", VehicleFeatures), createRoute("v_asm", "/manage_v_asm", VASMAnnotations), + createRoute("pip_sidecam", "/manage_pip_sidecam", PipSideCamera), ] router = createRouter({ diff --git a/starpilot/system/the_galaxy/assets/components/sidebar.js b/starpilot/system/the_galaxy/assets/components/sidebar.js index 14ffa4fdb..a86858d43 100644 --- a/starpilot/system/the_galaxy/assets/components/sidebar.js +++ b/starpilot/system/the_galaxy/assets/components/sidebar.js @@ -24,6 +24,7 @@ const MENU_ITEMS = { { name: "Testing Ground", link: "/testing_ground", icon: "bi-bezier2" }, { name: "Troubleshoot", link: "/troubleshoot", icon: "bi-tools" }, { name: "V-Adj Spot Monitor", link: "/manage_v_asm", icon: "bi-eye" }, + { name: "PiP Side Camera", link: "/manage_pip_sidecam", icon: "bi-badge-hd" }, { name: "Theme Maker", link: "/theme_maker", icon: "bi-palette-fill" }, { name: "Tmux Log", link: "/manage_tmux", icon: "bi-terminal" }, { name: "Backup and Restore", link: "/manage_toggles", icon: "bi-arrow-repeat" }, diff --git a/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json b/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json index b279e88b1..58e5616a9 100644 --- a/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json +++ b/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json @@ -2137,6 +2137,36 @@ "parent_key": "CustomUI", "settings_tier": "simple" }, + { + "key": "PIPPreviewEnabled", + "label": "Enable PiP Side Preview", + "description": "Show a temporary Picture-in-Picture bubble of the side window while the turn signal is on or a blind spot is detected.", + "data_type": "bool", + "ui_type": "toggle", + "parent_key": "CustomUI", + "is_parent_toggle": true, + "requires_nonempty_key": "PIPPreviewMask", + "disabled_reason": "Configure the PiP window mask first in Galaxy > PiP Side Camera", + "settings_tier": "simple" + }, + { + "key": "PIPPreviewShowOnBlinker", + "label": "Show on Turn Signal", + "description": "Display the side preview bubble while the corresponding turn signal is engaged.", + "data_type": "bool", + "ui_type": "toggle", + "parent_key": "PIPPreviewEnabled", + "settings_tier": "simple" + }, + { + "key": "PIPPreviewShowOnBSM", + "label": "Show on Blind Spot Detection", + "description": "Display the side preview bubble when a vehicle is detected in the blind spot, whether from factory BSM, V-ASM, or both.", + "data_type": "bool", + "ui_type": "toggle", + "parent_key": "PIPPreviewEnabled", + "settings_tier": "simple" + }, { "key": "Compass", "label": "Compass", diff --git a/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.css b/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.css new file mode 100644 index 000000000..32762e894 --- /dev/null +++ b/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.css @@ -0,0 +1,44 @@ +/* PiP Side Camera reuses the V-ASM tool styling. */ +@import url("./v_asm.css"); + +.pip-zoom-control { + margin-top: 18px; + padding: 14px 18px; + background: var(--card-bg, #16181d); + border: 1px solid var(--card-border, #2a2e37); + border-radius: 10px; +} + +.pip-zoom-header { + display: flex; + justify-content: space-between; + align-items: center; + font-size: 14px; + font-weight: 600; + color: #e6e6e6; +} + +.pip-zoom-value { + font-family: monospace; + color: #9ecbff; +} + +.pip-zoom-slider { + width: 100%; + margin-top: 12px; + accent-color: #0d6efd; +} + +.pip-zoom-labels { + display: flex; + justify-content: space-between; + font-size: 12px; + color: #8b8f98; + margin-top: 2px; +} + +.pip-zoom-hint { + margin-top: 8px; + font-size: 12px; + color: #8b8f98; +} diff --git a/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.js b/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.js new file mode 100644 index 000000000..4be5ee364 --- /dev/null +++ b/starpilot/system/the_galaxy/assets/components/tools/pip_sidecam.js @@ -0,0 +1,467 @@ +import { html, reactive } from "/assets/vendor/arrow-core.js"; + +const CANVAS_W = 640; +const CANVAS_H = 480; +const ZOOM_MIN = 60; +const ZOOM_MAX = 640; +const ZOOM_STEP = 5; +let initialLoadTriggered = false; + +const state = reactive({ + loading: false, + error: "", + success: "", + armSide: null, + leftCenter: null, + rightCenter: null, + zoom: 240, + image: false, + configSaved: false, + configExists: false, +}); + +let _loadedImage = null; +let _lastCanvas = null; +let loadedConfig = null; + +function getCanvas() { + return document.getElementById("pip-sidecam-canvas"); +} + +function canvasScale() { + const canvas = getCanvas(); + const img = _loadedImage; + const nativeW = img ? img.naturalWidth : 1920; + const nativeH = img ? img.naturalHeight : 1080; + const cw = canvas ? canvas.width || CANVAS_W : CANVAS_W; + const ch = canvas ? canvas.height || CANVAS_H : CANVAS_H; + return { cw, ch, nativeW, nativeH }; +} + +function redraw() { + const canvas = getCanvas(); + if (!canvas) return; + + if (canvas !== _lastCanvas) { + _lastCanvas = canvas; + if (_loadedImage) { + canvas._img = _loadedImage; + canvas.width = Math.min(_loadedImage.naturalWidth, 1280); + canvas.height = Math.round(canvas.width * (_loadedImage.naturalHeight / _loadedImage.naturalWidth)); + } else { + canvas.width = CANVAS_W; + canvas.height = CANVAS_H; + } + } + + const ctx = canvas.getContext("2d"); + ctx.clearRect(0, 0, canvas.width, canvas.height); + + const img = _loadedImage; + if (img) { + canvas._img = img; + ctx.drawImage(img, 0, 0, canvas.width, canvas.height); + } else { + ctx.fillStyle = "#222"; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.fillStyle = "#888"; + ctx.font = "16px monospace"; + ctx.textAlign = "center"; + ctx.fillText("Loading camera snapshot...", canvas.width / 2, canvas.height / 2); + return; + } + + const half = state.zoom / 2; + const sides = [ + { key: "left", center: state.leftCenter, color: "#0d6efd", label: "LEFT WINDOW" }, + { key: "right", center: state.rightCenter, color: "#fd7e14", label: "RIGHT WINDOW" }, + ]; + + for (const side of sides) { + if (!side.center) continue; + + const [cx, cy] = side.center; + + // Crop square (what gets sampled) + circular bubble overlay. + ctx.strokeStyle = side.color; + ctx.lineWidth = 2; + ctx.setLineDash([]); + ctx.strokeRect(cx - half, cy - half, state.zoom, state.zoom); + + ctx.beginPath(); + ctx.arc(cx, cy, half, 0, Math.PI * 2); + ctx.fillStyle = side.color + "40"; + ctx.fill(); + ctx.lineWidth = 2; + ctx.strokeStyle = side.color; + ctx.stroke(); + + // Center dot + ctx.beginPath(); + ctx.arc(cx, cy, 4, 0, Math.PI * 2); + ctx.fillStyle = "#fff"; + ctx.fill(); + ctx.strokeStyle = side.color; + ctx.lineWidth = 1.5; + ctx.stroke(); + + // Label + ctx.fillStyle = side.color; + ctx.font = "bold 13px monospace"; + ctx.textAlign = "center"; + ctx.fillText(side.label, cx, cy - half - 8); + } + + if (state.armSide) { + ctx.fillStyle = "#fff"; + ctx.font = "bold 15px monospace"; + ctx.textAlign = "center"; + ctx.fillText( + state.armSide === "left" ? "Click to set LEFT window center" : "Click to set RIGHT window center", + canvas.width / 2, + canvas.height - 18, + ); + } +} + +async function loadSnapshot() { + state.error = ""; + state.success = ""; + state.image = false; + let blobUrl = null; + try { + const resp = await fetch("/api/pip_preview/snapshot"); + if (!resp.ok) { + const payload = await resp.json().catch(() => ({})); + throw new Error(payload.error || resp.statusText || "Failed to load snapshot"); + } + + let src; + const contentType = resp.headers.get("content-type") || ""; + if (contentType.includes("application/json")) { + const data = await resp.json(); + if (!data.jpeg) throw new Error("Snapshot missing image data"); + src = `data:image/jpeg;base64,${data.jpeg}`; + } else { + const blob = await resp.blob(); + blobUrl = URL.createObjectURL(blob); + src = blobUrl; + } + + const img = new Image(); + img.onload = () => { + _loadedImage = img; + const canvas = getCanvas(); + if (canvas) { + canvas._img = img; + canvas.width = Math.min(img.naturalWidth, 1280); + canvas.height = Math.round(canvas.width * (img.naturalHeight / img.naturalWidth)); + } + state.image = true; + state.success = "Camera snapshot loaded. Place a center point on each window, then adjust the zoom."; + applyConfigToCanvas(); + requestAnimationFrame(redraw); + if (blobUrl) { + URL.revokeObjectURL(blobUrl); + blobUrl = null; + } + }; + img.onerror = () => { + state.image = false; + state.error = "Failed to decode image"; + requestAnimationFrame(redraw); + if (blobUrl) { + URL.revokeObjectURL(blobUrl); + blobUrl = null; + } + }; + img.src = src; + } catch (e) { + state.image = false; + state.error = e.message; + requestAnimationFrame(redraw); + if (blobUrl) { + URL.revokeObjectURL(blobUrl); + blobUrl = null; + } + } +} + +function canvasClick(e) { + if (!state.image || !state.armSide || !e) return; + const canvas = getCanvas(); + if (!canvas) return; + + const rect = canvas.getBoundingClientRect(); + const x = Math.round((e.clientX - rect.left) * (canvas.width / rect.width)); + const y = Math.round((e.clientY - rect.top) * (canvas.height / rect.height)); + if (x < 0 || y < 0) return; + + if (state.armSide === "left") { + state.leftCenter = [x, y]; + } else { + state.rightCenter = [x, y]; + } + state.armSide = null; + state.error = ""; + requestAnimationFrame(redraw); +} + +function setArm(e) { + const side = e?.currentTarget?.value || e?.target?.value; + if (!side) return; + state.armSide = side; + state.error = ""; + state.success = ""; + requestAnimationFrame(redraw); +} + +function updateZoom(e) { + const value = Number(e?.currentTarget?.value ?? e?.target?.value ?? 0); + if (!Number.isFinite(value)) return; + state.zoom = value; + requestAnimationFrame(redraw); +} + +function clearCenter(side) { + if (side === "left") { + state.leftCenter = null; + } else { + state.rightCenter = null; + } + if (state.armSide === side) state.armSide = null; + requestAnimationFrame(redraw); +} + +function clearAll() { + state.leftCenter = null; + state.rightCenter = null; + state.armSide = null; + state.configSaved = false; + requestAnimationFrame(redraw); +} + +async function saveConfig() { + if (!state.leftCenter && !state.rightCenter) { + state.error = "Place at least one center point."; + return; + } + + const { cw, ch, nativeW, nativeH } = canvasScale(); + + function toNative(center) { + if (!center) return []; + return [Math.round(center[0] * nativeW / cw), Math.round(center[1] * nativeH / ch)]; + } + + const config = { + width: nativeW, + height: nativeH, + center_left: toNative(state.leftCenter), + center_right: toNative(state.rightCenter), + crop_size: Math.round(state.zoom * nativeW / cw), + }; + + state.loading = true; + state.error = ""; + state.success = ""; + try { + const resp = await fetch("/api/pip_preview/config", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(config), + }); + const data = await resp.json(); + if (!resp.ok) throw new Error(data.error || "Failed to save"); + state.configSaved = true; + state.configExists = true; + state.success = "PiP Preview mask saved!"; + await loadExistingConfig(); + } catch (e) { + state.error = e.message; + } + state.loading = false; +} + +async function loadExistingConfig() { + try { + const resp = await fetch("/api/pip_preview/config"); + if (!resp.ok) return; + const config = await resp.json(); + loadedConfig = config; + applyConfigToCanvas(); + } catch (e) { + console.error("PiP Preview config load failed", e); + } +} + +function applyConfigToCanvas() { + const config = loadedConfig; + if (!config) { + state.leftCenter = null; + state.rightCenter = null; + state.configExists = false; + redraw(); + return; + } + + const { cw, ch, nativeW, nativeH } = canvasScale(); + + function toCanvas(center) { + if (!Array.isArray(center) || center.length < 2) return null; + return [Math.round(center[0] * cw / nativeW), Math.round(center[1] * ch / nativeH)]; + } + + state.leftCenter = toCanvas(config.center_left); + state.rightCenter = toCanvas(config.center_right); + if (Number.isFinite(Number(config.crop_size))) { + state.zoom = Math.round(Number(config.crop_size) * cw / nativeW); + } + state.configExists = Boolean(state.leftCenter || state.rightCenter); + redraw(); +} + +async function deleteConfig() { + state.loading = true; + state.error = ""; + state.success = ""; + try { + const resp = await fetch("/api/pip_preview/config", { method: "DELETE" }); + const data = await resp.json(); + if (!resp.ok) throw new Error(data.error || "Failed to delete"); + state.leftCenter = null; + state.rightCenter = null; + state.armSide = null; + state.configSaved = false; + state.configExists = false; + loadedConfig = null; + state.success = "PiP Preview mask cleared."; + requestAnimationFrame(redraw); + } catch (e) { + state.error = e.message; + } + state.loading = false; +} + +function scheduleInitialLoad() { + if (!initialLoadTriggered) { + initialLoadTriggered = true; + loadExistingConfig(); + loadSnapshot(); + } + const attempt = () => { + if (getCanvas()) { + redraw(); + return; + } + requestAnimationFrame(attempt); + }; + requestAnimationFrame(attempt); +} + +function retrySnapshot() { + state.error = ""; + state.success = ""; + loadSnapshot(); +} + +export function PipSideCamera() { + const el = html` +
+
+
+

PiP Side Camera Preview

+ +
+
About PiP Preview
+
    +
  • Shows a temporary Picture-in-Picture bubble of the adjacent side window while the turn signal is on or a blind spot is detected
    • +
  • Place a single center point on each window, then pick a shared zoom level
    • +
  • This mask is separate from the V-ASM detection mask, so you can tune the visual crop independently
    • +
  • Works alongside factory blind spot monitoring and/or V-ASM, and with turn signals alone
    • +
+
+ +
+
Setup
+
    +
  • Click "Set Left Center", then click the driver's side window; repeat for the right window
    • +
  • From the driver camera, the car's LEFT window appears on the right side of the image and vice versa
    • +
  • The zoom slider applies to BOTH windows so the preview stays consistent
    • +
  • At least one window center is required to enable the preview
    • +
+
+ +
+ +
+ Use the preview to enhance lateral awareness. Always check manually before merging and be aware the driver camera view is from the cabin and does not reflect your blind spot. +
+ + ${state.error ? html`
${state.error}
` : ""} + ${state.success ? html`
${state.success}
` : ""} + ${state.configSaved ? html` +
+ PiP Preview mask saved! Configure the preview in Toggles. +
+ ` : ""} + +
+
+ + + + + ${state.configExists ? html`` : ""} + + +
+
+ + ${state.armSide ? html`
${state.armSide === "left" ? "⬅ Placing Left Center" : "➡ Placing Right Center"}Click on the window to place its center point
` : ""} + +
+ +
+ ${state.armSide ? "Click the window to place its center point." : "Place a center point on each window, then use the zoom slider below."} +
+
+ +
+
+ Zoom + ${() => state.zoom} px +
+ +
+ Wide + Close +
+
Applied to both windows so the preview stays consistent.
+
+
+ +
+
Preview Settings
+ +
+
+ `; + + scheduleInitialLoad(); + return el; +} diff --git a/starpilot/system/the_galaxy/templates/index.html b/starpilot/system/the_galaxy/templates/index.html index 846f84e0b..fbb18915b 100644 --- a/starpilot/system/the_galaxy/templates/index.html +++ b/starpilot/system/the_galaxy/templates/index.html @@ -43,6 +43,7 @@ +