diff --git a/starpilot/system/the_galaxy/assets/components/sidebar.js b/starpilot/system/the_galaxy/assets/components/sidebar.js
index 0de6930ea..14ffa4fdb 100644
--- a/starpilot/system/the_galaxy/assets/components/sidebar.js
+++ b/starpilot/system/the_galaxy/assets/components/sidebar.js
@@ -23,7 +23,7 @@ const MENU_ITEMS = {
{ name: "Plots", link: "/plots", icon: "bi-graph-up-arrow" },
{ name: "Testing Ground", link: "/testing_ground", icon: "bi-bezier2" },
{ name: "Troubleshoot", link: "/troubleshoot", icon: "bi-tools" },
- { name: "V-ASM Annotations", link: "/manage_v_asm", icon: "bi-eye" },
+ { name: "V-Adj Spot Monitor", link: "/manage_v_asm", icon: "bi-eye" },
{ name: "Theme Maker", link: "/theme_maker", icon: "bi-palette-fill" },
{ name: "Tmux Log", link: "/manage_tmux", icon: "bi-terminal" },
{ name: "Backup and Restore", link: "/manage_toggles", icon: "bi-arrow-repeat" },
diff --git a/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json b/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json
index a5b7e69cc..8a1b26770 100644
--- a/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json
+++ b/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json
@@ -323,7 +323,7 @@
"ui_type": "toggle",
"is_parent_toggle": true,
"requires_nonempty_key": "VASMAnnotationConfig",
- "disabled_reason": "Configure window annotations first in Galaxy > V-ASM Annotations",
+ "disabled_reason": "Configure window annotations first in Galaxy > V-Adj Spot Monitor",
"settings_tier": "advanced"
},
{
diff --git a/starpilot/system/the_galaxy/assets/components/tools/v_asm.js b/starpilot/system/the_galaxy/assets/components/tools/v_asm.js
index 51ed38f0a..9c1200c5d 100644
--- a/starpilot/system/the_galaxy/assets/components/tools/v_asm.js
+++ b/starpilot/system/the_galaxy/assets/components/tools/v_asm.js
@@ -67,8 +67,8 @@ function redraw() {
}
const sides = [
- { key: "left", points: state.leftPoints, fillColor: "rgba(13, 110, 253, 0.25)", borderColor: "#0d6efd", label: "LEFT" },
- { key: "right", points: state.rightPoints, fillColor: "rgba(253, 126, 20, 0.25)", borderColor: "#fd7e14", label: "RIGHT" },
+ { key: "left", points: state.leftPoints, fillColor: "rgba(13, 110, 253, 0.25)", borderColor: "#0d6efd", label: "LEFT OF IMAGE" },
+ { key: "right", points: state.rightPoints, fillColor: "rgba(253, 126, 20, 0.25)", borderColor: "#fd7e14", label: "RIGHT OF IMAGE" },
];
for (const side of sides) {
@@ -429,7 +429,7 @@ export function VASMAnnotations() {
Trace the visible glass (front and rear side windows on each side, as seen by the driver camera). Mask as much of the window area as possible.
Exclude A-pillars, door frames, and interior. Include the side mirror if visible through the glass.
- The B-pillar is fine to include if needed for a continuous mask. Your head or body being in frame is fine (that is part of the training data). Be consistent left vs right.
+ The B-pillar is fine to include as needed for a continuous mask. Your head or body being in frame is fine (that is part of the training data). Be consistent left vs right.
@@ -451,11 +451,11 @@ export function VASMAnnotations() {
- ${state.annotating === "left" ? "Annotating Left..." : "Annotate Left"}
+ ${state.annotating === "left" ? "Annotating Left (perspective of image)..." : "Annotate Left (as seen)"}
- ${state.annotating === "right" ? "Annotating Right..." : "Annotate Right"}
+ ${state.annotating === "right" ? "Annotating Right (perspective of image)..." : "Annotate Right (as seen)"}
${state.annotating ? html`Finish ${state.annotating === "left" ? "Left" : "Right"} ` : ""}
@@ -464,6 +464,9 @@ export function VASMAnnotations() {
${state.configExists ? html`Delete Config ` : ""}
Clear All
+
+ ${state.loading ? "Loading..." : "Get a new Snapshot"}
+
diff --git a/starpilot/system/the_galaxy/the_galaxy.py b/starpilot/system/the_galaxy/the_galaxy.py
index aeedda037..54522f4a8 100644
--- a/starpilot/system/the_galaxy/the_galaxy.py
+++ b/starpilot/system/the_galaxy/the_galaxy.py
@@ -23,6 +23,9 @@ import selectors
import shutil
import signal
import subprocess
+import numpy as np
+from msgq.visionipc import VisionIpcClient, VisionStreamType
+from PIL import Image
import threading
import time
import traceback
@@ -6381,10 +6384,10 @@ def setup(app):
pw_hash = hashlib.sha256(password.encode()).hexdigest()
GALAXY_DIR.mkdir(parents=True, exist_ok=True)
GALAXY_AUTH_FILE.write_text(pw_hash)
-
+
# Generate 256-bit secure session token
GALAXY_SESSION_FILE.write_text(secrets.token_hex(32))
-
+
# Generate 16-character alphanumeric routing slug
charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
slug = ''.join(secrets.choice(charset) for _ in range(16))
@@ -7446,31 +7449,61 @@ def setup(app):
@app.route("/api/v_asm/snapshot", methods=["GET"])
def v_asm_snapshot():
- if params.get_bool("IsOnroad"):
- return jsonify({"error": "Snapshot only available while offroad."}), 409
+ jpeg = _get_live_driver_jpeg()
+ if jpeg is not None:
+ return Response(jpeg, mimetype="image/jpeg")
+ return jsonify({"error": "Unable to capture live frame from driver camera."}), 503
- for footage_path in FOOTAGE_PATHS:
- if not os.path.isdir(footage_path):
- continue
+
+ def _get_live_driver_jpeg():
+ from openpilot.system.manager.process_config import managed_processes
+ started = False
+ try:
try:
- entries = sorted((e for e in os.listdir(footage_path) if utilities.SEGMENT_RE.fullmatch(e)), reverse=True)
- except OSError:
- continue
- for entry in entries[:3]:
- camera_file = os.path.join(footage_path, entry, "dcamera.hevc")
- if not os.path.isfile(camera_file):
- continue
- for seek_time in ("5", "2", None):
- command = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-nostdin", "-i", camera_file]
- if seek_time is not None:
- command.extend(["-ss", seek_time])
- command.extend(["-frames:v", "1", "-q:v", "2", "-f", "image2pipe", "-vcodec", "mjpeg", "-"])
- try:
- result = subprocess.run(command, capture_output=True, check=True, timeout=5, stdin=subprocess.DEVNULL)
- return Response(result.stdout, mimetype="image/jpeg")
- except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
- continue
- return jsonify({"error": "No driver camera footage available."}), 404
+ subprocess.check_call(["pgrep", "camerad"])
+ except subprocess.CalledProcessError:
+ managed_processes['camerad'].start()
+ started = True
+
+ client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True)
+ if not client.connect(True):
+ return None
+
+ if started:
+ settle_deadline = time.monotonic() + 4.0
+ while time.monotonic() < settle_deadline:
+ client.recv(timeout_ms=100)
+
+ buf = client.recv(timeout_ms=5000)
+ if buf is None:
+ return None
+
+ y = np.array(buf.data[:buf.uv_offset], dtype=np.uint8).reshape((-1, buf.stride))[:buf.height, :buf.width]
+ u = np.array(buf.data[buf.uv_offset::2], dtype=np.uint8).reshape((-1, buf.stride // 2))[:buf.height // 2, :buf.width // 2]
+ v = np.array(buf.data[buf.uv_offset + 1::2], dtype=np.uint8).reshape((-1, buf.stride // 2))[:buf.height // 2, :buf.width // 2]
+
+ ul = np.repeat(np.repeat(u, 2).reshape(u.shape[0], y.shape[1]), 2, axis=0).reshape(y.shape)
+ vl = np.repeat(np.repeat(v, 2).reshape(v.shape[0], y.shape[1]), 2, axis=0).reshape(y.shape)
+
+ yuv = np.dstack((y, ul, vl)).astype(np.int16)
+ yuv[:, :, 1:] -= 128
+
+ m = np.array([
+ [1.00000, 1.00000, 1.00000],
+ [0.00000, -0.39465, 2.03211],
+ [1.13983, -0.58060, 0.00000],
+ ])
+ rgb = np.dot(yuv, m).clip(0, 255).astype(np.uint8)
+
+ img = Image.fromarray(rgb)
+ buf_io = BytesIO()
+ img.save(buf_io, format="JPEG", quality=85)
+ return buf_io.getvalue()
+ except Exception:
+ return None
+ finally:
+ if started:
+ managed_processes['camerad'].stop()
@app.route("/api/v_asm/config", methods=["GET"])
def v_asm_get_config():