diff --git a/common/params_keys.h b/common/params_keys.h index 27646b56e..980c9865f 100644 --- a/common/params_keys.h +++ b/common/params_keys.h @@ -650,6 +650,16 @@ inline static std::unordered_map keys = { {"UseSI", {PERSISTENT, BOOL, "1", "1", 3}}, {"UserFavorites", {PERSISTENT, STRING, "", "", 1}}, {"UseVienna", {PERSISTENT, BOOL, "0", "0", 1, SETTINGS_SIMPLE}}, + {"VASMAnnotationConfig", {PERSISTENT, JSON, "{}", "{}", 2}}, + {"VASMConfidenceThreshold", {PERSISTENT, FLOAT, "0.85", "0.85", 2}}, + {"VASMEnabled", {PERSISTENT, BOOL, "0", "0", 1}}, + {"VASMLeftActive", {CLEAR_ON_MANAGER_START, STRING, "0", "0", 2}}, + {"VASMLeftConfidence", {CLEAR_ON_MANAGER_START, STRING, "0.0", "0.0", 2}}, + {"VASMLastUpdateMonoTime", {CLEAR_ON_MANAGER_START, STRING, "0", "0", 2}}, + {"VASMRightActive", {CLEAR_ON_MANAGER_START, STRING, "0", "0", 2}}, + {"VASMRightConfidence", {CLEAR_ON_MANAGER_START, STRING, "0.0", "0.0", 2}}, + {"VASMSmoothSeconds", {PERSISTENT, FLOAT, "0.2", "0.2", 2}}, + {"VASMTimestampEof", {CLEAR_ON_MANAGER_START, STRING, "0", "0", 2}}, {"VEgoStarting", {PERSISTENT, FLOAT, "0.0", "0.0", 3}}, {"VEgoStartingStock", {PERSISTENT, FLOAT, "0.0", "0.0", 3}}, {"VEgoStopping", {PERSISTENT, FLOAT, "0.0", "0.0", 3}}, diff --git a/selfdrive/selfdrived/selfdrived.py b/selfdrive/selfdrived/selfdrived.py index 6324018b6..626bbba04 100644 --- a/selfdrive/selfdrived/selfdrived.py +++ b/selfdrive/selfdrived/selfdrived.py @@ -32,6 +32,7 @@ from openpilot.system.hardware import HARDWARE from openpilot.starpilot.common.starpilot_utilities import contains_event_type from openpilot.starpilot.common.starpilot_variables import get_starpilot_toggles +from openpilot.starpilot.common.vision_bsm import get_fresh_vasm_state REPLAY = "REPLAY" in os.environ SIMULATION = "SIMULATION" in os.environ @@ -61,13 +62,15 @@ def commanded_torque_at_max_for_saturation(CP, output: float) -> bool: return torque_controller and abs(output) > 0.99 -def should_loud_blindspot_alert_without_lateral(CS, sm, starpilot_toggles) -> bool: +def should_loud_blindspot_alert_without_lateral(CS, sm, starpilot_toggles, combined_left_bsm=None, combined_right_bsm=None) -> bool: if not (getattr(starpilot_toggles, "loud_blindspot_alert", False) and getattr(starpilot_toggles, "loud_blindspot_alert_when_disengaged", False)): return False - left_signal_blocked = bool(CS.leftBlinker and CS.leftBlindspot) - right_signal_blocked = bool(CS.rightBlinker and CS.rightBlindspot) + combined_left_bsm = CS.leftBlindspot if combined_left_bsm is None else combined_left_bsm + combined_right_bsm = CS.rightBlindspot if combined_right_bsm is None else combined_right_bsm + left_signal_blocked = bool(CS.leftBlinker and combined_left_bsm) + right_signal_blocked = bool(CS.rightBlinker and combined_right_bsm) one_blinker = bool(CS.leftBlinker) != bool(CS.rightBlinker) if not (one_blinker and (left_signal_blocked or right_signal_blocked)): return False @@ -502,12 +505,17 @@ class SelfdriveD: self.events.add(EventName.excessiveActuation) # ****************************************************************************************** - # Handle lane change + # Handle lane change - combine OEM BSM with fresh V-ASM state. blindspot_alert_added = False + vasm_left, vasm_right = (False, False) + if getattr(self.starpilot_toggles, "v_asm_enabled", False): + vasm_left, vasm_right = get_fresh_vasm_state(self.params_memory) + combined_left_bsm = CS.leftBlindspot or vasm_left + combined_right_bsm = CS.rightBlindspot or vasm_right if self.sm['modelV2'].meta.laneChangeState == LaneChangeState.preLaneChange: direction = self.sm['modelV2'].meta.laneChangeDirection - if (CS.leftBlindspot and direction == LaneChangeDirection.left) or \ - (CS.rightBlindspot and direction == LaneChangeDirection.right): + if (combined_left_bsm and direction == LaneChangeDirection.left) or \ + (combined_right_bsm and direction == LaneChangeDirection.right): blindspot_alert_added = True if self.starpilot_toggles.loud_blindspot_alert: self.starpilot_events.add(StarPilotEventName.laneChangeBlockedLoud) @@ -529,7 +537,7 @@ class SelfdriveD: LaneChangeState.laneChangeFinishing): self.events.add(EventName.laneChange) - if not blindspot_alert_added and should_loud_blindspot_alert_without_lateral(CS, self.sm, self.starpilot_toggles): + if not blindspot_alert_added and should_loud_blindspot_alert_without_lateral(CS, self.sm, self.starpilot_toggles, combined_left_bsm, combined_right_bsm): self.starpilot_events.add(StarPilotEventName.laneChangeBlockedLoud) for i, pandaState in enumerate(self.sm['pandaStates']): diff --git a/selfdrive/selfdrived/tests/test_blindspot_alerts.py b/selfdrive/selfdrived/tests/test_blindspot_alerts.py index 520df905d..4f6e12436 100644 --- a/selfdrive/selfdrived/tests/test_blindspot_alerts.py +++ b/selfdrive/selfdrived/tests/test_blindspot_alerts.py @@ -45,6 +45,14 @@ def test_loud_blindspot_alert_without_lateral_for_matching_signal(): assert should_loud_blindspot_alert_without_lateral(CS, _sm(lat_active=True, lateral_check=True, pause_lateral=True), _toggles()) +def test_loud_blindspot_alert_accepts_combined_vision_state(): + CS = _car_state(left_blinker=True) + + assert should_loud_blindspot_alert_without_lateral( + CS, _sm(lat_active=False), _toggles(), combined_left_bsm=True, + ) + + def test_loud_blindspot_alert_without_lateral_ignores_active_lateral(): CS = _car_state(right_blinker=True, right_blindspot=True) diff --git a/selfdrive/ui/onroad/starpilot/path.py b/selfdrive/ui/onroad/starpilot/path.py index 94b5a3ad0..7b74423ea 100644 --- a/selfdrive/ui/onroad/starpilot/path.py +++ b/selfdrive/ui/onroad/starpilot/path.py @@ -8,6 +8,7 @@ import pyray as rl from openpilot.selfdrive.ui.lib.starpilot_state import starpilot_state from openpilot.selfdrive.ui.lib.starpilot_theme import get_param_color, get_theme_color, is_stock_color_scheme, with_alpha from openpilot.selfdrive.ui.ui_state import ui_state +from openpilot.starpilot.common.vision_bsm import get_fresh_vasm_state from openpilot.system.ui.lib.application import gui_app, FontWeight from openpilot.system.ui.lib.shader_polygon import draw_polygon, Gradient from openpilot.system.ui.lib.text_measure import measure_text_cached @@ -68,6 +69,10 @@ def render_adjacent_lanes(renderer) -> None: car_state = sm["carState"] blindspot_left = bool(car_state.leftBlindspot) blindspot_right = bool(car_state.rightBlindspot) + if ui_state.starpilot_toggles.get("v_asm_enabled", False): + vasm_left, vasm_right = get_fresh_vasm_state(ui_state.params_memory) + blindspot_left = blindspot_left or vasm_left + blindspot_right = blindspot_right or vasm_right # Fetch adjacent lane widths if adjacent path is enabled lane_width_left = 0.0 diff --git a/selfdrive/ui/onroad/starpilot/starpilot_border.py b/selfdrive/ui/onroad/starpilot/starpilot_border.py index 429570167..32de7e5d4 100644 --- a/selfdrive/ui/onroad/starpilot/starpilot_border.py +++ b/selfdrive/ui/onroad/starpilot/starpilot_border.py @@ -5,6 +5,7 @@ from collections.abc import Callable from enum import Enum import pyray as rl from openpilot.selfdrive.ui import UI_BORDER_SIZE +from openpilot.starpilot.common.vision_bsm import get_fresh_vasm_state from openpilot.selfdrive.ui.ui_state import ui_state from openpilot.selfdrive.ui.lib.starpilot_status import ( @@ -180,6 +181,10 @@ def render_background_effects(rect: rl.Rectangle, border_width: float): if show_signal or show_blindspot: left_blindspot = car_state.leftBlindspot right_blindspot = car_state.rightBlindspot + if ui_state.starpilot_toggles.get("v_asm_enabled", False): + vasm_left, vasm_right = get_fresh_vasm_state(ui_state.params_memory) + left_blindspot = left_blindspot or vasm_left + right_blindspot = right_blindspot or vasm_right left_blinker = car_state.leftBlinker right_blinker = car_state.rightBlinker diff --git a/starpilot/assets/vision_models/v_asm_model.onnx b/starpilot/assets/vision_models/v_asm_model.onnx new file mode 100644 index 000000000..4e268fcf2 Binary files /dev/null and b/starpilot/assets/vision_models/v_asm_model.onnx differ diff --git a/starpilot/common/cpu_throttle.py b/starpilot/common/cpu_throttle.py new file mode 100644 index 000000000..1d4b4cc29 --- /dev/null +++ b/starpilot/common/cpu_throttle.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import math +import time + + +DEVICE_BUSY_MAX_CPU_USAGE_PERCENT = 89.0 +DEVICE_BUSY_AVG_CPU_USAGE_PERCENT = 74.0 +DEVICE_BUSY_HOT_CORE_COUNT = 4 + +_throttle_state: dict[str, dict[str, float]] = {} + + +def device_cpu_throttle_factor(cpu_usage, name="vision"): + """Return a process-local, low-pass-filtered CPU throttle factor.""" + usage = list(cpu_usage) + if not usage: + return 1.0 + + now = time.monotonic() + state = _throttle_state.setdefault(name, { + "factor": 1.0, + "last_time": now, + "last_logged_factor": 1.0, + }) + dt = max(0.0, now - state["last_time"]) + state["last_time"] = now + + average = sum(usage) / len(usage) + hot_cores = sum(core >= DEVICE_BUSY_MAX_CPU_USAGE_PERCENT for core in usage) + target_factor = _compute_throttle_factor(average, hot_cores) + + alpha = min(1.0 - math.exp(-0.8 * dt), 1.0) + state["factor"] = min(max(target_factor * alpha + state["factor"] * (1.0 - alpha), 1.0), 4.0) + + if state["factor"] >= state["last_logged_factor"] + 0.6: + print(f"[{name}] CPU throttle factor={state['factor']:.1f}x (avg={average:.0f}%, hot={hot_cores})") + state["last_logged_factor"] = state["factor"] + elif state["factor"] <= 1.05 and state["last_logged_factor"] > 1.05: + print(f"[{name}] CPU recovered") + state["last_logged_factor"] = 1.0 + + return state["factor"] + + +def _compute_throttle_factor(average, hot_cores): + average_factor = 1.0 if average < DEVICE_BUSY_AVG_CPU_USAGE_PERCENT else 1.0 + (average - DEVICE_BUSY_AVG_CPU_USAGE_PERCENT) / 8.0 + hot_factor = 1.0 + max(0, hot_cores - DEVICE_BUSY_HOT_CORE_COUNT + 1) * 0.5 + return min(max(average_factor, hot_factor), 4.0) diff --git a/starpilot/common/safe_mode.py b/starpilot/common/safe_mode.py index 10b8902b9..422fa06f9 100644 --- a/starpilot/common/safe_mode.py +++ b/starpilot/common/safe_mode.py @@ -149,6 +149,7 @@ SAFE_MODE_MANAGED_KEYS = ( "Offset7", "SpeedLimitFiller", "VisionSpeedLimitDetection", + "VASMEnabled", "CustomPersonalities", "TrafficPersonalityProfile", "AggressivePersonalityProfile", diff --git a/starpilot/common/starpilot_variables.py b/starpilot/common/starpilot_variables.py index a72ee2ce4..68ff27724 100644 --- a/starpilot/common/starpilot_variables.py +++ b/starpilot/common/starpilot_variables.py @@ -1314,6 +1314,7 @@ class StarPilotVariables: toggle.speed_limit_filler = self.get_value("SpeedLimitFiller") toggle.vision_speed_limit_detection = self.get_value("VisionSpeedLimitDetection") + toggle.v_asm_enabled = self.get_value("VASMEnabled") toggle.startup_alert_top = self.get_value("StartupMessageTop", cast=str, default="") toggle.startup_alert_bottom = self.get_value("StartupMessageBottom", cast=str, default="") diff --git a/starpilot/common/tests/test_vision_bsm.py b/starpilot/common/tests/test_vision_bsm.py new file mode 100644 index 000000000..8753b932a --- /dev/null +++ b/starpilot/common/tests/test_vision_bsm.py @@ -0,0 +1,27 @@ +from openpilot.starpilot.common.vision_bsm import VASM_STATE_TIMEOUT_SECONDS, get_fresh_vasm_state + + +class FakeParams: + def __init__(self, values): + self.values = values + + def get(self, key): + return self.values.get(key) + + +def test_fresh_vasm_state_is_returned(): + params = FakeParams({ + "VASMLastUpdateMonoTime": "100.0", + "VASMLeftActive": "1", + "VASMRightActive": "0", + }) + + assert get_fresh_vasm_state(params, now=101.0) == (True, False) + + +def test_stale_or_invalid_vasm_state_fails_closed(): + stale = FakeParams({"VASMLastUpdateMonoTime": "100.0", "VASMLeftActive": "1"}) + invalid = FakeParams({"VASMLastUpdateMonoTime": "invalid", "VASMLeftActive": "1"}) + + assert get_fresh_vasm_state(stale, now=100.0 + VASM_STATE_TIMEOUT_SECONDS + 0.01) == (False, False) + assert get_fresh_vasm_state(invalid, now=100.0) == (False, False) diff --git a/starpilot/common/vision_bsm.py b/starpilot/common/vision_bsm.py new file mode 100644 index 000000000..af448470d --- /dev/null +++ b/starpilot/common/vision_bsm.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import time + + +VASM_STATE_TIMEOUT_SECONDS = 3.0 + + +def get_fresh_vasm_state(params_memory, now: float | None = None) -> tuple[bool, bool]: + """Return V-ASM state only while the vision daemon is updating it.""" + try: + updated_at = float(params_memory.get("VASMLastUpdateMonoTime") or 0) + except (TypeError, ValueError): + return False, False + + current_time = time.monotonic() if now is None else now + age = current_time - updated_at + if updated_at <= 0 or age < 0 or age > VASM_STATE_TIMEOUT_SECONDS: + return False, False + + active_values = ("1", b"1", True) + return params_memory.get("VASMLeftActive") in active_values, params_memory.get("VASMRightActive") in active_values diff --git a/starpilot/system/adj_spot_monitor_vision.py b/starpilot/system/adj_spot_monitor_vision.py new file mode 100644 index 000000000..7ad7f984e --- /dev/null +++ b/starpilot/system/adj_spot_monitor_vision.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["OPENBLAS_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" +os.environ["VECLIB_MAXIMUM_THREADS"] = "1" +os.environ["NUMEXPR_NUM_THREADS"] = "1" +import time +import cv2 +import numpy as np +from openpilot.common.params import Params +from openpilot.common.realtime import set_core_affinity, Ratekeeper +from openpilot.system.hardware import PC +from openpilot.starpilot.common.cpu_throttle import device_cpu_throttle_factor +from openpilot.starpilot.system.adj_spot_monitor_vision_inference import VASMInference, V_ASM_MODEL_PATH + +V_ASM_AFFINITY_CORES = [2] +V_ASM_SOLO_AFFINITY_CORES = [0, 1, 2] + +BASE_INTERVAL = 0.500 +FOLLOWUP_INTERVAL = 0.200 +FOLLOWUP_WINDOW = 1.5 + +PARAM_REFRESH_INTERVAL = 2.0 +STATUS_LOG_INTERVAL = 10.0 + + +class VASMDaemon: + def __init__(self): + from cereal import messaging + from msgq.visionipc import VisionIpcClient, VisionStreamType + + self.params = Params() + self.params_memory = Params(memory=True) + self.sm = messaging.SubMaster(["deviceState", "starpilotCarState"]) + + self.VisionIpcClient = VisionIpcClient + self.stream_type = VisionStreamType.VISION_STREAM_DRIVER + self.client = None + + self.inference = VASMInference(V_ASM_MODEL_PATH) + self.inference.load() + + self._cache_params() + + self.last_inference_at = 0.0 + self.last_inference_at_side = {"left": 0.0, "right": 0.0} + self.current_side = "left" + + self.followup_until = 0.0 + self.onroad_prev = False + + self._last_pub_left = False + self._last_pub_right = False + self._last_pub_left_conf = -1.0 + self._last_pub_right_conf = -1.0 + self._last_update_at = 0.0 + self._last_param_refresh = 0.0 + self._throttle_reason = "none" + self._last_status_log = 0.0 + self._inference_count = 0 + self._throttle_factor = 1.0 + + self._affinity_set = False + self._prev_other_running = None + + self._annotation_loaded = False + self._annotation_config = object() + self._load_annotation_config() + self._publish(False, False, 0.0, 0.0, 0, force=True, updated_at=0.0) + + print(f"[VASM] Started (model_valid={self.inference.valid})") + + def _cache_params(self): + self._enabled = self.params.get_bool("VASMEnabled") + self._slv_enabled = self.params.get_bool("VisionSpeedLimitDetection") + confidence_threshold = self.params.get_float("VASMConfidenceThreshold") or 0.85 + smooth_seconds = self.params.get_float("VASMSmoothSeconds") or 0.2 + self._conf_thresh = min(max(confidence_threshold, 0.25), 1.0) + self._smooth_sec = min(max(smooth_seconds, 0.1), 0.5) + + def _maybe_refresh_params(self, now): + if now - self._last_param_refresh >= PARAM_REFRESH_INTERVAL: + self._last_param_refresh = now + self._cache_params() + if self._load_annotation_config(): + self._update_inactive(reset_inference=True) + + def _load_annotation_config(self): + config = {} + try: + config = self.params.get("VASMAnnotationConfig") or {} + if isinstance(config, (bytes, str)): + config = json.loads(config) + if not isinstance(config, dict): + config = {} + if config == self._annotation_config: + return False + + self.inference.reset_state() + if config: + has_left = bool(config.get("poly_left")) + has_right = bool(config.get("poly_right")) + if has_left or has_right: + self.inference.load_config(config) + self._annotation_config = config + self._annotation_loaded = True + sides = [] + if has_left: + sides.append("left") + if has_right: + sides.append("right") + print(f"[VASM] Annotation config loaded (sides: {', '.join(sides)})") + return True + except (TypeError, ValueError, json.JSONDecodeError) as exc: + print(f"[VASM] Invalid annotation config: {exc}") + self._annotation_config = config + self._annotation_loaded = False + print("[VASM] No valid annotation config found; inference disabled until annotated via Galaxy") + return True + + def _connect_camera(self): + if self.client is not None and self.client.is_connected(): + return True + try: + available = self.VisionIpcClient.available_streams("camerad", block=False) + except Exception: + available = [] + + if self.stream_type not in available: + return False + + if self.client is None: + self.client = self.VisionIpcClient("camerad", self.stream_type, True) + + if not self.client.is_connected(): + self.client.connect(True) + return self.client.is_connected() + + def _update_inactive(self, reset_inference=False): + if reset_inference: + self.inference.reset_state() + if self._last_update_at != 0.0 or self._last_pub_left or self._last_pub_right or self._last_pub_left_conf != 0.0 or self._last_pub_right_conf != 0.0: + self._publish(False, False, 0.0, 0.0, 0, force=True, updated_at=0.0) + + def _inference_interval(self, now): + in_followup = now < self.followup_until + base = FOLLOWUP_INTERVAL if in_followup else BASE_INTERVAL + cpu_usage = list(self.sm["deviceState"].cpuUsagePercent) if self.sm.valid.get("deviceState", False) else [] + factor = device_cpu_throttle_factor(cpu_usage, name="VASM") + self._throttle_factor = factor + interval = base * factor + self._throttle_reason = f"cpu_{factor:.1f}x" if factor > 1.05 else ("followup" if in_followup else "steady") + return interval + + def _update_core_affinity(self, now): + other_running = self._slv_enabled + if other_running != self._prev_other_running: + self._affinity_set = False + self._prev_other_running = other_running + if not self._affinity_set: + set_core_affinity(V_ASM_AFFINITY_CORES if other_running else V_ASM_SOLO_AFFINITY_CORES) + self._affinity_set = True + + def run(self): + rk = Ratekeeper(10, None) + + while True: + try: + now = time.monotonic() + self._maybe_refresh_params(now) + self.sm.update(0) + + if not PC: + self._update_core_affinity(now) + onroad = self.sm["deviceState"].started if self.sm.valid.get("deviceState", False) else False + parked = self.sm["starpilotCarState"].isParked if self.sm.valid.get("starpilotCarState", False) else False + + if not onroad or not self._enabled or not self.inference.valid or not self._annotation_loaded: + self._update_inactive(reset_inference=(onroad != self.onroad_prev)) + if onroad != self.onroad_prev: + self.last_inference_at = 0.0 + self.onroad_prev = onroad + if now - self._last_status_log >= STATUS_LOG_INTERVAL and not onroad: + cpu = list(self.sm["deviceState"].cpuUsagePercent) if self.sm.valid.get("deviceState", False) else [] + cpu_str = f"avg={sum(cpu)/len(cpu):.0f}% cores={','.join(f'{c:.0f}' for c in cpu)}" if cpu else "?" + status = f"[VASM] idle | {cpu_str} | onroad={onroad} parked={parked}" + status += f" enabled={self._enabled} model={self.inference.valid} ann={self._annotation_loaded}" + print(status) + self._last_status_log = now + rk.keep_time() + continue + + self.onroad_prev = onroad + + if not self._connect_camera(): + self._update_inactive(reset_inference=True) + rk.keep_time() + continue + + inference_interval = self._inference_interval(now) + + if self.last_inference_at != 0.0 and (now - self.last_inference_at < inference_interval - 0.015): + self._publish(self.inference.left_active, self.inference.right_active, + self.inference.left_confidence, self.inference.right_confidence, + int(self.client.timestamp_sof)) + rk.keep_time() + continue + + buffer = None + while True: + b = self.client.recv(timeout_ms=0) + if b is None: + break + buffer = b + + if buffer is None: + rk.keep_time() + continue + + configured_sides = self.inference.configured_sides + if not configured_sides: + self._update_inactive(reset_inference=True) + rk.keep_time() + continue + if self.current_side not in configured_sides: + self.current_side = configured_sides[0] + + last_side_time = self.last_inference_at_side[self.current_side] + dt = (now - last_side_time) if last_side_time != 0.0 else inference_interval + + self.last_inference_at = now + self.last_inference_at_side[self.current_side] = now + + image = np.frombuffer(buffer.data, dtype=np.uint8).reshape( + (len(buffer.data) // self.client.stride, self.client.stride) + ) + + if self.client.stride != self.client.width: + image = image[:, :self.client.width] + + l_active, r_active = self.inference.update( + image, + self.client.width, + self.client.height, + dt=dt, + conf_thresh=self._conf_thresh, + smooth_sec=self._smooth_sec, + side_to_infer=self.current_side, + ) + + self._inference_count += 1 + if now - self._last_status_log >= STATUS_LOG_INTERVAL: + self._last_status_log = now + cpu = list(self.sm["deviceState"].cpuUsagePercent) if self.sm.valid.get("deviceState", False) else [] + cpu_str = f"avg={sum(cpu)/len(cpu):.0f}% cores={','.join(f'{c:.0f}' for c in cpu)}" if cpu else "?" + status = f"[VASM] {self._inference_count} inf {self._throttle_reason} | {cpu_str}" + status += f" | factor={self._throttle_factor:.1f}x | L={self.inference.left_confidence:.3f}" + status += f" R={self.inference.right_confidence:.3f}" + print(status) + self._inference_count = 0 + + self._publish(l_active, r_active, self.inference.left_confidence, + self.inference.right_confidence, int(self.client.timestamp_sof), updated_at=now) + + if l_active or r_active: + self.followup_until = now + FOLLOWUP_WINDOW + + side_index = configured_sides.index(self.current_side) + self.current_side = configured_sides[(side_index + 1) % len(configured_sides)] + + rk.keep_time() + + except Exception as e: + print(f"VASM Daemon Error: {e}") + self._update_inactive(reset_inference=True) + time.sleep(1.0) + + def _publish(self, left_active, right_active, left_conf, right_conf, ts_sof, force=False, updated_at=None): + if updated_at is not None: + self._last_update_at = updated_at + self.params_memory.put("VASMLastUpdateMonoTime", str(updated_at)) + + r_left_conf = round(left_conf, 3) + r_right_conf = round(right_conf, 3) + + if not force and \ + left_active == self._last_pub_left and \ + right_active == self._last_pub_right and \ + abs(r_left_conf - self._last_pub_left_conf) < 0.005 and \ + abs(r_right_conf - self._last_pub_right_conf) < 0.005: + return + + self._last_pub_left = left_active + self._last_pub_right = right_active + self._last_pub_left_conf = r_left_conf + self._last_pub_right_conf = r_right_conf + + ui_left_active, ui_right_active = right_active, left_active + ui_left_conf, ui_right_conf = right_conf, left_conf + + self.params_memory.put("VASMLeftActive", "1" if ui_left_active else "0") + self.params_memory.put("VASMRightActive", "1" if ui_right_active else "0") + self.params_memory.put("VASMLeftConfidence", str(ui_left_conf)) + self.params_memory.put("VASMRightConfidence", str(ui_right_conf)) + self.params_memory.put("VASMTimestampEof", str(ts_sof)) + + +def main(): + cv2.setNumThreads(1) + VASMDaemon().run() + + +if __name__ == "__main__": + main() diff --git a/starpilot/system/adj_spot_monitor_vision_inference.py b/starpilot/system/adj_spot_monitor_vision_inference.py new file mode 100644 index 000000000..02f6bfd43 --- /dev/null +++ b/starpilot/system/adj_spot_monitor_vision_inference.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +from pathlib import Path +import cv2 +import numpy as np + +_ASSETS = Path(__file__).resolve().parents[1] / "assets" / "vision_models" +# The bundled export keeps 299 final candidates so OpenCV's TopK importer can load it. +V_ASM_MODEL_PATH = _ASSETS / "v_asm_model.onnx" + +MODEL_INPUT_H = 256 +MODEL_INPUT_W = 352 +HYSTERESIS_ON = 0.65 +HYSTERESIS_OFF = 0.25 + + +class VASMInference: + def __init__(self, model_path: Path): + self.model_path = model_path + self.net = None + self._valid = False + self.last_error = "" + + self.reset_state() + + self.frame_res = (0, 0) + self.config_width = 0 + self.config_height = 0 + self.masks = {"left": None, "right": None} + self.bboxes = {"left": None, "right": None, "left_raw": None, "right_raw": None} + + def load(self) -> bool: + if not self.model_path.is_file(): + self.last_error = f"Missing model: {self.model_path}" + self._valid = False + return False + try: + self.net = cv2.dnn.readNetFromONNX(str(self.model_path)) + self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) + self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) + self._valid = True + self.last_error = "" + except Exception as e: + self.last_error = f"Failed to load model: {e}" + self._valid = False + return self._valid + + def reset_state(self): + self._l_score = 0.0 + self._r_score = 0.0 + self.left_active = False + self.right_active = False + self.left_confidence = 0.0 + self.right_confidence = 0.0 + + @property + def valid(self): + return self._valid + + @property + def configured_sides(self): + return tuple(side for side in ("left", "right") if self.bboxes.get(f"{side}_raw") is not None) + + def _prepare_geometry(self, h, w): + if (h, w) == self.frame_res: + return + self.frame_res = (h, w) + + scale_x = w / float(self.config_width) if self.config_width > 0 else 1.0 + scale_y = h / float(self.config_height) if self.config_height > 0 else 1.0 + + for side in ("left", "right"): + raw_pts = self.bboxes.get(f"{side}_raw") + if raw_pts is None: + self.bboxes[side] = None + self.masks[side] = None + continue + + pts = raw_pts.copy() + pts[:, 0] *= scale_x + pts[:, 1] *= scale_y + + bx, by, bw, bh = cv2.boundingRect(pts.astype(np.int32)) + + bx = (bx // 2) * 2 + by = (by // 2) * 2 + bw = ((bw + 1) // 2) * 2 + bh = ((bh + 1) // 2) * 2 + + bx = max(0, min(bx, w - 2)) + by = max(0, min(by, h - 2)) + bw = max(2, min(bw, w - bx)) + bh = max(2, min(bh, h - by)) + + bw = (bw // 2) * 2 + bh = (bh // 2) * 2 + + self.bboxes[side] = (bx, by, bw, bh) + + mask = np.zeros((bh, bw), dtype=np.uint8) + cv2.fillPoly(mask, [pts.astype(np.int32) - [bx, by]], 255) + self.masks[side] = mask + + def load_config(self, config: dict): + self.frame_res = (0, 0) + self.config_width = config.get("width", 0) + self.config_height = config.get("height", 0) + + for side in ("left", "right"): + poly = config.get(f"poly_{side}", []) + if len(poly) >= 3: + self.bboxes[f"{side}_raw"] = np.array(poly, dtype=np.float32) + else: + self.bboxes[f"{side}_raw"] = None + self.bboxes[side] = None + + def _run_inference(self, raw_image, height, side): + bbox = self.bboxes[side] + if bbox is None or self.net is None: + return 0.0 + + x, y, w, h = bbox + + # Slice NV12 directly + y_crop = raw_image[y: y + h, x: x + w] + uv_crop = raw_image[height + y // 2: height + (y + h) // 2, x: x + w] + nv12_crop = np.vstack([y_crop, uv_crop]) + + # Convert cropped area directly from YUV NV12 to RGB (1-step, avoids double conversion) + crop_rgb = cv2.cvtColor(nv12_crop, cv2.COLOR_YUV2RGB_NV12) + if self.masks[side] is not None: + crop_rgb = cv2.bitwise_and(crop_rgb, crop_rgb, mask=self.masks[side]) + + # Preprocess -> NCHW Float32 [0.0 - 1.0] + resized = cv2.resize(crop_rgb, (MODEL_INPUT_W, MODEL_INPUT_H), interpolation=cv2.INTER_LINEAR) + blob = resized.astype(np.float32) / 255.0 + blob = np.transpose(blob, (2, 0, 1)) + blob = np.expand_dims(blob, axis=0) + + self.net.setInput(blob) + out = self.net.forward() + + preds = np.squeeze(out) + if preds.ndim == 2: + if preds.shape[0] < preds.shape[1]: + preds = preds.T + if preds.shape[1] >= 6: + is_class_0 = (np.round(preds[:, 5]).astype(int) == 0) + relevant = preds[is_class_0] + if len(relevant) == 0: + return 0.0 + return float(np.max(relevant[:, 4])) + elif preds.shape[1] >= 5: + return float(np.max(preds[:, 4])) + else: + return float(np.max(preds[:, 0])) + elif preds.ndim == 1 and preds.size > 0: + return float(np.max(preds)) + return 0.0 + + def update(self, raw_image, width, height, dt, conf_thresh, smooth_sec, side_to_infer): + if not self._valid: + return False, False + + self._prepare_geometry(height, width) + alpha = min(1.0, dt / max(smooth_sec, 0.001)) + + raw_conf = self._run_inference(raw_image, height, side_to_infer) + if side_to_infer == "left": + if raw_conf >= conf_thresh: + self._l_score = min(1.0, self._l_score + alpha) + else: + self._l_score = max(0.0, self._l_score - alpha) + self.left_confidence = raw_conf + if self._l_score >= HYSTERESIS_ON: + self.left_active = True + elif self._l_score <= HYSTERESIS_OFF: + self.left_active = False + else: + if raw_conf >= conf_thresh: + self._r_score = min(1.0, self._r_score + alpha) + else: + self._r_score = max(0.0, self._r_score - alpha) + self.right_confidence = raw_conf + if self._r_score >= HYSTERESIS_ON: + self.right_active = True + elif self._r_score <= HYSTERESIS_OFF: + self.right_active = False + + return self.left_active, self.right_active diff --git a/starpilot/system/speed_limit_vision.py b/starpilot/system/speed_limit_vision.py index 3bdc74746..058bc620a 100644 --- a/starpilot/system/speed_limit_vision.py +++ b/starpilot/system/speed_limit_vision.py @@ -15,6 +15,7 @@ import numpy as np from openpilot.common.constants import CV from openpilot.common.realtime import set_core_affinity +from openpilot.starpilot.common.cpu_throttle import device_cpu_throttle_factor from openpilot.system.hardware import PC RUNTIME_LOOP_HZ = 20 @@ -237,6 +238,13 @@ DEBUG_RUNTIME_STATUS_PATH = DEBUG_BASE_DIR / "runtime_status.json" DEBUG_CAPTURE_DIRNAME = "captures" SNAPSHOT_JPEG_QUALITY = 85 SPEED_LIMIT_VISION_AFFINITY_CORES = [0, 1, 2] +SPEED_LIMIT_VISION_COEXISTENCE_AFFINITY_CORES = [0, 1] +COEXISTENCE_PARAM_REFRESH_SECONDS = 2.0 +COEXISTENCE_TRACK_DETECTOR_INTERVAL = 0.80 +COEXISTENCE_DETECTOR_CLASSIFIER_EXPANSIONS = ( + (0.00, 0.00, 0.00, 0.00, 1.10), + (0.10, 0.06, 0.10, 0.12, 1.00), +) def device_cpu_usage_busy(cpu_usage): @@ -380,6 +388,11 @@ class SpeedLimitVisionDaemon: self.last_inference_interval = INFERENCE_INTERVAL self.last_inference_interval_reason = "steady" self.last_cpu_busy = False + self.coexistence_mode = False + self.last_coexistence_param_refresh_at = -float("inf") + self.temporal_tracking_enabled = TEMPORAL_TRACKING_ENABLED + self.track_detector_interval = TRACK_DETECTOR_INTERVAL + self.detector_classifier_expansions = DETECTOR_CLASSIFIER_EXPANSIONS self.last_frame_process_duration_s = 0.0 self.last_detector_forward_count = 0 self.last_detector_forward_duration_s = 0.0 @@ -838,6 +851,32 @@ class SpeedLimitVisionDaemon: return False return device_cpu_usage_busy(self.sm["deviceState"].cpuUsagePercent) + def _update_coexistence_mode(self, now): + if self.params is None or now - self.last_coexistence_param_refresh_at < COEXISTENCE_PARAM_REFRESH_SECONDS: + return + self.last_coexistence_param_refresh_at = now + + coexistence_mode = self.params.get_bool("VASMEnabled") + if coexistence_mode == self.coexistence_mode: + return + + self.coexistence_mode = coexistence_mode + self.latest_detector_proposal = None + self._clear_proposal_track() + if coexistence_mode: + self.temporal_tracking_enabled = True + self.track_detector_interval = COEXISTENCE_TRACK_DETECTOR_INTERVAL + self.detector_classifier_expansions = COEXISTENCE_DETECTOR_CLASSIFIER_EXPANSIONS + affinity_cores = SPEED_LIMIT_VISION_COEXISTENCE_AFFINITY_CORES + else: + self.temporal_tracking_enabled = TEMPORAL_TRACKING_ENABLED + self.track_detector_interval = TRACK_DETECTOR_INTERVAL + self.detector_classifier_expansions = DETECTOR_CLASSIFIER_EXPANSIONS + affinity_cores = SPEED_LIMIT_VISION_AFFINITY_CORES + + if not PC: + set_core_affinity(affinity_cores) + def _inference_interval(self, now): in_followup = now < self.followup_until interval = FOLLOWUP_INFERENCE_INTERVAL if in_followup else INFERENCE_INTERVAL @@ -846,6 +885,13 @@ class SpeedLimitVisionDaemon: if now - self.last_live_pose_inputs_not_ok_at < LIVE_POSE_RECOVERY_THROTTLE_SECONDS: interval = max(interval, LIVE_POSE_RECOVERY_INFERENCE_INTERVAL) reason = "live_pose_recovery" + elif self.coexistence_mode: + cpu_usage = list(self.sm["deviceState"].cpuUsagePercent) if self.sm is not None and self.sm.valid.get("deviceState", False) else [] + factor = device_cpu_throttle_factor(cpu_usage, name="SpeedLimit") + if factor > 1.05: + self.last_cpu_busy = True + interval *= factor + reason = f"cpu_{factor:.1f}x" elif self._device_cpu_busy(): self.last_cpu_busy = True interval = max(interval, BUSY_INFERENCE_INTERVAL) @@ -1145,7 +1191,7 @@ class SpeedLimitVisionDaemon: def _remember_detector_proposal(self, confidence, class_id, bbox, speed_limit_mph=0, preferred=False): min_confidence = TRACK_MIN_PROPOSAL_CONFIDENCE if speed_limit_mph else TRACK_UNREADABLE_MIN_PROPOSAL_CONFIDENCE - if not TEMPORAL_TRACKING_ENABLED or class_id == 1 or confidence < min_confidence: + if not getattr(self, "temporal_tracking_enabled", TEMPORAL_TRACKING_ENABLED) or class_id == 1 or confidence < min_confidence: return proposal = DetectorProposal(float(confidence), int(class_id), bbox, int(speed_limit_mph)) latest_proposal = getattr(self, "latest_detector_proposal", None) @@ -1156,7 +1202,7 @@ class SpeedLimitVisionDaemon: proposal = self.latest_detector_proposal self.latest_detector_proposal = None if ( - not TEMPORAL_TRACKING_ENABLED or + not getattr(self, "temporal_tracking_enabled", TEMPORAL_TRACKING_ENABLED) or proposal is None or (proposal.speed_limit_mph and not TRACK_CONFIRMED_PROPOSALS_ENABLED) ): @@ -1187,7 +1233,11 @@ class SpeedLimitVisionDaemon: interval = TRACK_CLASSIFICATION_INTERVAL if now - self.last_live_pose_inputs_not_ok_at < LIVE_POSE_RECOVERY_THROTTLE_SECONDS: return max(interval, LIVE_POSE_RECOVERY_INFERENCE_INTERVAL) - if self._device_cpu_busy(): + if self.coexistence_mode: + cpu_usage = list(self.sm["deviceState"].cpuUsagePercent) if self.sm is not None and self.sm.valid.get("deviceState", False) else [] + if device_cpu_throttle_factor(cpu_usage, name="SpeedLimit") > 1.05: + return max(interval, TRACK_BUSY_CLASSIFICATION_INTERVAL) + elif self._device_cpu_busy(): return max(interval, TRACK_BUSY_CLASSIFICATION_INTERVAL) return interval @@ -1200,6 +1250,13 @@ class SpeedLimitVisionDaemon: return False return now - track.last_classified_at >= self._track_classification_interval(now) + def _detector_interval(self, inference_interval): + if self.coexistence_mode: + return max(inference_interval, self.track_detector_interval) + if self.proposal_track is not None: + return max(inference_interval, TRACK_DETECTOR_INTERVAL) + return inference_interval + def _classify_proposal_track(self, frame_bgr, now): track = self.proposal_track if track is None: @@ -1748,7 +1805,8 @@ class SpeedLimitVisionDaemon: speed_direct_model_support: dict[int, int] = {} speed_strong_model_support: dict[int, int] = {} - for expand_left, expand_top, expand_right, expand_bottom, expansion_weight in DETECTOR_CLASSIFIER_EXPANSIONS: + expansions = getattr(self, "detector_classifier_expansions", DETECTOR_CLASSIFIER_EXPANSIONS) + for expand_left, expand_top, expand_right, expand_bottom, expansion_weight in expansions: expanded_x1 = max(int(x1 - box_width * expand_left), 0) expanded_y1 = max(int(y1 - box_height * expand_top), 0) expanded_x2 = min(int(x2 + box_width * expand_right), frame_width) @@ -2541,13 +2599,14 @@ class SpeedLimitVisionDaemon: ratekeeper.keep_time() continue + self._update_coexistence_mode(now) inference_interval = self._inference_interval(now) track_due = self._track_classification_due(now) - detector_interval = max(inference_interval, TRACK_DETECTOR_INTERVAL) if self.proposal_track is not None else inference_interval + detector_interval = self._detector_interval(inference_interval) detector_due = now - self.last_inference_at >= detector_interval if not track_due and not detector_due: self.interval_skip_count += 1 - if self.last_inference_interval_reason == "cpu_busy": + if self.last_cpu_busy: self.busy_skip_count += 1 stale_cleared = self._clear_published_detection_if_stale(now, "inference_interval") if self.published_speed_limit_mph > 0 and not stale_cleared: diff --git a/starpilot/system/tests/test_adj_spot_monitor_vision.py b/starpilot/system/tests/test_adj_spot_monitor_vision.py new file mode 100644 index 000000000..5903d0f34 --- /dev/null +++ b/starpilot/system/tests/test_adj_spot_monitor_vision.py @@ -0,0 +1,107 @@ +from pathlib import Path + +import numpy as np + +from starpilot.system.adj_spot_monitor_vision import VASMDaemon +from starpilot.system.adj_spot_monitor_vision_inference import MODEL_INPUT_H, MODEL_INPUT_W, V_ASM_MODEL_PATH, VASMInference + + +class FakeParams: + def __init__(self, config=None): + self.config = config or {} + + def get(self, key): + assert key == "VASMAnnotationConfig" + return self.config + + +class FakeMemoryParams: + def __init__(self): + self.values = {} + + def put(self, key, value): + self.values[key] = value + + +class FakeInference: + def __init__(self): + self.loaded = [] + self.reset_count = 0 + + def load_config(self, config): + self.loaded.append(config) + + def reset_state(self): + self.reset_count += 1 + + +def test_inference_geometry_supports_single_annotated_side(): + inference = VASMInference(Path("unused.onnx")) + inference.load_config({ + "width": 200, + "height": 100, + "poly_left": [[10, 10], [80, 10], [80, 80], [10, 80]], + "poly_right": [], + }) + + inference._prepare_geometry(100, 200) + + assert inference.configured_sides == ("left",) + assert inference.bboxes["left"] is not None + assert inference.bboxes["right"] is None + + +def test_model_loads_with_repo_inference_backend(): + inference = VASMInference(V_ASM_MODEL_PATH) + + assert inference.load(), inference.last_error + inference.net.setInput(np.zeros((1, 3, MODEL_INPUT_H, MODEL_INPUT_W), dtype=np.float32)) + + assert inference.net.forward().shape == (1, 299, 6) + + +def test_model_runs_from_nv12_camera_frame(): + inference = VASMInference(V_ASM_MODEL_PATH) + assert inference.load(), inference.last_error + inference.load_config({ + "width": MODEL_INPUT_W, + "height": MODEL_INPUT_H, + "poly_left": [[0, 0], [MODEL_INPUT_W, 0], [MODEL_INPUT_W, MODEL_INPUT_H], [0, MODEL_INPUT_H]], + "poly_right": [], + }) + nv12 = np.zeros((MODEL_INPUT_H * 3 // 2, MODEL_INPUT_W), dtype=np.uint8) + + assert inference.update(nv12, MODEL_INPUT_W, MODEL_INPUT_H, 0.5, 1.0, 0.2, "left") == (False, False) + + +def test_annotation_changes_reload_without_process_restart(): + first = {"width": 200, "height": 100, "poly_left": [[1, 1], [10, 1], [10, 10]], "poly_right": []} + second = {"width": 200, "height": 100, "poly_left": [], "poly_right": [[20, 1], [30, 1], [30, 10]]} + daemon = VASMDaemon.__new__(VASMDaemon) + daemon.params = FakeParams(first) + daemon.inference = FakeInference() + daemon._annotation_config = object() + daemon._annotation_loaded = False + + assert daemon._load_annotation_config() + assert not daemon._load_annotation_config() + daemon.params.config = second + assert daemon._load_annotation_config() + + assert daemon.inference.loaded == [first, second] + + +def test_publish_writes_freshness_and_maps_camera_sides_to_ui_sides(): + daemon = VASMDaemon.__new__(VASMDaemon) + daemon.params_memory = FakeMemoryParams() + daemon._last_pub_left = False + daemon._last_pub_right = False + daemon._last_pub_left_conf = -1.0 + daemon._last_pub_right_conf = -1.0 + daemon._last_update_at = 0.0 + + daemon._publish(True, False, 0.9, 0.1, 123, updated_at=50.0) + + assert daemon.params_memory.values["VASMLastUpdateMonoTime"] == "50.0" + assert daemon.params_memory.values["VASMLeftActive"] == "0" + assert daemon.params_memory.values["VASMRightActive"] == "1" diff --git a/starpilot/system/tests/test_speed_limit_vision.py b/starpilot/system/tests/test_speed_limit_vision.py index 8c0385371..2e6cc665e 100644 --- a/starpilot/system/tests/test_speed_limit_vision.py +++ b/starpilot/system/tests/test_speed_limit_vision.py @@ -34,6 +34,15 @@ class StaticClassifierNet: return self.probabilities +class ToggleParams: + def __init__(self, enabled): + self.enabled = enabled + + def get_bool(self, key): + assert key == "VASMEnabled" + return self.enabled + + def daemon_with_history(current_speed, entries): daemon = SpeedLimitVisionDaemon.__new__(SpeedLimitVisionDaemon) daemon.published_speed_limit_mph = current_speed @@ -71,6 +80,37 @@ def test_disconnect_camera_releases_client_state(): assert daemon.stream_name == "" +def test_vasm_coexistence_mode_is_conditional(monkeypatch): + daemon = SpeedLimitVisionDaemon.__new__(SpeedLimitVisionDaemon) + daemon.params = ToggleParams(False) + daemon.coexistence_mode = False + daemon.last_coexistence_param_refresh_at = -float("inf") + daemon.temporal_tracking_enabled = slv.TEMPORAL_TRACKING_ENABLED + daemon.track_detector_interval = slv.TRACK_DETECTOR_INTERVAL + daemon.detector_classifier_expansions = slv.DETECTOR_CLASSIFIER_EXPANSIONS + daemon.latest_detector_proposal = None + daemon.proposal_track = None + monkeypatch.setattr(slv, "PC", True) + + daemon._update_coexistence_mode(0.0) + assert not daemon.coexistence_mode + assert daemon.detector_classifier_expansions == slv.DETECTOR_CLASSIFIER_EXPANSIONS + assert daemon._detector_interval(slv.INFERENCE_INTERVAL) == slv.INFERENCE_INTERVAL + + daemon.params.enabled = True + daemon._update_coexistence_mode(slv.COEXISTENCE_PARAM_REFRESH_SECONDS + 0.1) + assert daemon.coexistence_mode + assert daemon.temporal_tracking_enabled + assert daemon.detector_classifier_expansions == slv.COEXISTENCE_DETECTOR_CLASSIFIER_EXPANSIONS + assert daemon._detector_interval(slv.INFERENCE_INTERVAL) == slv.COEXISTENCE_TRACK_DETECTOR_INTERVAL + + daemon.params.enabled = False + daemon._update_coexistence_mode(2 * slv.COEXISTENCE_PARAM_REFRESH_SECONDS + 0.2) + assert not daemon.coexistence_mode + assert daemon.temporal_tracking_enabled == slv.TEMPORAL_TRACKING_ENABLED + assert daemon.detector_classifier_expansions == slv.DETECTOR_CLASSIFIER_EXPANSIONS + + def test_receive_frame_does_not_retain_vision_buffer(monkeypatch): buffer_refs = [] diff --git a/starpilot/system/the_galaxy/assets/components/router.js b/starpilot/system/the_galaxy/assets/components/router.js index bd702bb33..8a14db9ad 100644 --- a/starpilot/system/the_galaxy/assets/components/router.js +++ b/starpilot/system/the_galaxy/assets/components/router.js @@ -23,6 +23,7 @@ import { Tuning } from "/assets/components/tools/tuning.js?v=flm-workspace-9" import { Troubleshoot } from "/assets/components/tools/troubleshoot.js" import { TmuxLog } from "/assets/components/tools/tmux.js" import { ToggleControl } from "/assets/components/tools/toggles.js" +import { VASMAnnotations } from "/assets/components/tools/v_asm.js" import { UpdateManager } from "/assets/components/tools/update_manager.js" let router, routerState @@ -84,6 +85,7 @@ function Root() { createRoute("toggles", "/manage_toggles", ToggleControl), createRoute("updates", "/manage_updates", UpdateManager), createRoute("vehicle_features", "/vehicle_features", VehicleFeatures), + createRoute("v_asm", "/manage_v_asm", VASMAnnotations), ] router = createRouter({ diff --git a/starpilot/system/the_galaxy/assets/components/sidebar.js b/starpilot/system/the_galaxy/assets/components/sidebar.js index 4e1339a20..0de6930ea 100644 --- a/starpilot/system/the_galaxy/assets/components/sidebar.js +++ b/starpilot/system/the_galaxy/assets/components/sidebar.js @@ -23,6 +23,7 @@ const MENU_ITEMS = { { name: "Plots", link: "/plots", icon: "bi-graph-up-arrow" }, { name: "Testing Ground", link: "/testing_ground", icon: "bi-bezier2" }, { name: "Troubleshoot", link: "/troubleshoot", icon: "bi-tools" }, + { name: "V-ASM Annotations", link: "/manage_v_asm", icon: "bi-eye" }, { name: "Theme Maker", link: "/theme_maker", icon: "bi-palette-fill" }, { name: "Tmux Log", link: "/manage_tmux", icon: "bi-terminal" }, { name: "Backup and Restore", link: "/manage_toggles", icon: "bi-arrow-repeat" }, diff --git a/starpilot/system/the_galaxy/assets/components/tools/device_settings.js b/starpilot/system/the_galaxy/assets/components/tools/device_settings.js index c2e440285..cdef0643b 100644 --- a/starpilot/system/the_galaxy/assets/components/tools/device_settings.js +++ b/starpilot/system/the_galaxy/assets/components/tools/device_settings.js @@ -1180,6 +1180,12 @@ function getSettingLockReason(param) { if (param?.disabled_when_key_true && state.values[param.disabled_when_key_true]) { return param.disabled_reason || "Disabled by another setting." } + if (param?.requires_nonempty_key) { + const val = state.values[param.requires_nonempty_key] + if (!val || val === "{}" || val === "") { + return param.disabled_reason || "Required configuration missing." + } + } return "" } @@ -1530,6 +1536,7 @@ function renderSettingRow(p) { type="checkbox" class="ds-toggle" id="ds-${p.key}" + disabled="${() => isLocked()}" @change="${() => updateParam(p.key, "checkbox")}" /> ` } diff --git a/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json b/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json index 6cd4ed263..935e45d33 100644 --- a/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json +++ b/starpilot/system/the_galaxy/assets/components/tools/device_settings_layout.json @@ -314,6 +314,43 @@ "step": 0.1, "parent_key": "QOLLateral", "settings_tier": "simple" + }, + { + "key": "VASMEnabled", + "label": "Enable V-ASM", + "description": "Camera-based adjacent spot monitoring using the driver camera. Supplements factory blind spot monitoring.", + "data_type": "bool", + "ui_type": "toggle", + "is_parent_toggle": true, + "requires_nonempty_key": "VASMAnnotationConfig", + "disabled_reason": "Configure window annotations first in Galaxy > V-ASM Annotations", + "settings_tier": "advanced" + }, + { + "key": "VASMConfidenceThreshold", + "label": "Confidence Threshold", + "description": "Minimum vehicle-detection confidence (0.25-1.00, default 0.85). Higher values reduce false positives but may miss detections.", + "data_type": "float", + "ui_type": "numeric", + "min": 0.25, + "max": 1.00, + "step": 0.05, + "precision": 2, + "parent_key": "VASMEnabled", + "settings_tier": "advanced" + }, + { + "key": "VASMSmoothSeconds", + "label": "Smoothing Duration", + "description": "Temporal smoothing duration (0.1-0.5 seconds, default 0.2). Higher values reduce flicker but add latency.", + "data_type": "float", + "ui_type": "numeric", + "min": 0.1, + "max": 0.5, + "step": 0.1, + "precision": 1, + "parent_key": "VASMEnabled", + "settings_tier": "advanced" } ] }, diff --git a/starpilot/system/the_galaxy/assets/components/tools/v_asm.css b/starpilot/system/the_galaxy/assets/components/tools/v_asm.css new file mode 100644 index 000000000..68d4daece --- /dev/null +++ b/starpilot/system/the_galaxy/assets/components/tools/v_asm.css @@ -0,0 +1,337 @@ +.v-asm-wrapper { + display: flex; + flex-direction: column; + gap: var(--gap-base); +} +.v-asm-note { + background: rgba(13, 110, 253, 0.08); + border-left: 4px solid #0d6efd; + padding: 12px 16px; + border-radius: 6px; + font-size: 0.85rem; + line-height: 1.5; + margin: 12px 0; + color: var(--text-muted); +} +.v-asm-header { + margin-bottom: var(--margin-base); + display: flex; + flex-direction: column; + gap: var(--gap-sm); +} +.v-asm-header h2 { + margin: 0 0 var(--margin-xs); + font-size: var(--font-size-xl); +} +.v-asm-card { + background: var(--card-bg); + border: 1px solid var(--sidebar-border-color); + border-radius: var(--border-radius-md); + padding: var(--padding-base); + display: flex; + flex-direction: column; + gap: var(--gap-sm); +} +.v-asm-card-info { + border-left: 4px solid var(--main-fg); +} +.v-asm-card-warning { + border-left: 4px solid #fd7e14; +} +.v-asm-card-danger { + border-left: 4px solid var(--danger-fg); +} +.v-asm-card-title { + color: var(--text-color); + font-weight: var(--font-weight-bold); + font-size: var(--font-size-base); +} +.v-asm-card-list { + margin: 0; + padding: 0; + list-style: none; + display: flex; + flex-direction: column; + gap: var(--gap-xs); +} +.v-asm-card-list li { + color: var(--text-muted); + font-size: var(--font-size-sm); + padding-left: 1.2rem; + position: relative; + line-height: 1.4; +} +.v-asm-card-list li::before { + content: "-"; + position: absolute; + left: 0; + color: var(--text-muted); +} +.v-asm-card-list li.v-asm-card-action { + border-top: 1px solid var(--sidebar-border-color); + margin-top: var(--margin-xs); + padding-top: var(--padding-xs); + padding-left: 0; + list-style-type: none; +} +.v-asm-card-list li.v-asm-card-action::before { + content: ""; +} +.v-asm-card-list a { + color: var(--main-fg); + text-decoration: none; + font-weight: var(--font-weight-demi-bold); +} +.v-asm-card-list a:hover { + text-decoration: underline; +} +.v-asm-section { + background: var(--sidebar-bg); + border: 1px solid var(--sidebar-border-color); + border-radius: var(--border-radius-lg); + padding: var(--padding-lg); + margin-bottom: var(--margin-lg); +} +.v-asm-section h3 { + margin: 0 0 var(--margin-base); + font-size: var(--font-size-lg); +} +.v-asm-desc { + color: var(--text-muted); + font-size: var(--font-size-sm); + margin: 4px 0 var(--margin-base); +} +.v-asm-toolbar { + display: flex; + flex-wrap: wrap; + align-items: center; + justify-content: space-between; + gap: var(--gap-sm); + background: var(--card-bg); + border: 1px solid var(--sidebar-border-color); + border-radius: var(--border-radius-md); + padding: var(--padding-sm) var(--padding-base); + margin-bottom: var(--margin-base); +} +.v-asm-btn-group { + display: flex; + flex-wrap: wrap; + gap: var(--gap-xs); +} +.v-asm-btn { + display: inline-flex; + align-items: center; + justify-content: center; + padding: var(--padding-sm) var(--padding-base); + background: linear-gradient(135deg, #7a62b8, #8b6cc5); + color: #fff; + border: none; + border-radius: var(--border-radius-md); + font-size: var(--font-size-xs); + font-weight: var(--font-weight-demi-bold); + cursor: pointer; + transition: opacity var(--transition-fast); + height: 32px; +} +.v-asm-btn:hover { + opacity: var(--hover-opacity); +} +.v-asm-btn:disabled { + opacity: var(--disabled-opacity); + cursor: not-allowed; +} +.v-asm-btn-primary { + background: linear-gradient(135deg, #3a9e5c, #28a745); +} +.v-asm-btn-secondary { + background: linear-gradient(135deg, #495057, #6c757d); +} +.v-asm-btn-danger { + background: linear-gradient(135deg, #b14a6b, #dc3545); +} +.v-asm-btn-outline-left { + background: transparent; + border: 1px solid #0d6efd; + color: #0d6efd; +} +.v-asm-btn-outline-left:hover { + background: rgba(13, 110, 253, 0.1); +} +.v-asm-btn-left-active { + background: #0d6efd; + border: 1px solid #0d6efd; + color: #fff; + outline: 2px solid #fff; + outline-offset: 2px; +} +.v-asm-btn-outline-right { + background: transparent; + border: 1px solid #fd7e14; + color: #fd7e14; +} +.v-asm-btn-outline-right:hover { + background: rgba(253, 126, 20, 0.1); +} +.v-asm-btn-right-active { + background: #fd7e14; + border: 1px solid #fd7e14; + color: #fff; + outline: 2px solid #fff; + outline-offset: 2px; +} +.v-asm-points-summary { + display: flex; + gap: var(--gap-xs); + align-items: center; +} +.v-asm-summary-badge { + display: flex; + align-items: center; + gap: var(--gap-xs); + font-size: var(--font-size-xs); + font-weight: var(--font-weight-bold); + padding: 4px 10px; + border-radius: 12px; + background: var(--sidebar-bg); + border: 1px solid var(--sidebar-border-color); +} +.v-asm-summary-badge.badge-left { + border-color: rgba(13, 110, 253, 0.4); + color: #0d6efd; +} +.v-asm-summary-badge.badge-right { + border-color: rgba(253, 126, 20, 0.4); + color: #fd7e14; +} +.v-asm-summary-badge .badge-done { + color: var(--success-fg); +} +.v-asm-mini-clear { + background: none; + border: none; + color: inherit; + cursor: pointer; + font-size: 14px; + padding: 0 0 0 4px; + line-height: 1; +} +.v-asm-mini-clear:hover { + opacity: 0.7; +} +.v-asm-error-banner, .v-asm-success-banner { + padding: var(--padding-sm) var(--padding-base); + border-radius: var(--border-radius-md); + font-size: var(--font-size-sm); + font-weight: var(--font-weight-demi-bold); + margin-bottom: var(--margin-sm); + border: 1px solid transparent; +} +.v-asm-error-banner { + background: rgba(217, 90, 123, 0.1); + border-color: var(--danger-fg); + color: var(--danger-fg); +} +.v-asm-success-banner { + background: rgba(40, 167, 69, 0.1); + border-color: var(--success-fg); + color: var(--success-fg); +} +.v-asm-success-banner a { + color: var(--main-fg); + text-decoration: underline; +} +.v-asm-canvas-wrapper { + position: relative; + margin: var(--margin-base) 0; + border: 1px solid var(--sidebar-border-color); + border-radius: var(--border-radius-md); + overflow: hidden; + background: #000; +} +.v-asm-canvas-wrapper canvas { + display: block; + width: 100%; + height: auto; + cursor: crosshair; +} +.v-asm-mode-banner { + display: flex; + align-items: center; + justify-content: space-between; + padding: var(--padding-sm) var(--padding-base); + border-radius: var(--border-radius-md); + font-size: var(--font-size-sm); + font-weight: var(--font-weight-bold); + margin-bottom: var(--margin-xs); + gap: var(--gap-sm); + background: var(--sidebar-bg); + border: 1px solid var(--sidebar-border-color); + color: var(--text-muted); +} +.v-asm-mode-banner.v-asm-mode-left { + background: rgba(13, 110, 253, 0.15); + border-color: #0d6efd; + color: #0d6efd; +} +.v-asm-mode-banner.v-asm-mode-right { + background: rgba(253, 126, 20, 0.15); + border-color: #fd7e14; + color: #fd7e14; +} +.v-asm-mode-banner.v-asm-mode-idle { + background: rgba(255, 255, 255, 0.03); + border-color: var(--sidebar-border-color); + color: var(--text-muted); +} +.v-asm-mode-banner.v-asm-mode-banner-error { + background: rgba(217, 90, 123, 0.12); + border-color: var(--danger-fg); + flex-wrap: wrap; +} +.v-asm-mode-banner span:last-child:not(button) { + font-weight: var(--font-weight-normal); + font-size: var(--font-size-xs); + opacity: 0.8; +} +.v-asm-btn-retry { + background: var(--danger-fg); + padding: 4px 14px; + font-size: var(--font-size-xs); + flex-shrink: 0; +} +.v-asm-instructions { + text-align: center; + padding: var(--padding-xs); + font-size: var(--font-size-sm); + color: var(--text-muted); + background: var(--sidebar-bg); + border-radius: 0 0 var(--border-radius-md) var(--border-radius-md); +} +.v-asm-status-grid { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: var(--gap-sm); + margin: var(--margin-base) 0; +} +.v-asm-status-card { + background: var(--card-bg); + border: 1px solid var(--sidebar-border-color); + border-radius: var(--border-radius-md); + padding: var(--padding-base); + text-align: center; +} +.v-asm-status-label { + font-size: var(--font-size-xs); + color: var(--text-muted); + margin-bottom: 4px; +} +.v-asm-status-value { + font-size: var(--font-size-base); + font-weight: var(--font-weight-bold); +} +.v-asm-status-value.active { + color: var(--success-fg); +} +.v-asm-status-value.inactive { + color: var(--text-muted); +} diff --git a/starpilot/system/the_galaxy/assets/components/tools/v_asm.js b/starpilot/system/the_galaxy/assets/components/tools/v_asm.js new file mode 100644 index 000000000..51ed38f0a --- /dev/null +++ b/starpilot/system/the_galaxy/assets/components/tools/v_asm.js @@ -0,0 +1,572 @@ +import { html, reactive } from "/assets/vendor/arrow-core.js"; + +const CANVAS_W = 640; +const CANVAS_H = 480; +let pollInterval = null; +let pollHasBeenActive = false; +let initialLoadTriggered = false; + +const state = reactive({ + loading: false, + error: "", + success: "", + annotating: null, + leftPoints: [], + rightPoints: [], + image: false, + configSaved: false, + configExists: false, +}); + +let _loadedImage = null; +let _lastCanvas = null; +let loadedConfig = null; + +function polyCenter(pts) { + if (!pts || pts.length === 0) return null; + const cx = pts.reduce((s, p) => s + p[0], 0) / pts.length; + const cy = pts.reduce((s, p) => s + p[1], 0) / pts.length; + return [cx, cy]; +} + +function getCanvas() { + return document.getElementById("v-asm-canvas"); +} + +function redraw() { + const canvas = getCanvas(); + if (!canvas) return; + + if (canvas !== _lastCanvas) { + _lastCanvas = canvas; + if (_loadedImage) { + canvas._img = _loadedImage; + canvas.width = Math.min(_loadedImage.naturalWidth, 1280); + canvas.height = Math.round(canvas.width * (_loadedImage.naturalHeight / _loadedImage.naturalWidth)); + } else { + canvas.width = CANVAS_W; + canvas.height = CANVAS_H; + } + } + + const ctx = canvas.getContext("2d"); + ctx.clearRect(0, 0, canvas.width, canvas.height); + + const img = _loadedImage; + if (img) { + canvas._img = img; + ctx.drawImage(img, 0, 0, canvas.width, canvas.height); + } else { + ctx.fillStyle = "#222"; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.fillStyle = "#888"; + ctx.font = "16px monospace"; + ctx.textAlign = "center"; + ctx.fillText("Loading camera snapshot...", canvas.width / 2, canvas.height / 2); + return; + } + + const sides = [ + { key: "left", points: state.leftPoints, fillColor: "rgba(13, 110, 253, 0.25)", borderColor: "#0d6efd", label: "LEFT" }, + { key: "right", points: state.rightPoints, fillColor: "rgba(253, 126, 20, 0.25)", borderColor: "#fd7e14", label: "RIGHT" }, + ]; + + for (const side of sides) { + if (side.points.length < 2) { + for (const pt of side.points) { + ctx.beginPath(); + ctx.arc(pt[0], pt[1], 5, 0, Math.PI * 2); + ctx.fillStyle = side.borderColor; + ctx.fill(); + } + continue; + } + + ctx.beginPath(); + ctx.moveTo(side.points[0][0], side.points[0][1]); + for (let i = 1; i < side.points.length; i++) { + ctx.lineTo(side.points[i][0], side.points[i][1]); + } + if (side.points.length >= 3) { + ctx.closePath(); + ctx.fillStyle = side.fillColor; + ctx.fill(); + } + ctx.strokeStyle = side.borderColor; + ctx.lineWidth = 2; + ctx.stroke(); + + for (const pt of side.points) { + ctx.beginPath(); + ctx.arc(pt[0], pt[1], 4, 0, Math.PI * 2); + ctx.fillStyle = "#fff"; + ctx.fill(); + ctx.strokeStyle = side.borderColor; + ctx.lineWidth = 1.5; + ctx.stroke(); + } + + if (side.points.length >= 3) { + const center = polyCenter(side.points); + if (center) { + ctx.fillStyle = "#fff"; + ctx.font = "bold 14px monospace"; + ctx.textAlign = "center"; + ctx.fillText(side.label, center[0], center[1] + 5); + } + } + } +} + +async function loadSnapshot() { + state.error = ""; + state.success = ""; + state.image = false; + let blobUrl = null; + try { + const resp = await fetch("/api/v_asm/snapshot"); + if (!resp.ok) { + const payload = await resp.json().catch(() => ({})); + throw new Error(payload.error || resp.statusText || "Failed to load snapshot"); + } + + let src; + const contentType = resp.headers.get("content-type") || ""; + if (contentType.includes("application/json")) { + const data = await resp.json(); + if (!data.jpeg) throw new Error("Snapshot missing image data"); + src = `data:image/jpeg;base64,${data.jpeg}`; + } else { + const blob = await resp.blob(); + blobUrl = URL.createObjectURL(blob); + src = blobUrl; + } + + const img = new Image(); + img.onload = () => { + _loadedImage = img; + const canvas = getCanvas(); + if (canvas) { + canvas._img = img; + canvas.width = Math.min(img.naturalWidth, 1280); + canvas.height = Math.round(canvas.width * (img.naturalHeight / img.naturalWidth)); + } + state.image = true; + state.success = "Camera snapshot loaded. Click a window button above to start annotating."; + applyConfigToCanvas(); + requestAnimationFrame(redraw); + if (blobUrl) { + URL.revokeObjectURL(blobUrl); + blobUrl = null; + } + }; + img.onerror = () => { + state.image = false; + state.error = "Failed to decode image"; + requestAnimationFrame(redraw); + if (blobUrl) { + URL.revokeObjectURL(blobUrl); + blobUrl = null; + } + }; + img.src = src; + } catch (e) { + state.image = false; + state.error = e.message; + requestAnimationFrame(redraw); + if (blobUrl) { + URL.revokeObjectURL(blobUrl); + blobUrl = null; + } + } +} + +function canvasClick(e) { + if (!state.annotating || !e || !state.image) return; + const canvas = getCanvas(); + if (!canvas) return; + + const rect = canvas.getBoundingClientRect(); + const x = Math.round((e.clientX - rect.left) * (canvas.width / rect.width)); + const y = Math.round((e.clientY - rect.top) * (canvas.height / rect.height)); + + if (x < 0 || y < 0) return; + + const points = state.annotating === "left" ? [...state.leftPoints] : [...state.rightPoints]; + + if (points.length >= 3) { + const first = points[0]; + const dist = Math.sqrt((x - first[0]) ** 2 + (y - first[1]) ** 2); + if (dist < 12) { + finishSide(); + return; + } + } + + points.push([x, y]); + + if (state.annotating === "left") { + state.leftPoints = points; + } else { + state.rightPoints = points; + } + requestAnimationFrame(redraw); +} + +function canvasRightClick(e) { + if (!state.annotating || !e) return; + e.preventDefault(); + const pts = state.annotating === "left" ? [...state.leftPoints] : [...state.rightPoints]; + if (!pts.length) return; + pts.pop(); + if (state.annotating === "left") { + state.leftPoints = pts; + } else { + state.rightPoints = pts; + } + requestAnimationFrame(redraw); +} + +function startAnnotate(e) { + const side = e?.currentTarget?.value || e?.target?.value; + if (!side) return; + state.annotating = side; + state.error = ""; + state.success = ""; + requestAnimationFrame(redraw); +} + +function finishSide() { + if (!state.annotating) return; + const side = state.annotating; + const points = side === "left" ? state.leftPoints : state.rightPoints; + if (points.length < 3) { + state.error = "Need at least 3 points to define a region."; + return; + } + state.annotating = null; + state.success = `${side === "left" ? "Left" : "Right"} window annotated (${points.length} points)!`; + requestAnimationFrame(redraw); +} + +function clearSide(side) { + if (side === "left") { + state.leftPoints = []; + } else { + state.rightPoints = []; + } + state.annotating = null; + requestAnimationFrame(redraw); +} + +function clearAll() { + state.leftPoints = []; + state.rightPoints = []; + state.annotating = null; + state.configSaved = false; + requestAnimationFrame(redraw); +} + +async function saveConfig() { + if (state.leftPoints.length < 3 && state.rightPoints.length < 3) { + state.error = "Annotate at least one window with 3+ points."; + return; + } + + const canvas = getCanvas(); + const img = _loadedImage; + const nativeW = img ? img.naturalWidth : 1920; + const nativeH = img ? img.naturalHeight : 1080; + const cw = canvas ? canvas.width : nativeW; + const ch = canvas ? canvas.height : nativeH; + + function scalePoints(pts) { + return pts.map(([x, y]) => [Math.round(x * nativeW / cw), Math.round(y * nativeH / ch)]); + } + + const config = { + width: nativeW, + height: nativeH, + }; + if (state.leftPoints.length >= 3) { + config.poly_left = scalePoints(state.leftPoints); + } else { + config.poly_left = []; + } + if (state.rightPoints.length >= 3) { + config.poly_right = scalePoints(state.rightPoints); + } else { + config.poly_right = []; + } + + state.loading = true; + state.error = ""; + state.success = ""; + try { + const resp = await fetch("/api/v_asm/config", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(config), + }); + const data = await resp.json(); + if (!resp.ok) throw new Error(data.error || "Failed to save"); + state.configSaved = true; + state.configExists = true; + state.success = "Annotation config saved! V-ASM is now enabled."; + await loadExistingConfig(); + } catch (e) { + state.error = e.message; + } + state.loading = false; +} + +async function loadExistingConfig() { + try { + const resp = await fetch("/api/v_asm/config"); + if (!resp.ok) return; + const config = await resp.json(); + loadedConfig = config; + applyConfigToCanvas(); + } catch (e) { + console.error("V-ASM config load failed", e); + } +} + +function applyConfigToCanvas() { + const config = loadedConfig; + if (!config) { + state.leftPoints = []; + state.rightPoints = []; + state.configExists = false; + redraw(); + return; + } + + const canvas = getCanvas(); + const cw = canvas ? canvas.width || CANVAS_W : CANVAS_W; + const ch = canvas ? canvas.height || CANVAS_H : CANVAS_H; + const nativeW = _loadedImage ? _loadedImage.naturalWidth : (config.width || 1920); + const nativeH = _loadedImage ? _loadedImage.naturalHeight : (config.height || 1080); + const scaleX = cw / nativeW; + const scaleY = ch / nativeH; + + const left = Array.isArray(config.poly_left) && config.poly_left.length >= 3 + ? config.poly_left.map(([x, y]) => [Math.round(x * scaleX), Math.round(y * scaleY)]) + : []; + const right = Array.isArray(config.poly_right) && config.poly_right.length >= 3 + ? config.poly_right.map(([x, y]) => [Math.round(x * scaleX), Math.round(y * scaleY)]) + : []; + + state.leftPoints = left; + state.rightPoints = right; + state.configExists = Boolean(left.length || right.length); + redraw(); +} + +async function deleteConfig() { + state.loading = true; + state.error = ""; + state.success = ""; + try { + const resp = await fetch("/api/v_asm/config", { method: "DELETE" }); + const data = await resp.json(); + if (!resp.ok) throw new Error(data.error || "Failed to delete"); + state.leftPoints = []; + state.rightPoints = []; + state.annotating = null; + state.configSaved = false; + state.configExists = false; + loadedConfig = null; + state.success = "Annotation config cleared."; + requestAnimationFrame(redraw); + } catch (e) { + state.error = e.message; + } + state.loading = false; +} + +function scheduleInitialLoad() { + if (!initialLoadTriggered) { + initialLoadTriggered = true; + loadExistingConfig(); + loadSnapshot(); + } + const attempt = () => { + if (getCanvas()) { + redraw(); + return; + } + requestAnimationFrame(attempt); + }; + requestAnimationFrame(attempt); +} + +function retrySnapshot() { + state.error = ""; + state.success = ""; + loadSnapshot(); +} + +export function VASMAnnotations() { + const el = html` +
+
+
+

Vision Adjacent Spot Monitoring (V-ASM)

+ +
+
About V-ASM
+
    +
  • Camera-based adjacent spot monitoring using the driver camera, works alongside factory blind spot monitoring or standalone
    • +
  • Annotate window areas so V-ASM knows where to look
    • +
  • Massive thanks to those who contributed to our total of 65GB of training data (over 36GB from your submissions). If performance is lacking, submit edge case routes via the repo form or give feedback in the StarPilot Discord.
    • +
  • Submit edge cases, learn about training pipeline, and data handling within the form at github.com/prabhaavp/vasm-op
    • +
+
+ +
+
Tracing Guidelines
+
    +
  • Trace the visible glass (front and rear side windows on each side, as seen by the driver camera). Mask as much of the window area as possible.
    • +
  • Exclude A-pillars, door frames, and interior. Include the side mirror if visible through the glass.
    • +
  • The B-pillar is fine to include if needed for a continuous mask. Your head or body being in frame is fine (that is part of the training data). Be consistent left vs right.
    • +
+
+ +
+ +
+ V-ASM is like a friend saying, "Hol' up, I don't think you're clear." Always check before merging and be aware false positives/negatives are possible. +
+ + ${state.error ? html`
${state.error}
` : ""} + ${state.success ? html`
${state.success}
` : ""} + ${state.configSaved ? html` +
+ Annotations saved! V-ASM is now enabled. Configure sensitivity in Toggles -> Lateral. +
+ ` : ""} + +
+
+ + + ${state.annotating ? html`` : ""} + + + ${state.configExists ? html`` : ""} + +
+ +
+
+ Left: ${state.leftPoints.length} pt${state.leftPoints.length !== 1 ? "s" : ""} + ${state.leftPoints.length >= 3 ? html`✔` : ""} + ${state.leftPoints.length > 0 ? html`` : ""} +
+
+ Right: ${state.rightPoints.length} pt${state.rightPoints.length !== 1 ? "s" : ""} + ${state.rightPoints.length >= 3 ? html`✔` : ""} + ${state.rightPoints.length > 0 ? html`` : ""} +
+
+
+ + ${state.annotating === "left" ? html`
⬅ Annotating Left WindowClick the canvas to place points around the visible glass
` + : state.annotating === "right" ? html`
➡ Annotating Right WindowClick the canvas to place points around the visible glass
` + : state.error ? html`
Snapshot unavailable${state.error}
` + : !state.image ? html`
Loading camera snapshot...The snapshot will load automatically
` + : html`
Idle ModeClick "Annotate" above to configure your active canvas boundaries
`} + +
+ +
+ ${state.annotating ? "Click near the first point to close the polygon. Right-click to undo last point." : "Use the controls above to annotate each side's window area."} +
+
+
+ +
+
Sensitivity Settings
+
    +
  • Confidence Threshold: minimum confidence for a detection (higher means fewer false positives)
    • +
  • Smoothing Duration: time constant for signal smoothing (higher means less flickering)
    • +
  • Adjust these in Toggles -> Lateral
    • +
+
+ +
+

Current Status

+

Real-time V-ASM detection state from the daemon.

+
+
+
Left Active
+
-
+
+
+
Right Active
+
-
+
+
+
Left Confidence
+
-
+
+
+
Right Confidence
+
-
+
+
+
+
+ `; + + scheduleInitialLoad(); + if (!window.__vasmPollStarted) { + window.__vasmPollStarted = true; + pollInterval = setInterval(async () => { + try { + const elLeft = document.getElementById("vasm-left-status"); + if (!elLeft) { + if (pollHasBeenActive) { + clearInterval(pollInterval); + window.__vasmPollStarted = false; + pollHasBeenActive = false; + } + return; + } + pollHasBeenActive = true; + const keys = ["VASMLeftActive", "VASMRightActive", "VASMLeftConfidence", "VASMRightConfidence"]; + const results = {}; + for (const key of keys) { + const resp = await fetch(`/api/params_memory?key=${encodeURIComponent(key)}`); + if (resp.ok) results[key] = await resp.text(); + } + const elRight = document.getElementById("vasm-right-status"); + const elLeftConf = document.getElementById("vasm-left-conf"); + const elRightConf = document.getElementById("vasm-right-conf"); + if (elLeft) { + const active = results.VASMLeftActive === "1"; + elLeft.textContent = active ? "YES" : "no"; + elLeft.className = "v-asm-status-value " + (active ? "active" : "inactive"); + } + if (elRight) { + const active = results.VASMRightActive === "1"; + elRight.textContent = active ? "YES" : "no"; + elRight.className = "v-asm-status-value " + (active ? "active" : "inactive"); + } + if (elLeftConf) elLeftConf.textContent = parseFloat(results.VASMLeftConfidence || "0").toFixed(3); + if (elRightConf) elRightConf.textContent = parseFloat(results.VASMRightConfidence || "0").toFixed(3); + } catch (e) { + } + }, 3000); + } + return el; +} \ No newline at end of file diff --git a/starpilot/system/the_galaxy/templates/index.html b/starpilot/system/the_galaxy/templates/index.html index 789f93bbd..e0c9cbcea 100644 --- a/starpilot/system/the_galaxy/templates/index.html +++ b/starpilot/system/the_galaxy/templates/index.html @@ -42,6 +42,7 @@ +