Anti Burn In

This commit is contained in:
firestar5683
2026-07-20 12:33:41 -05:00
parent 4ac7fcaf5c
commit 7b124faad2
6 changed files with 188 additions and 32 deletions
+34 -4
View File
@@ -12,7 +12,7 @@ import numpy as np
if __package__ in (None, ""):
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import ( # type: ignore
from common import ( # type: ignore # noqa: TID251
CLASSIFIER_EXPORT_NAME,
DEFAULT_WORKSPACE,
DETECTOR_EXPORT_NAME,
@@ -39,17 +39,40 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--output-dir", type=Path, help="Where exported ONNX models should be written. Defaults to <workspace>/exports.")
parser.add_argument("--detector-imgsz", type=int, default=640, help="Detector export image size.")
parser.add_argument("--classifier-imgsz", type=int, default=128, help="Classifier export image size.")
parser.add_argument(
"--detector-end2end",
action=argparse.BooleanOptionalAction,
default=None,
help="Override end-to-end detector export. Use --no-detector-end2end for a YOLO26 raw output compatible with the runtime parser.",
)
parser.add_argument("--opset", type=int, default=12, help="ONNX opset.")
parser.add_argument("--install-repo-assets", action="store_true", help="Copy exported ONNX files into starpilot/assets/vision_models.")
parser.add_argument("--skip-verify", action="store_true", help="Skip the OpenCV DNN load/forward smoke test after export.")
return parser.parse_args()
def export_yolo(weights_path: Path, output_path: Path, imgsz: int, opset: int, nms: bool) -> None:
def export_yolo(
weights_path: Path,
output_path: Path,
imgsz: int,
opset: int,
nms: bool,
end2end: bool | None = None,
) -> None:
from ultralytics import YOLO
model = YOLO(str(weights_path))
exported_path = Path(model.export(format="onnx", imgsz=imgsz, opset=opset, simplify=False, dynamic=False, nms=nms))
export_kwargs = {
"format": "onnx",
"imgsz": imgsz,
"opset": opset,
"simplify": False,
"dynamic": False,
"nms": nms,
}
if end2end is not None:
export_kwargs["end2end"] = end2end
exported_path = Path(model.export(**export_kwargs))
ensure_dir(output_path.parent)
shutil.copy2(exported_path, output_path)
@@ -79,7 +102,14 @@ def main() -> int:
if args.detector_weights:
detector_weights = args.detector_weights.resolve()
detector_output = output_dir / DETECTOR_EXPORT_NAME
export_yolo(detector_weights, detector_output, args.detector_imgsz, args.opset, nms=False)
export_yolo(
detector_weights,
detector_output,
args.detector_imgsz,
args.opset,
nms=False,
end2end=args.detector_end2end,
)
if not args.skip_verify:
verify_onnx_with_opencv(detector_output, args.detector_imgsz)
exported_paths.append(detector_output)
@@ -89,14 +89,18 @@ def load_route_bookmarks(clip_root: Path, log_id: str, event_types: str = "bookm
if event_types == "vision"
else ("rlog.zst", "rlog.bz2", "qlog.zst", "qlog.bz2")
)
log_path = next((segment_dir / name for name in log_names if (segment_dir / name).exists()), None)
if log_path is None:
log_paths = [segment_dir / name for name in log_names if (segment_dir / name).exists()]
if not log_paths:
continue
try:
events = list(log.Event.read_multiple_bytes(read_log_bytes(log_path)))
except Exception as exc:
print(f"{segment_dir.name}: skipping unreadable log {log_path.name}: {exc}")
events = None
for log_path in log_paths:
try:
events = list(log.Event.read_multiple_bytes(read_log_bytes(log_path)))
break
except Exception as exc:
print(f"{segment_dir.name}: skipping unreadable log {log_path.name}: {exc}")
if events is None:
continue
if not events:
continue
@@ -229,7 +233,7 @@ def main() -> int:
session_id = f"connect_{dongle_id}_{log_id}"
bookmarks = load_route_bookmarks(clip_root, log_id, args.event_types)
if not bookmarks:
print(f"{raw_route}: no bookmark events found in downloaded rlogs")
print(f"{raw_route}: no {args.event_types} events found in downloaded logs")
continue
print(f"{raw_route}: found {len(bookmarks)} event(s)")
+3
View File
@@ -8,6 +8,9 @@ Quick start:
* set `STRICT_MODE=1` to kill the app if it drops too much below 60fps
* set `SCALE=1.5` to scale the entire UI by 1.5x
* set `BURN_IN=1` to get a burn-in heatmap version of the UI
* burn-in prevention shifts the final UI by 2 pixels every 3 minutes on device; set `BURN_IN_PREVENTION=0` to disable it
or tune it with `BURN_IN_SHIFT_PIXELS` and `BURN_IN_SHIFT_INTERVAL` (seconds); near-white pixels are also softly capped
at 95% luminance and can be tuned or disabled with `WHITE_LUMINANCE_CAP` (set it to `1.0` to disable)
* set `GRID=50` to show a 50-pixel alignment grid overlay
* set `MAGIC_DEBUG=1` to show every dropped frames (only on device)
* set `RECORD=1` to record the screen, output defaults to `output.mp4` but can be set with `RECORD_OUTPUT`
+67 -2
View File
@@ -37,6 +37,10 @@ BIG_UI = os.getenv("BIG", "0") == "1"
MACOS = platform.system() == "Darwin"
ENABLE_VSYNC = os.getenv("ENABLE_VSYNC", "0") == "1"
MICI_FORCE_RENDER_TEXTURE = os.getenv("MICI_FORCE_RENDER_TEXTURE", "1" if DEVICE_TYPE == "mici" else "0") == "1"
BURN_IN_PREVENTION = os.getenv("BURN_IN_PREVENTION", "0" if PC else "1") == "1"
BURN_IN_SHIFT_INTERVAL = max(1.0, float(os.getenv("BURN_IN_SHIFT_INTERVAL", "180")))
BURN_IN_SHIFT_PIXELS = max(0, int(os.getenv("BURN_IN_SHIFT_PIXELS", "2")))
WHITE_LUMINANCE_CAP = min(1.0, max(0.0, float(os.getenv("WHITE_LUMINANCE_CAP", "0.95" if BURN_IN_PREVENTION else "1.0"))))
SHOW_FPS = os.getenv("SHOW_FPS") == "1"
SHOW_TOUCHES = os.getenv("SHOW_TOUCHES") == "1"
STRICT_MODE = os.getenv("STRICT_MODE") == "1"
@@ -61,6 +65,17 @@ if platform.system() == "Darwin":
"""
BURN_IN_MODE = "BURN_IN" in os.environ
BURN_IN_SHIFT_PATTERN = (
(0, 0),
(-1, 0),
(-1, -1),
(0, -1),
(1, -1),
(1, 0),
(1, 1),
(0, 1),
(-1, 1),
)
BURN_IN_VERTEX_SHADER = GL_VERSION + """
in vec3 vertexPosition;
in vec2 vertexTexCoord;
@@ -87,6 +102,29 @@ void main() {
fragColor = vec4(gradient, sampled.a);
}
"""
WHITE_LUMINANCE_FRAGMENT_SHADER = GL_VERSION + """
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform float whiteLuminanceCap;
out vec4 fragColor;
void main() {
vec4 sampled = texture(texture0, fragTexCoord);
float luminance = dot(sampled.rgb, vec3(0.2126, 0.7152, 0.0722));
float chroma = max(max(sampled.r, sampled.g), sampled.b) - min(min(sampled.r, sampled.g), sampled.b);
// Gently compress only near-white, low-saturation pixels. Saturated alert colors
// and the vast majority of camera pixels pass through unchanged.
float knee = max(0.0, whiteLuminanceCap - 0.05);
if (luminance > knee) {
float kneeRange = max(0.0001, 1.0 - knee);
float targetLuminance = knee + (whiteLuminanceCap - knee) * ((luminance - knee) / kneeRange);
float neutralAmount = 1.0 - smoothstep(0.08, 0.25, chroma);
sampled.rgb *= mix(1.0, targetLuminance / max(luminance, 0.0001), neutralAmount);
}
fragColor = sampled;
}
"""
DEFAULT_TEXT_SIZE = 60
DEFAULT_TEXT_COLOR = rl.Color(255, 255, 255, int(255 * 0.9))
@@ -451,6 +489,7 @@ class GuiApplication:
self._render_texture: rl.RenderTexture | None = None
self._burn_in_shader: rl.Shader | None = None
self._white_luminance_shader: rl.Shader | None = None
self._ffmpeg_proc: subprocess.Popen | None = None
self._ffmpeg_queue: queue.Queue | None = None
self._ffmpeg_thread: threading.Thread | None = None
@@ -459,6 +498,7 @@ class GuiApplication:
self._textures: dict[str, rl.Texture] = {}
self._target_fps: int = _DEFAULT_FPS
self._last_fps_log_time: float = time.monotonic()
self._burn_in_start_time = time.monotonic()
self._frame = 0
self._window_close_requested = False
self._nav_stack: list[object] = []
@@ -521,7 +561,8 @@ class GuiApplication:
self._render_texture_width = max(1, int(round(self._scaled_width * self._pixel_scale_x)))
self._render_texture_height = max(1, int(round(self._scaled_height * self._pixel_scale_y)))
needs_render_texture = (self._scale != 1.0 and not PC) or BURN_IN_MODE or RECORD or MICI_FORCE_RENDER_TEXTURE
needs_render_texture = ((self._scale != 1.0 and not PC) or BURN_IN_MODE or RECORD or
MICI_FORCE_RENDER_TEXTURE or BURN_IN_PREVENTION or WHITE_LUMINANCE_CAP < 1.0)
if PC and self._scale != 1.0:
rl.set_mouse_scale(1 / self._scale, 1 / self._scale)
if PC:
@@ -573,6 +614,12 @@ class GuiApplication:
self._patch_scissor_mode()
if BURN_IN_MODE and self._burn_in_shader is None:
self._burn_in_shader = rl.load_shader_from_memory(BURN_IN_VERTEX_SHADER, BURN_IN_FRAGMENT_SHADER)
if WHITE_LUMINANCE_CAP < 1.0 and self._white_luminance_shader is None:
self._white_luminance_shader = rl.load_shader_from_memory(BURN_IN_VERTEX_SHADER, WHITE_LUMINANCE_FRAGMENT_SHADER)
cap_location = rl.get_shader_location(self._white_luminance_shader, "whiteLuminanceCap")
cap_value = rl.ffi.new("float[]", [WHITE_LUMINANCE_CAP])
rl.set_shader_value(self._white_luminance_shader, cap_location, cap_value,
rl.ShaderUniformDataType.SHADER_UNIFORM_FLOAT)
if not PC:
self._mouse.start()
@@ -815,6 +862,10 @@ class GuiApplication:
rl.unload_shader(self._burn_in_shader)
self._burn_in_shader = None
if self._white_luminance_shader:
rl.unload_shader(self._white_luminance_shader)
self._white_luminance_shader = None
self._mouse.stop()
self.close_ffmpeg()
@@ -907,7 +958,8 @@ class GuiApplication:
rl.clear_background(rl.BLACK)
self._mark_progress("gui_app.after_present_clear_background")
src_rect = rl.Rectangle(0, 0, float(self._render_texture_width), -float(self._render_texture_height))
dst_rect = rl.Rectangle(0, 0, float(self._scaled_width), float(self._scaled_height))
shift_x, shift_y = self._burn_in_shift()
dst_rect = rl.Rectangle(shift_x, shift_y, float(self._scaled_width), float(self._scaled_height))
texture = self._render_texture.texture
if texture:
self._mark_progress("gui_app.before_present_draw_texture")
@@ -915,6 +967,10 @@ class GuiApplication:
rl.begin_shader_mode(self._burn_in_shader)
rl.draw_texture_pro(texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
elif self._white_luminance_shader:
rl.begin_shader_mode(self._white_luminance_shader)
rl.draw_texture_pro(texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
else:
rl.draw_texture_pro(texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
self._mark_progress("gui_app.after_present_draw_texture")
@@ -948,6 +1004,15 @@ class GuiApplication:
except KeyboardInterrupt:
pass
def _burn_in_shift(self, now: float | None = None) -> tuple[int, int]:
if not BURN_IN_PREVENTION or BURN_IN_SHIFT_PIXELS == 0:
return 0, 0
elapsed = (time.monotonic() if now is None else now) - self._burn_in_start_time
pattern_index = int(max(0.0, elapsed) // BURN_IN_SHIFT_INTERVAL) % len(BURN_IN_SHIFT_PATTERN)
x, y = BURN_IN_SHIFT_PATTERN[pattern_index]
return x * BURN_IN_SHIFT_PIXELS, y * BURN_IN_SHIFT_PIXELS
def font(self, font_weight: FontWeight = FontWeight.NORMAL) -> rl.Font:
return self._fonts[font_weight]
@@ -7,6 +7,7 @@ import math
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
import numpy as np
@@ -59,6 +60,7 @@ class LongitudinalSample:
curve_controller_active: bool = False
cruise_target: float = 0.0
desired_follow_distance: float = 0.0
personality: str = "unknown"
@dataclass
@@ -89,6 +91,7 @@ class SegmentData:
bookmarks: list[float]
car_params: dict[str, Any] | None
settings: dict[str, Any]
software: dict[str, Any] | None
def safe_float(value: Any, default: float = 0.0) -> float:
@@ -305,6 +308,7 @@ def analyze_samples(samples: list[LongitudinalSample], label: str, event_time_s:
round(max(sample.lead_distance for sample in lead_samples if sample.lead_distance is not None), 3),
],
"sources": sorted(set(sources)),
"personalities": sorted({sample.personality for sample in relevant}),
"minimumTtc": None if minimum_ttc is None else round(minimum_ttc, 3),
"maxPlanJerk": round(max_plan_jerk, 3),
"maxCommandJerk": round(max_command_jerk, 3),
@@ -331,10 +335,22 @@ def parse_settings(serialized: str) -> dict[str, Any]:
if not isinstance(settings, dict):
return {}
keywords = ("accel", "decel", "follow", "jerk", "longitudinal", "personality", "smooth", "truck")
exact_keys = {
"acceleration_profile",
"custom_accel_profile",
"deceleration_profile",
"longitudinalActuatorDelay",
"startAccel",
"stopAccel",
"stoppingDecelRate",
"taco_tune",
"truck_tuning",
}
return {
key: value for key, value in sorted(settings.items())
if any(keyword in key.lower() for keyword in keywords)
if key in exact_keys or key.endswith("_follow") or (
key.startswith(("aggressive_jerk_", "relaxed_jerk_", "standard_jerk_")) and not key.endswith("_via")
)
}
@@ -353,6 +369,17 @@ def snapshot_car_params(CP: Any) -> dict[str, Any]:
}
def snapshot_software(init_data: Any) -> dict[str, Any]:
return {
"version": str(safe_attr(init_data, "version", "unknown")),
"gitCommit": str(safe_attr(init_data, "gitCommit", "unknown")),
"gitSrcCommit": str(safe_attr(init_data, "gitSrcCommit", "")),
"gitBranch": str(safe_attr(init_data, "gitBranch", "unknown")),
"gitRemote": str(safe_attr(init_data, "gitRemote", "unknown")),
"dirty": bool(safe_attr(init_data, "dirty", False)),
}
def make_sample(segment: int, segment_start_ns: int, mono_time: int, latest: dict[str, Any]) -> LongitudinalSample | None:
required = ("carState", "carControl", "controlsState", "radarState", "starpilotPlan", "longitudinalPlan")
if not all(service in latest for service in required):
@@ -365,6 +392,7 @@ def make_sample(segment: int, segment_start_ns: int, mono_time: int, latest: dic
starpilot_plan = latest["starpilotPlan"]
long_plan = latest["longitudinalPlan"]
car_output = latest.get("carOutput")
selfdrive_state = latest.get("selfdriveState")
lead = radar_state.leadOne
lead_two = radar_state.leadTwo
command_accel = safe_float(safe_attr(safe_attr(car_control, "actuators"), "accel", 0.0))
@@ -410,33 +438,38 @@ def make_sample(segment: int, segment_start_ns: int, mono_time: int, latest: dic
curve_controller_active=bool(safe_attr(starpilot_plan, "cscControllingSpeed", False)),
cruise_target=safe_float(safe_attr(starpilot_plan, "vCruise", 0.0)),
desired_follow_distance=safe_float(safe_attr(starpilot_plan, "desiredFollowDistance", 0.0)),
personality=str(safe_attr(selfdrive_state, "personality", "unknown")),
)
def analyze_segment(identifier: str, segment: int, mode: ReadMode) -> SegmentData:
reader = LogReader(identifier, default_mode=mode, sort_by_time=True)
source = ",".join(Path(path).name or path.rsplit("/", 1)[-1] for path in reader.logreader_identifiers)
source = ",".join(Path(urlparse(path).path).name or path.rsplit("/", 1)[-1] for path in reader.logreader_identifiers)
latest: dict[str, Any] = {}
samples: list[LongitudinalSample] = []
bookmarks: list[float] = []
bookmark_mono_times: list[int] = []
car_params = None
settings: dict[str, Any] = {}
software = None
segment_start_ns: int | None = None
for msg in reader:
mono_time = int(msg.logMonoTime)
if segment_start_ns is None:
segment_start_ns = mono_time
which = msg.which()
if segment_start_ns is None and which in ("carState", "longitudinalPlan"):
segment_start_ns = mono_time
if which in ("userBookmark", "bookmarkButton"):
bookmark_time = (mono_time - segment_start_ns) / 1e9
if not bookmarks or bookmark_time - bookmarks[-1] > 0.5:
bookmarks.append(bookmark_time)
if not bookmark_mono_times or (mono_time - bookmark_mono_times[-1]) / 1e9 > 0.5:
bookmark_mono_times.append(mono_time)
continue
if which == "carParams" and car_params is None:
car_params = snapshot_car_params(msg.carParams)
if which in ("carState", "carControl", "carOutput", "controlsState", "radarState", "starpilotPlan", "longitudinalPlan"):
if which == "initData" and software is None:
software = snapshot_software(msg.initData)
if which in (
"carState", "carControl", "carOutput", "controlsState", "radarState", "selfdriveState", "starpilotPlan", "longitudinalPlan",
):
latest[which] = getattr(msg, which)
if which == "starpilotPlan":
settings = parse_settings(str(safe_attr(msg.starpilotPlan, "starpilotToggles", ""))) or settings
@@ -445,6 +478,10 @@ def analyze_segment(identifier: str, segment: int, mode: ReadMode) -> SegmentDat
if sample is not None:
samples.append(sample)
bookmarks = [] if segment_start_ns is None else [
(mono_time - segment_start_ns) / 1e9 for mono_time in bookmark_mono_times
]
return SegmentData(
segment=segment,
source=source,
@@ -452,6 +489,7 @@ def analyze_segment(identifier: str, segment: int, mode: ReadMode) -> SegmentDat
bookmarks=bookmarks,
car_params=car_params,
settings=settings,
software=software,
)
@@ -514,29 +552,34 @@ def resolve_segments(identifier: str, mode: ReadMode) -> tuple[str, list[tuple[i
def analyze_route(identifier: str, mode: ReadMode, before: float, after: float, top: int) -> dict[str, Any]:
route, segment_requests = resolve_segments(identifier, mode)
segments = [analyze_segment(request, segment, mode) for segment, request in segment_requests]
reports: list[WindowReport] = []
bookmark_reports: list[WindowReport] = []
anomaly_reports: list[WindowReport] = []
for segment_data in segments:
for bookmark_number, bookmark_time in enumerate(segment_data.bookmarks, start=1):
samples = window_samples(segment_data.samples, bookmark_time, before, after)
if samples:
reports.append(analyze_samples(samples, f"bookmark {bookmark_number}", bookmark_time))
bookmark_reports.append(analyze_samples(samples, f"bookmark {bookmark_number}", bookmark_time))
episode_times = anomaly_episode_times(segment_data.samples, top)
for episode_number, event_time in enumerate(episode_times, start=1):
if any(abs(event_time - bookmark) <= before for bookmark in segment_data.bookmarks):
if any(bookmark - before <= event_time <= bookmark + after for bookmark in segment_data.bookmarks):
continue
samples = window_samples(segment_data.samples, event_time, min(before, 5.0), min(after, 2.0))
if samples:
reports.append(analyze_samples(samples, f"route anomaly {episode_number}", event_time))
anomaly_reports.append(analyze_samples(samples, f"route anomaly {episode_number}", event_time))
anomaly_reports.sort(key=lambda report: report.findings[0].score if report.findings else 0.0, reverse=True)
reports = bookmark_reports + anomaly_reports[:top]
reports.sort(key=lambda report: (report.segment, report.event_time_s, report.label))
car_params = next((segment.car_params for segment in segments if segment.car_params), None)
settings = next((segment.settings for segment in reversed(segments) if segment.settings), {})
software = next((segment.software for segment in segments if segment.software), None)
return {
"route": route,
"carParams": car_params,
"settings": settings,
"software": software,
"segments": [
{
"segment": segment.segment,
@@ -552,6 +595,13 @@ def analyze_route(identifier: str, mode: ReadMode, before: float, after: float,
def print_report(payload: dict[str, Any]) -> None:
print(f"route={payload['route']}")
software = payload.get("software")
if software:
source_commit = software["gitSrcCommit"] or software["gitCommit"]
print(
f"software={source_commit[:10]} branch={software['gitBranch']} version={software['version']} "
+ f"dirty={software['dirty']}"
)
car_params = payload.get("carParams")
if car_params:
vehicle_line = f"vehicle={car_params['carFingerprint']} brand={car_params['brand']}"
@@ -583,10 +633,14 @@ def print_report(payload: dict[str, Any]) -> None:
f"speed={report['summary']['speedRangeMps']}m/s plan={report['summary']['planAccelRange']} "
+ f"cmd={report['summary']['commandAccelRange']} actual={report['summary']['actualAccelRange']}"
)
lead_line = f" leadRange={report['summary']['leadDistanceRange']}m sources={report['summary']['sources']}"
lead_line = (
f" leadRange={report['summary']['leadDistanceRange']}m sources={report['summary']['sources']} "
+ f"personalities={report['summary']['personalities']}"
)
print(event_line)
print(f"{window_line} {accel_line}")
print(f"{lead_line} minTTC={report['summary']['minimumTtc']}s")
minimum_ttc = report["summary"]["minimumTtc"]
print(f"{lead_line} minTTC={'n/a' if minimum_ttc is None else f'{minimum_ttc}s'}")
for finding in report["findings"][:3]:
print(f" {finding['kind']}: {finding['severity']} score={finding['score']:.2f}")
for item in finding["evidence"]:
@@ -599,7 +653,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--mode", choices=("auto", "qlog", "rlog"), default="auto")
parser.add_argument("--before", type=float, default=DEFAULT_BOOKMARK_BEFORE, help="Seconds before each bookmark to inspect")
parser.add_argument("--after", type=float, default=DEFAULT_BOOKMARK_AFTER, help="Seconds after each bookmark to inspect")
parser.add_argument("--top", type=int, default=5, help="Maximum route-wide anomaly episodes per segment")
parser.add_argument("--top", type=int, default=5, help="Maximum route-wide anomaly episodes across the route")
parser.add_argument("--json-out", type=Path, help="Optional JSON report path")
return parser.parse_args()
@@ -131,6 +131,6 @@ def test_anomaly_episodes_group_nearby_points():
def test_parse_settings_keeps_only_longitudinal_context():
settings = parse_settings('{"FollowDistance": 1.5, "AccelerationProfile": "eco", "LaneWidth": 3.5}')
settings = parse_settings('{"standard_follow": [1.5, 1.2], "acceleration_profile": "eco", "LaneWidth": 3.5}')
assert settings == {"AccelerationProfile": "eco", "FollowDistance": 1.5}
assert settings == {"acceleration_profile": "eco", "standard_follow": [1.5, 1.2]}