diff --git a/opendbc_repo/opendbc/car/volkswagen/interface.py b/opendbc_repo/opendbc/car/volkswagen/interface.py index eaa60f4e2..4d51f5f3a 100644 --- a/opendbc_repo/opendbc/car/volkswagen/interface.py +++ b/opendbc_repo/opendbc/car/volkswagen/interface.py @@ -94,6 +94,9 @@ class CarInterface(CarInterfaceBase): if candidate == CAR.PORSCHE_MACAN_MK1: ret.steerActuatorDelay = 0.07 + elif candidate == CAR.VOLKSWAGEN_TAOS_MK1: + # Logged Taos braking response aligns about 0.1 s later than the MQB default. + ret.longitudinalActuatorDelay = 0.25 ret.pcmCruise = not ret.openpilotLongitudinalControl ret.stopAccel = -0.55 diff --git a/opendbc_repo/opendbc/car/volkswagen/tests/test_volkswagen.py b/opendbc_repo/opendbc/car/volkswagen/tests/test_volkswagen.py index c0a7a6b65..538d4bd0c 100644 --- a/opendbc_repo/opendbc/car/volkswagen/tests/test_volkswagen.py +++ b/opendbc_repo/opendbc/car/volkswagen/tests/test_volkswagen.py @@ -2,6 +2,7 @@ import random import re from opendbc.car.structs import CarParams +from opendbc.car.volkswagen.interface import CarInterface from opendbc.car.volkswagen.values import CAR, FW_QUERY_CONFIG, WMI from opendbc.car.volkswagen.fingerprints import FW_VERSIONS @@ -13,6 +14,13 @@ SPARE_PART_FW_PATTERN = re.compile(b'\xf1\x87(?P[0-9][0-9A-Z]{2})(?P.+_bookmark_\d+_rank_\d+)$") +DETECTOR_CLASSES = { + "0": "regulatory_speed_limit", + "1": "advisory_speed_limit", + "2": "school_zone_speed_limit", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Convert inspected bookmark localizations into the standard manual-review format.") + parser.add_argument("--localized-manifest", type=Path, required=True, help="localized_bookmarks.csv to convert.") + parser.add_argument("--annotations", type=Path, required=True, help="CSV containing one reviewed row per localized record_key.") + parser.add_argument("--output-dir", type=Path, required=True, help="Directory for queue, labels, and conversion summary.") + parser.add_argument("--route", action="append", default=[], help="Optional log id or dongle/log id to include. Repeatable.") + parser.add_argument("--allow-unreviewed", action="store_true", help="Allow localized rows without a matching annotation.") + return parser.parse_args() + + +def read_csv(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, str]]) -> None: + ensure_dir(path.parent) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + +def record_key(row: dict[str, str]) -> str: + match = FRAME_KEY_RE.match(Path(row.get("frame_path", "")).stem) + if match is None: + raise ValueError(f"Cannot derive record key from frame path: {row.get('frame_path', '')}") + return match.group("key") + + +def route_identity(row: dict[str, str]) -> tuple[str, str, str]: + session_id = row.get("session_id", "") + prefix = "connect_" + if not session_id.startswith(prefix): + raise ValueError(f"Unsupported bookmark session id: {session_id}") + identity = session_id[len(prefix):] + dongle_id, separator, log_id = identity.partition("_") + if not separator or not dongle_id or not log_id: + raise ValueError(f"Cannot parse route identity from session id: {session_id}") + return f"{dongle_id}/{log_id}", dongle_id, log_id + + +def source_position(row: dict[str, str]) -> tuple[str, str]: + if row.get("source_segment", "") and row.get("source_time_s", ""): + return row["source_segment"], row["source_time_s"] + + bookmark_segment = int(row["segment"]) + relative_time_s = float(row["relative_time_s"]) + if relative_time_s < 0.0: + return str(bookmark_segment - 1), f"{relative_time_s + 60.0:.3f}" + return str(bookmark_segment), f"{relative_time_s:.3f}" + + +def parse_read(text: str) -> tuple[str, str]: + speed, separator, confidence = (text or "").partition("@") + return (speed, confidence) if separator else ("", "") + + +def queue_row(row: dict[str, str]) -> dict[str, str]: + key = record_key(row) + route, dongle_id, log_id = route_identity(row) + segment, frame_time_s = source_position(row) + candidate_speed, candidate_confidence = parse_read(row.get("model_read", "")) + detector_class = DETECTOR_CLASSES.get(row.get("class_id", ""), "regulatory_speed_limit") + read_sources = "model" + if row.get("full_detection", ""): + read_sources += ";full_detection" + + item = dict.fromkeys(FIELDNAMES, "") + item.update({ + "record_key": key, + "mining_fingerprint": "localized_bookmark_review_v1", + "route": route, + "dongle_id": dongle_id, + "log_id": log_id, + "segment": segment, + "frame_time_s": frame_time_s, + "frame_path": row.get("frame_path", ""), + "crop_path": row.get("crop_path", ""), + "source_video_path": row.get("source_video_path", ""), + "bbox": row.get("box", ""), + "crop_bbox": row.get("box", ""), + "class_id": row.get("class_id", ""), + "detector_class": detector_class, + "proposal_confidence": row.get("proposal_confidence", ""), + "candidate_speed_limit_mph": candidate_speed, + "candidate_confidence": candidate_confidence, + "model_read": row.get("model_read", ""), + "ocr_read": row.get("ocr_read", ""), + "full_detection": row.get("full_detection", ""), + "read_sources": read_sources, + "read_support_count": "1", + "is_regulatory": row.get("is_regulatory", ""), + "review_priority": row.get("score", ""), + "review_reasons": "route_bookmark;corrected_source_timing", + }) + return item + + +def main() -> int: + args = parse_args() + localized_path = args.localized_manifest.expanduser().resolve() + annotations_path = args.annotations.expanduser().resolve() + output_dir = ensure_dir(args.output_dir.expanduser().resolve()) + + annotations = read_csv(annotations_path) + annotations_by_key = {row["record_key"]: row for row in annotations if row.get("record_key")} + if len(annotations_by_key) != len(annotations): + raise ValueError("Annotations contain an empty or duplicate record_key") + + selected_routes = set(args.route) + queue_rows: list[dict[str, str]] = [] + label_rows: list[dict[str, str]] = [] + localized_keys: set[str] = set() + missing_annotations: list[str] = [] + for localized_row in read_csv(localized_path): + route, _, log_id = route_identity(localized_row) + if selected_routes and route not in selected_routes and log_id not in selected_routes: + continue + key = record_key(localized_row) + localized_keys.add(key) + annotation = annotations_by_key.get(key) + if annotation is None: + missing_annotations.append(key) + if not args.allow_unreviewed: + continue + queue_rows.append(queue_row(localized_row)) + if annotation is not None: + label = {field: annotation.get(field, "") for field in LABEL_FIELDNAMES} + label["record_key"] = key + label_rows.append(label) + + unused_annotations = sorted(set(annotations_by_key) - localized_keys) + if missing_annotations and not args.allow_unreviewed: + preview = ", ".join(missing_annotations[:5]) + raise ValueError(f"Missing annotations for {len(missing_annotations)} localized row(s): {preview}") + if unused_annotations and not selected_routes: + preview = ", ".join(unused_annotations[:5]) + raise ValueError(f"Annotations reference {len(unused_annotations)} unknown row(s): {preview}") + + queue_path = output_dir / "manual_review_queue.csv" + labels_path = output_dir / "manual_review_labels.csv" + write_csv(queue_path, FIELDNAMES, queue_rows) + write_csv(labels_path, LABEL_FIELDNAMES, label_rows) + + summary = { + "localized_manifest": str(localized_path), + "annotations": str(annotations_path), + "selected_routes": sorted(selected_routes), + "queue_rows": len(queue_rows), + "label_rows": len(label_rows), + "missing_annotations": missing_annotations, + "unused_annotations": unused_annotations, + "queue": str(queue_path), + "labels": str(labels_path), + } + summary_path = output_dir / "localized_review_conversion_summary.json" + summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"Wrote {len(queue_rows)} queue row(s) and {len(label_rows)} label row(s) to {output_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/build_manual_review_queue.py b/scripts/speed_limit_vision/build_manual_review_queue.py index 14f560e43..c2b420cd8 100644 --- a/scripts/speed_limit_vision/build_manual_review_queue.py +++ b/scripts/speed_limit_vision/build_manual_review_queue.py @@ -17,7 +17,7 @@ import starpilot.system.speed_limit_vision as slv if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) - from common import ensure_dir, preferred_clip_root, resolve_workspace # type: ignore # noqa: TID251 + from common import ensure_dir, preferred_clip_root, resolve_workspace, source_video_fps # type: ignore # noqa: TID251 from localize_bookmark_signs import configure_models # type: ignore from mine_route_training_samples import ( # type: ignore MapContext, @@ -35,7 +35,7 @@ if __package__ in (None, ""): transition_times, ) else: - from .common import ensure_dir, preferred_clip_root, resolve_workspace + from .common import ensure_dir, preferred_clip_root, resolve_workspace, source_video_fps from .localize_bookmark_signs import configure_models from .mine_route_training_samples import ( MapContext, @@ -458,7 +458,7 @@ def mine_route( break contexts = load_segment_map_context(segment.path) capture = cv2.VideoCapture(str(segment.video_path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(segment.video_path, capture.get(cv2.CAP_PROP_FPS)) frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0 duration_s = frame_count / fps if frame_count > 0 else 60.0 times = sample_times(duration_s, args.sample_every, transition_times(contexts), args.transition_radius, args.transition_step) diff --git a/scripts/speed_limit_vision/build_review_classifier_dataset.py b/scripts/speed_limit_vision/build_review_classifier_dataset.py index 8d48fc18e..192ff6f0d 100644 --- a/scripts/speed_limit_vision/build_review_classifier_dataset.py +++ b/scripts/speed_limit_vision/build_review_classifier_dataset.py @@ -10,6 +10,9 @@ import shutil from collections import Counter from pathlib import Path +import cv2 +import numpy as np + VALID_SPEEDS = frozenset((15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75)) @@ -26,6 +29,13 @@ def parse_args() -> argparse.Namespace: default=[], help="Remove inherited samples whose staged filename contains this corrected record key. Repeat as needed.", ) + parser.add_argument( + "--repeat-positive-record", + action="append", + default=[], + metavar="RECORD_KEY=COUNT", + help="Stage a reviewed positive COUNT times to give a hard example more training weight.", + ) parser.add_argument( "--repeat-reject-record", action="append", @@ -49,6 +59,7 @@ def parse_args() -> argparse.Namespace: default=1.0, help="Deterministic fraction of training advisories staged as reject; validation advisories are always retained.", ) + parser.add_argument("--input-size", type=int, default=128, help="Square letterbox size used by the runtime classifier.") return parser.parse_args() @@ -106,23 +117,40 @@ def remove_inherited_records(root: Path, record_keys: list[str]) -> int: return removed -def parse_reject_repeat_counts(specs: list[str]) -> dict[str, int]: +def parse_record_repeat_counts(specs: list[str], sample_kind: str) -> dict[str, int]: repeat_counts: dict[str, int] = {} for spec in specs: record_key, separator, count_text = spec.rpartition("=") if not separator or not record_key: - raise ValueError(f"Invalid --repeat-reject-record value: {spec!r}") + raise ValueError(f"Invalid --repeat-{sample_kind}-record value: {spec!r}") try: count = int(count_text) except ValueError as exc: - raise ValueError(f"Invalid reject repeat count: {spec!r}") from exc + raise ValueError(f"Invalid {sample_kind} repeat count: {spec!r}") from exc if count < 1: - raise ValueError(f"Reject repeat count must be at least 1: {spec!r}") + raise ValueError(f"{sample_kind.capitalize()} repeat count must be at least 1: {spec!r}") repeat_counts[record_key] = count return repeat_counts -def stage_crop(source: Path, destination_dir: Path, record_key: str) -> bool: +def parse_reject_repeat_counts(specs: list[str]) -> dict[str, int]: + return parse_record_repeat_counts(specs, "reject") + + +def square_resize(image: np.ndarray, size: int, color: tuple[int, int, int] = (114, 114, 114)) -> np.ndarray: + image_height, image_width = image.shape[:2] + ratio = min(size / max(image_height, 1), size / max(image_width, 1)) + resized_width = max(int(round(image_width * ratio)), 1) + resized_height = max(int(round(image_height * ratio)), 1) + resized = cv2.resize(image, (resized_width, resized_height), interpolation=cv2.INTER_LINEAR) + canvas = np.full((size, size, image.shape[2]), color, dtype=image.dtype) + offset_x = (size - resized_width) // 2 + offset_y = (size - resized_height) // 2 + canvas[offset_y:offset_y + resized_height, offset_x:offset_x + resized_width] = resized + return canvas + + +def stage_crop(source: Path, destination_dir: Path, record_key: str, input_size: int) -> bool: if not source.is_file(): return False digest = hashlib.sha256(source.read_bytes()).hexdigest()[:16] @@ -131,7 +159,12 @@ def stage_crop(source: Path, destination_dir: Path, record_key: str) -> bool: destination_dir.mkdir(parents=True, exist_ok=True) destination = destination_dir / f"review_{safe_key}_{digest}{suffix}" if not destination.exists(): - shutil.copyfile(source, destination) + image = cv2.imread(str(source)) + if image is None or image.size == 0: + return False + normalized = square_resize(image, input_size) + if not cv2.imwrite(str(destination), normalized, [cv2.IMWRITE_JPEG_QUALITY, 95]): + return False return True @@ -150,6 +183,7 @@ def main() -> int: shutil.copytree(base, output, copy_function=shutil.copyfile) appledouble_removed = remove_appledouble_files(output) inherited_records_removed = remove_inherited_records(output, args.exclude_base_record_key) + positive_repeat_counts = parse_record_repeat_counts(args.repeat_positive_record, "positive") reject_repeat_counts = parse_reject_repeat_counts(args.repeat_reject_record) positive_counts: Counter[str] = Counter() @@ -162,7 +196,9 @@ def main() -> int: elif args.advisory_as_reject and keep_advisory_reject(row, args.advisory_reject_fraction): split = row.get("split", "") source = Path(row.get("crop_path", "")).expanduser() - if split in ("train", "val") and stage_crop(source, output / split / "reject", row.get("record_key", "advisory")): + if split in ("train", "val") and stage_crop( + source, output / split / "reject", row.get("record_key", "advisory"), args.input_size, + ): reject_counts[f"advisory_{split}"] += 1 else: skipped += 1 @@ -172,10 +208,16 @@ def main() -> int: split = row.get("split", "") speed = parse_speed(row.get("speed_limit_mph", "")) source = Path(row.get("crop_path", "")).expanduser() - if split not in ("train", "val") or not speed or not stage_crop(source, output / split / str(speed), row.get("record_key", "positive")): + record_key = row.get("record_key", "positive") + repeat_count = positive_repeat_counts.get(record_key, 1) if split == "train" else 1 + staged = split in ("train", "val") and bool(speed) + for repeat_index in range(repeat_count): + staged_key = record_key if repeat_index == 0 else f"{record_key}_repeat_{repeat_index:03d}" + staged = staged and stage_crop(source, output / split / str(speed), staged_key, args.input_size) + if not staged: skipped += 1 continue - positive_counts[f"{split}/{speed}"] += 1 + positive_counts[f"{split}/{speed}"] += repeat_count for row in read_rows(args.reject_manifest): split = row.get("split", "") @@ -185,7 +227,7 @@ def main() -> int: staged = split in ("train", "val") for repeat_index in range(repeat_count): staged_key = record_key if repeat_index == 0 else f"{record_key}_repeat_{repeat_index:03d}" - staged = staged and stage_crop(source, output / split / "reject", staged_key) + staged = staged and stage_crop(source, output / split / "reject", staged_key, args.input_size) if not staged: skipped += 1 continue diff --git a/scripts/speed_limit_vision/common.py b/scripts/speed_limit_vision/common.py index 6d6b67f54..23f0cb872 100644 --- a/scripts/speed_limit_vision/common.py +++ b/scripts/speed_limit_vision/common.py @@ -26,6 +26,14 @@ DEFAULT_SPEED_VALUES = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75) DETECTOR_EXPORT_NAME = "speed_limit_us_detector.onnx" CLASSIFIER_EXPORT_NAME = "speed_limit_us_value_classifier.onnx" +COMMA_ROAD_CAMERA_FPS = 20.0 + + +def source_video_fps(video_path: str | Path, reported_fps: float) -> float: + # Raw comma HEVC streams have no timing metadata, so OpenCV invents 25 FPS. + if Path(video_path).name == "fcamera.hevc": + return COMMA_ROAD_CAMERA_FPS + return float(reported_fps) if reported_fps > 0.0 else COMMA_ROAD_CAMERA_FPS def resolve_workspace(path: str | Path | None) -> Path: diff --git a/scripts/speed_limit_vision/evaluate_bookmark_leadins.py b/scripts/speed_limit_vision/evaluate_bookmark_leadins.py index 2060688ab..9d97e3554 100644 --- a/scripts/speed_limit_vision/evaluate_bookmark_leadins.py +++ b/scripts/speed_limit_vision/evaluate_bookmark_leadins.py @@ -13,7 +13,12 @@ import cv2 import starpilot.system.speed_limit_vision as slv -from scripts.speed_limit_vision import common +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import common # type: ignore # noqa: TID251 +else: + from . import common DEFAULT_SESSION_ROOT = Path(".tmp/live_drive_debug") @@ -133,7 +138,7 @@ def locate_window(route: str, event: dict, route_mtimes: dict[str, dict[int, int def iter_video_window(path: Path, start_s: float, end_s: float, sample_fps: float | None = None): capture = cv2.VideoCapture(str(path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = common.source_video_fps(path, capture.get(cv2.CAP_PROP_FPS)) start_frame = max(int(start_s * fps), 0) end_frame = max(int(end_s * fps), start_frame) frame_step = 1 diff --git a/scripts/speed_limit_vision/evaluate_detector_classifier.py b/scripts/speed_limit_vision/evaluate_detector_classifier.py index ccf028375..cc1f610e5 100644 --- a/scripts/speed_limit_vision/evaluate_detector_classifier.py +++ b/scripts/speed_limit_vision/evaluate_detector_classifier.py @@ -13,10 +13,10 @@ from ultralytics import YOLO if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) - from common import DEFAULT_SPEED_VALUES # type: ignore + from common import DEFAULT_SPEED_VALUES, source_video_fps # type: ignore # noqa: TID251 from generate_value_roi_classifier_dataset import extract_value_mask # type: ignore else: - from .common import DEFAULT_SPEED_VALUES + from .common import DEFAULT_SPEED_VALUES, source_video_fps from .generate_value_roi_classifier_dataset import extract_value_mask @@ -50,7 +50,7 @@ def iter_frames(path: Path): return cap = cv2.VideoCapture(str(path)) - fps = cap.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(path, cap.get(cv2.CAP_PROP_FPS)) frame_index = 0 while True: ok, frame = cap.read() diff --git a/scripts/speed_limit_vision/evaluate_direct_value_detector.py b/scripts/speed_limit_vision/evaluate_direct_value_detector.py index 4950fd86e..60f454d56 100644 --- a/scripts/speed_limit_vision/evaluate_direct_value_detector.py +++ b/scripts/speed_limit_vision/evaluate_direct_value_detector.py @@ -17,10 +17,12 @@ import starpilot.system.speed_limit_vision as slv if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import source_video_fps # type: ignore # noqa: TID251 from evaluate_runtime_manifest import expected_value, first_present, is_negative, load_rows # type: ignore from evaluate_reviewed_route_events import load_cases # type: ignore from replay_route_runtime import RouteReplayDaemon # type: ignore else: + from .common import source_video_fps from .evaluate_runtime_manifest import expected_value, first_present, is_negative, load_rows from .evaluate_reviewed_route_events import load_cases from .replay_route_runtime import RouteReplayDaemon @@ -175,7 +177,7 @@ def evaluate_manifest(args: argparse.Namespace, detector: DirectValueDetector) - def replay_video_cases(cases, detector: DirectValueDetector, args: argparse.Namespace): daemons = {case.record_key: DirectRouteReplayDaemon(detector, args.measured_inference_seconds) for case in cases} capture = cv2.VideoCapture(str(cases[0].source_video_path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(cases[0].source_video_path, capture.get(cv2.CAP_PROP_FPS)) windows = { case.record_key: (max(case.frame_time_s - args.window_before, 0.0), case.frame_time_s + args.window_after) for case in cases diff --git a/scripts/speed_limit_vision/evaluate_reviewed_route_events.py b/scripts/speed_limit_vision/evaluate_reviewed_route_events.py index 760d14058..d35f5e621 100644 --- a/scripts/speed_limit_vision/evaluate_reviewed_route_events.py +++ b/scripts/speed_limit_vision/evaluate_reviewed_route_events.py @@ -16,9 +16,11 @@ import starpilot.system.speed_limit_vision as slv if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import source_video_fps # type: ignore # noqa: TID251 from import_manual_review_queue import merged_review_rows, parse_speed # type: ignore from replay_route_runtime import RouteReplayDaemon, configure_models # type: ignore else: + from .common import source_video_fps from .import_manual_review_queue import merged_review_rows, parse_speed from .replay_route_runtime import RouteReplayDaemon, configure_models @@ -164,7 +166,7 @@ def replay_video_cases(cases: list[ReviewedCase], args: argparse.Namespace) -> d daemon.published_speed_limit_mph = args.initial_speed_limit daemon.last_published_support_at = 0.0 capture = cv2.VideoCapture(str(cases[0].source_video_path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(cases[0].source_video_path, capture.get(cv2.CAP_PROP_FPS)) frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) duration_s = frame_count / fps if frame_count > 0 else 60.0 windows = { diff --git a/scripts/speed_limit_vision/import_bookmark_leadins.py b/scripts/speed_limit_vision/import_bookmark_leadins.py index 469b429fd..ad641c293 100644 --- a/scripts/speed_limit_vision/import_bookmark_leadins.py +++ b/scripts/speed_limit_vision/import_bookmark_leadins.py @@ -14,12 +14,13 @@ import numpy as np if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) - from common import ( # type: ignore + from common import ( # type: ignore # noqa: TID251 BOOKMARK_LEADIN_MANIFEST_FIELDS, DEFAULT_WORKSPACE, ensure_dir, preferred_clip_root, resolve_workspace, + source_video_fps, write_csv_header, ) else: @@ -29,6 +30,7 @@ else: ensure_dir, preferred_clip_root, resolve_workspace, + source_video_fps, write_csv_header, ) @@ -64,7 +66,7 @@ def load_existing_rows(manifest_path: Path) -> dict[str, dict[str, str]]: def read_frames_at(video_path: Path, target_times_s: list[float]): capture = cv2.VideoCapture(str(video_path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(video_path, capture.get(cv2.CAP_PROP_FPS)) targets = sorted((max(int(round(target_time_s * fps)), 0), target_time_s) for target_time_s in target_times_s) results = {} frame_index = 0 diff --git a/scripts/speed_limit_vision/import_localized_bookmark_detector_examples.py b/scripts/speed_limit_vision/import_localized_bookmark_detector_examples.py index ce54ccde1..cdabeffe5 100644 --- a/scripts/speed_limit_vision/import_localized_bookmark_detector_examples.py +++ b/scripts/speed_limit_vision/import_localized_bookmark_detector_examples.py @@ -12,9 +12,14 @@ import cv2 if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) - from common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore + from common import ( # type: ignore # noqa: TID251 + DEFAULT_WORKSPACE, + ensure_dir, + resolve_workspace, + source_video_fps, + ) else: - from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace, source_video_fps LOCALIZED_MANIFEST = Path(".tmp/bookmark_sign_localization/localized_bookmarks.csv") @@ -96,7 +101,7 @@ def expand_bbox(x1: int, y1: int, x2: int, y2: int, image_shape: tuple[int, int, def read_frame_at(video_path: Path, target_time_s: float): capture = cv2.VideoCapture(str(video_path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(video_path, capture.get(cv2.CAP_PROP_FPS)) frame_index = max(int(round(target_time_s * fps)), 0) capture.set(cv2.CAP_PROP_POS_FRAMES, frame_index) ok, frame_bgr = capture.read() diff --git a/scripts/speed_limit_vision/localize_bookmark_signs.py b/scripts/speed_limit_vision/localize_bookmark_signs.py index f8e5a6940..ff7940345 100644 --- a/scripts/speed_limit_vision/localize_bookmark_signs.py +++ b/scripts/speed_limit_vision/localize_bookmark_signs.py @@ -10,7 +10,12 @@ import cv2 import starpilot.system.speed_limit_vision as slv -from scripts.speed_limit_vision import common +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import common # type: ignore # noqa: TID251 +else: + from . import common from scripts.speed_limit_vision import evaluate_bookmark_leadins as ebl @@ -22,7 +27,12 @@ def parse_args(): parser.add_argument("--clip-root", type=Path, default=ebl.DEFAULT_CLIP_ROOT, help="Copied route clip root.") parser.add_argument("--qlog-mtimes", type=Path, default=ebl.DEFAULT_QLOG_MTIMES, help="Text file with ' ' lines.") parser.add_argument("--session-root", type=Path, default=ebl.DEFAULT_SESSION_ROOT, help="Directory containing debug session folders.") - parser.add_argument("--session-route-map", type=Path, default=common.preferred_session_route_map_path(), help="JSON file mapping debug session ids to route log ids.") + parser.add_argument( + "--session-route-map", + type=Path, + default=common.preferred_session_route_map_path(), + help="JSON file mapping debug session ids to route log ids.", + ) parser.add_argument("--models-dir", type=Path, help="Directory containing speed_limit_us_detector.onnx and speed_limit_us_value_classifier.onnx.") parser.add_argument("--search-before", type=float, default=18.0, help="Seconds before the bookmark to scan.") parser.add_argument("--search-after", type=float, default=2.0, help="Seconds after the bookmark to scan.") @@ -48,7 +58,7 @@ def configure_models(models_dir: Path | None): def iter_video_samples(clip_path: Path, start_s: float, end_s: float, sample_every: float): capture = cv2.VideoCapture(str(clip_path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = common.source_video_fps(clip_path, capture.get(cv2.CAP_PROP_FPS)) start_frame = max(int(start_s * fps), 0) end_frame = max(int(end_s * fps), start_frame) @@ -216,6 +226,8 @@ def write_manifest(rows: list[dict], path: Path): "route", "segment", "relative_time_s", + "source_segment", + "source_time_s", "source_video_path", "score", "proposal_confidence", @@ -270,7 +282,9 @@ def main(): ranked.append((scored["score"], relative_time_s, source_video_path, source_time_s, frame_bgr, scored)) ranked.sort(key=lambda item: item[0], reverse=True) - for rank_index, (_, relative_time_s, source_video_path, _, frame_bgr, scored) in enumerate(ranked[:max(args.top_k, 1)], start=1): + for rank_index, (_, relative_time_s, source_video_path, source_time_s, frame_bgr, scored) in enumerate( + ranked[:max(args.top_k, 1)], start=1, + ): x1, y1, x2, y2 = scored["box"] crop = frame_bgr[y1:y2, x1:x2] @@ -293,6 +307,8 @@ def main(): "route": route, "segment": window.segment, "relative_time_s": f"{relative_time_s:.3f}", + "source_segment": window.segment - int(relative_time_s < 0.0), + "source_time_s": f"{source_time_s:.3f}", "source_video_path": str(source_video_path), "score": f"{scored['score']:.4f}", "proposal_confidence": f"{scored['proposal_confidence']:.4f}", diff --git a/scripts/speed_limit_vision/mine_connect_route_bookmarks.py b/scripts/speed_limit_vision/mine_connect_route_bookmarks.py index 70c88a842..e0027a1e3 100644 --- a/scripts/speed_limit_vision/mine_connect_route_bookmarks.py +++ b/scripts/speed_limit_vision/mine_connect_route_bookmarks.py @@ -16,7 +16,7 @@ import starpilot.system.speed_limit_vision as slv if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) - from common import ensure_dir, preferred_clip_root, resolve_workspace # type: ignore + from common import ensure_dir, preferred_clip_root, resolve_workspace # type: ignore # noqa: TID251 from evaluate_bookmark_leadins import BookmarkWindow # type: ignore from import_bookmark_leadins import extract_window_frames, write_contact_sheet # type: ignore from localize_bookmark_signs import configure_models, iter_context_frames, score_frame # type: ignore @@ -126,6 +126,8 @@ def write_localized_manifest(path: Path, rows: list[dict]) -> None: "route", "segment", "relative_time_s", + "source_segment", + "source_time_s", "source_video_path", "score", "proposal_confidence", @@ -221,7 +223,7 @@ def main() -> int: write_contact_sheet(contact_sheet_path, contact_sheet_frames, contact_sheet_labels, args.overwrite) ranked = [] - for relative_time_s, source_video_path, _, frame_bgr in iter_context_frames( + for relative_time_s, source_video_path, source_time_s, frame_bgr in iter_context_frames( clip_root, window, args.search_before, @@ -231,10 +233,12 @@ def main() -> int: scored = score_frame(daemon, frame_bgr, use_ocr=not args.model_only) if scored is None: continue - ranked.append((scored["score"], relative_time_s, source_video_path, frame_bgr, scored)) + ranked.append((scored["score"], relative_time_s, source_video_path, source_time_s, frame_bgr, scored)) ranked.sort(key=lambda item: item[0], reverse=True) - for rank_index, (_, relative_time_s, source_video_path, frame_bgr, scored) in enumerate(ranked[:max(args.top_k, 1)], start=1): + for rank_index, (_, relative_time_s, source_video_path, source_time_s, frame_bgr, scored) in enumerate( + ranked[:max(args.top_k, 1)], start=1, + ): x1, y1, x2, y2 = scored["box"] crop = frame_bgr[y1:y2, x1:x2] frame_name = f"{session_id}_bookmark_{bookmark_number:03d}_rank_{rank_index:02d}.jpg" @@ -253,6 +257,8 @@ def main() -> int: "route": log_id, "segment": window.segment, "relative_time_s": f"{relative_time_s:.3f}", + "source_segment": window.segment - int(relative_time_s < 0.0), + "source_time_s": f"{source_time_s:.3f}", "source_video_path": str(source_video_path), "score": f"{scored['score']:.4f}", "proposal_confidence": f"{scored['proposal_confidence']:.4f}", diff --git a/scripts/speed_limit_vision/mine_reviewed_sign_tracks.py b/scripts/speed_limit_vision/mine_reviewed_sign_tracks.py index ae04497e5..fd7e81931 100644 --- a/scripts/speed_limit_vision/mine_reviewed_sign_tracks.py +++ b/scripts/speed_limit_vision/mine_reviewed_sign_tracks.py @@ -18,9 +18,11 @@ import starpilot.system.speed_limit_vision as slv if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import source_video_fps # type: ignore # noqa: TID251 from import_manual_review_queue import merged_review_rows, parse_speed # type: ignore from replay_route_runtime import configure_models # type: ignore else: + from .common import source_video_fps from .import_manual_review_queue import merged_review_rows, parse_speed from .replay_route_runtime import configure_models @@ -372,7 +374,7 @@ def mine_backward_samples( def mine_case(case: TrackCase, daemon: slv.SpeedLimitVisionDaemon, args: argparse.Namespace) -> list[TrackSample]: capture = cv2.VideoCapture(str(case.video_path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(case.video_path, capture.get(cv2.CAP_PROP_FPS)) anchor_frame_index = max(int(round(case.frame_time_s * fps)), 0) before_frame_count = max(int(round(args.window_before * fps)), 0) earlier_frames: deque[tuple[int, np.ndarray]] = deque(maxlen=before_frame_count) diff --git a/scripts/speed_limit_vision/mine_route_training_samples.py b/scripts/speed_limit_vision/mine_route_training_samples.py index 0599ecff0..45526825b 100644 --- a/scripts/speed_limit_vision/mine_route_training_samples.py +++ b/scripts/speed_limit_vision/mine_route_training_samples.py @@ -20,10 +20,16 @@ import starpilot.system.speed_limit_vision as slv if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) - from common import VALUE_LABEL_FIELDS, ensure_dir, preferred_clip_root, resolve_workspace # type: ignore + from common import ( # type: ignore # noqa: TID251 + VALUE_LABEL_FIELDS, + ensure_dir, + preferred_clip_root, + resolve_workspace, + source_video_fps, + ) from localize_bookmark_signs import configure_models, score_frame # type: ignore else: - from .common import VALUE_LABEL_FIELDS, ensure_dir, preferred_clip_root, resolve_workspace + from .common import VALUE_LABEL_FIELDS, ensure_dir, preferred_clip_root, resolve_workspace, source_video_fps from .localize_bookmark_signs import configure_models, score_frame @@ -542,7 +548,7 @@ def mine_route( break contexts = load_segment_map_context(segment.path) capture = cv2.VideoCapture(str(segment.video_path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(segment.video_path, capture.get(cv2.CAP_PROP_FPS)) frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0 duration_s = frame_count / fps if frame_count > 0 else 60.0 times = sample_times(duration_s, args.sample_every, transition_times(contexts), args.transition_radius, args.transition_step) diff --git a/scripts/speed_limit_vision/replay_route_runtime.py b/scripts/speed_limit_vision/replay_route_runtime.py index 2e37ba840..074ffb01f 100644 --- a/scripts/speed_limit_vision/replay_route_runtime.py +++ b/scripts/speed_limit_vision/replay_route_runtime.py @@ -15,6 +15,13 @@ from cereal import log import starpilot.system.speed_limit_vision as slv +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import source_video_fps # type: ignore # noqa: TID251 +else: + from .common import source_video_fps + @dataclass(frozen=True) class RouteSummary: @@ -419,7 +426,7 @@ def replay_route( for segment_path in segments: segment = segment_index(segment_path) capture = cv2.VideoCapture(str(segment_path)) - fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(segment_path, capture.get(cv2.CAP_PROP_FPS)) total_frames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) segment_start_s = segment * 60.0 frame_index = max(int(round(max(start_s - segment_start_s, 0.0) * fps)), 0) diff --git a/scripts/speed_limit_vision/sample_route_backgrounds.py b/scripts/speed_limit_vision/sample_route_backgrounds.py index afdf3ca45..9267f7a40 100644 --- a/scripts/speed_limit_vision/sample_route_backgrounds.py +++ b/scripts/speed_limit_vision/sample_route_backgrounds.py @@ -10,9 +10,9 @@ import cv2 if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) - from common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore + from common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace, source_video_fps # type: ignore # noqa: TID251 else: - from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace, source_video_fps IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp"} @@ -49,7 +49,7 @@ def sample_images(source_files: list[Path], output_dir: Path, max_per_file: int) def sample_video(video_path: Path, output_dir: Path, seconds_between_frames: float, max_frames: int): cap = cv2.VideoCapture(str(video_path)) - fps = cap.get(cv2.CAP_PROP_FPS) or 20.0 + fps = source_video_fps(video_path, cap.get(cv2.CAP_PROP_FPS)) total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0) frame_step = max(int(round(seconds_between_frames * fps)), 1) frame_indices = range(0, total_frames if total_frames > 0 else frame_step * max_frames, frame_step) diff --git a/scripts/speed_limit_vision/test_review_pipeline.py b/scripts/speed_limit_vision/test_review_pipeline.py index a16d22ebc..fac52d2e8 100644 --- a/scripts/speed_limit_vision/test_review_pipeline.py +++ b/scripts/speed_limit_vision/test_review_pipeline.py @@ -5,7 +5,6 @@ import pytest from argparse import Namespace from pathlib import Path - def load_local_module(name: str): path = Path(__file__).resolve().with_name(f"{name}.py") spec = importlib.util.spec_from_file_location(f"test_local_{name}", path) @@ -16,7 +15,9 @@ def load_local_module(name: str): import_queue = load_local_module("import_manual_review_queue") +common = load_local_module("common") build_review_classifier = load_local_module("build_review_classifier_dataset") +localized_review = load_local_module("build_localized_bookmark_review_queue") select_queue = load_local_module("select_manual_review_queue") compare_queues = load_local_module("compare_manual_review_queues") rescore_queue = load_local_module("rescore_manual_review_queue") @@ -26,6 +27,22 @@ split_group_key = import_queue.split_group_key select_rows = select_queue.select_rows +def test_raw_comma_camera_uses_real_frame_rate(): + assert common.source_video_fps(Path("route/fcamera.hevc"), 25.0) == 20.0 + assert common.source_video_fps(Path("clip.mp4"), 29.97) == 29.97 + assert common.source_video_fps(Path("clip.mp4"), 0.0) == 20.0 + + +def test_localized_bookmark_source_position_normalizes_previous_segment(): + previous = {"segment": "26", "relative_time_s": "-18.950"} + current = {"segment": "26", "relative_time_s": "12.500"} + explicit = {"segment": "26", "relative_time_s": "-18.950", "source_segment": "25", "source_time_s": "41.050"} + + assert localized_review.source_position(previous) == ("25", "41.050") + assert localized_review.source_position(current) == ("26", "12.500") + assert localized_review.source_position(explicit) == ("25", "41.050") + + def review_row(key: str, route: str, speed: int, priority: float) -> dict[str, str]: return { "record_key": key, @@ -253,13 +270,31 @@ def test_corrected_record_removes_inherited_classifier_sample(tmp_path): def test_reject_repeat_spec_preserves_record_key_punctuation(): counts = build_review_classifier.parse_reject_repeat_counts(["route/sign=track:55=32"]) + positive_counts = build_review_classifier.parse_record_repeat_counts(["route/sign=track:75=16"], "positive") assert counts == {"route/sign=track:55": 32} + assert positive_counts == {"route/sign=track:75": 16} with pytest.raises(ValueError, match="at least 1"): build_review_classifier.parse_reject_repeat_counts(["bad-record=0"]) +def test_review_crop_staging_matches_runtime_letterbox(tmp_path): + import cv2 + import numpy as np + + source = tmp_path / "portrait.jpg" + image = np.full((80, 40, 3), 255, dtype=np.uint8) + cv2.imwrite(str(source), image) + + assert build_review_classifier.stage_crop(source, tmp_path / "train" / "75", "portrait", 128) + staged = cv2.imread(str(next((tmp_path / "train" / "75").iterdir()))) + + assert staged is not None and staged.shape == (128, 128, 3) + assert staged[:, :20].mean() == pytest.approx(114, abs=2) + assert staged[:, 32:96].mean() > 245 + + def test_conditional_reject_generates_runtime_crop_expansions(tmp_path): import cv2 import numpy as np diff --git a/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx b/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx index c55a576c2..45b023cef 100755 Binary files a/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx and b/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx differ