From e577502f4be967bc3faf3e18e69f047115c335ba Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Sun, 12 Jul 2026 17:53:20 -0500 Subject: [PATCH] VACATION --- docs/how-to/train-speed-limit-vision.md | 109 ++++++ opendbc_repo/opendbc/car/gm/carcontroller.py | 44 ++- .../car/gm/tests/test_carcontroller.py | 18 +- .../build_manual_review_queue.py | 143 ++++++-- .../build_review_classifier_dataset.py | 161 +++++++++ .../compare_manual_review_queues.py | 141 ++++++++ .../evaluate_review_classifier.py | 122 +++++++ .../evaluate_reviewed_route_events.py | 267 +++++++++++++++ .../evaluate_runtime_manifest.py | 34 +- .../import_manual_review_queue.py | 127 ++++++- .../interpolate_yolo_checkpoints.py | 69 ++++ .../localize_bookmark_signs.py | 33 +- .../merge_manual_review_queues.py | 89 +++++ .../mine_classifier_reject_crops.py | 157 +++++++++ .../mine_route_training_samples.py | 123 ++++++- .../replay_route_runtime.py | 33 +- .../rescore_manual_review_queue.py | 138 ++++++++ .../select_manual_review_queue.py | 206 ++++++++++++ .../serve_manual_review_queue.py | 50 ++- .../test_review_pipeline.py | 193 +++++++++++ .../ui/onroad/starpilot/developer_sidebar.py | 61 +++- .../speed_limit_us_value_classifier.onnx | Bin 6204247 -> 6204255 bytes starpilot/system/speed_limit_vision.py | 64 +++- .../system/tests/test_speed_limit_vision.py | 71 +++- .../the_galaxy/assets/components/router.js | 4 +- .../components/tools/device_settings.css | 45 +++ .../components/tools/device_settings.js | 108 +++++- .../assets/components/tools/tuning.css | 195 ++++++++++- .../assets/components/tools/tuning.js | 289 +++++++++++++++- starpilot/system/the_galaxy/ftm_workspace.py | 309 ++++++++++++++++-- .../system/the_galaxy/templates/index.html | 4 +- .../the_galaxy/tests/test_ftm_workspace.py | 204 +++++++++++- starpilot/system/the_galaxy/the_galaxy.py | 2 + system/ui/lib/application.py | 14 +- 34 files changed, 3456 insertions(+), 171 deletions(-) create mode 100644 scripts/speed_limit_vision/build_review_classifier_dataset.py create mode 100644 scripts/speed_limit_vision/compare_manual_review_queues.py create mode 100644 scripts/speed_limit_vision/evaluate_review_classifier.py create mode 100644 scripts/speed_limit_vision/evaluate_reviewed_route_events.py create mode 100644 scripts/speed_limit_vision/interpolate_yolo_checkpoints.py create mode 100644 scripts/speed_limit_vision/merge_manual_review_queues.py create mode 100644 scripts/speed_limit_vision/mine_classifier_reject_crops.py create mode 100644 scripts/speed_limit_vision/rescore_manual_review_queue.py create mode 100644 scripts/speed_limit_vision/select_manual_review_queue.py create mode 100644 scripts/speed_limit_vision/test_review_pipeline.py diff --git a/docs/how-to/train-speed-limit-vision.md b/docs/how-to/train-speed-limit-vision.md index 9ba62c1a4..ece1550cb 100644 --- a/docs/how-to/train-speed-limit-vision.md +++ b/docs/how-to/train-speed-limit-vision.md @@ -302,3 +302,112 @@ For temporal behavior on a saved frame directory or route extract, replay the ru ```bash .venv/bin/python scripts/replay_speed_limit_vision.py .tmp/vision_iter/seg10_5fps --frames-fps 5 ``` + +The detector/classifier runtime is model-only by default. Use `--crop-ocr` with +`evaluate_runtime_manifest.py` or `replay_route_runtime.py` only for an explicit +legacy comparison. A model-only release must match reviewed-manifest accuracy +and pass representative route replays at measured on-device cadence. Evaluate +candidate recognition and temporal publish behavior separately: a correct +single-frame candidate can still be suppressed by the history and speed-change +confirmation policy. + +Ignored review rows label the proposed crop, not the entire camera frame. +Consequently, negative-window candidate and publish counts from +`evaluate_reviewed_route_events.py` are an upper bound until the full frame is +audited; another valid sign can be present outside the rejected crop. Use the +per-row output and frame image to audit any regression delta before treating it +as a runtime false positive. + +## Promotion Gate + +Do not promote a checkpoint from classifier validation accuracy alone. Export it +to an isolated model directory and run the complete runtime pipeline against the +reviewed positive, hard-negative, and failed-drive manifests. A candidate must +preserve exact-value recall, avoid new wrong-value reads, and remain within the +accepted false-positive budget before route replay. + +Mine detector proposals that fool an integrated-reject classifier into a new +reject class before retraining: + +```bash +.venv/bin/python scripts/speed_limit_vision/mine_classifier_reject_crops.py \ + --models-dir /path/to/candidate/models \ + --dataset /path/to/versioned/classifier \ + --manifest /path/to/reviewed-negative-manifest.csv +``` + +Keep the resulting dataset version separate from the current training set. If a +hard-negative retrain lowers reviewed recall, reject the checkpoint even when it +improves aggregate validation accuracy or removes a known false positive. + +## Active-Learning Review Pass + +Keep parallel miners in separate directories and merge them only when their +model and mining fingerprints match: + +```bash +.venv/bin/python scripts/speed_limit_vision/merge_manual_review_queues.py \ + /path/to/shard0 /path/to/shard1 /path/to/shard2 /path/to/shard3 \ + --output-dir /path/to/merged +``` + +When rescanning with a new model, compare the fingerprinted queues before +selecting another batch. The optional review output retains the full queue +schema so it can be passed directly to the selector and review server: + +```bash +.venv/bin/python scripts/speed_limit_vision/compare_manual_review_queues.py \ + --before /path/to/baseline/manual_review_queue.csv \ + --after /path/to/candidate/manual_review_queue.csv \ + --output-csv /path/to/comparison.csv \ + --review-output /path/to/disagreements/manual_review_queue.csv + +.venv/bin/python scripts/speed_limit_vision/select_manual_review_queue.py \ + --input /path/to/disagreements/manual_review_queue.csv \ + --output /path/to/review/manual_review_queue.csv \ + --max-rows 1200 \ + --min-seconds-per-route-speed 3 +``` + +The selector prioritizes value changes and gained/lost reads, balances routes +and speed classes, and removes adjacent same-speed frames from one scene. Start +the reviewer and import its labels without moving route media off the training +volume: + +```bash +.venv/bin/python scripts/speed_limit_vision/serve_manual_review_queue.py \ + --manifest /path/to/review/manual_review_queue.csv \ + --port 8765 + +.venv/bin/python scripts/speed_limit_vision/import_manual_review_queue.py \ + --queue /path/to/review/manual_review_queue.csv +``` + +## Re-mine the Route Backlog + +Re-run the backlog after a candidate passes the reviewed-manifest and route +replay gates. Use a model fingerprinted run so new pseudo-labels are staged next +to, rather than merged into, the original route-mining data: + +```bash +.venv/bin/python scripts/speed_limit_vision/mine_route_training_samples.py \ + --workspace /Volumes/T5/starpilot_speed_limit/workspace/speed_limit_training_clean \ + --models-dir /path/to/promoted/models \ + --model-only \ + --run-id auto \ + --sample-every 2.0 \ + --transition-step 0.5 \ + --max-frames-per-route 720 \ + --max-positives-per-route 120 \ + --max-negatives-per-route 200 +``` + +The output is written under +`staging/route_mining/model__run_/` with +its own detector images, classifier labels, review manifest, and per-route +completion state. The mining fingerprint includes the model-only mode, +thresholds, sampling configuration, and relevant source code. Review and +deduplicate that staged run before merging it into a training dataset. Never +overwrite the canonical route samples or automatically train on every mined +positive; map agreement and human review remain required because a stronger +model can still reproduce its own mistakes at larger scale. diff --git a/opendbc_repo/opendbc/car/gm/carcontroller.py b/opendbc_repo/opendbc/car/gm/carcontroller.py index 1f67d7dbf..1b9e055fd 100644 --- a/opendbc_repo/opendbc/car/gm/carcontroller.py +++ b/opendbc_repo/opendbc/car/gm/carcontroller.py @@ -66,6 +66,9 @@ TRUCK_LONG_SMOOTH_CARS = { CAR.CHEVROLET_SILVERADO, CAR.CHEVROLET_SILVERADO_CC, } +TRUCK_FRICTION_BRAKE_ENGAGE = 25 +TRUCK_FRICTION_BRAKE_RELEASE = 8 +TRUCK_FRICTION_BRAKE_IMMEDIATE_ACCEL = -0.65 ACC_DASHBOARD_ZERO_RESERVED_CARS = { CAR.CHEVROLET_BLAZER, CAR.CHEVROLET_EQUINOX, @@ -212,11 +215,11 @@ def shape_truck_positive_accel(accel: float, v_ego: float, enabled: bool, if not enabled or accel <= 0.0 or v_ego < 12.0: return accel - low_scale = float(np.interp(v_ego, [12.0, 18.0, 25.0, 35.0], [0.95, 0.88, 0.82, 0.76])) - mid_scale = float(np.interp(v_ego, [12.0, 18.0, 25.0, 35.0], [0.98, 0.94, 0.89, 0.84])) + low_scale = float(np.interp(v_ego, [12.0, 18.0, 25.0, 35.0], [0.93, 0.84, 0.76, 0.70])) + mid_scale = float(np.interp(v_ego, [12.0, 18.0, 25.0, 35.0], [0.97, 0.91, 0.85, 0.79])) if lead_visible and set_speed_error > 0.0: - follow_relief = float(np.interp(set_speed_error, [0.0, 1.0, 2.5, 4.0, 6.0], [0.0, 0.08, 0.18, 0.35, 0.55])) + follow_relief = float(np.interp(set_speed_error, [0.0, 1.0, 2.5, 4.0, 6.0], [0.0, 0.04, 0.10, 0.18, 0.30])) low_scale += (1.0 - low_scale) * follow_relief mid_scale += (1.0 - mid_scale) * follow_relief @@ -229,6 +232,27 @@ def shape_truck_positive_accel(accel: float, v_ego: float, enabled: bool, return accel +def shape_truck_friction_brake(apply_brake: int, accel_cmd: float, stopping: bool, active: bool) -> tuple[int, bool]: + if apply_brake <= 0: + return 0, False + + # Preserve full brake response for stop control and meaningful deceleration. + if stopping or accel_cmd <= TRUCK_FRICTION_BRAKE_IMMEDIATE_ACCEL: + return apply_brake, True + + if active: + if apply_brake <= TRUCK_FRICTION_BRAKE_RELEASE: + return 0, False + return apply_brake, True + + if apply_brake >= TRUCK_FRICTION_BRAKE_ENGAGE: + return apply_brake, True + + # Keep tiny corrections in the continuous gas/regen torque path. Switching + # to friction also forces max regen, which makes a small request perceptible. + return 0, False + + def get_lka_steering_cmd_counter(next_counter: int, CS) -> int: if getattr(CS, "loopback_lka_steering_cmd_updated", False): return (getattr(CS, "loopback_lka_steering_cmd_counter", next_counter) + 1) % 4 @@ -515,6 +539,7 @@ class CarController(CarControllerBase): self.gm_auto_hold_enabled = False self.bolt_acc_pedal_friction_release_frames = 0 self.bolt_acc_pedal_friction_low_speed_active = False + self.truck_friction_brake_active = False def _reset_volt_one_pedal(self): self.volt_one_pedal_pid.reset() @@ -968,13 +993,14 @@ class CarController(CarControllerBase): if testing_ground.use_1: accel_max = min(accel_max, np.interp(CS.out.vEgo, [0.0, 4.0, 12.0], [1.25, 1.6, self.params.ACCEL_MAX])) - accel_input = actuators.accel + accel_due_to_pitch - if ( + truck_long_smoothing = ( getattr(starpilot_toggles, "truck_tuning", False) and self.CP.carFingerprint in TRUCK_LONG_SMOOTH_CARS and getattr(self.CP, "transmissionType", None) == TransmissionType.automatic and not self.CP.enableGasInterceptorDEPRECATED - ): + ) + accel_input = actuators.accel + accel_due_to_pitch + if truck_long_smoothing: accel_input = shape_truck_positive_accel( accel_input, CS.out.vEgo, @@ -999,6 +1025,12 @@ class CarController(CarControllerBase): brake_accel = min((scaled_torque - brake_switch) / (self.tireRadius * self.mass), 0) self.apply_gas = int(round(apply_gas_torque)) self.apply_brake = int(round(np.interp(brake_accel, self.params.BRAKE_LOOKUP_BP, self.params.BRAKE_LOOKUP_V))) + if truck_long_smoothing: + self.apply_brake, self.truck_friction_brake_active = shape_truck_friction_brake( + self.apply_brake, accel_cmd, stopping, self.truck_friction_brake_active, + ) + else: + self.truck_friction_brake_active = False if bolt_acc_pedal_friction_main_on: if self.apply_brake > 0: full_brake_accel = min( diff --git a/opendbc_repo/opendbc/car/gm/tests/test_carcontroller.py b/opendbc_repo/opendbc/car/gm/tests/test_carcontroller.py index f44c1aede..bdf636cf5 100644 --- a/opendbc_repo/opendbc/car/gm/tests/test_carcontroller.py +++ b/opendbc_repo/opendbc/car/gm/tests/test_carcontroller.py @@ -54,6 +54,7 @@ from opendbc.car.gm.carcontroller import ( get_acc_dashboard_status_active, get_stock_cc_active_for_cancel, shape_bolt_acc_pedal_low_speed_friction, + shape_truck_friction_brake, shape_truck_positive_accel, should_use_fixed_stopping_brake, should_activate_auto_hold, @@ -827,7 +828,7 @@ def test_calc_pedal_command_keeps_strong_positive_requests_responsive(): def test_shape_truck_positive_accel_softens_small_highway_requests(): shaped = shape_truck_positive_accel(0.12, 26.0, True) - assert 0.09 < shaped < 0.10 + assert 0.08 < shaped < 0.095 def test_shape_truck_positive_accel_keeps_mid_follow_requests_available(): @@ -860,6 +861,21 @@ def test_shape_truck_positive_accel_does_not_relax_without_speed_error(): assert no_error == base +def test_shape_truck_friction_brake_suppresses_boundary_chatter(): + assert shape_truck_friction_brake(14, -0.3, False, False) == (0, False) + + +def test_shape_truck_friction_brake_uses_hysteresis_once_engaged(): + assert shape_truck_friction_brake(25, -0.3, False, False) == (25, True) + assert shape_truck_friction_brake(14, -0.3, False, True) == (14, True) + assert shape_truck_friction_brake(8, -0.3, False, True) == (0, False) + + +def test_shape_truck_friction_brake_never_delays_meaningful_braking(): + assert shape_truck_friction_brake(5, -0.65, False, False) == (5, True) + assert shape_truck_friction_brake(5, -0.2, True, False) == (5, True) + + def test_use_interceptor_sng_launch_requires_actual_near_stop(): CP = SimpleNamespace(vEgoStarting=0.25) diff --git a/scripts/speed_limit_vision/build_manual_review_queue.py b/scripts/speed_limit_vision/build_manual_review_queue.py index 30ca813fe..14f560e43 100644 --- a/scripts/speed_limit_vision/build_manual_review_queue.py +++ b/scripts/speed_limit_vision/build_manual_review_queue.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import csv +import hashlib import json import math @@ -16,7 +17,7 @@ import starpilot.system.speed_limit_vision as slv if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) - from common import ensure_dir, preferred_clip_root, resolve_workspace # type: ignore + from common import ensure_dir, preferred_clip_root, resolve_workspace # type: ignore # noqa: TID251 from localize_bookmark_signs import configure_models # type: ignore from mine_route_training_samples import ( # type: ignore MapContext, @@ -25,6 +26,7 @@ if __package__ in (None, ""): iter_frames_at_times, load_segment_map_context, nearest_context, + model_bundle_fingerprint, parse_route_id, read_frame_at, route_segments, @@ -42,6 +44,7 @@ else: iter_frames_at_times, load_segment_map_context, nearest_context, + model_bundle_fingerprint, parse_route_id, read_frame_at, route_segments, @@ -57,6 +60,8 @@ DEFAULT_OUTPUT_NAME = "manual_review_queue_v1" PRIORITY_SPEED_VALUES = frozenset((30, 35, 40, 45, 50, 55, 60, 65)) FIELDNAMES = [ "record_key", + "mining_fingerprint", + "model_fingerprint", "route", "dongle_id", "log_id", @@ -112,14 +117,15 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--clip-root", type=Path, default=preferred_clip_root(), help="Route realdata root.") parser.add_argument("--bundle-state-dir", type=Path, default=DEFAULT_ROUTE_BUNDLE_STATE_DIR, help="Completed extraction marker directory.") parser.add_argument("--models-dir", type=Path, help="Optional model directory for mining with non-repo ONNXs.") + parser.add_argument("--model-only", action="store_true", help="Do not run crop OCR while discovering review candidates.") parser.add_argument("--output-dir", type=Path, help=f"Defaults to /review/{DEFAULT_OUTPUT_NAME}.") parser.add_argument("--manifest-out", type=Path, help="Defaults to /manual_review_queue.csv.") parser.add_argument("--sample-every", type=float, default=2.0, help="Seconds between regular video samples.") parser.add_argument("--seek-sampling", action="store_true", help="Seek directly to sampled frames instead of sequential grabbing.") parser.add_argument("--transition-radius", type=float, default=18.0, help="Extra seconds around map speed transitions to sample densely.") parser.add_argument("--transition-step", type=float, default=0.75, help="Seconds between transition-window samples.") - parser.add_argument("--max-frames-per-route", type=int, default=1200, help="Maximum frames to score per route.") - parser.add_argument("--max-candidates-per-route", type=int, default=500, help="Maximum review candidates to keep per route.") + parser.add_argument("--max-frames-per-route", type=int, default=1200, help="Maximum frames to score per route. 0 scans the full route.") + parser.add_argument("--max-candidates-per-route", type=int, default=500, help="Maximum review candidates to keep per route. 0 keeps all.") parser.add_argument("--max-candidates-per-frame", type=int, default=1, help="Maximum detector candidates to keep from a single video frame. 0 keeps all.") parser.add_argument("--max-negatives-per-route", type=int, default=60, help="Maximum empty/no-candidate frames to keep per route.") parser.add_argument("--min-proposal-confidence", type=float, default=0.025, help="Loose detector confidence floor for review candidates.") @@ -132,6 +138,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--include-advisory", action=argparse.BooleanOptionalAction, default=True, help="Include advisory-speed detector class candidates.") parser.add_argument("--include-full-detection", action="store_true", help="Also run the full runtime detector on each frame for extra context. Slower.") parser.add_argument("--overwrite-images", action="store_true", help="Rewrite existing review images.") + parser.add_argument("--resume", action=argparse.BooleanOptionalAction, default=True, help="Resume a matching fingerprinted queue.") parser.add_argument("--dry-run", action="store_true", help="Score frames and print counts without writing images/CSV.") return parser.parse_args() @@ -160,6 +167,33 @@ def read_routes(args: argparse.Namespace) -> list[str]: return deduped +def review_mining_fingerprint(args: argparse.Namespace, model_fingerprint: str) -> str: + config = { + "schema_version": 1, + "model_fingerprint": model_fingerprint, + "model_only": args.model_only, + "sample_every": args.sample_every, + "transition_radius": args.transition_radius, + "transition_step": args.transition_step, + "max_frames_per_route": args.max_frames_per_route, + "max_candidates_per_route": args.max_candidates_per_route, + "max_candidates_per_frame": args.max_candidates_per_frame, + "max_negatives_per_route": args.max_negatives_per_route, + "min_proposal_confidence": args.min_proposal_confidence, + "no_read_min_proposal_confidence": args.no_read_min_proposal_confidence, + "school_zone_min_proposal_confidence": args.school_zone_min_proposal_confidence, + "min_width": args.min_width, + "min_height": args.min_height, + "dedupe_seconds": args.dedupe_seconds, + "include_advisory": args.include_advisory, + "include_full_detection": args.include_full_detection, + } + digest = hashlib.sha256(json.dumps(config, sort_keys=True).encode("utf-8")) + for source_path in (Path(__file__), Path(slv.__file__)): + digest.update(source_path.resolve().read_bytes()) + return digest.hexdigest() + + def clamp_box(box: tuple[int, int, int, int], frame_shape: tuple[int, int, int]) -> tuple[int, int, int, int] | None: frame_height, frame_width = frame_shape[:2] x1, y1, x2, y2 = box @@ -245,7 +279,14 @@ def classify_map_relation(speed_limit_mph: int, context: MapContext, next_limit_ return "no_map" -def score_review_priority(class_id: int, proposal_confidence: float, chosen_vote: ReadVote | None, support_count: int, map_relation: str, reasons: set[str]) -> float: +def score_review_priority( + class_id: int, + proposal_confidence: float, + chosen_vote: ReadVote | None, + support_count: int, + map_relation: str, + reasons: set[str], +) -> float: score = proposal_confidence * 2.0 if chosen_vote is not None: score += chosen_vote.confidence * 2.0 @@ -276,7 +317,14 @@ def summarize_votes(votes: list[ReadVote]) -> str: return "|".join(compact) -def analyze_proposal(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, proposal, full_detection, context: MapContext, args: argparse.Namespace): +def analyze_proposal( + daemon: slv.SpeedLimitVisionDaemon, + frame_bgr, + proposal, + full_detection, + context: MapContext, + args: argparse.Namespace, +): proposal_confidence, class_id, raw_box = proposal if class_id == 1 and not args.include_advisory: return None @@ -306,7 +354,8 @@ def analyze_proposal(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, proposal, fu is_regulatory = daemon._is_regulatory_speed_sign(crop) or class_id == 2 any_regulatory = any_regulatory or is_regulatory add_vote(votes, daemon._classify_speed_limit_from_model(crop), "model", expansion_index, crop_box, is_regulatory, weight) - add_vote(votes, daemon._read_speed_limit_from_crop(crop), "ocr", expansion_index, crop_box, is_regulatory, weight) + if not args.model_only: + add_vote(votes, daemon._read_speed_limit_from_crop(crop), "ocr", expansion_index, crop_box, is_regulatory, weight) chosen_vote, support_count = choose_vote(votes) if chosen_vote is None and proposal_confidence < args.no_read_min_proposal_confidence and class_id != 2: @@ -368,15 +417,10 @@ def write_image(path: Path, image, quality: int, overwrite: bool) -> None: cv2.imwrite(str(path), image, [cv2.IMWRITE_JPEG_QUALITY, quality]) -def cluster_key(route_id: str, segment: int, time_s: float, frame_shape: tuple[int, int, int], candidate: dict, dedupe_seconds: float) -> str: - x1, y1, x2, y2 = candidate["bbox"] - frame_height, frame_width = frame_shape[:2] - center_x = ((x1 + x2) / 2) / max(frame_width, 1) - center_y = ((y1 + y2) / 2) / max(frame_height, 1) +def cluster_key(route_id: str, segment: int, time_s: float, candidate: dict, dedupe_seconds: float) -> str: time_bucket = int(math.floor(time_s / max(dedupe_seconds, 0.1))) - grid_x = int(center_x * 12) - grid_y = int(center_y * 8) - return f"{route_id}|{segment}|{time_bucket}|{candidate['class_id']}|{grid_x}|{grid_y}" + candidate_speed = candidate.get("candidate_speed_limit_mph") or "none" + return f"{route_id}|{segment}|{time_bucket}|{candidate['class_id']}|{candidate_speed}" def candidate_record_key(route_key: str, segment: int, time_s: float, index: int) -> str: @@ -384,7 +428,14 @@ def candidate_record_key(route_key: str, segment: int, time_s: float, index: int return f"manual_review_{route_key}_{sample_index}" -def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse.Namespace, output_dir: Path) -> tuple[list[dict[str, object]], dict[str, object]]: +def mine_route( + route_id: str, + daemon: slv.SpeedLimitVisionDaemon, + args: argparse.Namespace, + output_dir: Path, + mining_fingerprint: str, + model_fingerprint: str, +) -> tuple[list[dict[str, object]], dict[str, object]]: route_id, dongle_id, log_id = parse_route_id(route_id) route_key = safe_key(route_id) clip_root = args.clip_root.expanduser().resolve() @@ -400,7 +451,10 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse frames_scored = 0 for segment in segments: - if frames_scored >= args.max_frames_per_route or route_candidates >= args.max_candidates_per_route: + if ( + (args.max_frames_per_route > 0 and frames_scored >= args.max_frames_per_route) or + (args.max_candidates_per_route > 0 and route_candidates >= args.max_candidates_per_route) + ): break contexts = load_segment_map_context(segment.path) capture = cv2.VideoCapture(str(segment.video_path)) @@ -414,7 +468,10 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse frame_iter = iter_frames_at_times(capture, fps, times) for time_s, frame_bgr in frame_iter: - if frames_scored >= args.max_frames_per_route or route_candidates >= args.max_candidates_per_route: + if ( + (args.max_frames_per_route > 0 and frames_scored >= args.max_frames_per_route) or + (args.max_candidates_per_route > 0 and route_candidates >= args.max_candidates_per_route) + ): break if frame_bgr is None: continue @@ -444,6 +501,8 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse crop_path = crop_dir / f"{record_key}_crop.jpg" row = { "record_key": record_key, + "mining_fingerprint": mining_fingerprint, + "model_fingerprint": model_fingerprint, "route": route_id, "dongle_id": dongle_id, "log_id": log_id, @@ -478,7 +537,7 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse "review_ignore_reason": "", "review_notes": "", } - key = cluster_key(route_id, segment.segment, time_s, frame_bgr.shape, candidate, args.dedupe_seconds) + key = cluster_key(route_id, segment.segment, time_s, candidate, args.dedupe_seconds) existing = rows_by_cluster.get(key) if existing is None or float(row["review_priority"]) > float(existing["review_priority"]): if not args.dry_run: @@ -494,6 +553,8 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse frame_path = frame_dir / f"{record_key}.jpg" row = { "record_key": record_key, + "mining_fingerprint": mining_fingerprint, + "model_fingerprint": model_fingerprint, "route": route_id, "dongle_id": dongle_id, "log_id": log_id, @@ -557,8 +618,17 @@ def write_manifest(path: Path, rows: list[dict[str, object]]) -> None: writer.writerows(rows) -def write_summary(path: Path, manifest_path: Path, rows: list[dict[str, object]], summaries: list[dict[str, object]]) -> None: +def write_summary( + path: Path, + manifest_path: Path, + rows: list[dict[str, object]], + summaries: list[dict[str, object]], + mining_fingerprint: str, + model_fingerprint: str, +) -> None: path.write_text(json.dumps({ + "mining_fingerprint": mining_fingerprint, + "model_fingerprint": model_fingerprint, "routes": summaries, "manifest": str(manifest_path), "rows": len(rows), @@ -575,6 +645,9 @@ def main() -> int: args = parse_args() configure_models(args.models_dir) + slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = not args.model_only + model_fingerprint = model_bundle_fingerprint() + mining_fingerprint = review_mining_fingerprint(args, model_fingerprint) workspace = resolve_workspace(args.workspace) output_dir = args.output_dir.expanduser().resolve() if args.output_dir else ensure_dir(workspace / "review" / DEFAULT_OUTPUT_NAME) manifest_path = args.manifest_out.expanduser().resolve() if args.manifest_out else output_dir / "manual_review_queue.csv" @@ -586,25 +659,43 @@ def main() -> int: all_rows: list[dict[str, object]] = [] summaries = [] summary_path = output_dir / "manual_review_summary.json" + completed_routes: set[str] = set() + if args.resume and summary_path.is_file(): + prior_summary = json.loads(summary_path.read_text(encoding="utf-8")) + if prior_summary.get("mining_fingerprint") != mining_fingerprint: + raise RuntimeError("Existing review queue fingerprint does not match this run. Use a new --output-dir or --no-resume.") + if manifest_path.is_file(): + with manifest_path.open("r", encoding="utf-8", newline="") as handle: + all_rows = list(csv.DictReader(handle)) + summaries = list(prior_summary.get("routes", [])) + completed_routes = {str(summary["route"]) for summary in summaries if summary.get("status") in ("mined", "missing_segments")} + elif args.resume and (manifest_path.exists() or summary_path.exists()): + raise RuntimeError("Existing review queue is missing fingerprinted resume state. Use a new --output-dir or --no-resume.") + for index, route_id in enumerate(routes, start=1): - rows, summary = mine_route(route_id, daemon, args, output_dir) + normalized_route, _, _ = parse_route_id(route_id) + if normalized_route in completed_routes: + print(f"[{index}/{len(routes)}] {normalized_route}: skipped (already mined)") + continue + rows, summary = mine_route(route_id, daemon, args, output_dir, mining_fingerprint, model_fingerprint) all_rows.extend(rows) summaries.append(summary) - print( - f"[{index}/{len(routes)}] {summary['route']}: {summary['status']} " - f"frames={summary['frames']} candidates={summary['candidates']} negatives={summary['negatives']}" - ) + progress = f"[{index}/{len(routes)}] {summary['route']}: {summary['status']}" + counts = f"frames={summary['frames']} candidates={summary['candidates']} negatives={summary['negatives']}" + print(f"{progress} {counts}") if not args.dry_run: all_rows.sort(key=lambda row: (-float(row["review_priority"]), str(row["record_key"]))) write_manifest(manifest_path, all_rows) - write_summary(summary_path, manifest_path, all_rows, summaries) + write_summary(summary_path, manifest_path, all_rows, summaries, mining_fingerprint, model_fingerprint) all_rows.sort(key=lambda row: (-float(row["review_priority"]), str(row["record_key"]))) if not args.dry_run: write_manifest(manifest_path, all_rows) - write_summary(summary_path, manifest_path, all_rows, summaries) + write_summary(summary_path, manifest_path, all_rows, summaries, mining_fingerprint, model_fingerprint) print(f"Wrote {len(all_rows)} review rows to {manifest_path}") print(f"Summary: {summary_path}") + print(f"Model fingerprint: {model_fingerprint}") + print(f"Mining fingerprint: {mining_fingerprint}") else: print(f"Dry run rows={len(all_rows)} candidates={sum(1 for row in all_rows if row['detector_class'] != 'negative_empty')}") diff --git a/scripts/speed_limit_vision/build_review_classifier_dataset.py b/scripts/speed_limit_vision/build_review_classifier_dataset.py new file mode 100644 index 000000000..4158579e9 --- /dev/null +++ b/scripts/speed_limit_vision/build_review_classifier_dataset.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import shutil + +from collections import Counter +from pathlib import Path + + +VALID_SPEEDS = frozenset((15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build an isolated classifier dataset from a base corpus and reviewed crops.") + parser.add_argument("--base", type=Path, required=True, help="Existing Ultralytics classification dataset root.") + parser.add_argument("--output", type=Path, required=True, help="New isolated dataset root.") + parser.add_argument("--positive-manifest", type=Path, action="append", default=[], help="Reviewed positive crop manifest. Repeat as needed.") + parser.add_argument("--reject-manifest", type=Path, action="append", default=[], help="Reviewed classifier reject manifest. Repeat as needed.") + parser.add_argument( + "--advisory-as-reject", + action="store_true", + help="Stage reviewed advisory-speed crops in the reject class instead of omitting them.", + ) + parser.add_argument( + "--include-advisory-positives", + action="store_true", + help="Train reviewed advisory crops as their numeric speed classes for recall-first models.", + ) + parser.add_argument( + "--advisory-reject-fraction", + type=float, + default=1.0, + help="Deterministic fraction of training advisories staged as reject; validation advisories are always retained.", + ) + return parser.parse_args() + + +def read_rows(paths: list[Path]): + for path in paths: + resolved = path.expanduser().resolve() + with resolved.open("r", encoding="utf-8", newline="") as handle: + yield from csv.DictReader(handle) + + +def remove_appledouble_files(root: Path) -> int: + removed = 0 + for path in root.rglob("._*"): + if path.is_file(): + path.unlink() + removed += 1 + return removed + + +def parse_speed(text: str) -> int: + try: + value = int(float((text or "").strip())) + except ValueError: + return 0 + return value if value in VALID_SPEEDS else 0 + + +def is_advisory(row: dict[str, str]) -> bool: + return row.get("review_sign_type", "").strip().lower() == "advisory" + + +def keep_advisory_reject(row: dict[str, str], fraction: float) -> bool: + if row.get("split") == "val" or fraction >= 1.0: + return True + if fraction <= 0.0: + return False + digest = hashlib.sha256(row.get("record_key", "").encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big") / 2**64 < fraction + + +def stage_crop(source: Path, destination_dir: Path, record_key: str) -> bool: + if not source.is_file(): + return False + digest = hashlib.sha256(source.read_bytes()).hexdigest()[:16] + suffix = source.suffix.lower() if source.suffix.lower() in (".jpg", ".jpeg", ".png") else ".jpg" + safe_key = "".join(char if char.isalnum() or char in "._-" else "_" for char in record_key)[:100] + destination_dir.mkdir(parents=True, exist_ok=True) + destination = destination_dir / f"review_{safe_key}_{digest}{suffix}" + if not destination.exists(): + shutil.copyfile(source, destination) + return True + + +def main() -> int: + args = parse_args() + if args.advisory_as_reject and args.include_advisory_positives: + raise ValueError("--advisory-as-reject and --include-advisory-positives are mutually exclusive") + if not 0.0 <= args.advisory_reject_fraction <= 1.0: + raise ValueError("--advisory-reject-fraction must be between 0 and 1") + base = args.base.expanduser().resolve() + output = args.output.expanduser().resolve() + if not base.is_dir(): + raise FileNotFoundError(base) + if output.exists(): + raise FileExistsError(f"Output dataset already exists: {output}") + shutil.copytree(base, output, copy_function=shutil.copyfile) + appledouble_removed = remove_appledouble_files(output) + + positive_counts: Counter[str] = Counter() + reject_counts: Counter[str] = Counter() + skipped = 0 + for row in read_rows(args.positive_manifest): + if is_advisory(row): + if args.include_advisory_positives: + pass + elif args.advisory_as_reject and keep_advisory_reject(row, args.advisory_reject_fraction): + split = row.get("split", "") + source = Path(row.get("crop_path", "")).expanduser() + if split in ("train", "val") and stage_crop(source, output / split / "reject", row.get("record_key", "advisory")): + reject_counts[f"advisory_{split}"] += 1 + else: + skipped += 1 + continue + else: + continue + split = row.get("split", "") + speed = parse_speed(row.get("speed_limit_mph", "")) + source = Path(row.get("crop_path", "")).expanduser() + if split not in ("train", "val") or not speed or not stage_crop(source, output / split / str(speed), row.get("record_key", "positive")): + skipped += 1 + continue + positive_counts[f"{split}/{speed}"] += 1 + + for row in read_rows(args.reject_manifest): + split = row.get("split", "") + source = Path(row.get("crop_path", "")).expanduser() + if split not in ("train", "val") or not stage_crop(source, output / split / "reject", row.get("record_key", "reject")): + skipped += 1 + continue + reject_counts[split] += 1 + + appledouble_removed += remove_appledouble_files(output) + for split in ("train", "val"): + cache_path = output / f"{split}.cache" + if cache_path.is_file(): + cache_path.unlink() + + summary = { + "base": str(base), + "output": str(output), + "positive_counts": dict(sorted(positive_counts.items())), + "reject_counts": dict(sorted(reject_counts.items())), + "skipped": skipped, + "appledouble_removed": appledouble_removed, + } + summary_path = output / "review_dataset_summary.json" + summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/compare_manual_review_queues.py b/scripts/speed_limit_vision/compare_manual_review_queues.py new file mode 100644 index 000000000..fdacf870d --- /dev/null +++ b/scripts/speed_limit_vision/compare_manual_review_queues.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json + +from collections import Counter +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Compare two fingerprinted manual-review queues by stable record key.") + parser.add_argument("--before", type=Path, required=True, help="Baseline manual_review_queue.csv.") + parser.add_argument("--after", type=Path, required=True, help="Candidate manual_review_queue.csv.") + parser.add_argument("--output-csv", type=Path, required=True, help="Changed-row output CSV.") + parser.add_argument("--review-output", type=Path, help="Optional review-compatible manifest containing the changed source rows.") + parser.add_argument("--confidence-delta", type=float, default=0.05, help="Minimum confidence-only change to report.") + return parser.parse_args() + + +def read_rows(path: Path) -> dict[str, dict[str, str]]: + with path.expanduser().resolve().open("r", encoding="utf-8", newline="") as handle: + return {row["record_key"]: row for row in csv.DictReader(handle) if row.get("record_key")} + + +def parse_float(text: str) -> float: + try: + return float(text) + except (TypeError, ValueError): + return 0.0 + + +def classify_change(before: dict[str, str] | None, after: dict[str, str] | None, confidence_delta: float) -> str: + if before is None: + return "added_proposal" + if after is None: + return "removed_proposal" + + before_speed = before.get("candidate_speed_limit_mph", "") + after_speed = after.get("candidate_speed_limit_mph", "") + if not before_speed and after_speed: + return "gained_read" + if before_speed and not after_speed: + return "lost_read" + if before_speed != after_speed: + return "value_changed" + + confidence_change = abs( + parse_float(after.get("candidate_confidence", "")) - parse_float(before.get("candidate_confidence", "")) + ) + if confidence_change >= confidence_delta: + return "confidence_changed" + return "" + + +def comparison_row(record_key: str, change: str, before: dict[str, str] | None, after: dict[str, str] | None) -> dict[str, str]: + source = after or before or {} + return { + "record_key": record_key, + "change": change, + "route": source.get("route", ""), + "segment": source.get("segment", ""), + "frame_time_s": source.get("frame_time_s", ""), + "detector_class": source.get("detector_class", ""), + "proposal_confidence": source.get("proposal_confidence", ""), + "before_speed_limit_mph": (before or {}).get("candidate_speed_limit_mph", ""), + "before_confidence": (before or {}).get("candidate_confidence", ""), + "after_speed_limit_mph": (after or {}).get("candidate_speed_limit_mph", ""), + "after_confidence": (after or {}).get("candidate_confidence", ""), + "before_support": (before or {}).get("read_support_count", ""), + "after_support": (after or {}).get("read_support_count", ""), + "frame_path": source.get("frame_path", ""), + "crop_path": source.get("crop_path", ""), + "source_video_path": source.get("source_video_path", ""), + } + + +def main() -> int: + args = parse_args() + before = read_rows(args.before) + after = read_rows(args.after) + rows: list[dict[str, str]] = [] + review_rows: list[dict[str, str]] = [] + change_counts: Counter[str] = Counter() + transition_counts: Counter[str] = Counter() + + for record_key in sorted(before.keys() | after.keys()): + before_row = before.get(record_key) + after_row = after.get(record_key) + change = classify_change(before_row, after_row, args.confidence_delta) + if not change: + continue + row = comparison_row(record_key, change, before_row, after_row) + rows.append(row) + source_row = dict(after_row or before_row or {}) + source_row.update({ + "comparison_change": change, + "before_speed_limit_mph": row["before_speed_limit_mph"], + "before_confidence": row["before_confidence"], + }) + review_rows.append(source_row) + change_counts[change] += 1 + before_speed = row["before_speed_limit_mph"] or "none" + after_speed = row["after_speed_limit_mph"] or "none" + transition_counts[f"{before_speed}->{after_speed}"] += 1 + + output_path = args.output_csv.expanduser().resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = list(rows[0]) if rows else list(comparison_row("", "", None, None)) + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + if args.review_output: + review_output = args.review_output.expanduser().resolve() + review_output.parent.mkdir(parents=True, exist_ok=True) + review_fieldnames = list(review_rows[0]) if review_rows else [] + with review_output.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=review_fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(review_rows) + + summary = { + "before": str(args.before.expanduser().resolve()), + "after": str(args.after.expanduser().resolve()), + "before_rows": len(before), + "after_rows": len(after), + "changed_rows": len(rows), + "review_output": str(args.review_output.expanduser().resolve()) if args.review_output else "", + "changes": dict(sorted(change_counts.items())), + "transitions": dict(sorted(transition_counts.items(), key=lambda item: (-item[1], item[0]))), + } + output_path.with_suffix(".json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/evaluate_review_classifier.py b/scripts/speed_limit_vision/evaluate_review_classifier.py new file mode 100644 index 000000000..5d402076b --- /dev/null +++ b/scripts/speed_limit_vision/evaluate_review_classifier.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv + +from collections import Counter +from pathlib import Path + +import cv2 + +import starpilot.system.speed_limit_vision as slv + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate the integrated value/reject classifier on reviewed crops.") + parser.add_argument("--models-dir", type=Path, default=Path("starpilot/assets/vision_models")) + parser.add_argument("--positive-manifest", type=Path, required=True) + parser.add_argument("--reject-manifest", type=Path, required=True) + parser.add_argument("--split", choices=("train", "val"), help="Optional source split filter.") + parser.add_argument("--output-csv", type=Path) + return parser.parse_args() + + +def read_rows(path: Path) -> list[dict[str, str]]: + with path.expanduser().resolve().open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def parse_speed(text: str) -> int | None: + try: + return int(float((text or "").strip())) + except ValueError: + return None + + +def main() -> int: + args = parse_args() + models_dir = args.models_dir.expanduser().resolve() + slv.US_DETECTOR_MODEL_PATH = models_dir / "speed_limit_us_detector.onnx" + slv.US_CLASSIFIER_MODEL_PATH = models_dir / "speed_limit_us_value_classifier.onnx" + reject_path = models_dir / "speed_limit_us_reject_classifier.onnx" + if reject_path.is_file(): + slv.US_REJECT_CLASSIFIER_MODEL_PATH = reject_path + daemon = slv.SpeedLimitVisionDaemon(use_runtime=False) + + cases: list[tuple[str, dict[str, str], int | None]] = [] + for row in read_rows(args.positive_manifest): + if args.split and row.get("split") != args.split: + continue + kind = "advisory" if row.get("review_sign_type", "").strip().lower() == "advisory" else "regulatory" + cases.append((kind, row, parse_speed(row.get("speed_limit_mph", "")) if kind == "regulatory" else None)) + for row in read_rows(args.reject_manifest): + if not args.split or row.get("split") == args.split: + cases.append(("hard_negative", row, None)) + + counts: Counter[str] = Counter() + output_rows = [] + for kind, row, expected in cases: + crop_path = Path(row.get("crop_path", "")).expanduser() + crop = cv2.imread(str(crop_path)) + if crop is None: + counts[f"{kind}_unreadable"] += 1 + continue + result = daemon._classify_speed_limit_from_model(crop) + predicted = result[0] if result is not None else None + confidence = result[1] if result is not None else None + counts[f"{kind}_total"] += 1 + if kind == "regulatory": + counts[f"regulatory_speed_{expected}_total"] += 1 + if predicted is not None: + counts["regulatory_any"] += 1 + if predicted == expected: + counts["regulatory_exact"] += 1 + counts[f"regulatory_speed_{expected}_exact"] += 1 + elif predicted is not None: + counts["regulatory_wrong"] += 1 + elif predicted is None: + counts[f"{kind}_rejected"] += 1 + else: + counts[f"{kind}_false_read"] += 1 + output_rows.append({ + "record_key": row.get("record_key", ""), + "split": row.get("split", ""), + "kind": kind, + "crop_path": str(crop_path), + "expected_speed_limit_mph": "" if expected is None else expected, + "predicted_speed_limit_mph": "" if predicted is None else predicted, + "confidence": "" if confidence is None else f"{confidence:.6f}", + }) + + regulatory_total = counts["regulatory_total"] + advisory_total = counts["advisory_total"] + hard_negative_total = counts["hard_negative_total"] + exact_rate = counts["regulatory_exact"] / regulatory_total if regulatory_total else 0.0 + advisory_reject_rate = counts["advisory_rejected"] / advisory_total if advisory_total else 0.0 + hard_negative_reject_rate = counts["hard_negative_rejected"] / hard_negative_total if hard_negative_total else 0.0 + regulatory_summary = f"Regulatory exact: {counts['regulatory_exact']}/{regulatory_total} ({exact_rate:.3f})" + print(f"{regulatory_summary}; wrong reads: {counts['regulatory_wrong']}") + print(f"Advisory rejected: {counts['advisory_rejected']}/{advisory_total} ({advisory_reject_rate:.3f})") + hard_negative_summary = f"Hard negatives rejected: {counts['hard_negative_rejected']}/{hard_negative_total}" + print(f"{hard_negative_summary} ({hard_negative_reject_rate:.3f})") + speed_parts = [] + for speed in slv.US_CLASSIFIER_SPEED_VALUES: + total = counts[f"regulatory_speed_{speed}_total"] + if total: + speed_parts.append(f"{speed}:{counts[f'regulatory_speed_{speed}_exact']}/{total}") + print("Exact by speed: " + " ".join(speed_parts)) + + if args.output_csv: + output = args.output_csv.expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=tuple(output_rows[0]) if output_rows else ("record_key",)) + writer.writeheader() + writer.writerows(output_rows) + print(f"Wrote {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/evaluate_reviewed_route_events.py b/scripts/speed_limit_vision/evaluate_reviewed_route_events.py new file mode 100644 index 000000000..06cd19a74 --- /dev/null +++ b/scripts/speed_limit_vision/evaluate_reviewed_route_events.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json + +from collections import Counter, defaultdict +from dataclasses import dataclass +from pathlib import Path + +import cv2 + +import starpilot.system.speed_limit_vision as slv + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from import_manual_review_queue import merged_review_rows, parse_speed # type: ignore + from replay_route_runtime import RouteReplayDaemon, configure_models # type: ignore +else: + from .import_manual_review_queue import merged_review_rows, parse_speed + from .replay_route_runtime import RouteReplayDaemon, configure_models + + +POSITIVE_STATUSES = frozenset(("accepted", "corrected")) + + +@dataclass(frozen=True) +class ReviewedCase: + record_key: str + route: str + segment: int + frame_time_s: float + source_video_path: Path + expected_speed_limit_mph: int + negative: bool + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate reviewed sign events through realistic runtime cadence and publish logic.") + parser.add_argument("--queue", type=Path, required=True, help="Reviewed manual_review_queue.csv.") + parser.add_argument("--labels", type=Path, help="Defaults to manual_review_labels.csv beside the queue.") + parser.add_argument("--models-dir", type=Path, default=Path("starpilot/assets/vision_models"), help="Candidate ONNX model directory.") + parser.add_argument("--output-csv", type=Path, required=True, help="Per-event evaluation output.") + parser.add_argument("--window-before", type=float, default=4.0, help="Seconds replayed before the reviewed frame.") + parser.add_argument("--window-after", type=float, default=3.0, help="Seconds replayed after the reviewed frame.") + parser.add_argument("--dedupe-seconds", type=float, default=3.0, help="Collapse nearby reviewed rows with the same expected value.") + parser.add_argument("--measured-base-inference-seconds", type=float, default=0.44, help="Measured no-proposal comma inference cost.") + parser.add_argument("--measured-classifier-forward-seconds", type=float, default=0.066, help="Measured comma cost per classifier forward.") + parser.add_argument("--crop-ocr", action="store_true", help="Evaluate with crop OCR confirmation enabled.") + parser.add_argument("--classifier-min-confidence", type=float, help="Override the value classifier confidence threshold.") + parser.add_argument("--trusted-model-min-confidence", type=float, help="Override tiny-box trusted model confidence.") + parser.add_argument("--strong-rescue-min-proposal-confidence", type=float, help="Override single-frame tiny-sign proposal confidence.") + parser.add_argument("--strong-rescue-min-read-confidence", type=float, help="Override single-frame tiny-sign classifier confidence.") + parser.add_argument("--low-speed-change-consistent-detections", type=int, help="Override reads required to change from 30+ mph to below 30 mph.") + parser.add_argument( + "--allow-low-speed-strong-consensus", + action="store_true", + help="Permit a strong multi-crop consensus to publish a low-speed change from one frame.", + ) + parser.add_argument( + "--enable-strong-model-consensus", + action="store_true", + help="Mark three agreeing high-confidence regulatory model crops as strong consensus.", + ) + parser.add_argument("--initial-speed-limit", type=int, default=0, help="Seed each replay window with a currently published speed limit.") + parser.add_argument("--positive-only", action="store_true", help="Replay only reviewed speed signs, omitting ignored-crop windows.") + parser.add_argument("--max-cases", type=int, default=0, help="Optional evaluation cap after deduplication.") + return parser.parse_args() + + +def load_cases(queue_path: Path, labels_path: Path, dedupe_seconds: float) -> list[ReviewedCase]: + rows = merged_review_rows(queue_path, labels_path) + cases: list[ReviewedCase] = [] + seen_buckets: set[tuple[str, int, int, int, bool]] = set() + for row in rows: + status = row.get("review_status", "") + positive = status in POSITIVE_STATUSES + negative = status == "ignore" and row.get("review_sign_type") == "not_speed_limit" + if not positive and not negative: + continue + speed = parse_speed(row.get("review_speed_limit_mph", "")) if positive else 0 + if positive and not speed: + continue + try: + segment = int(row.get("segment", "")) + frame_time_s = float(row.get("frame_time_s", "")) + except ValueError: + continue + source_video = Path(row.get("source_video_path", "")).expanduser() + if not source_video.is_file(): + continue + bucket = int(frame_time_s / max(dedupe_seconds, 0.1)) + dedupe_key = (row.get("route", ""), segment, bucket, speed, negative) + if dedupe_key in seen_buckets: + continue + seen_buckets.add(dedupe_key) + cases.append(ReviewedCase( + record_key=row.get("record_key", ""), + route=row.get("route", ""), + segment=segment, + frame_time_s=frame_time_s, + source_video_path=source_video.resolve(), + expected_speed_limit_mph=speed, + negative=negative, + )) + return cases + + +def replay_video_cases(cases: list[ReviewedCase], args: argparse.Namespace) -> dict[str, tuple[list[int], list[int], int]]: + daemons = { + case.record_key: RouteReplayDaemon( + runtime_context=None, + measured_inference_seconds=0.0, + measured_base_inference_seconds=args.measured_base_inference_seconds, + measured_classifier_forward_seconds=args.measured_classifier_forward_seconds, + ) + for case in cases + } + for daemon in daemons.values(): + daemon.published_speed_limit_mph = args.initial_speed_limit + daemon.last_published_support_at = 0.0 + capture = cv2.VideoCapture(str(cases[0].source_video_path)) + fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + duration_s = frame_count / fps if frame_count > 0 else 60.0 + windows = { + case.record_key: ( + max(case.frame_time_s - args.window_before, 0.0), + min(case.frame_time_s + args.window_after, duration_s), + ) + for case in cases + } + first_frame = max(int(min(window[0] for window in windows.values()) * fps), 0) + end_frame = max(int(max(window[1] for window in windows.values()) * fps), first_frame) + capture.set(cv2.CAP_PROP_POS_FRAMES, first_frame) + frame_index = first_frame + + while frame_index <= end_frame: + ok, frame_bgr = capture.read() + if not ok: + break + frame_time_s = frame_index / fps + for case in cases: + start_s, end_s = windows[case.record_key] + if start_s <= frame_time_s <= end_s: + daemons[case.record_key].process_frame(frame_time_s - start_s, frame_bgr) + frame_index += 1 + capture.release() + + results = {} + for case in cases: + daemon = daemons[case.record_key] + candidates = [int(event["candidateSpeedLimitMph"]) for event in daemon.events if event["event"] == "candidate"] + publishes = [int(event["speedLimitMph"]) for event in daemon.events if event["event"] == "publish"] + results[case.record_key] = candidates, publishes, daemon.inference_frames + return results + + +def main() -> int: + args = parse_args() + queue_path = args.queue.expanduser().resolve() + labels_path = args.labels.expanduser().resolve() if args.labels else queue_path.with_name("manual_review_labels.csv") + configure_models(args.models_dir) + slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = args.crop_ocr + if args.classifier_min_confidence is not None: + slv.US_CLASSIFIER_MIN_CONFIDENCE = args.classifier_min_confidence + if args.trusted_model_min_confidence is not None: + slv.DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE = args.trusted_model_min_confidence + if args.strong_rescue_min_proposal_confidence is not None: + slv.DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_PROPOSAL_CONFIDENCE = args.strong_rescue_min_proposal_confidence + if args.strong_rescue_min_read_confidence is not None: + slv.DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_READ_CONFIDENCE = args.strong_rescue_min_read_confidence + if args.low_speed_change_consistent_detections is not None: + slv.LOW_SPEED_CHANGE_CONSISTENT_DETECTIONS = args.low_speed_change_consistent_detections + if args.allow_low_speed_strong_consensus: + slv.LOW_SPEED_CHANGE_ALLOW_STRONG_CONSENSUS = True + if args.enable_strong_model_consensus: + slv.DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_ENABLED = True + cases = load_cases(queue_path, labels_path, args.dedupe_seconds) + if args.positive_only: + cases = [case for case in cases if not case.negative] + if args.max_cases > 0: + cases = cases[:args.max_cases] + + output_rows: list[dict[str, object]] = [] + positive_by_speed: dict[int, Counter[str]] = defaultdict(Counter) + negative_counts: Counter[str] = Counter() + results: dict[str, tuple[list[int], list[int], int]] = {} + cases_by_video: dict[Path, list[ReviewedCase]] = defaultdict(list) + for case in cases: + cases_by_video[case.source_video_path].append(case) + for index, video_cases in enumerate(cases_by_video.values(), start=1): + results.update(replay_video_cases(video_cases, args)) + if index % 10 == 0: + print(f"Replayed {index}/{len(cases_by_video)} video segments", flush=True) + + for case in cases: + candidates, publishes, inference_frames = results.get(case.record_key, ([], [], 0)) + candidate_hit = case.expected_speed_limit_mph in candidates if not case.negative else False + publish_hit = case.expected_speed_limit_mph in publishes if not case.negative else False + false_candidate = bool(candidates) if case.negative else False + false_publish = bool(publishes) if case.negative else False + if case.negative: + negative_counts.update(total=1, candidate_fp=int(false_candidate), publish_fp=int(false_publish)) + else: + positive_by_speed[case.expected_speed_limit_mph].update( + total=1, + candidate_hit=int(candidate_hit), + publish_hit=int(publish_hit), + ) + output_rows.append({ + "record_key": case.record_key, + "route": case.route, + "segment": case.segment, + "frame_time_s": f"{case.frame_time_s:.3f}", + "expected_speed_limit_mph": "" if case.negative else case.expected_speed_limit_mph, + "negative": case.negative, + "candidate_values": "|".join(str(value) for value in candidates), + "publish_values": "|".join(str(value) for value in publishes), + "candidate_hit": candidate_hit, + "publish_hit": publish_hit, + "false_candidate": false_candidate, + "false_publish": false_publish, + "inference_frames": inference_frames, + "source_video_path": str(case.source_video_path), + }) + output_path = args.output_csv.expanduser().resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = list(output_rows[0]) if output_rows else [] + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(output_rows) + + totals = Counter() + for counts in positive_by_speed.values(): + totals.update(counts) + negative_scope = " ".join(( + "Reviewed negative labels apply to the proposed crop.", + "Candidate/publish counts replay the full video window and are an upper bound, not confirmed false positives,", + "because another valid sign may be visible in that window.", + )) + summary = { + "models_dir": str(args.models_dir.expanduser().resolve()), + "crop_ocr": args.crop_ocr, + "classifier_min_confidence": slv.US_CLASSIFIER_MIN_CONFIDENCE, + "measured_base_inference_seconds": args.measured_base_inference_seconds, + "measured_classifier_forward_seconds": args.measured_classifier_forward_seconds, + "initial_speed_limit_mph": args.initial_speed_limit, + "low_speed_change_consistent_detections": slv.LOW_SPEED_CHANGE_CONSISTENT_DETECTIONS, + "low_speed_change_allow_strong_consensus": slv.LOW_SPEED_CHANGE_ALLOW_STRONG_CONSENSUS, + "strong_model_consensus_enabled": slv.DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_ENABLED, + "positive": dict(totals), + "positive_by_speed": {str(speed): dict(counts) for speed, counts in sorted(positive_by_speed.items())}, + "negative": dict(negative_counts), + "negative_scope": negative_scope, + } + summary_path = output_path.with_suffix(".json") + summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/evaluate_runtime_manifest.py b/scripts/speed_limit_vision/evaluate_runtime_manifest.py index 466ed7b01..6d12080b4 100644 --- a/scripts/speed_limit_vision/evaluate_runtime_manifest.py +++ b/scripts/speed_limit_vision/evaluate_runtime_manifest.py @@ -20,7 +20,7 @@ def parse_args() -> argparse.Namespace: default=Path("starpilot/assets/vision_models"), help="Directory containing speed_limit_us_detector.onnx and speed_limit_us_value_classifier.onnx.", ) - parser.add_argument("--manifest", type=Path, required=True, help="CSV manifest with dataset_image/frame_path and labels.") + parser.add_argument("--manifest", type=Path, required=True, help="CSV manifest with dataset_image/frame_path/image_path and labels.") parser.add_argument("--split", action="append", help="Optional split filter. Repeat for multiple splits.") parser.add_argument("--max-rows", type=int, default=0, help="Optional cap after filtering.") parser.add_argument("--seed", type=int, default=0, help="Sampling seed used with --max-rows.") @@ -28,6 +28,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--detector-min-confidence", type=float, help="Override runtime US detector confidence threshold.") parser.add_argument("--classifier-min-confidence", type=float, help="Override runtime US classifier confidence threshold.") parser.add_argument("--classifier-reject-min-confidence", type=float, help="Override runtime reject-class confidence threshold.") + parser.add_argument("--trusted-model-min-confidence", type=float, help="Override tiny-box trusted model confidence for evaluation.") + parser.add_argument("--strong-rescue-min-proposal-confidence", type=float, help="Override single-frame tiny-sign proposal confidence.") + parser.add_argument("--strong-rescue-min-read-confidence", type=float, help="Override single-frame tiny-sign classifier confidence.") parser.add_argument( "--detector-region-mode", choices=("full", "right_roi", "full_and_right_roi"), @@ -36,7 +39,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--right-roi-bounds", help="Override the right ROI as left,top,right,bottom ratios, for example 0.45,0,1,0.82.") parser.add_argument("--right-roi-min-confidence", type=float, help="Override the right ROI detector minimum confidence.") parser.add_argument("--full-frame-ocr", action="store_true", help="Enable the expensive full-frame OCR fallback during eval.") + crop_ocr_group = parser.add_mutually_exclusive_group() + crop_ocr_group.add_argument("--crop-ocr", action="store_true", dest="crop_ocr", default=None, help="Enable crop OCR confirmation.") + crop_ocr_group.add_argument("--no-crop-ocr", action="store_false", dest="crop_ocr", help="Evaluate the model-only detector/classifier path.") + parser.add_argument("--separate-reject-classifier", action="store_true", help="Enable the optional second-stage reject classifier during eval.") parser.add_argument("--include-uncertain", action="store_true", help="Include uncertain_positive review rows in positive metrics.") + parser.add_argument("--advisory-positive", action="store_true", help="Score reviewed advisory rows as readable speed positives.") parser.add_argument("--strict-positive-recall", type=float, help="Exit non-zero if positive exact recall is below this value.") parser.add_argument("--strict-negative-fpr", type=float, help="Exit non-zero if negative false-positive rate is above this value.") return parser.parse_args() @@ -48,6 +56,10 @@ def configure_runtime_options(args: argparse.Namespace) -> None: if args.full_frame_ocr: slv.FULL_FRAME_OCR_FALLBACK_ENABLED = True + if args.crop_ocr is not None: + slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = args.crop_ocr + if args.separate_reject_classifier: + slv.SEPARATE_REJECT_CLASSIFIER_ENABLED = True if args.right_roi_bounds: parts = [float(part.strip()) for part in args.right_roi_bounds.split(",")] @@ -81,7 +93,7 @@ def first_present(row: dict[str, str], keys: tuple[str, ...]) -> str: def expected_value(row: dict[str, str]) -> int | None: - value_text = first_present(row, ("speed_limit_mph", "dominant_value")) + value_text = first_present(row, ("expected_speed_limit_mph", "speed_limit_mph", "dominant_value")) if value_text: try: return int(float(value_text)) @@ -151,6 +163,12 @@ def main() -> int: if args.classifier_reject_min_confidence is not None: slv.US_CLASSIFIER_REJECT_MIN_CONFIDENCE = args.classifier_reject_min_confidence slv.US_REJECT_CLASSIFIER_MIN_CONFIDENCE = args.classifier_reject_min_confidence + if args.trusted_model_min_confidence is not None: + slv.DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE = args.trusted_model_min_confidence + if args.strong_rescue_min_proposal_confidence is not None: + slv.DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_PROPOSAL_CONFIDENCE = args.strong_rescue_min_proposal_confidence + if args.strong_rescue_min_read_confidence is not None: + slv.DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_READ_CONFIDENCE = args.strong_rescue_min_read_confidence configure_runtime_options(args) daemon = slv.SpeedLimitVisionDaemon(use_runtime=False) @@ -163,7 +181,7 @@ def main() -> int: unreadable_count = 0 for row in rows: - image_text = first_present(row, ("dataset_image", "frame_path", "source_frame")) + image_text = first_present(row, ("dataset_image", "frame_path", "source_frame", "image_path")) if not image_text: unreadable_count += 1 continue @@ -178,7 +196,8 @@ def main() -> int: predicted_value = detection.speed_limit_mph if detection is not None else None confidence = detection.confidence if detection is not None else None expected = expected_value(row) - negative = is_negative(row) + advisory_positive = args.advisory_positive and row.get("review_sign_type", "").strip().lower() == "advisory" + negative = False if advisory_positive else is_negative(row) if negative: negative_count += 1 @@ -211,10 +230,9 @@ def main() -> int: if uncertain_count and not args.include_uncertain: print(f"Skipped uncertain rows: {uncertain_count}") print(f"Unreadable rows: {unreadable_count}") - print( - f"Positive exact: {positive_exact}/{positive_count} " - f"({positive_exact_recall:.3f}); any detection: {positive_detected}/{positive_count} ({positive_any_recall:.3f})" - ) + positive_summary = f"Positive exact: {positive_exact}/{positive_count} ({positive_exact_recall:.3f})" + detection_summary = f"any detection: {positive_detected}/{positive_count} ({positive_any_recall:.3f})" + print(f"{positive_summary}; {detection_summary}") print(f"Negative false positives: {negative_false_positive}/{negative_count} ({negative_fpr:.3f})") if args.output_csv: diff --git a/scripts/speed_limit_vision/import_manual_review_queue.py b/scripts/speed_limit_vision/import_manual_review_queue.py index a2971a95a..75b6c3ae0 100644 --- a/scripts/speed_limit_vision/import_manual_review_queue.py +++ b/scripts/speed_limit_vision/import_manual_review_queue.py @@ -14,13 +14,16 @@ import cv2 if __package__ in (None, ""): import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) - from common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore + from common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore # noqa: TID251 else: from .common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace CLASSIFIER_FIELDNAMES = [ "record_key", + "route", + "log_id", + "segment", "split", "speed_limit_mph", "review_sign_type", @@ -36,6 +39,9 @@ CLASSIFIER_FIELDNAMES = [ RUNTIME_FIELDNAMES = [ "record_key", + "route", + "log_id", + "segment", "split", "sample_type", "dataset_image", @@ -49,6 +55,9 @@ RUNTIME_FIELDNAMES = [ DETECTOR_MANIFEST_FIELDNAMES = [ "record_key", + "route", + "log_id", + "segment", "split", "sample_type", "speed_limit_mph", @@ -62,6 +71,23 @@ DETECTOR_MANIFEST_FIELDNAMES = [ "detector_class", ] +REJECT_FIELDNAMES = [ + "record_key", + "route", + "log_id", + "segment", + "split", + "crop_path", + "frame_path", + "bbox", + "crop_bbox", + "candidate_speed_limit_mph", + "candidate_confidence", + "detector_class", + "review_ignore_reason", + "review_notes", +] + POSITIVE_STATUSES = {"accepted", "corrected"} UNCERTAIN_STATUS = "uncertain" NEGATIVE_STATUS = "ignore" @@ -81,6 +107,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--classifier-manifest-out", type=Path, help="Positive crop manifest for import_manifest_classifier_masks.py.") parser.add_argument("--runtime-manifest-out", type=Path, help="Full-frame eval manifest including positives and true negatives.") parser.add_argument("--detector-manifest-out", type=Path, help="Manifest of imported detector examples.") + parser.add_argument("--reject-manifest-out", type=Path, help="Reviewed proposal crops that should train the classifier reject class.") parser.add_argument("--source-name", default="manual_review", help="Filename prefix for detector dataset imports.") parser.add_argument("--mode", choices=("symlink", "copy"), default="symlink", help="How to place detector images.") parser.add_argument("--val-modulo", type=int, default=5, help="Hash modulo for validation split. 0 sends everything to train.") @@ -110,6 +137,10 @@ def split_for_key(key: str, val_modulo: int, val_remainder: int) -> str: return "val" if int(digest[:8], 16) % val_modulo == val_remainder else "train" +def split_group_key(row: dict[str, str]) -> str: + return row.get("route") or row.get("log_id") or row["record_key"] + + def parse_speed(text: str) -> int: text = (text or "").strip() if not text: @@ -227,6 +258,10 @@ def is_positive(row: dict[str, str]) -> bool: return Path(row.get("crop_path", "")).is_file() and Path(row.get("frame_path", "")).is_file() +def is_advisory_positive(row: dict[str, str]) -> bool: + return is_positive(row) and effective_sign_type(row) == "advisory" + + def is_uncertain_positive(row: dict[str, str]) -> bool: if row.get("review_status") != UNCERTAIN_STATUS: return False @@ -243,10 +278,40 @@ def is_true_negative(row: dict[str, str]) -> bool: return Path(row.get("frame_path", "")).is_file() +def is_classifier_reject(row: dict[str, str]) -> bool: + if row.get("review_status") != NEGATIVE_STATUS or row.get("detector_class") == "negative_empty": + return False + if row.get("review_sign_type") != "not_speed_limit": + return False + return Path(row.get("crop_path", "")).is_file() + + +def classifier_reject_row(row: dict[str, str], split: str) -> dict[str, object]: + return { + "record_key": row["record_key"], + "route": row.get("route", ""), + "log_id": row.get("log_id", ""), + "segment": row.get("segment", ""), + "split": split, + "crop_path": row.get("crop_path", ""), + "frame_path": row.get("frame_path", ""), + "bbox": row.get("bbox", ""), + "crop_bbox": row.get("crop_bbox", ""), + "candidate_speed_limit_mph": row.get("candidate_speed_limit_mph", ""), + "candidate_confidence": row.get("candidate_confidence", ""), + "detector_class": row.get("detector_class", ""), + "review_ignore_reason": row.get("review_ignore_reason", ""), + "review_notes": row.get("review_notes", ""), + } + + def positive_classifier_row(row: dict[str, str], split: str) -> dict[str, object]: speed = parse_speed(row.get("review_speed_limit_mph", "")) return { "record_key": row["record_key"], + "route": row.get("route", ""), + "log_id": row.get("log_id", ""), + "segment": row.get("segment", ""), "split": split, "speed_limit_mph": speed, "review_sign_type": effective_sign_type(row), @@ -262,9 +327,13 @@ def positive_classifier_row(row: dict[str, str], split: str) -> dict[str, object def runtime_row(row: dict[str, str], split: str, sample_type: str) -> dict[str, object]: - speed = parse_speed(row.get("review_speed_limit_mph", "")) if sample_type in ("positive", "uncertain_positive") else 0 + positive_sample_types = ("positive", "uncertain_positive", "advisory_negative") + speed = parse_speed(row.get("review_speed_limit_mph", "")) if sample_type in positive_sample_types else 0 return { "record_key": row["record_key"], + "route": row.get("route", ""), + "log_id": row.get("log_id", ""), + "segment": row.get("segment", ""), "split": split, "sample_type": sample_type, "dataset_image": row.get("frame_path", ""), @@ -314,6 +383,9 @@ def import_detector_example( return { "record_key": row["record_key"], + "route": row.get("route", ""), + "log_id": row.get("log_id", ""), + "segment": row.get("segment", ""), "split": split, "sample_type": sample_type, "speed_limit_mph": parse_speed(row.get("review_speed_limit_mph", "")) if sample_type == "positive" else "", @@ -334,67 +406,94 @@ def main() -> int: queue_path = args.queue.expanduser().resolve() labels_path = args.labels.expanduser().resolve() if args.labels else queue_path.with_name("manual_review_labels.csv") output_dir = queue_path.parent - classifier_manifest = args.classifier_manifest_out.expanduser().resolve() if args.classifier_manifest_out else output_dir / "manual_review_classifier_manifest.csv" - runtime_manifest = args.runtime_manifest_out.expanduser().resolve() if args.runtime_manifest_out else output_dir / "manual_review_runtime_eval_manifest.csv" - detector_manifest = args.detector_manifest_out.expanduser().resolve() if args.detector_manifest_out else output_dir / "manual_review_detector_import_manifest.csv" + classifier_manifest = ( + args.classifier_manifest_out.expanduser().resolve() + if args.classifier_manifest_out else output_dir / "manual_review_classifier_manifest.csv" + ) + runtime_manifest = ( + args.runtime_manifest_out.expanduser().resolve() + if args.runtime_manifest_out else output_dir / "manual_review_runtime_eval_manifest.csv" + ) + detector_manifest = ( + args.detector_manifest_out.expanduser().resolve() + if args.detector_manifest_out else output_dir / "manual_review_detector_import_manifest.csv" + ) + reject_manifest = ( + args.reject_manifest_out.expanduser().resolve() + if args.reject_manifest_out else output_dir / "manual_review_classifier_reject_manifest.csv" + ) rows = merged_review_rows(queue_path, labels_path) positive_rows = [row for row in rows if is_positive(row)] + advisory_positive_rows = [row for row in positive_rows if is_advisory_positive(row)] uncertain_positive_rows = [row for row in rows if is_uncertain_positive(row)] true_negative_rows = [row for row in rows if is_true_negative(row)] + classifier_reject_rows = [row for row in rows if is_classifier_reject(row)] if args.max_detector_negatives > 0: true_negative_rows = true_negative_rows[:args.max_detector_negatives] classifier_rows: list[dict[str, object]] = [] runtime_rows: list[dict[str, object]] = [] detector_rows: list[dict[str, object]] = [] + reject_rows: list[dict[str, object]] = [] for row in positive_rows: - split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder) + split = split_for_key(split_group_key(row), args.val_modulo, args.val_remainder) classifier_rows.append(positive_classifier_row(row, split)) - runtime_rows.append(runtime_row(row, split, "positive")) + sample_type = "advisory_negative" if is_advisory_positive(row) else "positive" + runtime_rows.append(runtime_row(row, split, sample_type)) detector_row = import_detector_example(workspace, row, split, args.source_name, "positive", args.mode, args.overwrite) if detector_row is not None: detector_rows.append(detector_row) for row in uncertain_positive_rows: - split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder) + split = split_for_key(split_group_key(row), args.val_modulo, args.val_remainder) runtime_rows.append(runtime_row(row, split, "uncertain_positive")) for row in true_negative_rows: - split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder) + split = split_for_key(split_group_key(row), args.val_modulo, args.val_remainder) runtime_rows.append(runtime_row(row, split, "negative_empty")) detector_row = import_detector_example(workspace, row, split, args.source_name, "negative_empty", args.mode, args.overwrite) if detector_row is not None: detector_rows.append(detector_row) + for row in classifier_reject_rows: + split = split_for_key(split_group_key(row), args.val_modulo, args.val_remainder) + reject_rows.append(classifier_reject_row(row, split)) + write_csv(classifier_manifest, CLASSIFIER_FIELDNAMES, classifier_rows) write_csv(runtime_manifest, RUNTIME_FIELDNAMES, runtime_rows) write_csv(detector_manifest, DETECTOR_MANIFEST_FIELDNAMES, detector_rows) + write_csv(reject_manifest, REJECT_FIELDNAMES, reject_rows) summary = { "queue": str(queue_path), "labels": str(labels_path), "reviewed_rows": len(rows), "positive_rows": len(positive_rows), + "advisory_positive_rows": len(advisory_positive_rows), "uncertain_positive_rows": len(uncertain_positive_rows), "true_negative_rows": len(true_negative_rows), + "classifier_reject_rows": len(reject_rows), "classifier_manifest": str(classifier_manifest), "runtime_manifest": str(runtime_manifest), "detector_manifest": str(detector_manifest), "detector_imported": len(detector_rows), + "reject_manifest": str(reject_manifest), } summary_path = output_dir / "manual_review_import_summary.json" summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") - print( - "Imported manual review queue: " - f"reviewed={len(rows)} positives={len(positive_rows)} uncertain_positives={len(uncertain_positive_rows)} true_negatives={len(true_negative_rows)} " - f"detector_imported={len(detector_rows)}" - ) + review_counts = " ".join(( + f"reviewed={len(rows)} positives={len(positive_rows)}", + f"uncertain_positives={len(uncertain_positive_rows)} true_negatives={len(true_negative_rows)}", + )) + import_counts = f"classifier_rejects={len(reject_rows)} detector_imported={len(detector_rows)}" + print(f"Imported manual review queue: {review_counts} {import_counts}") print(f"Classifier manifest: {classifier_manifest}") print(f"Runtime eval manifest: {runtime_manifest}") print(f"Detector import manifest: {detector_manifest}") + print(f"Classifier reject manifest: {reject_manifest}") print(f"Summary: {summary_path}") return 0 diff --git a/scripts/speed_limit_vision/interpolate_yolo_checkpoints.py b/scripts/speed_limit_vision/interpolate_yolo_checkpoints.py new file mode 100644 index 000000000..65cc3ea6e --- /dev/null +++ b/scripts/speed_limit_vision/interpolate_yolo_checkpoints.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import copy + +from pathlib import Path + +import torch + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Linearly interpolate compatible Ultralytics checkpoints.") + parser.add_argument("--base", type=Path, required=True, help="Baseline checkpoint used at alpha=0.") + parser.add_argument("--candidate", type=Path, required=True, help="Candidate checkpoint used at alpha=1.") + parser.add_argument("--alpha", type=float, required=True, help="Candidate weight in [0, 1].") + parser.add_argument("--output", type=Path, required=True, help="Interpolated checkpoint path.") + return parser.parse_args() + + +def checkpoint_model(checkpoint: dict): + model = checkpoint.get("ema") or checkpoint.get("model") + if model is None: + raise ValueError("Checkpoint contains neither model nor ema weights") + return model.float() + + +def main() -> int: + args = parse_args() + if not 0.0 <= args.alpha <= 1.0: + raise ValueError("--alpha must be between 0 and 1") + base_checkpoint = torch.load(args.base.expanduser().resolve(), map_location="cpu", weights_only=False) + candidate_checkpoint = torch.load(args.candidate.expanduser().resolve(), map_location="cpu", weights_only=False) + base_model = checkpoint_model(base_checkpoint) + candidate_model = checkpoint_model(candidate_checkpoint) + base_state = base_model.state_dict() + candidate_state = candidate_model.state_dict() + if base_state.keys() != candidate_state.keys(): + raise ValueError("Checkpoint model state keys differ") + + interpolated_state = {} + for key, base_value in base_state.items(): + candidate_value = candidate_state[key] + if base_value.shape != candidate_value.shape: + raise ValueError(f"Checkpoint tensor shape differs for {key}") + if torch.is_floating_point(base_value): + interpolated_state[key] = base_value * (1.0 - args.alpha) + candidate_value * args.alpha + else: + interpolated_state[key] = candidate_value if args.alpha >= 0.5 else base_value + + interpolated_model = copy.deepcopy(base_model) + interpolated_model.load_state_dict(interpolated_state) + output_checkpoint = dict(base_checkpoint) + output_checkpoint.update({ + "model": interpolated_model, + "ema": None, + "optimizer": None, + "epoch": -1, + "best_fitness": None, + }) + output = args.output.expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + torch.save(output_checkpoint, output) + print(f"Wrote alpha={args.alpha:.4f} interpolated checkpoint to {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/localize_bookmark_signs.py b/scripts/speed_limit_vision/localize_bookmark_signs.py index 18cc79fb1..f8e5a6940 100644 --- a/scripts/speed_limit_vision/localize_bookmark_signs.py +++ b/scripts/speed_limit_vision/localize_bookmark_signs.py @@ -100,7 +100,15 @@ def iter_context_frames(clip_root: Path, window: ebl.BookmarkWindow, search_befo yield relative_time_s, clip_path, source_time_s, frame_bgr -def _score_expanded_candidate(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, class_id: int, proposal_confidence: float, box, full_detection): +def _score_expanded_candidate( + daemon: slv.SpeedLimitVisionDaemon, + frame_bgr, + class_id: int, + proposal_confidence: float, + box, + full_detection, + use_ocr: bool = True, +): frame_height, frame_width = frame_bgr.shape[:2] x1, y1, x2, y2 = box box_width = x2 - x1 @@ -123,7 +131,7 @@ def _score_expanded_candidate(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, cla is_regulatory = daemon._is_regulatory_speed_sign(sign_crop) or class_id == 2 model_read = daemon._classify_speed_limit_from_model(sign_crop) - ocr_read = daemon._read_speed_limit_from_crop(sign_crop) + ocr_read = daemon._read_speed_limit_from_crop(sign_crop) if use_ocr else None if model_read is None and ocr_read is None: continue @@ -132,9 +140,14 @@ def _score_expanded_candidate(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, cla if read_result is None or read_result[0] not in slv.SCHOOL_ZONE_SPEED_VALUES: continue elif not is_regulatory: - if model_read is None or ocr_read is None or model_read[0] != ocr_read[0]: + if use_ocr: + if model_read is None or ocr_read is None or model_read[0] != ocr_read[0]: + continue + read_result = (model_read[0], min(model_read[1], ocr_read[1])) + elif model_read is None or model_read[1] < slv.DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE: continue - read_result = (model_read[0], min(model_read[1], ocr_read[1])) + else: + read_result = model_read else: if model_read is not None and ocr_read is not None and model_read[0] == ocr_read[0]: read_result = (model_read[0], max(model_read[1], ocr_read[1])) @@ -169,7 +182,7 @@ def _score_expanded_candidate(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, cla return best -def score_frame(daemon: slv.SpeedLimitVisionDaemon, frame_bgr): +def score_frame(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, use_ocr: bool = True): full_detection = daemon._detect_sign(frame_bgr) best = None @@ -177,7 +190,15 @@ def score_frame(daemon: slv.SpeedLimitVisionDaemon, frame_bgr): if class_id == 1: continue - candidate = _score_expanded_candidate(daemon, frame_bgr, class_id, proposal_confidence, (x1, y1, x2, y2), full_detection) + candidate = _score_expanded_candidate( + daemon, + frame_bgr, + class_id, + proposal_confidence, + (x1, y1, x2, y2), + full_detection, + use_ocr=use_ocr, + ) if candidate is None: continue if best is None or candidate["score"] > best["score"]: diff --git a/scripts/speed_limit_vision/merge_manual_review_queues.py b/scripts/speed_limit_vision/merge_manual_review_queues.py new file mode 100644 index 000000000..370a1a969 --- /dev/null +++ b/scripts/speed_limit_vision/merge_manual_review_queues.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json + +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Merge fingerprint-compatible speed-limit manual review queues.") + parser.add_argument("inputs", type=Path, nargs="+", help="Queue directories or manual_review_queue.csv files.") + parser.add_argument("--output-dir", type=Path, required=True, help="Destination queue directory.") + return parser.parse_args() + + +def queue_paths(input_path: Path) -> tuple[Path, Path]: + resolved = input_path.expanduser().resolve() + if resolved.is_dir(): + return resolved / "manual_review_queue.csv", resolved / "manual_review_summary.json" + return resolved, resolved.with_name("manual_review_summary.json") + + +def main() -> int: + args = parse_args() + rows_by_key: dict[str, dict[str, str]] = {} + summaries_by_route: dict[str, dict[str, object]] = {} + fieldnames: list[str] | None = None + mining_fingerprint = "" + model_fingerprint = "" + + for input_path in args.inputs: + queue_path, summary_path = queue_paths(input_path) + if not queue_path.is_file() or not summary_path.is_file(): + raise FileNotFoundError(f"Queue or summary missing for {input_path}") + summary = json.loads(summary_path.read_text(encoding="utf-8")) + current_mining = str(summary.get("mining_fingerprint", "")) + current_model = str(summary.get("model_fingerprint", "")) + if not current_mining or not current_model: + raise RuntimeError(f"Queue is not fingerprinted: {input_path}") + if mining_fingerprint and current_mining != mining_fingerprint: + raise RuntimeError(f"Mining fingerprint mismatch: {input_path}") + if model_fingerprint and current_model != model_fingerprint: + raise RuntimeError(f"Model fingerprint mismatch: {input_path}") + mining_fingerprint = current_mining + model_fingerprint = current_model + + with queue_path.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + current_fields = list(reader.fieldnames or []) + if fieldnames is not None and current_fields != fieldnames: + raise RuntimeError(f"Queue fields differ: {input_path}") + fieldnames = current_fields + for row in reader: + key = row.get("record_key", "") + if key: + rows_by_key[key] = row + for route_summary in summary.get("routes", []): + route = str(route_summary.get("route", "")) + if route: + summaries_by_route[route] = route_summary + + rows = sorted(rows_by_key.values(), key=lambda row: (-float(row.get("review_priority") or 0.0), row["record_key"])) + route_summaries = sorted(summaries_by_route.values(), key=lambda item: str(item["route"])) + output_dir = args.output_dir.expanduser().resolve() + output_dir.mkdir(parents=True, exist_ok=True) + queue_output = output_dir / "manual_review_queue.csv" + with queue_output.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames or []) + writer.writeheader() + writer.writerows(rows) + + summary_output = output_dir / "manual_review_summary.json" + summary_output.write_text(json.dumps({ + "mining_fingerprint": mining_fingerprint, + "model_fingerprint": model_fingerprint, + "manifest": str(queue_output), + "rows": len(rows), + "candidates": sum(row.get("detector_class") != "negative_empty" for row in rows), + "negatives": sum(row.get("detector_class") == "negative_empty" for row in rows), + "routes": route_summaries, + }, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"Merged {len(rows)} rows from {len(route_summaries)} routes into {queue_output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/mine_classifier_reject_crops.py b/scripts/speed_limit_vision/mine_classifier_reject_crops.py new file mode 100644 index 000000000..01e58d2d2 --- /dev/null +++ b/scripts/speed_limit_vision/mine_classifier_reject_crops.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import hashlib + +from pathlib import Path + +import cv2 + +import starpilot.system.speed_limit_vision as slv + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from localize_bookmark_signs import configure_models # type: ignore +else: + from .localize_bookmark_signs import configure_models + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Mine model-confusing negative crops into an integrated classifier reject class.") + parser.add_argument("--manifest", type=Path, action="append", required=True, help="Reviewed runtime manifest. Repeat for multiple sets.") + parser.add_argument("--models-dir", type=Path, required=True, help="Detector/classifier ONNX bundle used to mine hard negatives.") + parser.add_argument("--dataset", type=Path, required=True, help="Classifier dataset containing train/reject and val/reject.") + parser.add_argument("--split", choices=("train", "val"), default="train", help="Destination dataset split.") + parser.add_argument("--classifier-min-confidence", type=float, default=0.55, help="Minimum wrong speed confidence to mine.") + parser.add_argument("--max-crops", type=int, default=2000, help="Maximum reject crops to add.") + parser.add_argument("--overwrite", action="store_true", help="Overwrite an existing crop with the same content hash.") + return parser.parse_args() + + +def first_present(row: dict[str, str], keys: tuple[str, ...]) -> str: + for key in keys: + value = row.get(key, "").strip() + if value: + return value + return "" + + +def expected_value(row: dict[str, str]) -> int | None: + value = first_present(row, ("expected_speed_limit_mph", "speed_limit_mph", "dominant_value")) + if not value: + return None + try: + return int(float(value)) + except ValueError: + return None + + +def is_reviewed_negative(row: dict[str, str]) -> bool: + sample_type = row.get("sample_type", "").lower() + review_status = row.get("review_status", "").lower() + explicit_negative = row.get("negative", "").strip().lower() in ("1", "true", "yes") + return explicit_negative or "negative" in sample_type or review_status in ("negative", "reject") + + +def iter_negative_images(manifests: list[Path]): + seen: set[Path] = set() + for manifest in manifests: + with manifest.expanduser().resolve().open("r", encoding="utf-8", newline="") as handle: + for row in csv.DictReader(handle): + if not is_reviewed_negative(row): + continue + image_text = first_present(row, ("dataset_image", "frame_path", "source_frame", "image_path")) + if not image_text: + continue + image_path = Path(image_text).expanduser().resolve() + if image_path in seen or not image_path.is_file(): + continue + seen.add(image_path) + yield image_path + + +def crop_hash(crop) -> str: + ok, encoded = cv2.imencode(".jpg", crop, [cv2.IMWRITE_JPEG_QUALITY, 94]) + if not ok: + return "" + return hashlib.sha256(encoded.tobytes()).hexdigest()[:20] + + +def remove_appledouble_files(root: Path) -> int: + removed = 0 + for path in root.rglob("._*"): + if path.is_file(): + path.unlink() + removed += 1 + return removed + + +def main() -> int: + args = parse_args() + configure_models(args.models_dir) + slv.US_CLASSIFIER_MIN_CONFIDENCE = args.classifier_min_confidence + daemon = slv.SpeedLimitVisionDaemon(use_runtime=False) + dataset_root = args.dataset.expanduser().resolve() + appledouble_removed = remove_appledouble_files(dataset_root) + output_dir = dataset_root / args.split / "reject" + output_dir.mkdir(parents=True, exist_ok=True) + + added = 0 + frames = 0 + proposals = 0 + for image_path in iter_negative_images(args.manifest): + if added >= args.max_crops: + break + frame_bgr = cv2.imread(str(image_path)) + if frame_bgr is None: + continue + frames += 1 + frame_height, frame_width = frame_bgr.shape[:2] + + for _proposal_confidence, class_id, (x1, y1, x2, y2) in daemon._collect_detector_classifier_proposals(frame_bgr): + if class_id == 1: + continue + proposals += 1 + box_width = x2 - x1 + box_height = y2 - y1 + if box_width <= 0 or box_height <= 0: + continue + + for expansion_index, (left, top, right, bottom, _weight) in enumerate(slv.DETECTOR_CLASSIFIER_EXPANSIONS): + crop_x1 = max(int(x1 - box_width * left), 0) + crop_y1 = max(int(y1 - box_height * top), 0) + crop_x2 = min(int(x2 + box_width * right), frame_width) + crop_y2 = min(int(y2 + box_height * bottom), frame_height) + crop = frame_bgr[crop_y1:crop_y2, crop_x1:crop_x2] + if crop.size == 0 or daemon._classify_speed_limit_from_model(crop) is None: + continue + + digest = crop_hash(crop) + if not digest: + continue + output_path = output_dir / f"hardneg_{digest}_e{expansion_index}.jpg" + if output_path.exists() and not args.overwrite: + continue + cv2.imwrite(str(output_path), crop, [cv2.IMWRITE_JPEG_QUALITY, 94]) + added += 1 + if added >= args.max_crops: + break + if added >= args.max_crops: + break + + if added: + cache_path = dataset_root / f"{args.split}.cache" + if cache_path.is_file(): + cache_path.unlink() + summary = f"Hard-negative mining complete: frames={frames} proposals={proposals} added={added}" + summary += f" appledouble_removed={appledouble_removed}" + print(summary) + print(f"Reject dataset: {output_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/mine_route_training_samples.py b/scripts/speed_limit_vision/mine_route_training_samples.py index e1c2bedfa..0599ecff0 100644 --- a/scripts/speed_limit_vision/mine_route_training_samples.py +++ b/scripts/speed_limit_vision/mine_route_training_samples.py @@ -30,10 +30,14 @@ else: DEFAULT_ROUTE_BUNDLE_STATE_DIR = Path("/Volumes/T5/starpilot_speed_limit/analysis/route_bundles/state") DEFAULT_WORKSPACE = Path("/Volumes/T5/starpilot_speed_limit/workspace/speed_limit_training_clean") DEFAULT_REVIEW_MANIFEST_NAME = "route_training_samples.csv" +MINING_RUN_SCHEMA_VERSION = 2 MPH_PER_MS = 2.2369362920544 VALID_WEAK_LABEL_VALUES = set(slv.US_CLASSIFIER_SPEED_VALUES) POSITIVE_FIELDNAMES = [ "record_key", + "mining_run_id", + "mining_fingerprint", + "model_fingerprint", "route", "dongle_id", "log_id", @@ -83,6 +87,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--clip-root", type=Path, default=preferred_clip_root(), help="Route realdata root.") parser.add_argument("--bundle-state-dir", type=Path, default=DEFAULT_ROUTE_BUNDLE_STATE_DIR, help="Completed extraction marker directory.") parser.add_argument("--models-dir", type=Path, help="Optional model directory for mining with non-repo ONNXs.") + parser.add_argument("--model-only", action="store_true", help="Use detector/classifier output without OCR when weak-labeling signs.") + parser.add_argument("--run-id", help="Version this mining pass. Use 'auto' to derive an id from the ONNX bundle.") + parser.add_argument("--output-root", type=Path, help="Output root for this pass. Defaults to a versioned staging directory when --run-id is set.") parser.add_argument("--manifest-out", type=Path, help=f"Review manifest path. Defaults to /review/{DEFAULT_REVIEW_MANIFEST_NAME}.") parser.add_argument("--sample-every", type=float, default=4.0, help="Seconds between regular video samples.") parser.add_argument("--seek-sampling", action="store_true", help="Seek directly to sampled frames instead of sequentially grabbing through each segment.") @@ -110,6 +117,54 @@ def safe_key(text: str) -> str: return text.replace("/", "_").replace("|", "_").replace(":", "_") +def model_bundle_fingerprint() -> str: + digest = hashlib.sha256() + for path in (slv.US_DETECTOR_MODEL_PATH, slv.US_CLASSIFIER_MODEL_PATH): + resolved = Path(path).expanduser().resolve() + digest.update(resolved.name.encode("utf-8")) + with resolved.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def mining_configuration_fingerprint(args: argparse.Namespace, model_fingerprint: str) -> str: + config = { + "schema_version": MINING_RUN_SCHEMA_VERSION, + "model_fingerprint": model_fingerprint, + "model_only": args.model_only, + "sample_every": args.sample_every, + "transition_radius": args.transition_radius, + "transition_step": args.transition_step, + "max_frames_per_route": args.max_frames_per_route, + "max_positives_per_route": args.max_positives_per_route, + "max_negatives_per_route": args.max_negatives_per_route, + "positive_min_score": args.positive_min_score, + "no_map_min_score": args.no_map_min_score, + "min_proposal_confidence": args.min_proposal_confidence, + "min_width": args.min_width, + "min_height": args.min_height, + "next_limit_distance": args.next_limit_distance, + } + digest = hashlib.sha256(json.dumps(config, sort_keys=True).encode("utf-8")) + for source_path in (Path(__file__), Path(score_frame.__code__.co_filename), Path(slv.__file__)): + digest.update(source_path.resolve().read_bytes()) + return digest.hexdigest() + + +def resolve_run_id(requested: str | None, model_fingerprint: str, mining_fingerprint: str) -> str: + if not requested: + return "" + run_id = ( + f"model_{model_fingerprint[:12]}_run_{mining_fingerprint[:12]}" + if requested == "auto" + else safe_key(requested.strip()) + ) + if not run_id: + raise ValueError("--run-id must not be empty") + return run_id + + def parse_route_id(text: str) -> tuple[str, str, str]: normalized = text.strip().replace("|", "/") if "/" not in normalized: @@ -390,6 +445,8 @@ def should_keep_positive(scored: dict, speed_limit_mph: int, consistent_count: i return False if float(scored["proposal_confidence"]) < args.min_proposal_confidence: return False + if args.model_only and relation not in ("agree_current", "agree_next"): + return False if relation in ("agree_current", "agree_next"): return float(scored["score"]) >= args.positive_min_score and consistent_count >= 1 return float(scored["score"]) >= args.no_map_min_score and consistent_count >= 2 @@ -443,16 +500,33 @@ def write_sample(frame_bgr, image_path: Path, label_path: Path, label_text: str, return True -def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse.Namespace, workspace: Path, clip_root: Path, manifest_path: Path, route_state_dir: Path) -> dict[str, int | str | float]: +def mine_route( + route_id: str, + daemon: slv.SpeedLimitVisionDaemon, + args: argparse.Namespace, + output_root: Path, + clip_root: Path, + manifest_path: Path, + route_state_dir: Path, + run_id: str, + mining_fingerprint: str, + model_fingerprint: str, +) -> dict[str, int | str | float]: route_id, dongle_id, log_id = parse_route_id(route_id) route_key = safe_key(route_id) state_path = route_state_dir / f"{route_key}.json" + if run_id and state_path.exists(): + state = json.loads(state_path.read_text(encoding="utf-8")) + if state.get("model_fingerprint") != model_fingerprint or state.get("mining_fingerprint") != mining_fingerprint: + raise RuntimeError( + f"Mining state fingerprint mismatch for {route_id}. Use a new --run-id or output root instead of mixing runs." + ) if state_path.exists() and not args.force: return {"route": route_id, "status": "skipped", "positives": 0, "negatives": 0, "scored": 0} split = route_split(route_id, args.val_route_modulo, args.val_route_remainder) - image_dir = ensure_dir(workspace / "detector" / "images" / split) - label_dir = ensure_dir(workspace / "detector" / "labels" / split) + image_dir = ensure_dir(output_root / "detector" / "images" / split) + label_dir = ensure_dir(output_root / "detector" / "labels" / split) segments = route_segments(clip_root, log_id) if not segments: return {"route": route_id, "status": "missing_segments", "positives": 0, "negatives": 0, "scored": 0} @@ -487,7 +561,7 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse continue scored_frames += 1 - scored = score_frame(daemon, frame_bgr) + scored = score_frame(daemon, frame_bgr, use_ocr=not args.model_only) context = nearest_context(contexts, time_s) if scored is None: @@ -501,6 +575,9 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse negatives += 1 route_rows.append({ "record_key": record_key, + "mining_run_id": run_id, + "mining_fingerprint": mining_fingerprint, + "model_fingerprint": model_fingerprint, "route": route_id, "dongle_id": dongle_id, "log_id": log_id, @@ -548,6 +625,9 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse bbox = ",".join(str(value) for value in scored["box"]) route_rows.append({ "record_key": record_key, + "mining_run_id": run_id, + "mining_fingerprint": mining_fingerprint, + "model_fingerprint": model_fingerprint, "route": route_id, "dongle_id": dongle_id, "log_id": log_id, @@ -585,9 +665,13 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse if not args.dry_run: merge_review_rows(manifest_path, route_rows) - merge_value_labels(workspace / "classifier" / "value_labels.csv", value_rows) + merge_value_labels(output_root / "classifier" / "value_labels.csv", value_rows) state_path.write_text(json.dumps({ "route": route_id, + "mining_run_id": run_id, + "mining_fingerprint": mining_fingerprint, + "model_fingerprint": model_fingerprint, + "model_only": args.model_only, "status": "mined", "positives": positives, "negatives": negatives, @@ -613,20 +697,41 @@ def main() -> int: args = parse_args() workspace = resolve_workspace(args.workspace) clip_root = args.clip_root.expanduser().resolve() - manifest_path = args.manifest_out.expanduser().resolve() if args.manifest_out else (ensure_dir(workspace / "review") / DEFAULT_REVIEW_MANIFEST_NAME) - route_state_dir = ensure_dir(workspace / "review" / "route_training_samples_state") routes = read_routes(args) if not routes: raise SystemExit("No routes to mine. Pass route ids, --routes-file, or completed bundle markers.") configure_models(args.models_dir) + slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = not args.model_only + model_fingerprint = model_bundle_fingerprint() + mining_fingerprint = mining_configuration_fingerprint(args, model_fingerprint) + run_id = resolve_run_id(args.run_id, model_fingerprint, mining_fingerprint) + if args.output_root: + output_root = args.output_root.expanduser().resolve() + elif run_id: + output_root = workspace / "staging" / "route_mining" / run_id + else: + output_root = workspace + manifest_path = args.manifest_out.expanduser().resolve() if args.manifest_out else (ensure_dir(output_root / "review") / DEFAULT_REVIEW_MANIFEST_NAME) + route_state_dir = ensure_dir(output_root / "review" / "route_training_samples_state") daemon = slv.SpeedLimitVisionDaemon(use_runtime=False) total_positive = 0 total_negative = 0 total_scored = 0 for index, route in enumerate(routes, start=1): - result = mine_route(route, daemon, args, workspace, clip_root, manifest_path, route_state_dir) + result = mine_route( + route, + daemon, + args, + output_root, + clip_root, + manifest_path, + route_state_dir, + run_id, + mining_fingerprint, + model_fingerprint, + ) total_positive += int(result.get("positives", 0)) total_negative += int(result.get("negatives", 0)) total_scored += int(result.get("scored", 0)) @@ -641,6 +746,8 @@ def main() -> int: flush=True, ) print(f"Review manifest: {manifest_path}", flush=True) + print(f"Model fingerprint: {model_fingerprint}", flush=True) + print(f"Mining fingerprint: {mining_fingerprint}", flush=True) return 0 diff --git a/scripts/speed_limit_vision/replay_route_runtime.py b/scripts/speed_limit_vision/replay_route_runtime.py index e651c29b2..af1280a95 100644 --- a/scripts/speed_limit_vision/replay_route_runtime.py +++ b/scripts/speed_limit_vision/replay_route_runtime.py @@ -193,6 +193,21 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--right-roi-min-confidence", type=float, help="Override the right ROI detector minimum confidence.") parser.add_argument("--classifier-min-confidence", type=float, help="Override the value classifier confidence threshold.") parser.add_argument("--full-frame-ocr", action="store_true", help="Enable the expensive full-frame OCR fallback during replay.") + crop_ocr_group = parser.add_mutually_exclusive_group() + crop_ocr_group.add_argument("--crop-ocr", action="store_true", dest="crop_ocr", default=None, help="Enable crop OCR confirmation during replay.") + crop_ocr_group.add_argument("--no-crop-ocr", action="store_false", dest="crop_ocr", help="Replay the model-only detector/classifier path.") + parser.add_argument("--low-speed-change-consistent-detections", type=int, help="Override reads required to change from 30+ mph to below 30 mph.") + parser.add_argument( + "--allow-low-speed-strong-consensus", + action="store_true", + help="Permit a strong multi-crop consensus to publish a low-speed change from one frame.", + ) + parser.add_argument( + "--enable-strong-model-consensus", + action="store_true", + help="Mark three agreeing high-confidence regulatory model crops as strong consensus.", + ) + parser.add_argument("--initial-speed-limit", type=int, default=0, help="Seed route replay with a currently published speed limit.") return parser.parse_args() @@ -299,6 +314,14 @@ def configure_runtime_options(args: argparse.Namespace) -> None: if args.full_frame_ocr: slv.FULL_FRAME_OCR_FALLBACK_ENABLED = True + if args.crop_ocr is not None: + slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = args.crop_ocr + if args.low_speed_change_consistent_detections is not None: + slv.LOW_SPEED_CHANGE_CONSISTENT_DETECTIONS = args.low_speed_change_consistent_detections + if args.allow_low_speed_strong_consensus: + slv.LOW_SPEED_CHANGE_ALLOW_STRONG_CONSENSUS = True + if args.enable_strong_model_consensus: + slv.DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_ENABLED = True if args.right_roi_bounds: parts = [float(part.strip()) for part in args.right_roi_bounds.split(",")] @@ -347,6 +370,7 @@ def replay_route( measured_inference_seconds: float, measured_base_inference_seconds: float | None = None, measured_classifier_forward_seconds: float = 0.0, + initial_speed_limit_mph: int = 0, ) -> tuple[RouteSummary, list[dict[str, str]]]: daemon = RouteReplayDaemon( runtime_context, @@ -354,6 +378,7 @@ def replay_route( measured_base_inference_seconds, measured_classifier_forward_seconds, ) + daemon.published_speed_limit_mph = initial_speed_limit_mph for segment_path in segments: segment = segment_index(segment_path) capture = cv2.VideoCapture(str(segment_path)) @@ -397,11 +422,8 @@ def replay_route( capture.release() if progress: - print( - f" seg {segment:02d}: sampled={daemon.sampled_frames} inference={daemon.inference_frames} " - f"events={len(daemon.events)}", - flush=True, - ) + counts = f"sampled={daemon.sampled_frames} inference={daemon.inference_frames} events={len(daemon.events)}" + print(f" seg {segment:02d}: {counts}", flush=True) return summarize(log_id, len(segments), runtime_context is not None, daemon), daemon.events @@ -490,6 +512,7 @@ def main() -> int: args.measured_inference_seconds, args.measured_base_inference_seconds, args.measured_classifier_forward_seconds, + args.initial_speed_limit, ) all_events.extend((log_id, event) for event in events) summary_line = "".join(( diff --git a/scripts/speed_limit_vision/rescore_manual_review_queue.py b/scripts/speed_limit_vision/rescore_manual_review_queue.py new file mode 100644 index 000000000..b5fc42e0a --- /dev/null +++ b/scripts/speed_limit_vision/rescore_manual_review_queue.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json + +from collections import Counter +from pathlib import Path + +import cv2 + +import starpilot.system.speed_limit_vision as slv + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from compare_manual_review_queues import classify_change # type: ignore + from mine_route_training_samples import model_bundle_fingerprint # type: ignore + from replay_route_runtime import configure_models # type: ignore +else: + from .compare_manual_review_queues import classify_change + from .mine_route_training_samples import model_bundle_fingerprint + from .replay_route_runtime import configure_models + + +EXTRA_FIELDS = ( + "comparison_change", + "before_speed_limit_mph", + "before_confidence", + "rescore_status", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Rescore stored manual-review crops with a new value classifier.") + parser.add_argument("--input", type=Path, required=True, help="Baseline manual_review_queue.csv.") + parser.add_argument("--models-dir", type=Path, required=True, help="Candidate detector/classifier ONNX directory.") + parser.add_argument("--output", type=Path, required=True, help="Rescored review-compatible output CSV.") + parser.add_argument("--confidence-delta", type=float, default=0.05, help="Minimum confidence-only change to report.") + parser.add_argument("--shard-count", type=int, default=1, help="Number of deterministic row shards.") + parser.add_argument("--shard-index", type=int, default=0, help="Zero-based row shard processed by this invocation.") + return parser.parse_args() + + +def rescore_row( + row: dict[str, str], + daemon: slv.SpeedLimitVisionDaemon, + model_fingerprint: str, + confidence_delta: float, +) -> dict[str, str]: + output = dict(row) + before_speed = row.get("candidate_speed_limit_mph", "") + before_confidence = row.get("candidate_confidence", "") + output["before_speed_limit_mph"] = before_speed + output["before_confidence"] = before_confidence + output["model_fingerprint"] = model_fingerprint + + crop_path = Path(row.get("crop_path", "")).expanduser() + crop = cv2.imread(str(crop_path)) if crop_path.is_file() else None + if crop is None: + output["comparison_change"] = "unreadable" + output["rescore_status"] = "unreadable" + return output + + result = daemon._classify_speed_limit_from_model(crop) + if result is None: + output["candidate_speed_limit_mph"] = "" + output["candidate_confidence"] = "" + output["model_read"] = "" + else: + speed_limit_mph, confidence = result + output["candidate_speed_limit_mph"] = str(int(speed_limit_mph)) + output["candidate_confidence"] = f"{float(confidence):.8f}" + output["model_read"] = f"{int(speed_limit_mph)}@{float(confidence):.3f}" + + change = classify_change(row, output, confidence_delta) + output["comparison_change"] = change or "unchanged" + output["rescore_status"] = "rescored_crop" + return output + + +def main() -> int: + args = parse_args() + if args.shard_count <= 0 or not 0 <= args.shard_index < args.shard_count: + raise ValueError("--shard-index must be within --shard-count") + + configure_models(args.models_dir) + slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = False + daemon = slv.SpeedLimitVisionDaemon(use_runtime=False) + fingerprint = model_bundle_fingerprint() + input_path = args.input.expanduser().resolve() + with input_path.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + input_fields = list(reader.fieldnames or []) + rows = [row for index, row in enumerate(reader) if index % args.shard_count == args.shard_index] + + output_rows: list[dict[str, str]] = [] + changes: Counter[str] = Counter() + transitions: Counter[str] = Counter() + for index, row in enumerate(rows, start=1): + rescored = rescore_row(row, daemon, fingerprint, args.confidence_delta) + output_rows.append(rescored) + change = rescored["comparison_change"] + changes[change] += 1 + before_speed = rescored["before_speed_limit_mph"] or "none" + after_speed = rescored.get("candidate_speed_limit_mph", "") or "none" + if change != "unchanged": + transitions[f"{before_speed}->{after_speed}"] += 1 + if index % 1000 == 0: + print(f"Rescored {index}/{len(rows)} crops", flush=True) + + output_path = args.output.expanduser().resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = [*input_fields, *(field for field in EXTRA_FIELDS if field not in input_fields)] + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(output_rows) + + summary = { + "input": str(input_path), + "output": str(output_path), + "models_dir": str(args.models_dir.expanduser().resolve()), + "model_fingerprint": fingerprint, + "shard_count": args.shard_count, + "shard_index": args.shard_index, + "rows": len(output_rows), + "changes": dict(sorted(changes.items())), + "transitions": dict(sorted(transitions.items(), key=lambda item: (-item[1], item[0]))), + } + output_path.with_suffix(".json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/select_manual_review_queue.py b/scripts/speed_limit_vision/select_manual_review_queue.py new file mode 100644 index 000000000..95cf496c3 --- /dev/null +++ b/scripts/speed_limit_vision/select_manual_review_queue.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json + +from collections import Counter, defaultdict, deque +from pathlib import Path + + +PRIORITY_SPEED_ORDER = (60, 65, 55, 50, 45, 40, 35, 30, 25, 20, 70, 15, 75) +COMPARISON_PRIORITY_BONUS = { + "value_changed": 4.0, + "gained_read": 3.0, + "lost_read": 3.0, + "added_proposal": 1.0, + "removed_proposal": 1.0, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Select a diverse, high-value subset from a raw speed-limit review queue.") + parser.add_argument("--input", type=Path, required=True, help="Raw manual_review_queue.csv.") + parser.add_argument("--output", type=Path, required=True, help="Selected manual_review_queue.csv.") + parser.add_argument("--max-rows", type=int, default=1000, help="Maximum selected rows.") + parser.add_argument("--max-per-route", type=int, default=30, help="Maximum selected rows from one route.") + parser.add_argument("--max-per-speed", type=int, default=140, help="Maximum rows for each predicted speed.") + parser.add_argument( + "--max-primary-speed", + type=int, + default=0, + help="Optional per-speed cap for the primary 30-65 mph range; defaults to --max-per-speed.", + ) + parser.add_argument( + "--max-speed-20", + type=int, + default=0, + help="Optional cap for 20 mph rows; defaults to --max-per-speed.", + ) + parser.add_argument("--max-no-read", type=int, default=220, help="Maximum detector proposals without a value read.") + parser.add_argument("--max-school", type=int, default=100, help="Maximum school-zone candidates.") + parser.add_argument("--max-advisory", type=int, default=100, help="Maximum advisory candidates.") + parser.add_argument( + "--min-seconds-per-route-speed", + type=float, + default=3.0, + help="Minimum spacing between selected rows from the same route, segment, and predicted-speed bucket.", + ) + return parser.parse_args() + + +def read_rows(path: Path) -> tuple[list[str], list[dict[str, str]]]: + with path.expanduser().resolve().open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + return list(reader.fieldnames or []), list(reader) + + +def predicted_speed(row: dict[str, str]) -> int: + text = (row.get("candidate_speed_limit_mph") or "").strip() + if not text and row.get("comparison_change") == "lost_read": + text = (row.get("before_speed_limit_mph") or "").strip() + try: + return int(float(text)) if text else 0 + except ValueError: + return 0 + + +def bucket_name(row: dict[str, str]) -> str: + detector_class = row.get("detector_class", "") + if detector_class == "school_zone_speed_limit": + return "school" + if detector_class == "advisory_speed_limit": + return "advisory" + speed = predicted_speed(row) + return f"speed_{speed}" if speed else "no_read" + + +def priority(row: dict[str, str]) -> tuple[float, float, float, str]: + review_priority = ( + float(row.get("review_priority") or 0.0) + + COMPARISON_PRIORITY_BONUS.get(row.get("comparison_change", ""), 0.0) + ) + proposal_confidence = float(row.get("proposal_confidence") or 0.0) + candidate_confidence = float(row.get("candidate_confidence") or 0.0) + return review_priority, proposal_confidence, candidate_confidence, row.get("record_key", "") + + +def temporal_key(row: dict[str, str]) -> tuple[str, str, str, float] | None: + try: + frame_time_s = float(row.get("frame_time_s", "")) + except ValueError: + return None + return row.get("route", ""), row.get("segment", ""), bucket_name(row), frame_time_s + + +def round_robin_routes(rows: list[dict[str, str]]): + by_route: dict[str, deque[dict[str, str]]] = defaultdict(deque) + for row in sorted(rows, key=priority, reverse=True): + by_route[row.get("route", "")].append(row) + route_order = deque(sorted(by_route, key=lambda route: priority(by_route[route][0]), reverse=True)) + while route_order: + route = route_order.popleft() + yield by_route[route].popleft() + if by_route[route]: + route_order.append(route) + + +def select_rows(rows: list[dict[str, str]], args: argparse.Namespace) -> list[dict[str, str]]: + buckets: dict[str, list[dict[str, str]]] = defaultdict(list) + for row in rows: + if row.get("detector_class") == "negative_empty": + continue + buckets[bucket_name(row)].append(row) + + max_primary_speed = int(getattr(args, "max_primary_speed", 0)) + max_speed_20 = int(getattr(args, "max_speed_20", 0)) + primary_limit = max_primary_speed if max_primary_speed > 0 else args.max_per_speed + speed_20_limit = max_speed_20 if max_speed_20 > 0 else args.max_per_speed + limits = { + f"speed_{speed}": ( + primary_limit if 30 <= speed <= 65 else speed_20_limit if speed == 20 else args.max_per_speed + ) + for speed in PRIORITY_SPEED_ORDER + } + limits.update({"school": args.max_school, "advisory": args.max_advisory, "no_read": args.max_no_read}) + ordered_buckets = [f"speed_{speed}" for speed in PRIORITY_SPEED_ORDER] + ["school", "advisory", "no_read"] + ordered_buckets.extend(sorted(set(buckets) - set(ordered_buckets))) + + selected: list[dict[str, str]] = [] + selected_keys: set[str] = set() + route_counts: Counter[str] = Counter() + bucket_counts: Counter[str] = Counter() + selected_times: dict[tuple[str, str, str], list[float]] = defaultdict(list) + min_spacing = max(float(getattr(args, "min_seconds_per_route_speed", 0.0)), 0.0) + + def try_add(row: dict[str, str], bucket: str) -> bool: + key = row.get("record_key", "") + route = row.get("route", "") + if not key or key in selected_keys or route_counts[route] >= args.max_per_route: + return False + if bucket_counts[bucket] >= limits.get(bucket, args.max_per_speed): + return False + time_key = temporal_key(row) + if time_key is not None and min_spacing > 0.0: + route_key = time_key[:3] + if any(abs(time_key[3] - selected_time) < min_spacing for selected_time in selected_times[route_key]): + return False + selected.append(row) + selected_keys.add(key) + route_counts[route] += 1 + bucket_counts[bucket] += 1 + if time_key is not None: + selected_times[time_key[:3]].append(time_key[3]) + return True + + iterators = {bucket: iter(round_robin_routes(buckets[bucket])) for bucket in ordered_buckets if buckets.get(bucket)} + active = deque(bucket for bucket in ordered_buckets if bucket in iterators) + while active and len(selected) < args.max_rows: + bucket = active.popleft() + iterator = iterators[bucket] + added = False + for row in iterator: + if try_add(row, bucket): + added = True + break + if added and bucket_counts[bucket] < limits.get(bucket, args.max_per_speed): + active.append(bucket) + + if len(selected) < args.max_rows: + for row in sorted(rows, key=priority, reverse=True): + if len(selected) >= args.max_rows: + break + try_add(row, bucket_name(row)) + + return sorted(selected, key=priority, reverse=True) + + +def main() -> int: + args = parse_args() + fieldnames, rows = read_rows(args.input) + selected = select_rows(rows, args) + output = args.output.expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(selected) + + summary = { + "input": str(args.input.expanduser().resolve()), + "output": str(output), + "input_rows": len(rows), + "selected_rows": len(selected), + "routes": len({row.get("route", "") for row in selected}), + "buckets": dict(sorted(Counter(bucket_name(row) for row in selected).items())), + "min_seconds_per_route_speed": args.min_seconds_per_route_speed, + } + summary_path = output.with_name("manual_review_selection_summary.json") + summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/serve_manual_review_queue.py b/scripts/speed_limit_vision/serve_manual_review_queue.py index 1f3e21d76..67c81f80f 100644 --- a/scripts/speed_limit_vision/serve_manual_review_queue.py +++ b/scripts/speed_limit_vision/serve_manual_review_queue.py @@ -4,8 +4,8 @@ from __future__ import annotations import argparse import csv import json -import time +from datetime import UTC, datetime from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -40,7 +40,8 @@ HTML = r""" @@ -79,7 +81,8 @@ HTML = r""" - Keys: Space/p accept model, type speed to correct, u uncertain, i/x ignore, Enter save correction, j/k next/prev, s school, r regulatory, a advisory + Keys: Space/p accept model, type speed to correct, u uncertain, i/x ignore, + Enter save correction, j/k next/prev, s school, r regulatory, a advisory
@@ -93,6 +96,7 @@ HTML = r"""