mawei/StarPilot
1
0
Fork 0
mirror of https://github.com/firestar5683/StarPilot.git synced 2026-07-14 22:02:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

VACATION

Browse Source
This commit is contained in:
firestar5683
2026-07-12 17:53:20 -05:00
parent 7b0c0784ed
commit e577502f4b
34 changed files with 3456 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/speed_limit_vision/build_manual_review_queue.py
+117 -26
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import argparse
import csv
import hashlib
import json
import math
@@ -16,7 +17,7 @@ import starpilot.system.speed_limit_vision as slv
if __package__ in (None, ""):
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import ensure_dir, preferred_clip_root, resolve_workspace # type: ignore
from common import ensure_dir, preferred_clip_root, resolve_workspace # type: ignore # noqa: TID251
from localize_bookmark_signs import configure_models # type: ignore
from mine_route_training_samples import ( # type: ignore
MapContext,
@@ -25,6 +26,7 @@ if __package__ in (None, ""):
iter_frames_at_times,
load_segment_map_context,
nearest_context,
model_bundle_fingerprint,
parse_route_id,
read_frame_at,
route_segments,
@@ -42,6 +44,7 @@ else:
iter_frames_at_times,
load_segment_map_context,
nearest_context,
model_bundle_fingerprint,
parse_route_id,
read_frame_at,
route_segments,
@@ -57,6 +60,8 @@ DEFAULT_OUTPUT_NAME = "manual_review_queue_v1"
PRIORITY_SPEED_VALUES = frozenset((30, 35, 40, 45, 50, 55, 60, 65))
FIELDNAMES = [
"record_key",
"mining_fingerprint",
"model_fingerprint",
"route",
"dongle_id",
"log_id",
@@ -112,14 +117,15 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--clip-root", type=Path, default=preferred_clip_root(), help="Route realdata root.")
parser.add_argument("--bundle-state-dir", type=Path, default=DEFAULT_ROUTE_BUNDLE_STATE_DIR, help="Completed extraction marker directory.")
parser.add_argument("--models-dir", type=Path, help="Optional model directory for mining with non-repo ONNXs.")
parser.add_argument("--model-only", action="store_true", help="Do not run crop OCR while discovering review candidates.")
parser.add_argument("--output-dir", type=Path, help=f"Defaults to <workspace>/review/{DEFAULT_OUTPUT_NAME}.")
parser.add_argument("--manifest-out", type=Path, help="Defaults to <output-dir>/manual_review_queue.csv.")
parser.add_argument("--sample-every", type=float, default=2.0, help="Seconds between regular video samples.")
parser.add_argument("--seek-sampling", action="store_true", help="Seek directly to sampled frames instead of sequential grabbing.")
parser.add_argument("--transition-radius", type=float, default=18.0, help="Extra seconds around map speed transitions to sample densely.")
parser.add_argument("--transition-step", type=float, default=0.75, help="Seconds between transition-window samples.")
parser.add_argument("--max-frames-per-route", type=int, default=1200, help="Maximum frames to score per route.")
parser.add_argument("--max-candidates-per-route", type=int, default=500, help="Maximum review candidates to keep per route.")
parser.add_argument("--max-frames-per-route", type=int, default=1200, help="Maximum frames to score per route. 0 scans the full route.")
parser.add_argument("--max-candidates-per-route", type=int, default=500, help="Maximum review candidates to keep per route. 0 keeps all.")
parser.add_argument("--max-candidates-per-frame", type=int, default=1, help="Maximum detector candidates to keep from a single video frame. 0 keeps all.")
parser.add_argument("--max-negatives-per-route", type=int, default=60, help="Maximum empty/no-candidate frames to keep per route.")
parser.add_argument("--min-proposal-confidence", type=float, default=0.025, help="Loose detector confidence floor for review candidates.")
@@ -132,6 +138,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--include-advisory", action=argparse.BooleanOptionalAction, default=True, help="Include advisory-speed detector class candidates.")
parser.add_argument("--include-full-detection", action="store_true", help="Also run the full runtime detector on each frame for extra context. Slower.")
parser.add_argument("--overwrite-images", action="store_true", help="Rewrite existing review images.")
parser.add_argument("--resume", action=argparse.BooleanOptionalAction, default=True, help="Resume a matching fingerprinted queue.")
parser.add_argument("--dry-run", action="store_true", help="Score frames and print counts without writing images/CSV.")
return parser.parse_args()
@@ -160,6 +167,33 @@ def read_routes(args: argparse.Namespace) -> list[str]:
return deduped
def review_mining_fingerprint(args: argparse.Namespace, model_fingerprint: str) -> str:
config = {
"schema_version": 1,
"model_fingerprint": model_fingerprint,
"model_only": args.model_only,
"sample_every": args.sample_every,
"transition_radius": args.transition_radius,
"transition_step": args.transition_step,
"max_frames_per_route": args.max_frames_per_route,
"max_candidates_per_route": args.max_candidates_per_route,
"max_candidates_per_frame": args.max_candidates_per_frame,
"max_negatives_per_route": args.max_negatives_per_route,
"min_proposal_confidence": args.min_proposal_confidence,
"no_read_min_proposal_confidence": args.no_read_min_proposal_confidence,
"school_zone_min_proposal_confidence": args.school_zone_min_proposal_confidence,
"min_width": args.min_width,
"min_height": args.min_height,
"dedupe_seconds": args.dedupe_seconds,
"include_advisory": args.include_advisory,
"include_full_detection": args.include_full_detection,
}
digest = hashlib.sha256(json.dumps(config, sort_keys=True).encode("utf-8"))
for source_path in (Path(__file__), Path(slv.__file__)):
digest.update(source_path.resolve().read_bytes())
return digest.hexdigest()
def clamp_box(box: tuple[int, int, int, int], frame_shape: tuple[int, int, int]) -> tuple[int, int, int, int] | None:
frame_height, frame_width = frame_shape[:2]
x1, y1, x2, y2 = box
@@ -245,7 +279,14 @@ def classify_map_relation(speed_limit_mph: int, context: MapContext, next_limit_
return "no_map"
def score_review_priority(class_id: int, proposal_confidence: float, chosen_vote: ReadVote | None, support_count: int, map_relation: str, reasons: set[str]) -> float:
def score_review_priority(
class_id: int,
proposal_confidence: float,
chosen_vote: ReadVote | None,
support_count: int,
map_relation: str,
reasons: set[str],
) -> float:
score = proposal_confidence * 2.0
if chosen_vote is not None:
score += chosen_vote.confidence * 2.0
@@ -276,7 +317,14 @@ def summarize_votes(votes: list[ReadVote]) -> str:
return "|".join(compact)
def analyze_proposal(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, proposal, full_detection, context: MapContext, args: argparse.Namespace):
def analyze_proposal(
daemon: slv.SpeedLimitVisionDaemon,
frame_bgr,
proposal,
full_detection,
context: MapContext,
args: argparse.Namespace,
):
proposal_confidence, class_id, raw_box = proposal
if class_id == 1 and not args.include_advisory:
return None
@@ -306,7 +354,8 @@ def analyze_proposal(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, proposal, fu
is_regulatory = daemon._is_regulatory_speed_sign(crop) or class_id == 2
any_regulatory = any_regulatory or is_regulatory
add_vote(votes, daemon._classify_speed_limit_from_model(crop), "model", expansion_index, crop_box, is_regulatory, weight)
add_vote(votes, daemon._read_speed_limit_from_crop(crop), "ocr", expansion_index, crop_box, is_regulatory, weight)
if not args.model_only:
add_vote(votes, daemon._read_speed_limit_from_crop(crop), "ocr", expansion_index, crop_box, is_regulatory, weight)
chosen_vote, support_count = choose_vote(votes)
if chosen_vote is None and proposal_confidence < args.no_read_min_proposal_confidence and class_id != 2:
@@ -368,15 +417,10 @@ def write_image(path: Path, image, quality: int, overwrite: bool) -> None:
cv2.imwrite(str(path), image, [cv2.IMWRITE_JPEG_QUALITY, quality])
def cluster_key(route_id: str, segment: int, time_s: float, frame_shape: tuple[int, int, int], candidate: dict, dedupe_seconds: float) -> str:
x1, y1, x2, y2 = candidate["bbox"]
frame_height, frame_width = frame_shape[:2]
center_x = ((x1 + x2) / 2) / max(frame_width, 1)
center_y = ((y1 + y2) / 2) / max(frame_height, 1)
def cluster_key(route_id: str, segment: int, time_s: float, candidate: dict, dedupe_seconds: float) -> str:
time_bucket = int(math.floor(time_s / max(dedupe_seconds, 0.1)))
grid_x = int(center_x * 12)
grid_y = int(center_y * 8)
return f"{route_id}|{segment}|{time_bucket}|{candidate['class_id']}|{grid_x}|{grid_y}"
candidate_speed = candidate.get("candidate_speed_limit_mph") or "none"
return f"{route_id}|{segment}|{time_bucket}|{candidate['class_id']}|{candidate_speed}"
def candidate_record_key(route_key: str, segment: int, time_s: float, index: int) -> str:
@@ -384,7 +428,14 @@ def candidate_record_key(route_key: str, segment: int, time_s: float, index: int
return f"manual_review_{route_key}_{sample_index}"
def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse.Namespace, output_dir: Path) -> tuple[list[dict[str, object]], dict[str, object]]:
def mine_route(
route_id: str,
daemon: slv.SpeedLimitVisionDaemon,
args: argparse.Namespace,
output_dir: Path,
mining_fingerprint: str,
model_fingerprint: str,
) -> tuple[list[dict[str, object]], dict[str, object]]:
route_id, dongle_id, log_id = parse_route_id(route_id)
route_key = safe_key(route_id)
clip_root = args.clip_root.expanduser().resolve()
@@ -400,7 +451,10 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse
frames_scored = 0
for segment in segments:
if frames_scored >= args.max_frames_per_route or route_candidates >= args.max_candidates_per_route:
if (
(args.max_frames_per_route > 0 and frames_scored >= args.max_frames_per_route) or
(args.max_candidates_per_route > 0 and route_candidates >= args.max_candidates_per_route)
):
break
contexts = load_segment_map_context(segment.path)
capture = cv2.VideoCapture(str(segment.video_path))
@@ -414,7 +468,10 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse
frame_iter = iter_frames_at_times(capture, fps, times)
for time_s, frame_bgr in frame_iter:
if frames_scored >= args.max_frames_per_route or route_candidates >= args.max_candidates_per_route:
if (
(args.max_frames_per_route > 0 and frames_scored >= args.max_frames_per_route) or
(args.max_candidates_per_route > 0 and route_candidates >= args.max_candidates_per_route)
):
break
if frame_bgr is None:
continue
@@ -444,6 +501,8 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse
crop_path = crop_dir / f"{record_key}_crop.jpg"
row = {
"record_key": record_key,
"mining_fingerprint": mining_fingerprint,
"model_fingerprint": model_fingerprint,
"route": route_id,
"dongle_id": dongle_id,
"log_id": log_id,
@@ -478,7 +537,7 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse
"review_ignore_reason": "",
"review_notes": "",
}
key = cluster_key(route_id, segment.segment, time_s, frame_bgr.shape, candidate, args.dedupe_seconds)
key = cluster_key(route_id, segment.segment, time_s, candidate, args.dedupe_seconds)
existing = rows_by_cluster.get(key)
if existing is None or float(row["review_priority"]) > float(existing["review_priority"]):
if not args.dry_run:
@@ -494,6 +553,8 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse
frame_path = frame_dir / f"{record_key}.jpg"
row = {
"record_key": record_key,
"mining_fingerprint": mining_fingerprint,
"model_fingerprint": model_fingerprint,
"route": route_id,
"dongle_id": dongle_id,
"log_id": log_id,
@@ -557,8 +618,17 @@ def write_manifest(path: Path, rows: list[dict[str, object]]) -> None:
writer.writerows(rows)
def write_summary(path: Path, manifest_path: Path, rows: list[dict[str, object]], summaries: list[dict[str, object]]) -> None:
def write_summary(
path: Path,
manifest_path: Path,
rows: list[dict[str, object]],
summaries: list[dict[str, object]],
mining_fingerprint: str,
model_fingerprint: str,
) -> None:
path.write_text(json.dumps({
"mining_fingerprint": mining_fingerprint,
"model_fingerprint": model_fingerprint,
"routes": summaries,
"manifest": str(manifest_path),
"rows": len(rows),
@@ -575,6 +645,9 @@ def main() -> int:
args = parse_args()
configure_models(args.models_dir)
slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = not args.model_only
model_fingerprint = model_bundle_fingerprint()
mining_fingerprint = review_mining_fingerprint(args, model_fingerprint)
workspace = resolve_workspace(args.workspace)
output_dir = args.output_dir.expanduser().resolve() if args.output_dir else ensure_dir(workspace / "review" / DEFAULT_OUTPUT_NAME)
manifest_path = args.manifest_out.expanduser().resolve() if args.manifest_out else output_dir / "manual_review_queue.csv"
@@ -586,25 +659,43 @@ def main() -> int:
all_rows: list[dict[str, object]] = []
summaries = []
summary_path = output_dir / "manual_review_summary.json"
completed_routes: set[str] = set()
if args.resume and summary_path.is_file():
prior_summary = json.loads(summary_path.read_text(encoding="utf-8"))
if prior_summary.get("mining_fingerprint") != mining_fingerprint:
raise RuntimeError("Existing review queue fingerprint does not match this run. Use a new --output-dir or --no-resume.")
if manifest_path.is_file():
with manifest_path.open("r", encoding="utf-8", newline="") as handle:
all_rows = list(csv.DictReader(handle))
summaries = list(prior_summary.get("routes", []))
completed_routes = {str(summary["route"]) for summary in summaries if summary.get("status") in ("mined", "missing_segments")}
elif args.resume and (manifest_path.exists() or summary_path.exists()):
raise RuntimeError("Existing review queue is missing fingerprinted resume state. Use a new --output-dir or --no-resume.")
for index, route_id in enumerate(routes, start=1):
rows, summary = mine_route(route_id, daemon, args, output_dir)
normalized_route, _, _ = parse_route_id(route_id)
if normalized_route in completed_routes:
print(f"[{index}/{len(routes)}] {normalized_route}: skipped (already mined)")
continue
rows, summary = mine_route(route_id, daemon, args, output_dir, mining_fingerprint, model_fingerprint)
all_rows.extend(rows)
summaries.append(summary)
print(
f"[{index}/{len(routes)}] {summary['route']}: {summary['status']} "
f"frames={summary['frames']} candidates={summary['candidates']} negatives={summary['negatives']}"
)
progress = f"[{index}/{len(routes)}] {summary['route']}: {summary['status']}"
counts = f"frames={summary['frames']} candidates={summary['candidates']} negatives={summary['negatives']}"
print(f"{progress} {counts}")
if not args.dry_run:
all_rows.sort(key=lambda row: (-float(row["review_priority"]), str(row["record_key"])))
write_manifest(manifest_path, all_rows)
write_summary(summary_path, manifest_path, all_rows, summaries)
write_summary(summary_path, manifest_path, all_rows, summaries, mining_fingerprint, model_fingerprint)
all_rows.sort(key=lambda row: (-float(row["review_priority"]), str(row["record_key"])))
if not args.dry_run:
write_manifest(manifest_path, all_rows)
write_summary(summary_path, manifest_path, all_rows, summaries)
write_summary(summary_path, manifest_path, all_rows, summaries, mining_fingerprint, model_fingerprint)
print(f"Wrote {len(all_rows)} review rows to {manifest_path}")
print(f"Summary: {summary_path}")
print(f"Model fingerprint: {model_fingerprint}")
print(f"Mining fingerprint: {mining_fingerprint}")
else:
print(f"Dry run rows={len(all_rows)} candidates={sum(1 for row in all_rows if row['detector_class'] != 'negative_empty')}")
scripts/speed_limit_vision/build_review_classifier_dataset.py
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import shutil
from collections import Counter
from pathlib import Path
VALID_SPEEDS = frozenset((15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75))
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build an isolated classifier dataset from a base corpus and reviewed crops.")
parser.add_argument("--base", type=Path, required=True, help="Existing Ultralytics classification dataset root.")
parser.add_argument("--output", type=Path, required=True, help="New isolated dataset root.")
parser.add_argument("--positive-manifest", type=Path, action="append", default=[], help="Reviewed positive crop manifest. Repeat as needed.")
parser.add_argument("--reject-manifest", type=Path, action="append", default=[], help="Reviewed classifier reject manifest. Repeat as needed.")
parser.add_argument(
"--advisory-as-reject",
action="store_true",
help="Stage reviewed advisory-speed crops in the reject class instead of omitting them.",
)
parser.add_argument(
"--include-advisory-positives",
action="store_true",
help="Train reviewed advisory crops as their numeric speed classes for recall-first models.",
)
parser.add_argument(
"--advisory-reject-fraction",
type=float,
default=1.0,
help="Deterministic fraction of training advisories staged as reject; validation advisories are always retained.",
)
return parser.parse_args()
def read_rows(paths: list[Path]):
for path in paths:
resolved = path.expanduser().resolve()
with resolved.open("r", encoding="utf-8", newline="") as handle:
yield from csv.DictReader(handle)
def remove_appledouble_files(root: Path) -> int:
removed = 0
for path in root.rglob("._*"):
if path.is_file():
path.unlink()
removed += 1
return removed
def parse_speed(text: str) -> int:
try:
value = int(float((text or "").strip()))
except ValueError:
return 0
return value if value in VALID_SPEEDS else 0
def is_advisory(row: dict[str, str]) -> bool:
return row.get("review_sign_type", "").strip().lower() == "advisory"
def keep_advisory_reject(row: dict[str, str], fraction: float) -> bool:
if row.get("split") == "val" or fraction >= 1.0:
return True
if fraction <= 0.0:
return False
digest = hashlib.sha256(row.get("record_key", "").encode("utf-8")).digest()
return int.from_bytes(digest[:8], "big") / 2**64 < fraction
def stage_crop(source: Path, destination_dir: Path, record_key: str) -> bool:
if not source.is_file():
return False
digest = hashlib.sha256(source.read_bytes()).hexdigest()[:16]
suffix = source.suffix.lower() if source.suffix.lower() in (".jpg", ".jpeg", ".png") else ".jpg"
safe_key = "".join(char if char.isalnum() or char in "._-" else "_" for char in record_key)[:100]
destination_dir.mkdir(parents=True, exist_ok=True)
destination = destination_dir / f"review_{safe_key}_{digest}{suffix}"
if not destination.exists():
shutil.copyfile(source, destination)
return True
def main() -> int:
args = parse_args()
if args.advisory_as_reject and args.include_advisory_positives:
raise ValueError("--advisory-as-reject and --include-advisory-positives are mutually exclusive")
if not 0.0 <= args.advisory_reject_fraction <= 1.0:
raise ValueError("--advisory-reject-fraction must be between 0 and 1")
base = args.base.expanduser().resolve()
output = args.output.expanduser().resolve()
if not base.is_dir():
raise FileNotFoundError(base)
if output.exists():
raise FileExistsError(f"Output dataset already exists: {output}")
shutil.copytree(base, output, copy_function=shutil.copyfile)
appledouble_removed = remove_appledouble_files(output)
positive_counts: Counter[str] = Counter()
reject_counts: Counter[str] = Counter()
skipped = 0
for row in read_rows(args.positive_manifest):
if is_advisory(row):
if args.include_advisory_positives:
pass
elif args.advisory_as_reject and keep_advisory_reject(row, args.advisory_reject_fraction):
split = row.get("split", "")
source = Path(row.get("crop_path", "")).expanduser()
if split in ("train", "val") and stage_crop(source, output / split / "reject", row.get("record_key", "advisory")):
reject_counts[f"advisory_{split}"] += 1
else:
skipped += 1
continue
else:
continue
split = row.get("split", "")
speed = parse_speed(row.get("speed_limit_mph", ""))
source = Path(row.get("crop_path", "")).expanduser()
if split not in ("train", "val") or not speed or not stage_crop(source, output / split / str(speed), row.get("record_key", "positive")):
skipped += 1
continue
positive_counts[f"{split}/{speed}"] += 1
for row in read_rows(args.reject_manifest):
split = row.get("split", "")
source = Path(row.get("crop_path", "")).expanduser()
if split not in ("train", "val") or not stage_crop(source, output / split / "reject", row.get("record_key", "reject")):
skipped += 1
continue
reject_counts[split] += 1
appledouble_removed += remove_appledouble_files(output)
for split in ("train", "val"):
cache_path = output / f"{split}.cache"
if cache_path.is_file():
cache_path.unlink()
summary = {
"base": str(base),
"output": str(output),
"positive_counts": dict(sorted(positive_counts.items())),
"reject_counts": dict(sorted(reject_counts.items())),
"skipped": skipped,
"appledouble_removed": appledouble_removed,
}
summary_path = output / "review_dataset_summary.json"
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/speed_limit_vision/compare_manual_review_queues.py
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import json
from collections import Counter
from pathlib import Path
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Compare two fingerprinted manual-review queues by stable record key.")
parser.add_argument("--before", type=Path, required=True, help="Baseline manual_review_queue.csv.")
parser.add_argument("--after", type=Path, required=True, help="Candidate manual_review_queue.csv.")
parser.add_argument("--output-csv", type=Path, required=True, help="Changed-row output CSV.")
parser.add_argument("--review-output", type=Path, help="Optional review-compatible manifest containing the changed source rows.")
parser.add_argument("--confidence-delta", type=float, default=0.05, help="Minimum confidence-only change to report.")
return parser.parse_args()
def read_rows(path: Path) -> dict[str, dict[str, str]]:
with path.expanduser().resolve().open("r", encoding="utf-8", newline="") as handle:
return {row["record_key"]: row for row in csv.DictReader(handle) if row.get("record_key")}
def parse_float(text: str) -> float:
try:
return float(text)
except (TypeError, ValueError):
return 0.0
def classify_change(before: dict[str, str] | None, after: dict[str, str] | None, confidence_delta: float) -> str:
if before is None:
return "added_proposal"
if after is None:
return "removed_proposal"
before_speed = before.get("candidate_speed_limit_mph", "")
after_speed = after.get("candidate_speed_limit_mph", "")
if not before_speed and after_speed:
return "gained_read"
if before_speed and not after_speed:
return "lost_read"
if before_speed != after_speed:
return "value_changed"
confidence_change = abs(
parse_float(after.get("candidate_confidence", "")) - parse_float(before.get("candidate_confidence", ""))
)
if confidence_change >= confidence_delta:
return "confidence_changed"
return ""
def comparison_row(record_key: str, change: str, before: dict[str, str] | None, after: dict[str, str] | None) -> dict[str, str]:
source = after or before or {}
return {
"record_key": record_key,
"change": change,
"route": source.get("route", ""),
"segment": source.get("segment", ""),
"frame_time_s": source.get("frame_time_s", ""),
"detector_class": source.get("detector_class", ""),
"proposal_confidence": source.get("proposal_confidence", ""),
"before_speed_limit_mph": (before or {}).get("candidate_speed_limit_mph", ""),
"before_confidence": (before or {}).get("candidate_confidence", ""),
"after_speed_limit_mph": (after or {}).get("candidate_speed_limit_mph", ""),
"after_confidence": (after or {}).get("candidate_confidence", ""),
"before_support": (before or {}).get("read_support_count", ""),
"after_support": (after or {}).get("read_support_count", ""),
"frame_path": source.get("frame_path", ""),
"crop_path": source.get("crop_path", ""),
"source_video_path": source.get("source_video_path", ""),
}
def main() -> int:
args = parse_args()
before = read_rows(args.before)
after = read_rows(args.after)
rows: list[dict[str, str]] = []
review_rows: list[dict[str, str]] = []
change_counts: Counter[str] = Counter()
transition_counts: Counter[str] = Counter()
for record_key in sorted(before.keys() | after.keys()):
before_row = before.get(record_key)
after_row = after.get(record_key)
change = classify_change(before_row, after_row, args.confidence_delta)
if not change:
continue
row = comparison_row(record_key, change, before_row, after_row)
rows.append(row)
source_row = dict(after_row or before_row or {})
source_row.update({
"comparison_change": change,
"before_speed_limit_mph": row["before_speed_limit_mph"],
"before_confidence": row["before_confidence"],
})
review_rows.append(source_row)
change_counts[change] += 1
before_speed = row["before_speed_limit_mph"] or "none"
after_speed = row["after_speed_limit_mph"] or "none"
transition_counts[f"{before_speed}->{after_speed}"] += 1
output_path = args.output_csv.expanduser().resolve()
output_path.parent.mkdir(parents=True, exist_ok=True)
fieldnames = list(rows[0]) if rows else list(comparison_row("", "", None, None))
with output_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
if args.review_output:
review_output = args.review_output.expanduser().resolve()
review_output.parent.mkdir(parents=True, exist_ok=True)
review_fieldnames = list(review_rows[0]) if review_rows else []
with review_output.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=review_fieldnames, extrasaction="ignore")
writer.writeheader()
writer.writerows(review_rows)
summary = {
"before": str(args.before.expanduser().resolve()),
"after": str(args.after.expanduser().resolve()),
"before_rows": len(before),
"after_rows": len(after),
"changed_rows": len(rows),
"review_output": str(args.review_output.expanduser().resolve()) if args.review_output else "",
"changes": dict(sorted(change_counts.items())),
"transitions": dict(sorted(transition_counts.items(), key=lambda item: (-item[1], item[0]))),
}
output_path.with_suffix(".json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
print(json.dumps(summary, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/speed_limit_vision/evaluate_review_classifier.py
+122
View File
@@ -0,0 +1,122 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
from collections import Counter
from pathlib import Path
import cv2
import starpilot.system.speed_limit_vision as slv
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Evaluate the integrated value/reject classifier on reviewed crops.")
parser.add_argument("--models-dir", type=Path, default=Path("starpilot/assets/vision_models"))
parser.add_argument("--positive-manifest", type=Path, required=True)
parser.add_argument("--reject-manifest", type=Path, required=True)
parser.add_argument("--split", choices=("train", "val"), help="Optional source split filter.")
parser.add_argument("--output-csv", type=Path)
return parser.parse_args()
def read_rows(path: Path) -> list[dict[str, str]]:
with path.expanduser().resolve().open("r", encoding="utf-8", newline="") as handle:
return list(csv.DictReader(handle))
def parse_speed(text: str) -> int | None:
try:
return int(float((text or "").strip()))
except ValueError:
return None
def main() -> int:
args = parse_args()
models_dir = args.models_dir.expanduser().resolve()
slv.US_DETECTOR_MODEL_PATH = models_dir / "speed_limit_us_detector.onnx"
slv.US_CLASSIFIER_MODEL_PATH = models_dir / "speed_limit_us_value_classifier.onnx"
reject_path = models_dir / "speed_limit_us_reject_classifier.onnx"
if reject_path.is_file():
slv.US_REJECT_CLASSIFIER_MODEL_PATH = reject_path
daemon = slv.SpeedLimitVisionDaemon(use_runtime=False)
cases: list[tuple[str, dict[str, str], int | None]] = []
for row in read_rows(args.positive_manifest):
if args.split and row.get("split") != args.split:
continue
kind = "advisory" if row.get("review_sign_type", "").strip().lower() == "advisory" else "regulatory"
cases.append((kind, row, parse_speed(row.get("speed_limit_mph", "")) if kind == "regulatory" else None))
for row in read_rows(args.reject_manifest):
if not args.split or row.get("split") == args.split:
cases.append(("hard_negative", row, None))
counts: Counter[str] = Counter()
output_rows = []
for kind, row, expected in cases:
crop_path = Path(row.get("crop_path", "")).expanduser()
crop = cv2.imread(str(crop_path))
if crop is None:
counts[f"{kind}_unreadable"] += 1
continue
result = daemon._classify_speed_limit_from_model(crop)
predicted = result[0] if result is not None else None
confidence = result[1] if result is not None else None
counts[f"{kind}_total"] += 1
if kind == "regulatory":
counts[f"regulatory_speed_{expected}_total"] += 1
if predicted is not None:
counts["regulatory_any"] += 1
if predicted == expected:
counts["regulatory_exact"] += 1
counts[f"regulatory_speed_{expected}_exact"] += 1
elif predicted is not None:
counts["regulatory_wrong"] += 1
elif predicted is None:
counts[f"{kind}_rejected"] += 1
else:
counts[f"{kind}_false_read"] += 1
output_rows.append({
"record_key": row.get("record_key", ""),
"split": row.get("split", ""),
"kind": kind,
"crop_path": str(crop_path),
"expected_speed_limit_mph": "" if expected is None else expected,
"predicted_speed_limit_mph": "" if predicted is None else predicted,
"confidence": "" if confidence is None else f"{confidence:.6f}",
})
regulatory_total = counts["regulatory_total"]
advisory_total = counts["advisory_total"]
hard_negative_total = counts["hard_negative_total"]
exact_rate = counts["regulatory_exact"] / regulatory_total if regulatory_total else 0.0
advisory_reject_rate = counts["advisory_rejected"] / advisory_total if advisory_total else 0.0
hard_negative_reject_rate = counts["hard_negative_rejected"] / hard_negative_total if hard_negative_total else 0.0
regulatory_summary = f"Regulatory exact: {counts['regulatory_exact']}/{regulatory_total} ({exact_rate:.3f})"
print(f"{regulatory_summary}; wrong reads: {counts['regulatory_wrong']}")
print(f"Advisory rejected: {counts['advisory_rejected']}/{advisory_total} ({advisory_reject_rate:.3f})")
hard_negative_summary = f"Hard negatives rejected: {counts['hard_negative_rejected']}/{hard_negative_total}"
print(f"{hard_negative_summary} ({hard_negative_reject_rate:.3f})")
speed_parts = []
for speed in slv.US_CLASSIFIER_SPEED_VALUES:
total = counts[f"regulatory_speed_{speed}_total"]
if total:
speed_parts.append(f"{speed}:{counts[f'regulatory_speed_{speed}_exact']}/{total}")
print("Exact by speed: " + " ".join(speed_parts))
if args.output_csv:
output = args.output_csv.expanduser().resolve()
output.parent.mkdir(parents=True, exist_ok=True)
with output.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=tuple(output_rows[0]) if output_rows else ("record_key",))
writer.writeheader()
writer.writerows(output_rows)
print(f"Wrote {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/speed_limit_vision/evaluate_reviewed_route_events.py
+267
View File
@@ -0,0 +1,267 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import json
from collections import Counter, defaultdict
from dataclasses import dataclass
from pathlib import Path
import cv2
import starpilot.system.speed_limit_vision as slv
if __package__ in (None, ""):
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent))
from import_manual_review_queue import merged_review_rows, parse_speed # type: ignore
from replay_route_runtime import RouteReplayDaemon, configure_models # type: ignore
else:
from .import_manual_review_queue import merged_review_rows, parse_speed
from .replay_route_runtime import RouteReplayDaemon, configure_models
POSITIVE_STATUSES = frozenset(("accepted", "corrected"))
@dataclass(frozen=True)
class ReviewedCase:
record_key: str
route: str
segment: int
frame_time_s: float
source_video_path: Path
expected_speed_limit_mph: int
negative: bool
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Evaluate reviewed sign events through realistic runtime cadence and publish logic.")
parser.add_argument("--queue", type=Path, required=True, help="Reviewed manual_review_queue.csv.")
parser.add_argument("--labels", type=Path, help="Defaults to manual_review_labels.csv beside the queue.")
parser.add_argument("--models-dir", type=Path, default=Path("starpilot/assets/vision_models"), help="Candidate ONNX model directory.")
parser.add_argument("--output-csv", type=Path, required=True, help="Per-event evaluation output.")
parser.add_argument("--window-before", type=float, default=4.0, help="Seconds replayed before the reviewed frame.")
parser.add_argument("--window-after", type=float, default=3.0, help="Seconds replayed after the reviewed frame.")
parser.add_argument("--dedupe-seconds", type=float, default=3.0, help="Collapse nearby reviewed rows with the same expected value.")
parser.add_argument("--measured-base-inference-seconds", type=float, default=0.44, help="Measured no-proposal comma inference cost.")
parser.add_argument("--measured-classifier-forward-seconds", type=float, default=0.066, help="Measured comma cost per classifier forward.")
parser.add_argument("--crop-ocr", action="store_true", help="Evaluate with crop OCR confirmation enabled.")
parser.add_argument("--classifier-min-confidence", type=float, help="Override the value classifier confidence threshold.")
parser.add_argument("--trusted-model-min-confidence", type=float, help="Override tiny-box trusted model confidence.")
parser.add_argument("--strong-rescue-min-proposal-confidence", type=float, help="Override single-frame tiny-sign proposal confidence.")
parser.add_argument("--strong-rescue-min-read-confidence", type=float, help="Override single-frame tiny-sign classifier confidence.")
parser.add_argument("--low-speed-change-consistent-detections", type=int, help="Override reads required to change from 30+ mph to below 30 mph.")
parser.add_argument(
"--allow-low-speed-strong-consensus",
action="store_true",
help="Permit a strong multi-crop consensus to publish a low-speed change from one frame.",
)
parser.add_argument(
"--enable-strong-model-consensus",
action="store_true",
help="Mark three agreeing high-confidence regulatory model crops as strong consensus.",
)
parser.add_argument("--initial-speed-limit", type=int, default=0, help="Seed each replay window with a currently published speed limit.")
parser.add_argument("--positive-only", action="store_true", help="Replay only reviewed speed signs, omitting ignored-crop windows.")
parser.add_argument("--max-cases", type=int, default=0, help="Optional evaluation cap after deduplication.")
return parser.parse_args()
def load_cases(queue_path: Path, labels_path: Path, dedupe_seconds: float) -> list[ReviewedCase]:
rows = merged_review_rows(queue_path, labels_path)
cases: list[ReviewedCase] = []
seen_buckets: set[tuple[str, int, int, int, bool]] = set()
for row in rows:
status = row.get("review_status", "")
positive = status in POSITIVE_STATUSES
negative = status == "ignore" and row.get("review_sign_type") == "not_speed_limit"
if not positive and not negative:
continue
speed = parse_speed(row.get("review_speed_limit_mph", "")) if positive else 0
if positive and not speed:
continue
try:
segment = int(row.get("segment", ""))
frame_time_s = float(row.get("frame_time_s", ""))
except ValueError:
continue
source_video = Path(row.get("source_video_path", "")).expanduser()
if not source_video.is_file():
continue
bucket = int(frame_time_s / max(dedupe_seconds, 0.1))
dedupe_key = (row.get("route", ""), segment, bucket, speed, negative)
if dedupe_key in seen_buckets:
continue
seen_buckets.add(dedupe_key)
cases.append(ReviewedCase(
record_key=row.get("record_key", ""),
route=row.get("route", ""),
segment=segment,
frame_time_s=frame_time_s,
source_video_path=source_video.resolve(),
expected_speed_limit_mph=speed,
negative=negative,
))
return cases
def replay_video_cases(cases: list[ReviewedCase], args: argparse.Namespace) -> dict[str, tuple[list[int], list[int], int]]:
daemons = {
case.record_key: RouteReplayDaemon(
runtime_context=None,
measured_inference_seconds=0.0,
measured_base_inference_seconds=args.measured_base_inference_seconds,
measured_classifier_forward_seconds=args.measured_classifier_forward_seconds,
)
for case in cases
}
for daemon in daemons.values():
daemon.published_speed_limit_mph = args.initial_speed_limit
daemon.last_published_support_at = 0.0
capture = cv2.VideoCapture(str(cases[0].source_video_path))
fps = capture.get(cv2.CAP_PROP_FPS) or 20.0
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
duration_s = frame_count / fps if frame_count > 0 else 60.0
windows = {
case.record_key: (
max(case.frame_time_s - args.window_before, 0.0),
min(case.frame_time_s + args.window_after, duration_s),
)
for case in cases
}
first_frame = max(int(min(window[0] for window in windows.values()) * fps), 0)
end_frame = max(int(max(window[1] for window in windows.values()) * fps), first_frame)
capture.set(cv2.CAP_PROP_POS_FRAMES, first_frame)
frame_index = first_frame
while frame_index <= end_frame:
ok, frame_bgr = capture.read()
if not ok:
break
frame_time_s = frame_index / fps
for case in cases:
start_s, end_s = windows[case.record_key]
if start_s <= frame_time_s <= end_s:
daemons[case.record_key].process_frame(frame_time_s - start_s, frame_bgr)
frame_index += 1
capture.release()
results = {}
for case in cases:
daemon = daemons[case.record_key]
candidates = [int(event["candidateSpeedLimitMph"]) for event in daemon.events if event["event"] == "candidate"]
publishes = [int(event["speedLimitMph"]) for event in daemon.events if event["event"] == "publish"]
results[case.record_key] = candidates, publishes, daemon.inference_frames
return results
def main() -> int:
args = parse_args()
queue_path = args.queue.expanduser().resolve()
labels_path = args.labels.expanduser().resolve() if args.labels else queue_path.with_name("manual_review_labels.csv")
configure_models(args.models_dir)
slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = args.crop_ocr
if args.classifier_min_confidence is not None:
slv.US_CLASSIFIER_MIN_CONFIDENCE = args.classifier_min_confidence
if args.trusted_model_min_confidence is not None:
slv.DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE = args.trusted_model_min_confidence
if args.strong_rescue_min_proposal_confidence is not None:
slv.DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_PROPOSAL_CONFIDENCE = args.strong_rescue_min_proposal_confidence
if args.strong_rescue_min_read_confidence is not None:
slv.DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_READ_CONFIDENCE = args.strong_rescue_min_read_confidence
if args.low_speed_change_consistent_detections is not None:
slv.LOW_SPEED_CHANGE_CONSISTENT_DETECTIONS = args.low_speed_change_consistent_detections
if args.allow_low_speed_strong_consensus:
slv.LOW_SPEED_CHANGE_ALLOW_STRONG_CONSENSUS = True
if args.enable_strong_model_consensus:
slv.DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_ENABLED = True
cases = load_cases(queue_path, labels_path, args.dedupe_seconds)
if args.positive_only:
cases = [case for case in cases if not case.negative]
if args.max_cases > 0:
cases = cases[:args.max_cases]
output_rows: list[dict[str, object]] = []
positive_by_speed: dict[int, Counter[str]] = defaultdict(Counter)
negative_counts: Counter[str] = Counter()
results: dict[str, tuple[list[int], list[int], int]] = {}
cases_by_video: dict[Path, list[ReviewedCase]] = defaultdict(list)
for case in cases:
cases_by_video[case.source_video_path].append(case)
for index, video_cases in enumerate(cases_by_video.values(), start=1):
results.update(replay_video_cases(video_cases, args))
if index % 10 == 0:
print(f"Replayed {index}/{len(cases_by_video)} video segments", flush=True)
for case in cases:
candidates, publishes, inference_frames = results.get(case.record_key, ([], [], 0))
candidate_hit = case.expected_speed_limit_mph in candidates if not case.negative else False
publish_hit = case.expected_speed_limit_mph in publishes if not case.negative else False
false_candidate = bool(candidates) if case.negative else False
false_publish = bool(publishes) if case.negative else False
if case.negative:
negative_counts.update(total=1, candidate_fp=int(false_candidate), publish_fp=int(false_publish))
else:
positive_by_speed[case.expected_speed_limit_mph].update(
total=1,
candidate_hit=int(candidate_hit),
publish_hit=int(publish_hit),
)
output_rows.append({
"record_key": case.record_key,
"route": case.route,
"segment": case.segment,
"frame_time_s": f"{case.frame_time_s:.3f}",
"expected_speed_limit_mph": "" if case.negative else case.expected_speed_limit_mph,
"negative": case.negative,
"candidate_values": "|".join(str(value) for value in candidates),
"publish_values": "|".join(str(value) for value in publishes),
"candidate_hit": candidate_hit,
"publish_hit": publish_hit,
"false_candidate": false_candidate,
"false_publish": false_publish,
"inference_frames": inference_frames,
"source_video_path": str(case.source_video_path),
})
output_path = args.output_csv.expanduser().resolve()
output_path.parent.mkdir(parents=True, exist_ok=True)
fieldnames = list(output_rows[0]) if output_rows else []
with output_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(output_rows)
totals = Counter()
for counts in positive_by_speed.values():
totals.update(counts)
negative_scope = " ".join((
"Reviewed negative labels apply to the proposed crop.",
"Candidate/publish counts replay the full video window and are an upper bound, not confirmed false positives,",
"because another valid sign may be visible in that window.",
))
summary = {
"models_dir": str(args.models_dir.expanduser().resolve()),
"crop_ocr": args.crop_ocr,
"classifier_min_confidence": slv.US_CLASSIFIER_MIN_CONFIDENCE,
"measured_base_inference_seconds": args.measured_base_inference_seconds,
"measured_classifier_forward_seconds": args.measured_classifier_forward_seconds,
"initial_speed_limit_mph": args.initial_speed_limit,
"low_speed_change_consistent_detections": slv.LOW_SPEED_CHANGE_CONSISTENT_DETECTIONS,
"low_speed_change_allow_strong_consensus": slv.LOW_SPEED_CHANGE_ALLOW_STRONG_CONSENSUS,
"strong_model_consensus_enabled": slv.DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_ENABLED,
"positive": dict(totals),
"positive_by_speed": {str(speed): dict(counts) for speed, counts in sorted(positive_by_speed.items())},
"negative": dict(negative_counts),
"negative_scope": negative_scope,
}
summary_path = output_path.with_suffix(".json")
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/speed_limit_vision/evaluate_runtime_manifest.py
+26 -8
View File
@@ -20,7 +20,7 @@ def parse_args() -> argparse.Namespace:
default=Path("starpilot/assets/vision_models"),
help="Directory containing speed_limit_us_detector.onnx and speed_limit_us_value_classifier.onnx.",
)
parser.add_argument("--manifest", type=Path, required=True, help="CSV manifest with dataset_image/frame_path and labels.")
parser.add_argument("--manifest", type=Path, required=True, help="CSV manifest with dataset_image/frame_path/image_path and labels.")
parser.add_argument("--split", action="append", help="Optional split filter. Repeat for multiple splits.")
parser.add_argument("--max-rows", type=int, default=0, help="Optional cap after filtering.")
parser.add_argument("--seed", type=int, default=0, help="Sampling seed used with --max-rows.")
@@ -28,6 +28,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--detector-min-confidence", type=float, help="Override runtime US detector confidence threshold.")
parser.add_argument("--classifier-min-confidence", type=float, help="Override runtime US classifier confidence threshold.")
parser.add_argument("--classifier-reject-min-confidence", type=float, help="Override runtime reject-class confidence threshold.")
parser.add_argument("--trusted-model-min-confidence", type=float, help="Override tiny-box trusted model confidence for evaluation.")
parser.add_argument("--strong-rescue-min-proposal-confidence", type=float, help="Override single-frame tiny-sign proposal confidence.")
parser.add_argument("--strong-rescue-min-read-confidence", type=float, help="Override single-frame tiny-sign classifier confidence.")
parser.add_argument(
"--detector-region-mode",
choices=("full", "right_roi", "full_and_right_roi"),
@@ -36,7 +39,12 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--right-roi-bounds", help="Override the right ROI as left,top,right,bottom ratios, for example 0.45,0,1,0.82.")
parser.add_argument("--right-roi-min-confidence", type=float, help="Override the right ROI detector minimum confidence.")
parser.add_argument("--full-frame-ocr", action="store_true", help="Enable the expensive full-frame OCR fallback during eval.")
crop_ocr_group = parser.add_mutually_exclusive_group()
crop_ocr_group.add_argument("--crop-ocr", action="store_true", dest="crop_ocr", default=None, help="Enable crop OCR confirmation.")
crop_ocr_group.add_argument("--no-crop-ocr", action="store_false", dest="crop_ocr", help="Evaluate the model-only detector/classifier path.")
parser.add_argument("--separate-reject-classifier", action="store_true", help="Enable the optional second-stage reject classifier during eval.")
parser.add_argument("--include-uncertain", action="store_true", help="Include uncertain_positive review rows in positive metrics.")
parser.add_argument("--advisory-positive", action="store_true", help="Score reviewed advisory rows as readable speed positives.")
parser.add_argument("--strict-positive-recall", type=float, help="Exit non-zero if positive exact recall is below this value.")
parser.add_argument("--strict-negative-fpr", type=float, help="Exit non-zero if negative false-positive rate is above this value.")
return parser.parse_args()
@@ -48,6 +56,10 @@ def configure_runtime_options(args: argparse.Namespace) -> None:
if args.full_frame_ocr:
slv.FULL_FRAME_OCR_FALLBACK_ENABLED = True
if args.crop_ocr is not None:
slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = args.crop_ocr
if args.separate_reject_classifier:
slv.SEPARATE_REJECT_CLASSIFIER_ENABLED = True
if args.right_roi_bounds:
parts = [float(part.strip()) for part in args.right_roi_bounds.split(",")]
@@ -81,7 +93,7 @@ def first_present(row: dict[str, str], keys: tuple[str, ...]) -> str:
def expected_value(row: dict[str, str]) -> int | None:
value_text = first_present(row, ("speed_limit_mph", "dominant_value"))
value_text = first_present(row, ("expected_speed_limit_mph", "speed_limit_mph", "dominant_value"))
if value_text:
try:
return int(float(value_text))
@@ -151,6 +163,12 @@ def main() -> int:
if args.classifier_reject_min_confidence is not None:
slv.US_CLASSIFIER_REJECT_MIN_CONFIDENCE = args.classifier_reject_min_confidence
slv.US_REJECT_CLASSIFIER_MIN_CONFIDENCE = args.classifier_reject_min_confidence
if args.trusted_model_min_confidence is not None:
slv.DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE = args.trusted_model_min_confidence
if args.strong_rescue_min_proposal_confidence is not None:
slv.DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_PROPOSAL_CONFIDENCE = args.strong_rescue_min_proposal_confidence
if args.strong_rescue_min_read_confidence is not None:
slv.DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_READ_CONFIDENCE = args.strong_rescue_min_read_confidence
configure_runtime_options(args)
daemon = slv.SpeedLimitVisionDaemon(use_runtime=False)
@@ -163,7 +181,7 @@ def main() -> int:
unreadable_count = 0
for row in rows:
image_text = first_present(row, ("dataset_image", "frame_path", "source_frame"))
image_text = first_present(row, ("dataset_image", "frame_path", "source_frame", "image_path"))
if not image_text:
unreadable_count += 1
continue
@@ -178,7 +196,8 @@ def main() -> int:
predicted_value = detection.speed_limit_mph if detection is not None else None
confidence = detection.confidence if detection is not None else None
expected = expected_value(row)
negative = is_negative(row)
advisory_positive = args.advisory_positive and row.get("review_sign_type", "").strip().lower() == "advisory"
negative = False if advisory_positive else is_negative(row)
if negative:
negative_count += 1
@@ -211,10 +230,9 @@ def main() -> int:
if uncertain_count and not args.include_uncertain:
print(f"Skipped uncertain rows: {uncertain_count}")
print(f"Unreadable rows: {unreadable_count}")
print(
f"Positive exact: {positive_exact}/{positive_count} "
f"({positive_exact_recall:.3f}); any detection: {positive_detected}/{positive_count} ({positive_any_recall:.3f})"
)
positive_summary = f"Positive exact: {positive_exact}/{positive_count} ({positive_exact_recall:.3f})"
detection_summary = f"any detection: {positive_detected}/{positive_count} ({positive_any_recall:.3f})"
print(f"{positive_summary}; {detection_summary}")
print(f"Negative false positives: {negative_false_positive}/{negative_count} ({negative_fpr:.3f})")
if args.output_csv:
scripts/speed_limit_vision/import_manual_review_queue.py
+113 -14
View File
@@ -14,13 +14,16 @@ import cv2
if __package__ in (None, ""):
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore
from common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore # noqa: TID251
else:
from .common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace
CLASSIFIER_FIELDNAMES = [
"record_key",
"route",
"log_id",
"segment",
"split",
"speed_limit_mph",
"review_sign_type",
@@ -36,6 +39,9 @@ CLASSIFIER_FIELDNAMES = [
RUNTIME_FIELDNAMES = [
"record_key",
"route",
"log_id",
"segment",
"split",
"sample_type",
"dataset_image",
@@ -49,6 +55,9 @@ RUNTIME_FIELDNAMES = [
DETECTOR_MANIFEST_FIELDNAMES = [
"record_key",
"route",
"log_id",
"segment",
"split",
"sample_type",
"speed_limit_mph",
@@ -62,6 +71,23 @@ DETECTOR_MANIFEST_FIELDNAMES = [
"detector_class",
]
REJECT_FIELDNAMES = [
"record_key",
"route",
"log_id",
"segment",
"split",
"crop_path",
"frame_path",
"bbox",
"crop_bbox",
"candidate_speed_limit_mph",
"candidate_confidence",
"detector_class",
"review_ignore_reason",
"review_notes",
]
POSITIVE_STATUSES = {"accepted", "corrected"}
UNCERTAIN_STATUS = "uncertain"
NEGATIVE_STATUS = "ignore"
@@ -81,6 +107,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--classifier-manifest-out", type=Path, help="Positive crop manifest for import_manifest_classifier_masks.py.")
parser.add_argument("--runtime-manifest-out", type=Path, help="Full-frame eval manifest including positives and true negatives.")
parser.add_argument("--detector-manifest-out", type=Path, help="Manifest of imported detector examples.")
parser.add_argument("--reject-manifest-out", type=Path, help="Reviewed proposal crops that should train the classifier reject class.")
parser.add_argument("--source-name", default="manual_review", help="Filename prefix for detector dataset imports.")
parser.add_argument("--mode", choices=("symlink", "copy"), default="symlink", help="How to place detector images.")
parser.add_argument("--val-modulo", type=int, default=5, help="Hash modulo for validation split. 0 sends everything to train.")
@@ -110,6 +137,10 @@ def split_for_key(key: str, val_modulo: int, val_remainder: int) -> str:
return "val" if int(digest[:8], 16) % val_modulo == val_remainder else "train"
def split_group_key(row: dict[str, str]) -> str:
return row.get("route") or row.get("log_id") or row["record_key"]
def parse_speed(text: str) -> int:
text = (text or "").strip()
if not text:
@@ -227,6 +258,10 @@ def is_positive(row: dict[str, str]) -> bool:
return Path(row.get("crop_path", "")).is_file() and Path(row.get("frame_path", "")).is_file()
def is_advisory_positive(row: dict[str, str]) -> bool:
return is_positive(row) and effective_sign_type(row) == "advisory"
def is_uncertain_positive(row: dict[str, str]) -> bool:
if row.get("review_status") != UNCERTAIN_STATUS:
return False
@@ -243,10 +278,40 @@ def is_true_negative(row: dict[str, str]) -> bool:
return Path(row.get("frame_path", "")).is_file()
def is_classifier_reject(row: dict[str, str]) -> bool:
if row.get("review_status") != NEGATIVE_STATUS or row.get("detector_class") == "negative_empty":
return False
if row.get("review_sign_type") != "not_speed_limit":
return False
return Path(row.get("crop_path", "")).is_file()
def classifier_reject_row(row: dict[str, str], split: str) -> dict[str, object]:
return {
"record_key": row["record_key"],
"route": row.get("route", ""),
"log_id": row.get("log_id", ""),
"segment": row.get("segment", ""),
"split": split,
"crop_path": row.get("crop_path", ""),
"frame_path": row.get("frame_path", ""),
"bbox": row.get("bbox", ""),
"crop_bbox": row.get("crop_bbox", ""),
"candidate_speed_limit_mph": row.get("candidate_speed_limit_mph", ""),
"candidate_confidence": row.get("candidate_confidence", ""),
"detector_class": row.get("detector_class", ""),
"review_ignore_reason": row.get("review_ignore_reason", ""),
"review_notes": row.get("review_notes", ""),
}
def positive_classifier_row(row: dict[str, str], split: str) -> dict[str, object]:
speed = parse_speed(row.get("review_speed_limit_mph", ""))
return {
"record_key": row["record_key"],
"route": row.get("route", ""),
"log_id": row.get("log_id", ""),
"segment": row.get("segment", ""),
"split": split,
"speed_limit_mph": speed,
"review_sign_type": effective_sign_type(row),
@@ -262,9 +327,13 @@ def positive_classifier_row(row: dict[str, str], split: str) -> dict[str, object
def runtime_row(row: dict[str, str], split: str, sample_type: str) -> dict[str, object]:
speed = parse_speed(row.get("review_speed_limit_mph", "")) if sample_type in ("positive", "uncertain_positive") else 0
positive_sample_types = ("positive", "uncertain_positive", "advisory_negative")
speed = parse_speed(row.get("review_speed_limit_mph", "")) if sample_type in positive_sample_types else 0
return {
"record_key": row["record_key"],
"route": row.get("route", ""),
"log_id": row.get("log_id", ""),
"segment": row.get("segment", ""),
"split": split,
"sample_type": sample_type,
"dataset_image": row.get("frame_path", ""),
@@ -314,6 +383,9 @@ def import_detector_example(
return {
"record_key": row["record_key"],
"route": row.get("route", ""),
"log_id": row.get("log_id", ""),
"segment": row.get("segment", ""),
"split": split,
"sample_type": sample_type,
"speed_limit_mph": parse_speed(row.get("review_speed_limit_mph", "")) if sample_type == "positive" else "",
@@ -334,67 +406,94 @@ def main() -> int:
queue_path = args.queue.expanduser().resolve()
labels_path = args.labels.expanduser().resolve() if args.labels else queue_path.with_name("manual_review_labels.csv")
output_dir = queue_path.parent
classifier_manifest = args.classifier_manifest_out.expanduser().resolve() if args.classifier_manifest_out else output_dir / "manual_review_classifier_manifest.csv"
runtime_manifest = args.runtime_manifest_out.expanduser().resolve() if args.runtime_manifest_out else output_dir / "manual_review_runtime_eval_manifest.csv"
detector_manifest = args.detector_manifest_out.expanduser().resolve() if args.detector_manifest_out else output_dir / "manual_review_detector_import_manifest.csv"
classifier_manifest = (
args.classifier_manifest_out.expanduser().resolve()
if args.classifier_manifest_out else output_dir / "manual_review_classifier_manifest.csv"
)
runtime_manifest = (
args.runtime_manifest_out.expanduser().resolve()
if args.runtime_manifest_out else output_dir / "manual_review_runtime_eval_manifest.csv"
)
detector_manifest = (
args.detector_manifest_out.expanduser().resolve()
if args.detector_manifest_out else output_dir / "manual_review_detector_import_manifest.csv"
)
reject_manifest = (
args.reject_manifest_out.expanduser().resolve()
if args.reject_manifest_out else output_dir / "manual_review_classifier_reject_manifest.csv"
)
rows = merged_review_rows(queue_path, labels_path)
positive_rows = [row for row in rows if is_positive(row)]
advisory_positive_rows = [row for row in positive_rows if is_advisory_positive(row)]
uncertain_positive_rows = [row for row in rows if is_uncertain_positive(row)]
true_negative_rows = [row for row in rows if is_true_negative(row)]
classifier_reject_rows = [row for row in rows if is_classifier_reject(row)]
if args.max_detector_negatives > 0:
true_negative_rows = true_negative_rows[:args.max_detector_negatives]
classifier_rows: list[dict[str, object]] = []
runtime_rows: list[dict[str, object]] = []
detector_rows: list[dict[str, object]] = []
reject_rows: list[dict[str, object]] = []
for row in positive_rows:
split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder)
split = split_for_key(split_group_key(row), args.val_modulo, args.val_remainder)
classifier_rows.append(positive_classifier_row(row, split))
runtime_rows.append(runtime_row(row, split, "positive"))
sample_type = "advisory_negative" if is_advisory_positive(row) else "positive"
runtime_rows.append(runtime_row(row, split, sample_type))
detector_row = import_detector_example(workspace, row, split, args.source_name, "positive", args.mode, args.overwrite)
if detector_row is not None:
detector_rows.append(detector_row)
for row in uncertain_positive_rows:
split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder)
split = split_for_key(split_group_key(row), args.val_modulo, args.val_remainder)
runtime_rows.append(runtime_row(row, split, "uncertain_positive"))
for row in true_negative_rows:
split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder)
split = split_for_key(split_group_key(row), args.val_modulo, args.val_remainder)
runtime_rows.append(runtime_row(row, split, "negative_empty"))
detector_row = import_detector_example(workspace, row, split, args.source_name, "negative_empty", args.mode, args.overwrite)
if detector_row is not None:
detector_rows.append(detector_row)
for row in classifier_reject_rows:
split = split_for_key(split_group_key(row), args.val_modulo, args.val_remainder)
reject_rows.append(classifier_reject_row(row, split))
write_csv(classifier_manifest, CLASSIFIER_FIELDNAMES, classifier_rows)
write_csv(runtime_manifest, RUNTIME_FIELDNAMES, runtime_rows)
write_csv(detector_manifest, DETECTOR_MANIFEST_FIELDNAMES, detector_rows)
write_csv(reject_manifest, REJECT_FIELDNAMES, reject_rows)
summary = {
"queue": str(queue_path),
"labels": str(labels_path),
"reviewed_rows": len(rows),
"positive_rows": len(positive_rows),
"advisory_positive_rows": len(advisory_positive_rows),
"uncertain_positive_rows": len(uncertain_positive_rows),
"true_negative_rows": len(true_negative_rows),
"classifier_reject_rows": len(reject_rows),
"classifier_manifest": str(classifier_manifest),
"runtime_manifest": str(runtime_manifest),
"detector_manifest": str(detector_manifest),
"detector_imported": len(detector_rows),
"reject_manifest": str(reject_manifest),
}
summary_path = output_dir / "manual_review_import_summary.json"
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(
"Imported manual review queue: "
f"reviewed={len(rows)} positives={len(positive_rows)} uncertain_positives={len(uncertain_positive_rows)} true_negatives={len(true_negative_rows)} "
f"detector_imported={len(detector_rows)}"
)
review_counts = " ".join((
f"reviewed={len(rows)} positives={len(positive_rows)}",
f"uncertain_positives={len(uncertain_positive_rows)} true_negatives={len(true_negative_rows)}",
))
import_counts = f"classifier_rejects={len(reject_rows)} detector_imported={len(detector_rows)}"
print(f"Imported manual review queue: {review_counts} {import_counts}")
print(f"Classifier manifest: {classifier_manifest}")
print(f"Runtime eval manifest: {runtime_manifest}")
print(f"Detector import manifest: {detector_manifest}")
print(f"Classifier reject manifest: {reject_manifest}")
print(f"Summary: {summary_path}")
return 0
scripts/speed_limit_vision/interpolate_yolo_checkpoints.py
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import copy
from pathlib import Path
import torch
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Linearly interpolate compatible Ultralytics checkpoints.")
parser.add_argument("--base", type=Path, required=True, help="Baseline checkpoint used at alpha=0.")
parser.add_argument("--candidate", type=Path, required=True, help="Candidate checkpoint used at alpha=1.")
parser.add_argument("--alpha", type=float, required=True, help="Candidate weight in [0, 1].")
parser.add_argument("--output", type=Path, required=True, help="Interpolated checkpoint path.")
return parser.parse_args()
def checkpoint_model(checkpoint: dict):
model = checkpoint.get("ema") or checkpoint.get("model")
if model is None:
raise ValueError("Checkpoint contains neither model nor ema weights")
return model.float()
def main() -> int:
args = parse_args()
if not 0.0 <= args.alpha <= 1.0:
raise ValueError("--alpha must be between 0 and 1")
base_checkpoint = torch.load(args.base.expanduser().resolve(), map_location="cpu", weights_only=False)
candidate_checkpoint = torch.load(args.candidate.expanduser().resolve(), map_location="cpu", weights_only=False)
base_model = checkpoint_model(base_checkpoint)
candidate_model = checkpoint_model(candidate_checkpoint)
base_state = base_model.state_dict()
candidate_state = candidate_model.state_dict()
if base_state.keys() != candidate_state.keys():
raise ValueError("Checkpoint model state keys differ")
interpolated_state = {}
for key, base_value in base_state.items():
candidate_value = candidate_state[key]
if base_value.shape != candidate_value.shape:
raise ValueError(f"Checkpoint tensor shape differs for {key}")
if torch.is_floating_point(base_value):
interpolated_state[key] = base_value * (1.0 - args.alpha) + candidate_value * args.alpha
else:
interpolated_state[key] = candidate_value if args.alpha >= 0.5 else base_value
interpolated_model = copy.deepcopy(base_model)
interpolated_model.load_state_dict(interpolated_state)
output_checkpoint = dict(base_checkpoint)
output_checkpoint.update({
"model": interpolated_model,
"ema": None,
"optimizer": None,
"epoch": -1,
"best_fitness": None,
})
output = args.output.expanduser().resolve()
output.parent.mkdir(parents=True, exist_ok=True)
torch.save(output_checkpoint, output)
print(f"Wrote alpha={args.alpha:.4f} interpolated checkpoint to {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/speed_limit_vision/localize_bookmark_signs.py
+27 -6
View File
@@ -100,7 +100,15 @@ def iter_context_frames(clip_root: Path, window: ebl.BookmarkWindow, search_befo
yield relative_time_s, clip_path, source_time_s, frame_bgr
def _score_expanded_candidate(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, class_id: int, proposal_confidence: float, box, full_detection):
def _score_expanded_candidate(
daemon: slv.SpeedLimitVisionDaemon,
frame_bgr,
class_id: int,
proposal_confidence: float,
box,
full_detection,
use_ocr: bool = True,
):
frame_height, frame_width = frame_bgr.shape[:2]
x1, y1, x2, y2 = box
box_width = x2 - x1
@@ -123,7 +131,7 @@ def _score_expanded_candidate(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, cla
is_regulatory = daemon._is_regulatory_speed_sign(sign_crop) or class_id == 2
model_read = daemon._classify_speed_limit_from_model(sign_crop)
ocr_read = daemon._read_speed_limit_from_crop(sign_crop)
ocr_read = daemon._read_speed_limit_from_crop(sign_crop) if use_ocr else None
if model_read is None and ocr_read is None:
continue
@@ -132,9 +140,14 @@ def _score_expanded_candidate(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, cla
if read_result is None or read_result[0] not in slv.SCHOOL_ZONE_SPEED_VALUES:
continue
elif not is_regulatory:
if model_read is None or ocr_read is None or model_read[0] != ocr_read[0]:
if use_ocr:
if model_read is None or ocr_read is None or model_read[0] != ocr_read[0]:
continue
read_result = (model_read[0], min(model_read[1], ocr_read[1]))
elif model_read is None or model_read[1] < slv.DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE:
continue
read_result = (model_read[0], min(model_read[1], ocr_read[1]))
else:
read_result = model_read
else:
if model_read is not None and ocr_read is not None and model_read[0] == ocr_read[0]:
read_result = (model_read[0], max(model_read[1], ocr_read[1]))
@@ -169,7 +182,7 @@ def _score_expanded_candidate(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, cla
return best
def score_frame(daemon: slv.SpeedLimitVisionDaemon, frame_bgr):
def score_frame(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, use_ocr: bool = True):
full_detection = daemon._detect_sign(frame_bgr)
best = None
@@ -177,7 +190,15 @@ def score_frame(daemon: slv.SpeedLimitVisionDaemon, frame_bgr):
if class_id == 1:
continue
candidate = _score_expanded_candidate(daemon, frame_bgr, class_id, proposal_confidence, (x1, y1, x2, y2), full_detection)
candidate = _score_expanded_candidate(
daemon,
frame_bgr,
class_id,
proposal_confidence,
(x1, y1, x2, y2),
full_detection,
use_ocr=use_ocr,
)
if candidate is None:
continue
if best is None or candidate["score"] > best["score"]:
scripts/speed_limit_vision/merge_manual_review_queues.py
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Merge fingerprint-compatible speed-limit manual review queues.")
parser.add_argument("inputs", type=Path, nargs="+", help="Queue directories or manual_review_queue.csv files.")
parser.add_argument("--output-dir", type=Path, required=True, help="Destination queue directory.")
return parser.parse_args()
def queue_paths(input_path: Path) -> tuple[Path, Path]:
resolved = input_path.expanduser().resolve()
if resolved.is_dir():
return resolved / "manual_review_queue.csv", resolved / "manual_review_summary.json"
return resolved, resolved.with_name("manual_review_summary.json")
def main() -> int:
args = parse_args()
rows_by_key: dict[str, dict[str, str]] = {}
summaries_by_route: dict[str, dict[str, object]] = {}
fieldnames: list[str] | None = None
mining_fingerprint = ""
model_fingerprint = ""
for input_path in args.inputs:
queue_path, summary_path = queue_paths(input_path)
if not queue_path.is_file() or not summary_path.is_file():
raise FileNotFoundError(f"Queue or summary missing for {input_path}")
summary = json.loads(summary_path.read_text(encoding="utf-8"))
current_mining = str(summary.get("mining_fingerprint", ""))
current_model = str(summary.get("model_fingerprint", ""))
if not current_mining or not current_model:
raise RuntimeError(f"Queue is not fingerprinted: {input_path}")
if mining_fingerprint and current_mining != mining_fingerprint:
raise RuntimeError(f"Mining fingerprint mismatch: {input_path}")
if model_fingerprint and current_model != model_fingerprint:
raise RuntimeError(f"Model fingerprint mismatch: {input_path}")
mining_fingerprint = current_mining
model_fingerprint = current_model
with queue_path.open("r", encoding="utf-8", newline="") as handle:
reader = csv.DictReader(handle)
current_fields = list(reader.fieldnames or [])
if fieldnames is not None and current_fields != fieldnames:
raise RuntimeError(f"Queue fields differ: {input_path}")
fieldnames = current_fields
for row in reader:
key = row.get("record_key", "")
if key:
rows_by_key[key] = row
for route_summary in summary.get("routes", []):
route = str(route_summary.get("route", ""))
if route:
summaries_by_route[route] = route_summary
rows = sorted(rows_by_key.values(), key=lambda row: (-float(row.get("review_priority") or 0.0), row["record_key"]))
route_summaries = sorted(summaries_by_route.values(), key=lambda item: str(item["route"]))
output_dir = args.output_dir.expanduser().resolve()
output_dir.mkdir(parents=True, exist_ok=True)
queue_output = output_dir / "manual_review_queue.csv"
with queue_output.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames or [])
writer.writeheader()
writer.writerows(rows)
summary_output = output_dir / "manual_review_summary.json"
summary_output.write_text(json.dumps({
"mining_fingerprint": mining_fingerprint,
"model_fingerprint": model_fingerprint,
"manifest": str(queue_output),
"rows": len(rows),
"candidates": sum(row.get("detector_class") != "negative_empty" for row in rows),
"negatives": sum(row.get("detector_class") == "negative_empty" for row in rows),
"routes": route_summaries,
}, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"Merged {len(rows)} rows from {len(route_summaries)} routes into {queue_output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/speed_limit_vision/mine_classifier_reject_crops.py
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import hashlib
from pathlib import Path
import cv2
import starpilot.system.speed_limit_vision as slv
if __package__ in (None, ""):
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent))
from localize_bookmark_signs import configure_models # type: ignore
else:
from .localize_bookmark_signs import configure_models
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Mine model-confusing negative crops into an integrated classifier reject class.")
parser.add_argument("--manifest", type=Path, action="append", required=True, help="Reviewed runtime manifest. Repeat for multiple sets.")
parser.add_argument("--models-dir", type=Path, required=True, help="Detector/classifier ONNX bundle used to mine hard negatives.")
parser.add_argument("--dataset", type=Path, required=True, help="Classifier dataset containing train/reject and val/reject.")
parser.add_argument("--split", choices=("train", "val"), default="train", help="Destination dataset split.")
parser.add_argument("--classifier-min-confidence", type=float, default=0.55, help="Minimum wrong speed confidence to mine.")
parser.add_argument("--max-crops", type=int, default=2000, help="Maximum reject crops to add.")
parser.add_argument("--overwrite", action="store_true", help="Overwrite an existing crop with the same content hash.")
return parser.parse_args()
def first_present(row: dict[str, str], keys: tuple[str, ...]) -> str:
for key in keys:
value = row.get(key, "").strip()
if value:
return value
return ""
def expected_value(row: dict[str, str]) -> int | None:
value = first_present(row, ("expected_speed_limit_mph", "speed_limit_mph", "dominant_value"))
if not value:
return None
try:
return int(float(value))
except ValueError:
return None
def is_reviewed_negative(row: dict[str, str]) -> bool:
sample_type = row.get("sample_type", "").lower()
review_status = row.get("review_status", "").lower()
explicit_negative = row.get("negative", "").strip().lower() in ("1", "true", "yes")
return explicit_negative or "negative" in sample_type or review_status in ("negative", "reject")
def iter_negative_images(manifests: list[Path]):
seen: set[Path] = set()
for manifest in manifests:
with manifest.expanduser().resolve().open("r", encoding="utf-8", newline="") as handle:
for row in csv.DictReader(handle):
if not is_reviewed_negative(row):
continue
image_text = first_present(row, ("dataset_image", "frame_path", "source_frame", "image_path"))
if not image_text:
continue
image_path = Path(image_text).expanduser().resolve()
if image_path in seen or not image_path.is_file():
continue
seen.add(image_path)
yield image_path
def crop_hash(crop) -> str:
ok, encoded = cv2.imencode(".jpg", crop, [cv2.IMWRITE_JPEG_QUALITY, 94])
if not ok:
return ""
return hashlib.sha256(encoded.tobytes()).hexdigest()[:20]
def remove_appledouble_files(root: Path) -> int:
removed = 0
for path in root.rglob("._*"):
if path.is_file():
path.unlink()
removed += 1
return removed
def main() -> int:
args = parse_args()
configure_models(args.models_dir)
slv.US_CLASSIFIER_MIN_CONFIDENCE = args.classifier_min_confidence
daemon = slv.SpeedLimitVisionDaemon(use_runtime=False)
dataset_root = args.dataset.expanduser().resolve()
appledouble_removed = remove_appledouble_files(dataset_root)
output_dir = dataset_root / args.split / "reject"
output_dir.mkdir(parents=True, exist_ok=True)
added = 0
frames = 0
proposals = 0
for image_path in iter_negative_images(args.manifest):
if added >= args.max_crops:
break
frame_bgr = cv2.imread(str(image_path))
if frame_bgr is None:
continue
frames += 1
frame_height, frame_width = frame_bgr.shape[:2]
for _proposal_confidence, class_id, (x1, y1, x2, y2) in daemon._collect_detector_classifier_proposals(frame_bgr):
if class_id == 1:
continue
proposals += 1
box_width = x2 - x1
box_height = y2 - y1
if box_width <= 0 or box_height <= 0:
continue
for expansion_index, (left, top, right, bottom, _weight) in enumerate(slv.DETECTOR_CLASSIFIER_EXPANSIONS):
crop_x1 = max(int(x1 - box_width * left), 0)
crop_y1 = max(int(y1 - box_height * top), 0)
crop_x2 = min(int(x2 + box_width * right), frame_width)
crop_y2 = min(int(y2 + box_height * bottom), frame_height)
crop = frame_bgr[crop_y1:crop_y2, crop_x1:crop_x2]
if crop.size == 0 or daemon._classify_speed_limit_from_model(crop) is None:
continue
digest = crop_hash(crop)
if not digest:
continue
output_path = output_dir / f"hardneg_{digest}_e{expansion_index}.jpg"
if output_path.exists() and not args.overwrite:
continue
cv2.imwrite(str(output_path), crop, [cv2.IMWRITE_JPEG_QUALITY, 94])
added += 1
if added >= args.max_crops:
break
if added >= args.max_crops:
break
if added:
cache_path = dataset_root / f"{args.split}.cache"
if cache_path.is_file():
cache_path.unlink()
summary = f"Hard-negative mining complete: frames={frames} proposals={proposals} added={added}"
summary += f" appledouble_removed={appledouble_removed}"
print(summary)
print(f"Reject dataset: {output_dir}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/speed_limit_vision/mine_route_training_samples.py
+115 -8
View File
@@ -30,10 +30,14 @@ else:
DEFAULT_ROUTE_BUNDLE_STATE_DIR = Path("/Volumes/T5/starpilot_speed_limit/analysis/route_bundles/state")
DEFAULT_WORKSPACE = Path("/Volumes/T5/starpilot_speed_limit/workspace/speed_limit_training_clean")
DEFAULT_REVIEW_MANIFEST_NAME = "route_training_samples.csv"
MINING_RUN_SCHEMA_VERSION = 2
MPH_PER_MS = 2.2369362920544
VALID_WEAK_LABEL_VALUES = set(slv.US_CLASSIFIER_SPEED_VALUES)
POSITIVE_FIELDNAMES = [
"record_key",
"mining_run_id",
"mining_fingerprint",
"model_fingerprint",
"route",
"dongle_id",
"log_id",
@@ -83,6 +87,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--clip-root", type=Path, default=preferred_clip_root(), help="Route realdata root.")
parser.add_argument("--bundle-state-dir", type=Path, default=DEFAULT_ROUTE_BUNDLE_STATE_DIR, help="Completed extraction marker directory.")
parser.add_argument("--models-dir", type=Path, help="Optional model directory for mining with non-repo ONNXs.")
parser.add_argument("--model-only", action="store_true", help="Use detector/classifier output without OCR when weak-labeling signs.")
parser.add_argument("--run-id", help="Version this mining pass. Use 'auto' to derive an id from the ONNX bundle.")
parser.add_argument("--output-root", type=Path, help="Output root for this pass. Defaults to a versioned staging directory when --run-id is set.")
parser.add_argument("--manifest-out", type=Path, help=f"Review manifest path. Defaults to <workspace>/review/{DEFAULT_REVIEW_MANIFEST_NAME}.")
parser.add_argument("--sample-every", type=float, default=4.0, help="Seconds between regular video samples.")
parser.add_argument("--seek-sampling", action="store_true", help="Seek directly to sampled frames instead of sequentially grabbing through each segment.")
@@ -110,6 +117,54 @@ def safe_key(text: str) -> str:
return text.replace("/", "_").replace("|", "_").replace(":", "_")
def model_bundle_fingerprint() -> str:
digest = hashlib.sha256()
for path in (slv.US_DETECTOR_MODEL_PATH, slv.US_CLASSIFIER_MODEL_PATH):
resolved = Path(path).expanduser().resolve()
digest.update(resolved.name.encode("utf-8"))
with resolved.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def mining_configuration_fingerprint(args: argparse.Namespace, model_fingerprint: str) -> str:
config = {
"schema_version": MINING_RUN_SCHEMA_VERSION,
"model_fingerprint": model_fingerprint,
"model_only": args.model_only,
"sample_every": args.sample_every,
"transition_radius": args.transition_radius,
"transition_step": args.transition_step,
"max_frames_per_route": args.max_frames_per_route,
"max_positives_per_route": args.max_positives_per_route,
"max_negatives_per_route": args.max_negatives_per_route,
"positive_min_score": args.positive_min_score,
"no_map_min_score": args.no_map_min_score,
"min_proposal_confidence": args.min_proposal_confidence,
"min_width": args.min_width,
"min_height": args.min_height,
"next_limit_distance": args.next_limit_distance,
}
digest = hashlib.sha256(json.dumps(config, sort_keys=True).encode("utf-8"))
for source_path in (Path(__file__), Path(score_frame.__code__.co_filename), Path(slv.__file__)):
digest.update(source_path.resolve().read_bytes())
return digest.hexdigest()
def resolve_run_id(requested: str | None, model_fingerprint: str, mining_fingerprint: str) -> str:
if not requested:
return ""
run_id = (
f"model_{model_fingerprint[:12]}_run_{mining_fingerprint[:12]}"
if requested == "auto"
else safe_key(requested.strip())
)
if not run_id:
raise ValueError("--run-id must not be empty")
return run_id
def parse_route_id(text: str) -> tuple[str, str, str]:
normalized = text.strip().replace("|", "/")
if "/" not in normalized:
@@ -390,6 +445,8 @@ def should_keep_positive(scored: dict, speed_limit_mph: int, consistent_count: i
return False
if float(scored["proposal_confidence"]) < args.min_proposal_confidence:
return False
if args.model_only and relation not in ("agree_current", "agree_next"):
return False
if relation in ("agree_current", "agree_next"):
return float(scored["score"]) >= args.positive_min_score and consistent_count >= 1
return float(scored["score"]) >= args.no_map_min_score and consistent_count >= 2
@@ -443,16 +500,33 @@ def write_sample(frame_bgr, image_path: Path, label_path: Path, label_text: str,
return True
def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse.Namespace, workspace: Path, clip_root: Path, manifest_path: Path, route_state_dir: Path) -> dict[str, int | str | float]:
def mine_route(
route_id: str,
daemon: slv.SpeedLimitVisionDaemon,
args: argparse.Namespace,
output_root: Path,
clip_root: Path,
manifest_path: Path,
route_state_dir: Path,
run_id: str,
mining_fingerprint: str,
model_fingerprint: str,
) -> dict[str, int | str | float]:
route_id, dongle_id, log_id = parse_route_id(route_id)
route_key = safe_key(route_id)
state_path = route_state_dir / f"{route_key}.json"
if run_id and state_path.exists():
state = json.loads(state_path.read_text(encoding="utf-8"))
if state.get("model_fingerprint") != model_fingerprint or state.get("mining_fingerprint") != mining_fingerprint:
raise RuntimeError(
f"Mining state fingerprint mismatch for {route_id}. Use a new --run-id or output root instead of mixing runs."
)
if state_path.exists() and not args.force:
return {"route": route_id, "status": "skipped", "positives": 0, "negatives": 0, "scored": 0}
split = route_split(route_id, args.val_route_modulo, args.val_route_remainder)
image_dir = ensure_dir(workspace / "detector" / "images" / split)
label_dir = ensure_dir(workspace / "detector" / "labels" / split)
image_dir = ensure_dir(output_root / "detector" / "images" / split)
label_dir = ensure_dir(output_root / "detector" / "labels" / split)
segments = route_segments(clip_root, log_id)
if not segments:
return {"route": route_id, "status": "missing_segments", "positives": 0, "negatives": 0, "scored": 0}
@@ -487,7 +561,7 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse
continue
scored_frames += 1
scored = score_frame(daemon, frame_bgr)
scored = score_frame(daemon, frame_bgr, use_ocr=not args.model_only)
context = nearest_context(contexts, time_s)
if scored is None:
@@ -501,6 +575,9 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse
negatives += 1
route_rows.append({
"record_key": record_key,
"mining_run_id": run_id,
"mining_fingerprint": mining_fingerprint,
"model_fingerprint": model_fingerprint,
"route": route_id,
"dongle_id": dongle_id,
"log_id": log_id,
@@ -548,6 +625,9 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse
bbox = ",".join(str(value) for value in scored["box"])
route_rows.append({
"record_key": record_key,
"mining_run_id": run_id,
"mining_fingerprint": mining_fingerprint,
"model_fingerprint": model_fingerprint,
"route": route_id,
"dongle_id": dongle_id,
"log_id": log_id,
@@ -585,9 +665,13 @@ def mine_route(route_id: str, daemon: slv.SpeedLimitVisionDaemon, args: argparse
if not args.dry_run:
merge_review_rows(manifest_path, route_rows)
merge_value_labels(workspace / "classifier" / "value_labels.csv", value_rows)
merge_value_labels(output_root / "classifier" / "value_labels.csv", value_rows)
state_path.write_text(json.dumps({
"route": route_id,
"mining_run_id": run_id,
"mining_fingerprint": mining_fingerprint,
"model_fingerprint": model_fingerprint,
"model_only": args.model_only,
"status": "mined",
"positives": positives,
"negatives": negatives,
@@ -613,20 +697,41 @@ def main() -> int:
args = parse_args()
workspace = resolve_workspace(args.workspace)
clip_root = args.clip_root.expanduser().resolve()
manifest_path = args.manifest_out.expanduser().resolve() if args.manifest_out else (ensure_dir(workspace / "review") / DEFAULT_REVIEW_MANIFEST_NAME)
route_state_dir = ensure_dir(workspace / "review" / "route_training_samples_state")
routes = read_routes(args)
if not routes:
raise SystemExit("No routes to mine. Pass route ids, --routes-file, or completed bundle markers.")
configure_models(args.models_dir)
slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = not args.model_only
model_fingerprint = model_bundle_fingerprint()
mining_fingerprint = mining_configuration_fingerprint(args, model_fingerprint)
run_id = resolve_run_id(args.run_id, model_fingerprint, mining_fingerprint)
if args.output_root:
output_root = args.output_root.expanduser().resolve()
elif run_id:
output_root = workspace / "staging" / "route_mining" / run_id
else:
output_root = workspace
manifest_path = args.manifest_out.expanduser().resolve() if args.manifest_out else (ensure_dir(output_root / "review") / DEFAULT_REVIEW_MANIFEST_NAME)
route_state_dir = ensure_dir(output_root / "review" / "route_training_samples_state")
daemon = slv.SpeedLimitVisionDaemon(use_runtime=False)
total_positive = 0
total_negative = 0
total_scored = 0
for index, route in enumerate(routes, start=1):
result = mine_route(route, daemon, args, workspace, clip_root, manifest_path, route_state_dir)
result = mine_route(
route,
daemon,
args,
output_root,
clip_root,
manifest_path,
route_state_dir,
run_id,
mining_fingerprint,
model_fingerprint,
)
total_positive += int(result.get("positives", 0))
total_negative += int(result.get("negatives", 0))
total_scored += int(result.get("scored", 0))
@@ -641,6 +746,8 @@ def main() -> int:
flush=True,
)
print(f"Review manifest: {manifest_path}", flush=True)
print(f"Model fingerprint: {model_fingerprint}", flush=True)
print(f"Mining fingerprint: {mining_fingerprint}", flush=True)
return 0
scripts/speed_limit_vision/replay_route_runtime.py
+28 -5
View File
@@ -193,6 +193,21 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--right-roi-min-confidence", type=float, help="Override the right ROI detector minimum confidence.")
parser.add_argument("--classifier-min-confidence", type=float, help="Override the value classifier confidence threshold.")
parser.add_argument("--full-frame-ocr", action="store_true", help="Enable the expensive full-frame OCR fallback during replay.")
crop_ocr_group = parser.add_mutually_exclusive_group()
crop_ocr_group.add_argument("--crop-ocr", action="store_true", dest="crop_ocr", default=None, help="Enable crop OCR confirmation during replay.")
crop_ocr_group.add_argument("--no-crop-ocr", action="store_false", dest="crop_ocr", help="Replay the model-only detector/classifier path.")
parser.add_argument("--low-speed-change-consistent-detections", type=int, help="Override reads required to change from 30+ mph to below 30 mph.")
parser.add_argument(
"--allow-low-speed-strong-consensus",
action="store_true",
help="Permit a strong multi-crop consensus to publish a low-speed change from one frame.",
)
parser.add_argument(
"--enable-strong-model-consensus",
action="store_true",
help="Mark three agreeing high-confidence regulatory model crops as strong consensus.",
)
parser.add_argument("--initial-speed-limit", type=int, default=0, help="Seed route replay with a currently published speed limit.")
return parser.parse_args()
@@ -299,6 +314,14 @@ def configure_runtime_options(args: argparse.Namespace) -> None:
if args.full_frame_ocr:
slv.FULL_FRAME_OCR_FALLBACK_ENABLED = True
if args.crop_ocr is not None:
slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = args.crop_ocr
if args.low_speed_change_consistent_detections is not None:
slv.LOW_SPEED_CHANGE_CONSISTENT_DETECTIONS = args.low_speed_change_consistent_detections
if args.allow_low_speed_strong_consensus:
slv.LOW_SPEED_CHANGE_ALLOW_STRONG_CONSENSUS = True
if args.enable_strong_model_consensus:
slv.DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_ENABLED = True
if args.right_roi_bounds:
parts = [float(part.strip()) for part in args.right_roi_bounds.split(",")]
@@ -347,6 +370,7 @@ def replay_route(
measured_inference_seconds: float,
measured_base_inference_seconds: float | None = None,
measured_classifier_forward_seconds: float = 0.0,
initial_speed_limit_mph: int = 0,
) -> tuple[RouteSummary, list[dict[str, str]]]:
daemon = RouteReplayDaemon(
runtime_context,
@@ -354,6 +378,7 @@ def replay_route(
measured_base_inference_seconds,
measured_classifier_forward_seconds,
)
daemon.published_speed_limit_mph = initial_speed_limit_mph
for segment_path in segments:
segment = segment_index(segment_path)
capture = cv2.VideoCapture(str(segment_path))
@@ -397,11 +422,8 @@ def replay_route(
capture.release()
if progress:
print(
f" seg {segment:02d}: sampled={daemon.sampled_frames} inference={daemon.inference_frames} "
f"events={len(daemon.events)}",
flush=True,
)
counts = f"sampled={daemon.sampled_frames} inference={daemon.inference_frames} events={len(daemon.events)}"
print(f" seg {segment:02d}: {counts}", flush=True)
return summarize(log_id, len(segments), runtime_context is not None, daemon), daemon.events
@@ -490,6 +512,7 @@ def main() -> int:
args.measured_inference_seconds,
args.measured_base_inference_seconds,
args.measured_classifier_forward_seconds,
args.initial_speed_limit,
)
all_events.extend((log_id, event) for event in events)
summary_line = "".join((
scripts/speed_limit_vision/rescore_manual_review_queue.py
+138
View File
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import json
from collections import Counter
from pathlib import Path
import cv2
import starpilot.system.speed_limit_vision as slv
if __package__ in (None, ""):
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent))
from compare_manual_review_queues import classify_change # type: ignore
from mine_route_training_samples import model_bundle_fingerprint # type: ignore
from replay_route_runtime import configure_models # type: ignore
else:
from .compare_manual_review_queues import classify_change
from .mine_route_training_samples import model_bundle_fingerprint
from .replay_route_runtime import configure_models
EXTRA_FIELDS = (
"comparison_change",
"before_speed_limit_mph",
"before_confidence",
"rescore_status",
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Rescore stored manual-review crops with a new value classifier.")
parser.add_argument("--input", type=Path, required=True, help="Baseline manual_review_queue.csv.")
parser.add_argument("--models-dir", type=Path, required=True, help="Candidate detector/classifier ONNX directory.")
parser.add_argument("--output", type=Path, required=True, help="Rescored review-compatible output CSV.")
parser.add_argument("--confidence-delta", type=float, default=0.05, help="Minimum confidence-only change to report.")
parser.add_argument("--shard-count", type=int, default=1, help="Number of deterministic row shards.")
parser.add_argument("--shard-index", type=int, default=0, help="Zero-based row shard processed by this invocation.")
return parser.parse_args()
def rescore_row(
row: dict[str, str],
daemon: slv.SpeedLimitVisionDaemon,
model_fingerprint: str,
confidence_delta: float,
) -> dict[str, str]:
output = dict(row)
before_speed = row.get("candidate_speed_limit_mph", "")
before_confidence = row.get("candidate_confidence", "")
output["before_speed_limit_mph"] = before_speed
output["before_confidence"] = before_confidence
output["model_fingerprint"] = model_fingerprint
crop_path = Path(row.get("crop_path", "")).expanduser()
crop = cv2.imread(str(crop_path)) if crop_path.is_file() else None
if crop is None:
output["comparison_change"] = "unreadable"
output["rescore_status"] = "unreadable"
return output
result = daemon._classify_speed_limit_from_model(crop)
if result is None:
output["candidate_speed_limit_mph"] = ""
output["candidate_confidence"] = ""
output["model_read"] = ""
else:
speed_limit_mph, confidence = result
output["candidate_speed_limit_mph"] = str(int(speed_limit_mph))
output["candidate_confidence"] = f"{float(confidence):.8f}"
output["model_read"] = f"{int(speed_limit_mph)}@{float(confidence):.3f}"
change = classify_change(row, output, confidence_delta)
output["comparison_change"] = change or "unchanged"
output["rescore_status"] = "rescored_crop"
return output
def main() -> int:
args = parse_args()
if args.shard_count <= 0 or not 0 <= args.shard_index < args.shard_count:
raise ValueError("--shard-index must be within --shard-count")
configure_models(args.models_dir)
slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = False
daemon = slv.SpeedLimitVisionDaemon(use_runtime=False)
fingerprint = model_bundle_fingerprint()
input_path = args.input.expanduser().resolve()
with input_path.open("r", encoding="utf-8", newline="") as handle:
reader = csv.DictReader(handle)
input_fields = list(reader.fieldnames or [])
rows = [row for index, row in enumerate(reader) if index % args.shard_count == args.shard_index]
output_rows: list[dict[str, str]] = []
changes: Counter[str] = Counter()
transitions: Counter[str] = Counter()
for index, row in enumerate(rows, start=1):
rescored = rescore_row(row, daemon, fingerprint, args.confidence_delta)
output_rows.append(rescored)
change = rescored["comparison_change"]
changes[change] += 1
before_speed = rescored["before_speed_limit_mph"] or "none"
after_speed = rescored.get("candidate_speed_limit_mph", "") or "none"
if change != "unchanged":
transitions[f"{before_speed}->{after_speed}"] += 1
if index % 1000 == 0:
print(f"Rescored {index}/{len(rows)} crops", flush=True)
output_path = args.output.expanduser().resolve()
output_path.parent.mkdir(parents=True, exist_ok=True)
fieldnames = [*input_fields, *(field for field in EXTRA_FIELDS if field not in input_fields)]
with output_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore")
writer.writeheader()
writer.writerows(output_rows)
summary = {
"input": str(input_path),
"output": str(output_path),
"models_dir": str(args.models_dir.expanduser().resolve()),
"model_fingerprint": fingerprint,
"shard_count": args.shard_count,
"shard_index": args.shard_index,
"rows": len(output_rows),
"changes": dict(sorted(changes.items())),
"transitions": dict(sorted(transitions.items(), key=lambda item: (-item[1], item[0]))),
}
output_path.with_suffix(".json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
print(json.dumps(summary, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/speed_limit_vision/select_manual_review_queue.py
+206
View File
@@ -0,0 +1,206 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import json
from collections import Counter, defaultdict, deque
from pathlib import Path
PRIORITY_SPEED_ORDER = (60, 65, 55, 50, 45, 40, 35, 30, 25, 20, 70, 15, 75)
COMPARISON_PRIORITY_BONUS = {
"value_changed": 4.0,
"gained_read": 3.0,
"lost_read": 3.0,
"added_proposal": 1.0,
"removed_proposal": 1.0,
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Select a diverse, high-value subset from a raw speed-limit review queue.")
parser.add_argument("--input", type=Path, required=True, help="Raw manual_review_queue.csv.")
parser.add_argument("--output", type=Path, required=True, help="Selected manual_review_queue.csv.")
parser.add_argument("--max-rows", type=int, default=1000, help="Maximum selected rows.")
parser.add_argument("--max-per-route", type=int, default=30, help="Maximum selected rows from one route.")
parser.add_argument("--max-per-speed", type=int, default=140, help="Maximum rows for each predicted speed.")
parser.add_argument(
"--max-primary-speed",
type=int,
default=0,
help="Optional per-speed cap for the primary 30-65 mph range; defaults to --max-per-speed.",
)
parser.add_argument(
"--max-speed-20",
type=int,
default=0,
help="Optional cap for 20 mph rows; defaults to --max-per-speed.",
)
parser.add_argument("--max-no-read", type=int, default=220, help="Maximum detector proposals without a value read.")
parser.add_argument("--max-school", type=int, default=100, help="Maximum school-zone candidates.")
parser.add_argument("--max-advisory", type=int, default=100, help="Maximum advisory candidates.")
parser.add_argument(
"--min-seconds-per-route-speed",
type=float,
default=3.0,
help="Minimum spacing between selected rows from the same route, segment, and predicted-speed bucket.",
)
return parser.parse_args()
def read_rows(path: Path) -> tuple[list[str], list[dict[str, str]]]:
with path.expanduser().resolve().open("r", encoding="utf-8", newline="") as handle:
reader = csv.DictReader(handle)
return list(reader.fieldnames or []), list(reader)
def predicted_speed(row: dict[str, str]) -> int:
text = (row.get("candidate_speed_limit_mph") or "").strip()
if not text and row.get("comparison_change") == "lost_read":
text = (row.get("before_speed_limit_mph") or "").strip()
try:
return int(float(text)) if text else 0
except ValueError:
return 0
def bucket_name(row: dict[str, str]) -> str:
detector_class = row.get("detector_class", "")
if detector_class == "school_zone_speed_limit":
return "school"
if detector_class == "advisory_speed_limit":
return "advisory"
speed = predicted_speed(row)
return f"speed_{speed}" if speed else "no_read"
def priority(row: dict[str, str]) -> tuple[float, float, float, str]:
review_priority = (
float(row.get("review_priority") or 0.0) +
COMPARISON_PRIORITY_BONUS.get(row.get("comparison_change", ""), 0.0)
)
proposal_confidence = float(row.get("proposal_confidence") or 0.0)
candidate_confidence = float(row.get("candidate_confidence") or 0.0)
return review_priority, proposal_confidence, candidate_confidence, row.get("record_key", "")
def temporal_key(row: dict[str, str]) -> tuple[str, str, str, float] | None:
try:
frame_time_s = float(row.get("frame_time_s", ""))
except ValueError:
return None
return row.get("route", ""), row.get("segment", ""), bucket_name(row), frame_time_s
def round_robin_routes(rows: list[dict[str, str]]):
by_route: dict[str, deque[dict[str, str]]] = defaultdict(deque)
for row in sorted(rows, key=priority, reverse=True):
by_route[row.get("route", "")].append(row)
route_order = deque(sorted(by_route, key=lambda route: priority(by_route[route][0]), reverse=True))
while route_order:
route = route_order.popleft()
yield by_route[route].popleft()
if by_route[route]:
route_order.append(route)
def select_rows(rows: list[dict[str, str]], args: argparse.Namespace) -> list[dict[str, str]]:
buckets: dict[str, list[dict[str, str]]] = defaultdict(list)
for row in rows:
if row.get("detector_class") == "negative_empty":
continue
buckets[bucket_name(row)].append(row)
max_primary_speed = int(getattr(args, "max_primary_speed", 0))
max_speed_20 = int(getattr(args, "max_speed_20", 0))
primary_limit = max_primary_speed if max_primary_speed > 0 else args.max_per_speed
speed_20_limit = max_speed_20 if max_speed_20 > 0 else args.max_per_speed
limits = {
f"speed_{speed}": (
primary_limit if 30 <= speed <= 65 else speed_20_limit if speed == 20 else args.max_per_speed
)
for speed in PRIORITY_SPEED_ORDER
}
limits.update({"school": args.max_school, "advisory": args.max_advisory, "no_read": args.max_no_read})
ordered_buckets = [f"speed_{speed}" for speed in PRIORITY_SPEED_ORDER] + ["school", "advisory", "no_read"]
ordered_buckets.extend(sorted(set(buckets) - set(ordered_buckets)))
selected: list[dict[str, str]] = []
selected_keys: set[str] = set()
route_counts: Counter[str] = Counter()
bucket_counts: Counter[str] = Counter()
selected_times: dict[tuple[str, str, str], list[float]] = defaultdict(list)
min_spacing = max(float(getattr(args, "min_seconds_per_route_speed", 0.0)), 0.0)
def try_add(row: dict[str, str], bucket: str) -> bool:
key = row.get("record_key", "")
route = row.get("route", "")
if not key or key in selected_keys or route_counts[route] >= args.max_per_route:
return False
if bucket_counts[bucket] >= limits.get(bucket, args.max_per_speed):
return False
time_key = temporal_key(row)
if time_key is not None and min_spacing > 0.0:
route_key = time_key[:3]
if any(abs(time_key[3] - selected_time) < min_spacing for selected_time in selected_times[route_key]):
return False
selected.append(row)
selected_keys.add(key)
route_counts[route] += 1
bucket_counts[bucket] += 1
if time_key is not None:
selected_times[time_key[:3]].append(time_key[3])
return True
iterators = {bucket: iter(round_robin_routes(buckets[bucket])) for bucket in ordered_buckets if buckets.get(bucket)}
active = deque(bucket for bucket in ordered_buckets if bucket in iterators)
while active and len(selected) < args.max_rows:
bucket = active.popleft()
iterator = iterators[bucket]
added = False
for row in iterator:
if try_add(row, bucket):
added = True
break
if added and bucket_counts[bucket] < limits.get(bucket, args.max_per_speed):
active.append(bucket)
if len(selected) < args.max_rows:
for row in sorted(rows, key=priority, reverse=True):
if len(selected) >= args.max_rows:
break
try_add(row, bucket_name(row))
return sorted(selected, key=priority, reverse=True)
def main() -> int:
args = parse_args()
fieldnames, rows = read_rows(args.input)
selected = select_rows(rows, args)
output = args.output.expanduser().resolve()
output.parent.mkdir(parents=True, exist_ok=True)
with output.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore")
writer.writeheader()
writer.writerows(selected)
summary = {
"input": str(args.input.expanduser().resolve()),
"output": str(output),
"input_rows": len(rows),
"selected_rows": len(selected),
"routes": len({row.get("route", "") for row in selected}),
"buckets": dict(sorted(Counter(bucket_name(row) for row in selected).items())),
"min_seconds_per_route_speed": args.min_seconds_per_route_speed,
}
summary_path = output.with_name("manual_review_selection_summary.json")
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/speed_limit_vision/serve_manual_review_queue.py
+39 -11
View File
@@ -4,8 +4,8 @@ from __future__ import annotations
import argparse
import csv
import json
import time
from datetime import UTC, datetime
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
@@ -40,7 +40,8 @@ HTML = r"""<!doctype html>
<style>
:root { color-scheme: dark; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
body { margin: 0; background: #111; color: #eee; }
header { display: flex; align-items: center; gap: 16px; padding: 10px 14px; background: #1b1b1b; border-bottom: 1px solid #333; position: sticky; top: 0; z-index: 2; }
header { display: flex; align-items: center; gap: 16px; padding: 10px 14px; background: #1b1b1b; }
header { border-bottom: 1px solid #333; position: sticky; top: 0; z-index: 2; }
button, select, input, textarea { background: #222; color: #eee; border: 1px solid #555; border-radius: 6px; padding: 7px 9px; font: inherit; }
button { cursor: pointer; }
button:hover { background: #333; }
@@ -63,6 +64,7 @@ HTML = r"""<!doctype html>
.speed button { min-width: 42px; }
.muted { color: #aaa; }
.status { white-space: nowrap; }
.hint { margin-bottom: 10px; padding: 8px; border-left: 3px solid #3f7ec8; background: #20252b; color: #dbeaff; }
@media (max-width: 980px) { main { grid-template-columns: 1fr; } img { max-height: 45vh; } }
</style>
</head>
@@ -79,7 +81,8 @@ HTML = r"""<!doctype html>
<option value="negative">Negatives</option>
</select>
<span class="status" id="status"></span>
<span class="muted">Keys: Space/p accept model, type speed to correct, u uncertain, i/x ignore, Enter save correction, j/k next/prev, s school, r regulatory, a advisory</span>
<span class="muted">Keys: Space/p accept model, type speed to correct, u uncertain, i/x ignore,
Enter save correction, j/k next/prev, s school, r regulatory, a advisory</span>
</header>
<main>
<section class="images">
@@ -93,6 +96,7 @@ HTML = r"""<!doctype html>
</div>
</section>
<aside class="panel">
<div class="hint" id="taskHint" hidden></div>
<div class="meta" id="meta"></div>
<h3>Speed</h3>
<div class="buttons speed" id="speedButtons"></div>
@@ -270,9 +274,26 @@ function inferredType(row) {
return "regulatory";
}
function auditHint(row) {
if (!row) return "";
const previous = row.before_speed_limit_mph || "no value";
const candidate = row.candidate_speed_limit_mph || "no value";
if (row.comparison_change === "advisory_type_reaudit") {
return `Recheck sign type for reviewed ${candidate}: Space = regulatory; A then Space = advisory; S then Space = school zone.`;
}
if (row.comparison_change === "lost_read") {
return `Current model rejected previous ${previous}. Type the speed if readable; press i if it is not a speed sign.`;
}
if (row.comparison_change === "gained_read") return `Current model gained ${candidate}. Press Space if correct, or type the correct speed.`;
if (row.comparison_change === "value_changed") {
return `Model changed ${previous} to ${candidate}. Press Space for the current value, or type the correct speed.`;
}
return "";
}
function ensureSpeedSignType() {
if (draft.review_sign_type === "not_speed_limit") {
draft.review_sign_type = inferredType(current);
draft.review_sign_type = "regulatory";
setActive("#typeButtons button", "type", draft.review_sign_type);
}
}
@@ -329,12 +350,15 @@ function render() {
draft = {
review_status: current.review_status || "",
review_speed_limit_mph: current.review_speed_limit_mph || current.candidate_speed_limit_mph || "",
review_sign_type: current.review_sign_type || inferredType(current),
review_sign_type: current.review_sign_type || "regulatory",
review_bbox: current.review_bbox || current.bbox || "",
review_ignore_reason: current.review_ignore_reason || "",
review_notes: current.review_notes || "",
};
qs("#status").textContent = `${index + 1}/${rows.length}`;
const hint = auditHint(current);
qs("#taskHint").textContent = hint;
qs("#taskHint").hidden = !hint;
qs("#cropImg").src = current.crop_path ? `/media/${current.record_key}/crop` : "";
qs("#frameImg").src = `/media/${current.record_key}/frame`;
setBBox(draft.review_bbox, false);
@@ -343,13 +367,18 @@ function render() {
setActive("#speedButtons button", "speed", draft.review_speed_limit_mph);
setActive("#typeButtons button", "type", draft.review_sign_type);
setActive("#statusButtons button", "status", draft.review_status);
qs("#acceptPredBtn").disabled = !current.candidate_speed_limit_mph;
const mapSummary = `${current.map_relation} current=${current.map_current_speed_limit_mph}`;
const nextMapSummary = `next=${current.map_next_speed_limit_mph} dist=${current.map_next_speed_limit_distance_m}`;
qs("#meta").innerHTML = [
["record", current.record_key],
["change", current.comparison_change],
["previous", `${current.before_speed_limit_mph || "none"} @ ${current.before_confidence || ""}`],
["candidate", `${current.candidate_speed_limit_mph || "none"} @ ${current.candidate_confidence || ""}`],
["class", `${current.detector_class} (${current.proposal_confidence})`],
["bbox", draft.review_bbox],
["reasons", current.review_reasons],
["map", `${current.map_relation} current=${current.map_current_speed_limit_mph} next=${current.map_next_speed_limit_mph} dist=${current.map_next_speed_limit_distance_m}`],
["map", `${mapSummary} ${nextMapSummary}`],
["reads", current.read_sources],
["route", current.route],
["time", `seg ${current.segment} @ ${current.frame_time_s}s`],
@@ -390,7 +419,6 @@ qs("#clearBBoxBtn").onclick = () => setBBox("");
qs("#acceptPredBtn").onclick = () => {
if (!current) return;
draft.review_speed_limit_mph = current.candidate_speed_limit_mph || "";
draft.review_sign_type = inferredType(current);
save(true, "accepted");
};
qsa("#typeButtons button").forEach(btn => btn.onclick = () => {
@@ -529,7 +557,7 @@ class ReviewServer(ThreadingHTTPServer):
class Handler(BaseHTTPRequestHandler):
server: ReviewServer
def log_message(self, format, *args): # noqa: A003
def log_message(self, _format, *args):
return
def send_json(self, data, status=HTTPStatus.OK):
@@ -548,7 +576,7 @@ class Handler(BaseHTTPRequestHandler):
self.end_headers()
self.wfile.write(body)
def do_GET(self): # noqa: N802
def do_GET(self):
parsed = urlparse(self.path)
if parsed.path == "/":
self.send_text(HTML)
@@ -582,7 +610,7 @@ class Handler(BaseHTTPRequestHandler):
return
self.send_error(HTTPStatus.NOT_FOUND)
def do_POST(self): # noqa: N802
def do_POST(self):
if urlparse(self.path).path != "/api/review":
self.send_error(HTTPStatus.NOT_FOUND)
return
@@ -596,7 +624,7 @@ class Handler(BaseHTTPRequestHandler):
if record_key not in self.server.row_by_key:
self.send_error(HTTPStatus.BAD_REQUEST, "Unknown record_key")
return
label = {"record_key": record_key, "reviewed_at_unix": f"{time.time():.3f}"}
label = {"record_key": record_key, "reviewed_at_unix": f"{datetime.now(UTC).timestamp():.3f}"}
for field in QUEUE_REVIEW_FIELDS:
label[field] = str(payload.get(field) or "")
self.server.labels[record_key] = label
scripts/speed_limit_vision/test_review_pipeline.py
+193
View File
@@ -0,0 +1,193 @@
import importlib.util
from argparse import Namespace
from pathlib import Path
def load_local_module(name: str):
path = Path(__file__).resolve().with_name(f"{name}.py")
spec = importlib.util.spec_from_file_location(f"test_local_{name}", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
import_queue = load_local_module("import_manual_review_queue")
build_review_classifier = load_local_module("build_review_classifier_dataset")
select_queue = load_local_module("select_manual_review_queue")
compare_queues = load_local_module("compare_manual_review_queues")
rescore_queue = load_local_module("rescore_manual_review_queue")
is_classifier_reject = import_queue.is_classifier_reject
split_for_key = import_queue.split_for_key
split_group_key = import_queue.split_group_key
select_rows = select_queue.select_rows
def review_row(key: str, route: str, speed: int, priority: float) -> dict[str, str]:
return {
"record_key": key,
"route": route,
"detector_class": "regulatory_speed_limit",
"candidate_speed_limit_mph": str(speed),
"review_priority": str(priority),
"proposal_confidence": "0.8",
"candidate_confidence": "0.99",
}
def test_review_selection_balances_routes_and_speeds():
rows = [
*(review_row(f"a-{index}", "route-a", 30, 10 - index) for index in range(5)),
*(review_row(f"b-{index}", "route-b", 65, 10 - index) for index in range(5)),
]
args = Namespace(
max_rows=4,
max_per_route=2,
max_per_speed=2,
max_no_read=2,
max_school=2,
max_advisory=2,
)
selected = select_rows(rows, args)
assert len(selected) == 4
assert sum(row["route"] == "route-a" for row in selected) == 2
assert sum(row["route"] == "route-b" for row in selected) == 2
def test_review_selection_prioritizes_model_disagreements():
unchanged = review_row("unchanged", "route-a", 30, 5.0)
changed = {**review_row("changed", "route-a", 30, 2.0), "comparison_change": "value_changed"}
args = Namespace(max_rows=1, max_per_route=2, max_per_speed=2, max_no_read=2, max_school=2, max_advisory=2)
assert select_rows([unchanged, changed], args)[0]["record_key"] == "changed"
def test_review_selection_deduplicates_adjacent_same_speed_frames():
first = {**review_row("first", "route-a", 40, 5.0), "segment": "1", "frame_time_s": "10.0"}
duplicate = {**review_row("duplicate", "route-a", 40, 4.0), "segment": "1", "frame_time_s": "11.0"}
different_speed = {**review_row("different", "route-a", 45, 3.0), "segment": "1", "frame_time_s": "11.0"}
args = Namespace(
max_rows=3,
max_per_route=3,
max_per_speed=3,
max_no_read=3,
max_school=3,
max_advisory=3,
min_seconds_per_route_speed=3.0,
)
selected_keys = {row["record_key"] for row in select_rows([first, duplicate, different_speed], args)}
assert selected_keys == {"first", "different"}
def test_lost_reads_remain_balanced_by_previous_speed():
row = {
**review_row("lost", "route-a", 0, 5.0),
"candidate_speed_limit_mph": "",
"before_speed_limit_mph": "55",
"comparison_change": "lost_read",
}
assert select_queue.predicted_speed(row) == 55
assert select_queue.bucket_name(row) == "speed_55"
def test_primary_speed_limits_override_general_limit():
rows = [
*(review_row(f"primary-{index}", f"route-p-{index}", 40, 10 - index) for index in range(3)),
*(review_row(f"low-{index}", f"route-l-{index}", 15, 10 - index) for index in range(3)),
]
args = Namespace(
max_rows=6,
max_per_route=1,
max_per_speed=1,
max_primary_speed=3,
max_speed_20=2,
max_no_read=1,
max_school=1,
max_advisory=1,
min_seconds_per_route_speed=0.0,
)
selected = select_rows(rows, args)
assert sum(select_queue.predicted_speed(row) == 40 for row in selected) == 3
assert sum(select_queue.predicted_speed(row) == 15 for row in selected) == 1
def test_manual_import_splits_adjacent_frames_by_route():
rows = [{"record_key": f"frame-{index}", "route": "dongle/route"} for index in range(8)]
splits = {split_for_key(split_group_key(row), 5, 0) for row in rows}
assert len(splits) == 1
def test_only_reviewed_proposal_crops_become_classifier_rejects(tmp_path):
crop_path = tmp_path / "crop.jpg"
crop_path.write_bytes(b"crop")
row = {
"review_status": "ignore",
"review_sign_type": "not_speed_limit",
"detector_class": "regulatory_speed_limit",
"crop_path": str(crop_path),
}
assert is_classifier_reject(row)
assert not is_classifier_reject({**row, "detector_class": "negative_empty"})
assert not is_classifier_reject({**row, "review_status": "uncertain"})
def test_advisory_positive_is_a_runtime_negative(tmp_path):
crop_path = tmp_path / "crop.jpg"
frame_path = tmp_path / "frame.jpg"
crop_path.write_bytes(b"crop")
frame_path.write_bytes(b"frame")
row = {
"record_key": "advisory",
"review_status": "corrected",
"review_sign_type": "advisory",
"review_speed_limit_mph": "40",
"crop_path": str(crop_path),
"frame_path": str(frame_path),
}
assert import_queue.is_advisory_positive(row)
runtime_row = import_queue.runtime_row(row, "val", "advisory_negative")
assert runtime_row["sample_type"] == "advisory_negative"
assert runtime_row["speed_limit_mph"] == 40
assert build_review_classifier.is_advisory(row)
assert build_review_classifier.keep_advisory_reject({**row, "split": "val"}, 0.0)
assert not build_review_classifier.keep_advisory_reject({**row, "split": "train"}, 0.0)
def test_queue_comparison_distinguishes_gained_lost_and_changed_reads():
no_read = {"candidate_speed_limit_mph": "", "candidate_confidence": ""}
speed_20 = {"candidate_speed_limit_mph": "20", "candidate_confidence": "0.99"}
speed_30 = {"candidate_speed_limit_mph": "30", "candidate_confidence": "0.98"}
assert compare_queues.classify_change(no_read, speed_20, 0.05) == "gained_read"
assert compare_queues.classify_change(speed_20, no_read, 0.05) == "lost_read"
assert compare_queues.classify_change(speed_20, speed_30, 0.05) == "value_changed"
assert compare_queues.classify_change(speed_20, {**speed_20, "candidate_confidence": "0.97"}, 0.05) == ""
def test_rescore_row_preserves_before_values_and_marks_gained_read(tmp_path):
crop_path = tmp_path / "crop.jpg"
import cv2
import numpy as np
cv2.imwrite(str(crop_path), np.zeros((32, 32, 3), dtype=np.uint8))
daemon = type("Daemon", (), {"_classify_speed_limit_from_model": lambda self, crop: (20, 0.99)})()
row = {
"record_key": "candidate",
"crop_path": str(crop_path),
"candidate_speed_limit_mph": "",
"candidate_confidence": "",
}
rescored = rescore_queue.rescore_row(row, daemon, "model", 0.05)
assert rescored["before_speed_limit_mph"] == ""
assert rescored["candidate_speed_limit_mph"] == "20"
assert rescored["comparison_change"] == "gained_read"