Omnioculars V1

scripts/speed_limit_vision/common.py

@@ -26,7 +26,7 @@ DEFAULT_SPEED_VALUES = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75)
 # Values the review and dataset tooling can accept. Keep DEFAULT_SPEED_VALUES
 # aligned with the currently deployed classifier until an expanded model is
 # promoted; adding a class changes every output index after it.
-SUPPORTED_SPEED_VALUES = (10, *DEFAULT_SPEED_VALUES, 80, 90, 100)
+SUPPORTED_SPEED_VALUES = (5, 10, *DEFAULT_SPEED_VALUES, 80, 90, 100)
 EXTENDED_CLASSIFIER_SPEED_VALUES = tuple(sorted(SUPPORTED_SPEED_VALUES, key=str))
 
 DETECTOR_EXPORT_NAME = "speed_limit_us_detector.onnx"

scripts/speed_limit_vision/evaluate_classifier_dataset.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+
+from collections import Counter
+from pathlib import Path
+
+
+IMAGE_SUFFIXES = frozenset((".jpg", ".jpeg", ".png", ".bmp", ".webp"))
+
+
+def parse_args() -> argparse.Namespace:
+  parser = argparse.ArgumentParser(description="Evaluate a YOLO classifier dataset with the runtime confidence threshold.")
+  parser.add_argument("--model", type=Path, required=True)
+  parser.add_argument("--data", type=Path, required=True, help="Classifier dataset root containing train/ and val/.")
+  parser.add_argument("--split", choices=("train", "val"), default="val")
+  parser.add_argument("--imgsz", type=int, default=128)
+  parser.add_argument("--batch", type=int, default=64)
+  parser.add_argument("--device", default="cpu")
+  parser.add_argument("--min-confidence", type=float, default=0.60)
+  parser.add_argument("--output-json", type=Path)
+  return parser.parse_args()
+
+
+def main() -> int:
+  args = parse_args()
+  split_root = args.data.expanduser().resolve() / args.split
+  if not split_root.is_dir():
+    raise FileNotFoundError(split_root)
+
+  samples: list[tuple[Path, str]] = []
+  for class_dir in sorted(split_root.iterdir()):
+    if not class_dir.is_dir() or class_dir.name.startswith("._"):
+      continue
+    for image_path in sorted(class_dir.iterdir()):
+      if image_path.name.startswith("._") or image_path.suffix.lower() not in IMAGE_SUFFIXES or not image_path.is_file():
+        continue
+      samples.append((image_path, class_dir.name))
+  if not samples:
+    raise RuntimeError(f"No classifier images found under {split_root}")
+
+  try:
+    from ultralytics import YOLO
+  except Exception as exc:
+    raise SystemExit("Ultralytics is required to evaluate classifier checkpoints.") from exc
+
+  model = YOLO(str(args.model.expanduser().resolve()))
+  predictions = model.predict(
+    source=[str(path) for path, _ in samples],
+    imgsz=args.imgsz,
+    batch=args.batch,
+    device=args.device,
+    verbose=False,
+    stream=True,
+  )
+
+  class_counts: dict[str, Counter[str]] = {}
+  total_counts: Counter[str] = Counter()
+  for (_, expected), result in zip(samples, predictions, strict=True):
+    probabilities = result.probs
+    if probabilities is None:
+      raise RuntimeError("Classifier prediction did not include probabilities")
+    predicted = str(result.names[int(probabilities.top1)])
+    confidence = float(probabilities.top1conf)
+    counts = class_counts.setdefault(expected, Counter())
+    counts["total"] += 1
+    total_counts["total"] += 1
+    if predicted == expected:
+      counts["top1_exact"] += 1
+      total_counts["top1_exact"] += 1
+    if confidence < args.min_confidence:
+      counts["rejected"] += 1
+      total_counts["rejected"] += 1
+      continue
+    counts["accepted"] += 1
+    total_counts["accepted"] += 1
+    if predicted == expected:
+      counts["accepted_exact"] += 1
+      total_counts["accepted_exact"] += 1
+    else:
+      counts["accepted_wrong"] += 1
+      total_counts["accepted_wrong"] += 1
+
+  def summarize(counts: Counter[str]) -> dict[str, float | int]:
+    total = counts["total"]
+    accepted = counts["accepted"]
+    return {
+      "total": total,
+      "top1_exact": counts["top1_exact"],
+      "top1_exact_rate": round(counts["top1_exact"] / total, 6) if total else 0.0,
+      "accepted": accepted,
+      "accepted_coverage": round(accepted / total, 6) if total else 0.0,
+      "accepted_exact": counts["accepted_exact"],
+      "accepted_wrong": counts["accepted_wrong"],
+      "accepted_precision": round(counts["accepted_exact"] / accepted, 6) if accepted else 0.0,
+      "rejected": counts["rejected"],
+    }
+
+  summary = {
+    "model": str(args.model.expanduser().resolve()),
+    "data": str(split_root),
+    "minimum_confidence": args.min_confidence,
+    "overall": summarize(total_counts),
+    "classes": {name: summarize(class_counts[name]) for name in sorted(class_counts)},
+  }
+  encoded = json.dumps(summary, indent=2, sort_keys=True) + "\n"
+  print(encoded, end="")
+  if args.output_json:
+    output_path = args.output_json.expanduser().resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(encoded, encoding="ascii")
+  return 0
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())

scripts/speed_limit_vision/evaluate_reviewed_route_events.py

@@ -59,6 +59,8 @@ def parse_args() -> argparse.Namespace:
   parser.add_argument("--track-max-age", type=float, help="Override the maximum proposal track lifetime.")
   parser.add_argument("--crop-ocr", action="store_true", help="Evaluate with crop OCR confirmation enabled.")
   parser.add_argument("--classifier-min-confidence", type=float, help="Override the value classifier confidence threshold.")
+  parser.add_argument("--classifier-speed-values", help="Comma-separated classifier classes for a candidate model.")
+  parser.add_argument("--extended-classifier-min-confidence", type=float, help="Override confidence for 5/10/80/90/100.")
   parser.add_argument("--trusted-model-min-confidence", type=float, help="Override tiny-box trusted model confidence.")
   parser.add_argument("--classifier-expansion-limit", type=int, help="Evaluate only the first N detector crop expansions.")
   parser.add_argument("--classifier-expansion-indices", help="Comma-separated detector crop expansion indices to evaluate.")
@@ -232,6 +234,10 @@ def main() -> int:
   slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = args.crop_ocr
   if args.classifier_min_confidence is not None:
     slv.US_CLASSIFIER_MIN_CONFIDENCE = args.classifier_min_confidence
+  if args.classifier_speed_values:
+    slv.US_CLASSIFIER_SPEED_VALUES = tuple(int(value) for value in args.classifier_speed_values.split(","))
+  if args.extended_classifier_min_confidence is not None:
+    slv.EXTENDED_CLASSIFIER_MIN_CONFIDENCE = args.extended_classifier_min_confidence
   if args.trusted_model_min_confidence is not None:
     slv.DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE = args.trusted_model_min_confidence
   if args.classifier_expansion_indices:

scripts/speed_limit_vision/merge_classifier_datasets.py

@@ -34,7 +34,9 @@ def link_or_copy(source: Path, destination: Path) -> None:
   try:
     os.link(source, destination)
   except OSError:
-    shutil.copy2(source, destination)
+    # copy2 preserves macOS metadata as AppleDouble `._` files on exFAT. Image
+    # loaders then mistake those sidecars for corrupt training images.
+    shutil.copyfile(source, destination)
 
 
 def main() -> int:

scripts/speed_limit_vision/serve_manual_review_queue.py

@@ -130,7 +130,8 @@ HTML = r"""<!doctype html>
     </aside>
   </main>
 <script>
-const speeds = [10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,90,100];
+const speeds = [5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,90,100];
+const shortcutSpeeds = speeds.filter((speed) => speed !== 5 && speed !== 100);
 let rows = [];
 let index = 0;
 let current = null;
@@ -339,7 +340,7 @@ function handleDigitShortcut(digit) {
   }
 
   const speed = Number(speedBuffer);
-  if (speeds.includes(speed)) {
+  if (shortcutSpeeds.includes(speed)) {
     clearSpeedBuffer();
     setSpeed(speed, true);
     return;

scripts/speed_limit_vision/test_review_pipeline.py

@@ -34,11 +34,11 @@ def test_raw_comma_camera_uses_real_frame_rate():
 
 
 def test_extended_classifier_order_matches_lexical_dataset_classes():
-  assert common.SUPPORTED_SPEED_VALUES == (10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 90, 100)
-  assert common.EXTENDED_CLASSIFIER_SPEED_VALUES == (10, 100, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 90)
+  assert common.SUPPORTED_SPEED_VALUES == (5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 90, 100)
+  assert common.EXTENDED_CLASSIFIER_SPEED_VALUES == (10, 100, 15, 20, 25, 30, 35, 40, 45, 5, 50, 55, 60, 65, 70, 75, 80, 90)
 
 
-@pytest.mark.parametrize("speed", (10, 80, 90, 100))
+@pytest.mark.parametrize("speed", (5, 10, 80, 90, 100))
 def test_manual_import_accepts_extended_speed_values(speed):
   assert import_queue.parse_speed(str(speed)) == speed