Oh What A Night!

scripts/speed_limit_vision/evaluate_runtime_manifest.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import csv
+import random
+
+from pathlib import Path
+
+import cv2
+
+import starpilot.system.speed_limit_vision as slv
+
+
+def parse_args() -> argparse.Namespace:
+  parser = argparse.ArgumentParser(description="Evaluate runtime speed-limit ONNX models on a mined frame manifest.")
+  parser.add_argument(
+    "--models-dir",
+    type=Path,
+    default=Path("starpilot/assets/vision_models"),
+    help="Directory containing speed_limit_us_detector.onnx and speed_limit_us_value_classifier.onnx.",
+  )
+  parser.add_argument("--manifest", type=Path, required=True, help="CSV manifest with dataset_image/frame_path and labels.")
+  parser.add_argument("--split", action="append", help="Optional split filter. Repeat for multiple splits.")
+  parser.add_argument("--max-rows", type=int, default=0, help="Optional cap after filtering.")
+  parser.add_argument("--seed", type=int, default=0, help="Sampling seed used with --max-rows.")
+  parser.add_argument("--output-csv", type=Path, help="Optional per-row prediction output.")
+  parser.add_argument("--strict-positive-recall", type=float, help="Exit non-zero if positive exact recall is below this value.")
+  parser.add_argument("--strict-negative-fpr", type=float, help="Exit non-zero if negative false-positive rate is above this value.")
+  return parser.parse_args()
+
+
+def first_present(row: dict[str, str], keys: tuple[str, ...]) -> str:
+  for key in keys:
+    value = row.get(key, "").strip()
+    if value:
+      return value
+  return ""
+
+
+def expected_value(row: dict[str, str]) -> int | None:
+  value_text = first_present(row, ("speed_limit_mph", "dominant_value"))
+  if value_text:
+    try:
+      return int(float(value_text))
+    except ValueError:
+      return None
+
+  for key in ("full_detection", "model_read", "ocr_read"):
+    read_text = row.get(key, "").strip()
+    if "@" in read_text:
+      try:
+        return int(float(read_text.split("@", 1)[0]))
+      except ValueError:
+        return None
+  return None
+
+
+def is_negative(row: dict[str, str]) -> bool:
+  sample_type = row.get("sample_type", "").lower()
+  if "negative" in sample_type:
+    return True
+  return expected_value(row) is None
+
+
+def load_rows(manifest_path: Path, splits: set[str] | None) -> list[dict[str, str]]:
+  with manifest_path.open("r", encoding="utf-8", newline="") as manifest_file:
+    reader = csv.DictReader(manifest_file)
+    rows = []
+    for row in reader:
+      if splits is not None and row.get("split", "") not in splits:
+        continue
+      rows.append(row)
+  return rows
+
+
+def main() -> int:
+  args = parse_args()
+  models_dir = args.models_dir.expanduser().resolve()
+  detector_path = models_dir / "speed_limit_us_detector.onnx"
+  classifier_path = models_dir / "speed_limit_us_value_classifier.onnx"
+  if not detector_path.is_file():
+    raise FileNotFoundError(detector_path)
+  if not classifier_path.is_file():
+    raise FileNotFoundError(classifier_path)
+
+  rows = load_rows(args.manifest.expanduser().resolve(), set(args.split) if args.split else None)
+  if args.max_rows > 0 and len(rows) > args.max_rows:
+    rng = random.Random(args.seed)
+    rows = rng.sample(rows, args.max_rows)
+
+  slv.US_DETECTOR_MODEL_PATH = detector_path
+  slv.US_CLASSIFIER_MODEL_PATH = classifier_path
+  daemon = slv.SpeedLimitVisionDaemon(use_runtime=False)
+
+  output_rows: list[dict[str, str]] = []
+  positive_count = 0
+  positive_exact = 0
+  positive_detected = 0
+  negative_count = 0
+  negative_false_positive = 0
+  unreadable_count = 0
+
+  for row in rows:
+    image_text = first_present(row, ("dataset_image", "frame_path", "source_frame"))
+    if not image_text:
+      unreadable_count += 1
+      continue
+
+    image_path = Path(image_text).expanduser().resolve()
+    frame_bgr = cv2.imread(str(image_path))
+    if frame_bgr is None:
+      unreadable_count += 1
+      continue
+
+    detection = daemon._detect_sign(frame_bgr)
+    predicted_value = detection.speed_limit_mph if detection is not None else None
+    confidence = detection.confidence if detection is not None else None
+    expected = expected_value(row)
+    negative = is_negative(row)
+
+    if negative:
+      negative_count += 1
+      if predicted_value is not None:
+        negative_false_positive += 1
+    else:
+      positive_count += 1
+      if predicted_value is not None:
+        positive_detected += 1
+      if predicted_value == expected:
+        positive_exact += 1
+
+    if args.output_csv:
+      output_rows.append({
+        "record_key": row.get("record_key", ""),
+        "split": row.get("split", ""),
+        "sample_type": row.get("sample_type", ""),
+        "image_path": str(image_path),
+        "expected_speed_limit_mph": "" if expected is None else str(expected),
+        "predicted_speed_limit_mph": "" if predicted_value is None else str(predicted_value),
+        "confidence": "" if confidence is None else f"{confidence:.6f}",
+        "negative": str(negative),
+      })
+
+  positive_exact_recall = positive_exact / positive_count if positive_count else 0.0
+  positive_any_recall = positive_detected / positive_count if positive_count else 0.0
+  negative_fpr = negative_false_positive / negative_count if negative_count else 0.0
+
+  print(f"Rows evaluated: {positive_count + negative_count}")
+  print(f"Unreadable rows: {unreadable_count}")
+  print(
+    f"Positive exact: {positive_exact}/{positive_count} "
+    f"({positive_exact_recall:.3f}); any detection: {positive_detected}/{positive_count} ({positive_any_recall:.3f})"
+  )
+  print(f"Negative false positives: {negative_false_positive}/{negative_count} ({negative_fpr:.3f})")
+
+  if args.output_csv:
+    args.output_csv.parent.mkdir(parents=True, exist_ok=True)
+    with args.output_csv.open("w", encoding="utf-8", newline="") as output_file:
+      fieldnames = (
+        "record_key",
+        "split",
+        "sample_type",
+        "image_path",
+        "expected_speed_limit_mph",
+        "predicted_speed_limit_mph",
+        "confidence",
+        "negative",
+      )
+      writer = csv.DictWriter(output_file, fieldnames=fieldnames)
+      writer.writeheader()
+      writer.writerows(output_rows)
+    print(f"Wrote {args.output_csv}")
+
+  failed = False
+  if args.strict_positive_recall is not None and positive_exact_recall < args.strict_positive_recall:
+    failed = True
+  if args.strict_negative_fpr is not None and negative_fpr > args.strict_negative_fpr:
+    failed = True
+  return 1 if failed else 0
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())

scripts/speed_limit_vision/import_manifest_classifier_masks.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import random
+import re
+
+from pathlib import Path
+
+import cv2
+
+if __package__ in (None, ""):
+  import sys
+  sys.path.insert(0, str(Path(__file__).resolve().parent))
+  from build_value_dataset import crop_box, parse_yolo_labels  # type: ignore
+  from common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace  # type: ignore
+  from generate_value_roi_classifier_dataset import augment_mask, extract_value_mask  # type: ignore
+else:
+  from .build_value_dataset import crop_box, parse_yolo_labels
+  from .common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace
+  from .generate_value_roi_classifier_dataset import augment_mask, extract_value_mask
+
+
+READ_RE = re.compile(r"^\s*(\d+)(?:@|$)")
+
+
+def parse_args() -> argparse.Namespace:
+  parser = argparse.ArgumentParser(description="Import mined comma speed-limit crops as ROI classifier digit masks.")
+  parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Target training workspace root.")
+  parser.add_argument("--manifest", type=Path, action="append", required=True, help="CSV manifest to import. May be passed more than once.")
+  parser.add_argument("--variants-per-example", type=int, default=3, help="Augmented mask variants to generate per imported crop.")
+  parser.add_argument("--default-padding", type=float, default=0.12, help="Crop padding when a manifest row does not specify one.")
+  parser.add_argument("--val-modulo", type=int, default=5, help="Hash modulo for rows that do not specify train/val. 0 sends them to train.")
+  parser.add_argument("--val-remainder", type=int, default=0, help="Hash remainder for rows that do not specify train/val.")
+  parser.add_argument("--max-rows", type=int, default=0, help="Optional maximum rows to attempt across all manifests.")
+  parser.add_argument("--seed", type=int, default=20260630, help="Random seed.")
+  return parser.parse_args()
+
+
+def safe_stem(text: str) -> str:
+  cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "_", text.strip())
+  return cleaned.strip("._")[:180] or "sample"
+
+
+def read_rows(path: Path) -> list[dict[str, str]]:
+  with path.open("r", encoding="utf-8", newline="") as csv_file:
+    return list(csv.DictReader(csv_file))
+
+
+def parse_speed_from_read(text: str) -> int:
+  match = READ_RE.match(text or "")
+  if not match:
+    return 0
+  value = int(match.group(1))
+  return value if value in DEFAULT_SPEED_VALUES else 0
+
+
+def row_speed(row: dict[str, str]) -> int:
+  for field in ("speed_limit_mph", "speed_limit", "posted_speed"):
+    text = (row.get(field) or "").strip()
+    if text.isdigit():
+      value = int(text)
+      if value in DEFAULT_SPEED_VALUES:
+        return value
+  for field in ("full_detection", "model_read", "ocr_read"):
+    value = parse_speed_from_read(row.get(field, ""))
+    if value:
+      return value
+  return 0
+
+
+def row_split(row: dict[str, str], key_text: str, val_modulo: int, val_remainder: int) -> str:
+  split = (row.get("split") or "").strip().lower()
+  if split in ("train", "val"):
+    return split
+  if val_modulo <= 0:
+    return "train"
+  digest = hashlib.sha1(key_text.encode("utf-8")).hexdigest()
+  return "val" if int(digest[:8], 16) % val_modulo == val_remainder else "train"
+
+
+def resolve_existing_path(path_text: str, manifest_path: Path) -> Path | None:
+  text = (path_text or "").strip()
+  if not text:
+    return None
+  path = Path(text).expanduser()
+  candidates = [path]
+  if not path.is_absolute():
+    candidates.append((manifest_path.parent / path).resolve())
+  for candidate in candidates:
+    if candidate.is_file():
+      return candidate.resolve()
+  return None
+
+
+def parse_xyxy(text: str) -> tuple[int, int, int, int] | None:
+  if not text:
+    return None
+  parts = [part.strip() for part in text.replace(";", ",").split(",") if part.strip()]
+  if len(parts) != 4:
+    return None
+  try:
+    x1, y1, x2, y2 = (int(round(float(part))) for part in parts)
+  except ValueError:
+    return None
+  if x2 <= x1 or y2 <= y1:
+    return None
+  return x1, y1, x2, y2
+
+
+def crop_from_xyxy(image, box: tuple[int, int, int, int], padding: float):
+  height, width = image.shape[:2]
+  x1, y1, x2, y2 = box
+  pad_x = int(round((x2 - x1) * padding))
+  pad_y = int(round((y2 - y1) * padding))
+  x1 = max(x1 - pad_x, 0)
+  y1 = max(y1 - pad_y, 0)
+  x2 = min(x2 + pad_x, width)
+  y2 = min(y2 + pad_y, height)
+  if x2 <= x1 or y2 <= y1:
+    return None
+  return image[y1:y2, x1:x2]
+
+
+def load_crop(row: dict[str, str], manifest_path: Path, default_padding: float):
+  crop_path = resolve_existing_path(row.get("crop_path", ""), manifest_path)
+  if crop_path is not None:
+    crop = cv2.imread(str(crop_path))
+    if crop is not None and crop.size:
+      return crop
+
+  image_path = (
+    resolve_existing_path(row.get("image_path", ""), manifest_path) or
+    resolve_existing_path(row.get("dataset_image", ""), manifest_path) or
+    resolve_existing_path(row.get("frame_path", ""), manifest_path)
+  )
+  if image_path is None:
+    return None
+
+  image = cv2.imread(str(image_path))
+  if image is None or not image.size:
+    return None
+
+  padding_text = (row.get("padding") or "").strip()
+  padding = float(padding_text) if padding_text else default_padding
+  box = parse_xyxy(row.get("bbox", "") or row.get("box", ""))
+  if box is not None:
+    return crop_from_xyxy(image, box, padding)
+
+  label_path = (
+    resolve_existing_path(row.get("label_path", ""), manifest_path) or
+    resolve_existing_path(row.get("dataset_label", ""), manifest_path)
+  )
+  if label_path is None:
+    return image
+
+  bbox_index = int((row.get("bbox_index") or "0").strip() or "0")
+  boxes = parse_yolo_labels(label_path)
+  if bbox_index >= len(boxes):
+    return None
+  return crop_box(image, boxes[bbox_index], padding)
+
+
+def write_mask(workspace: Path, split: str, speed_value: int, stem: str, image_bgr) -> None:
+  output_dir = ensure_dir(workspace / "classifier" / split / str(speed_value))
+  cv2.imwrite(str(output_dir / f"{stem}.png"), image_bgr)
+
+
+def main() -> int:
+  args = parse_args()
+  workspace = resolve_workspace(args.workspace)
+  rng = random.Random(args.seed)
+
+  attempted = 0
+  imported = 0
+  skipped_no_speed = 0
+  skipped_no_crop = 0
+  skipped_no_mask = 0
+  written = 0
+
+  for manifest_path in [path.expanduser().resolve() for path in args.manifest]:
+    rows = read_rows(manifest_path)
+    for row_index, row in enumerate(rows):
+      if args.max_rows > 0 and attempted >= args.max_rows:
+        break
+      attempted += 1
+
+      speed_value = row_speed(row)
+      if not speed_value:
+        skipped_no_speed += 1
+        continue
+
+      key_text = "|".join(
+        row.get(field, "")
+        for field in ("record_key", "image_path", "dataset_image", "crop_path", "frame_path", "session_id", "bookmark_number")
+      ) or f"{manifest_path}:{row_index}"
+      split = row_split(row, key_text, args.val_modulo, args.val_remainder)
+      crop = load_crop(row, manifest_path, args.default_padding)
+      if crop is None:
+        skipped_no_crop += 1
+        continue
+
+      mask = extract_value_mask(crop)
+      if mask is None:
+        skipped_no_mask += 1
+        continue
+
+      manifest_stem = safe_stem(manifest_path.stem)
+      source_stem = safe_stem(key_text)
+      base_stem = f"manifest_{manifest_stem}_{row_index:06d}_{source_stem}"
+      base_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)
+      write_mask(workspace, split, speed_value, f"{base_stem}_base", base_mask)
+      written += 1
+
+      for variant_index in range(max(args.variants_per_example, 0)):
+        augmented = augment_mask(mask, rng)
+        write_mask(workspace, split, speed_value, f"{base_stem}_var{variant_index:02d}", augmented)
+        written += 1
+      imported += 1
+    if args.max_rows > 0 and attempted >= args.max_rows:
+      break
+
+  print(
+    "Imported manifest classifier masks: "
+    f"attempted={attempted} imported={imported} written={written} "
+    f"skipped_no_speed={skipped_no_speed} skipped_no_crop={skipped_no_crop} skipped_no_mask={skipped_no_mask}"
+  )
+  return 0
+
+
+if __name__ == "__main__":
+  raise SystemExit(main())

scripts/speed_limit_vision/train_detector.py

@@ -28,6 +28,23 @@ def parse_args() -> argparse.Namespace:
   parser.add_argument("--patience", type=int, default=20, help="Early stopping patience.")
   parser.add_argument("--cache", action="store_true", help="Cache images in RAM if supported.")
   parser.add_argument("--exist-ok", action="store_true", help="Allow overwriting an existing run directory.")
+  parser.add_argument("--optimizer", help="Ultralytics optimizer name, for example SGD, Adam, or AdamW.")
+  parser.add_argument("--lr0", type=float, help="Initial learning rate passed to Ultralytics.")
+  parser.add_argument("--lrf", type=float, help="Final LR fraction passed to Ultralytics.")
+  parser.add_argument("--warmup-epochs", type=float, help="Warmup epochs passed to Ultralytics.")
+  parser.add_argument("--weight-decay", type=float, help="Weight decay passed to Ultralytics.")
+  parser.add_argument("--cos-lr", action="store_true", help="Use cosine LR scheduling.")
+  parser.add_argument("--close-mosaic", type=int, help="Disable mosaic augmentation for the final N epochs.")
+  parser.add_argument("--mosaic", type=float, help="Mosaic augmentation probability.")
+  parser.add_argument("--mixup", type=float, help="MixUp augmentation probability.")
+  parser.add_argument("--copy-paste", type=float, help="Copy-paste augmentation probability.")
+  parser.add_argument("--degrees", type=float, help="Rotation augmentation degrees.")
+  parser.add_argument("--translate", type=float, help="Translation augmentation fraction.")
+  parser.add_argument("--scale", type=float, help="Scale augmentation gain.")
+  parser.add_argument("--shear", type=float, help="Shear augmentation degrees.")
+  parser.add_argument("--perspective", type=float, help="Perspective augmentation fraction.")
+  parser.add_argument("--fliplr", type=float, help="Horizontal flip augmentation probability.")
+  parser.add_argument("--freeze", type=int, help="Freeze the first N model layers.")
   return parser.parse_args()
 
 
@@ -44,19 +61,44 @@ def main() -> int:
       "Ultralytics is not installed. Run `uv sync --extra speedvision` in the repo root before training."
     ) from exc
 
+  train_kwargs = {
+    "data": str(data_path),
+    "epochs": args.epochs,
+    "imgsz": args.imgsz,
+    "batch": args.batch,
+    "workers": args.workers,
+    "device": args.device,
+    "project": str(project_path),
+    "name": args.name,
+    "patience": args.patience,
+    "cache": args.cache,
+    "exist_ok": args.exist_ok,
+  }
+  optional_kwargs = {
+    "optimizer": args.optimizer,
+    "lr0": args.lr0,
+    "lrf": args.lrf,
+    "warmup_epochs": args.warmup_epochs,
+    "weight_decay": args.weight_decay,
+    "close_mosaic": args.close_mosaic,
+    "mosaic": args.mosaic,
+    "mixup": args.mixup,
+    "copy_paste": args.copy_paste,
+    "degrees": args.degrees,
+    "translate": args.translate,
+    "scale": args.scale,
+    "shear": args.shear,
+    "perspective": args.perspective,
+    "fliplr": args.fliplr,
+    "freeze": args.freeze,
+  }
+  train_kwargs.update({key: value for key, value in optional_kwargs.items() if value is not None})
+  if args.cos_lr:
+    train_kwargs["cos_lr"] = True
+
   model = YOLO(args.model)
   model.train(
-    data=str(data_path),
-    epochs=args.epochs,
-    imgsz=args.imgsz,
-    batch=args.batch,
-    workers=args.workers,
-    device=args.device,
-    project=str(project_path),
-    name=args.name,
-    patience=args.patience,
-    cache=args.cache,
-    exist_ok=args.exist_ok,
+    **train_kwargs,
   )
   print(f"Detector training complete under {project_path / args.name}")
   return 0

scripts/speed_limit_vision/train_value_classifier.py

@@ -28,6 +28,22 @@ def parse_args() -> argparse.Namespace:
   parser.add_argument("--patience", type=int, default=15, help="Early stopping patience.")
   parser.add_argument("--cache", action="store_true", help="Cache images in RAM if supported.")
   parser.add_argument("--exist-ok", action="store_true", help="Allow overwriting an existing run directory.")
+  parser.add_argument("--optimizer", help="Ultralytics optimizer name, for example SGD, Adam, or AdamW.")
+  parser.add_argument("--lr0", type=float, help="Initial learning rate passed to Ultralytics.")
+  parser.add_argument("--lrf", type=float, help="Final LR fraction passed to Ultralytics.")
+  parser.add_argument("--warmup-epochs", type=float, help="Warmup epochs passed to Ultralytics.")
+  parser.add_argument("--weight-decay", type=float, help="Weight decay passed to Ultralytics.")
+  parser.add_argument("--cos-lr", action="store_true", help="Use cosine LR scheduling.")
+  parser.add_argument("--degrees", type=float, help="Rotation augmentation degrees.")
+  parser.add_argument("--translate", type=float, help="Translation augmentation fraction.")
+  parser.add_argument("--scale", type=float, help="Scale augmentation gain.")
+  parser.add_argument("--shear", type=float, help="Shear augmentation degrees.")
+  parser.add_argument("--perspective", type=float, help="Perspective augmentation fraction.")
+  parser.add_argument("--fliplr", type=float, help="Horizontal flip augmentation probability.")
+  parser.add_argument("--erasing", type=float, help="Random erasing augmentation probability.")
+  parser.add_argument("--auto-augment", help="Ultralytics classification auto-augment policy.")
+  parser.add_argument("--dropout", type=float, help="Classification head dropout.")
+  parser.add_argument("--freeze", type=int, help="Freeze the first N model layers.")
   return parser.parse_args()
 
 
@@ -44,19 +60,45 @@ def main() -> int:
       "Ultralytics is not installed. Run `uv sync --extra speedvision` in the repo root before training."
     ) from exc
 
+  train_kwargs = {
+    "data": str(data_path),
+    "epochs": args.epochs,
+    "imgsz": args.imgsz,
+    "batch": args.batch,
+    "workers": args.workers,
+    "device": args.device,
+    "project": str(project_path),
+    "name": args.name,
+    "patience": args.patience,
+    "cache": args.cache,
+    "exist_ok": args.exist_ok,
+  }
+  optional_kwargs = {
+    "optimizer": args.optimizer,
+    "lr0": args.lr0,
+    "lrf": args.lrf,
+    "warmup_epochs": args.warmup_epochs,
+    "weight_decay": args.weight_decay,
+    "degrees": args.degrees,
+    "translate": args.translate,
+    "scale": args.scale,
+    "shear": args.shear,
+    "perspective": args.perspective,
+    "fliplr": args.fliplr,
+    "erasing": args.erasing,
+    "auto_augment": args.auto_augment,
+    "dropout": args.dropout,
+    "freeze": args.freeze,
+  }
+  train_kwargs.update({key: value for key, value in optional_kwargs.items() if value is not None})
+  if args.auto_augment is not None and args.auto_augment.lower() in ("none", "off", "false", "0"):
+    train_kwargs["auto_augment"] = None
+  if args.cos_lr:
+    train_kwargs["cos_lr"] = True
+
   model = YOLO(args.model)
   model.train(
-    data=str(data_path),
-    epochs=args.epochs,
-    imgsz=args.imgsz,
-    batch=args.batch,
-    workers=args.workers,
-    device=args.device,
-    project=str(project_path),
-    name=args.name,
-    patience=args.patience,
-    cache=args.cache,
-    exist_ok=args.exist_ok,
+    **train_kwargs,
   )
   print(f"Classifier training complete under {project_path / args.name}")
   return 0