diff --git a/launch_env.sh b/launch_env.sh index 8dde50a60..b474bc2a7 100755 --- a/launch_env.sh +++ b/launch_env.sh @@ -21,7 +21,7 @@ fi export QCOM_PRIORITY=12 if [ -z "$AGNOS_VERSION" ]; then - export AGNOS_VERSION="12.8.27" + export AGNOS_VERSION="12.8.28" fi if [ -z "$AGNOS_ACCEPTED_VERSIONS" ]; then diff --git a/scripts/speed_limit_vision/common.py b/scripts/speed_limit_vision/common.py index db93bad4e..b6984242e 100644 --- a/scripts/speed_limit_vision/common.py +++ b/scripts/speed_limit_vision/common.py @@ -26,7 +26,7 @@ DEFAULT_SPEED_VALUES = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75) # Values the review and dataset tooling can accept. Keep DEFAULT_SPEED_VALUES # aligned with the currently deployed classifier until an expanded model is # promoted; adding a class changes every output index after it. -SUPPORTED_SPEED_VALUES = (10, *DEFAULT_SPEED_VALUES, 80, 90, 100) +SUPPORTED_SPEED_VALUES = (5, 10, *DEFAULT_SPEED_VALUES, 80, 90, 100) EXTENDED_CLASSIFIER_SPEED_VALUES = tuple(sorted(SUPPORTED_SPEED_VALUES, key=str)) DETECTOR_EXPORT_NAME = "speed_limit_us_detector.onnx" diff --git a/scripts/speed_limit_vision/evaluate_classifier_dataset.py b/scripts/speed_limit_vision/evaluate_classifier_dataset.py new file mode 100644 index 000000000..ae1f2c698 --- /dev/null +++ b/scripts/speed_limit_vision/evaluate_classifier_dataset.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json + +from collections import Counter +from pathlib import Path + + +IMAGE_SUFFIXES = frozenset((".jpg", ".jpeg", ".png", ".bmp", ".webp")) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate a YOLO classifier dataset with the runtime confidence threshold.") + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--data", type=Path, required=True, help="Classifier dataset root containing train/ and val/.") + parser.add_argument("--split", choices=("train", "val"), default="val") + parser.add_argument("--imgsz", type=int, default=128) + parser.add_argument("--batch", type=int, default=64) + parser.add_argument("--device", default="cpu") + parser.add_argument("--min-confidence", type=float, default=0.60) + parser.add_argument("--output-json", type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + split_root = args.data.expanduser().resolve() / args.split + if not split_root.is_dir(): + raise FileNotFoundError(split_root) + + samples: list[tuple[Path, str]] = [] + for class_dir in sorted(split_root.iterdir()): + if not class_dir.is_dir() or class_dir.name.startswith("._"): + continue + for image_path in sorted(class_dir.iterdir()): + if image_path.name.startswith("._") or image_path.suffix.lower() not in IMAGE_SUFFIXES or not image_path.is_file(): + continue + samples.append((image_path, class_dir.name)) + if not samples: + raise RuntimeError(f"No classifier images found under {split_root}") + + try: + from ultralytics import YOLO + except Exception as exc: + raise SystemExit("Ultralytics is required to evaluate classifier checkpoints.") from exc + + model = YOLO(str(args.model.expanduser().resolve())) + predictions = model.predict( + source=[str(path) for path, _ in samples], + imgsz=args.imgsz, + batch=args.batch, + device=args.device, + verbose=False, + stream=True, + ) + + class_counts: dict[str, Counter[str]] = {} + total_counts: Counter[str] = Counter() + for (_, expected), result in zip(samples, predictions, strict=True): + probabilities = result.probs + if probabilities is None: + raise RuntimeError("Classifier prediction did not include probabilities") + predicted = str(result.names[int(probabilities.top1)]) + confidence = float(probabilities.top1conf) + counts = class_counts.setdefault(expected, Counter()) + counts["total"] += 1 + total_counts["total"] += 1 + if predicted == expected: + counts["top1_exact"] += 1 + total_counts["top1_exact"] += 1 + if confidence < args.min_confidence: + counts["rejected"] += 1 + total_counts["rejected"] += 1 + continue + counts["accepted"] += 1 + total_counts["accepted"] += 1 + if predicted == expected: + counts["accepted_exact"] += 1 + total_counts["accepted_exact"] += 1 + else: + counts["accepted_wrong"] += 1 + total_counts["accepted_wrong"] += 1 + + def summarize(counts: Counter[str]) -> dict[str, float | int]: + total = counts["total"] + accepted = counts["accepted"] + return { + "total": total, + "top1_exact": counts["top1_exact"], + "top1_exact_rate": round(counts["top1_exact"] / total, 6) if total else 0.0, + "accepted": accepted, + "accepted_coverage": round(accepted / total, 6) if total else 0.0, + "accepted_exact": counts["accepted_exact"], + "accepted_wrong": counts["accepted_wrong"], + "accepted_precision": round(counts["accepted_exact"] / accepted, 6) if accepted else 0.0, + "rejected": counts["rejected"], + } + + summary = { + "model": str(args.model.expanduser().resolve()), + "data": str(split_root), + "minimum_confidence": args.min_confidence, + "overall": summarize(total_counts), + "classes": {name: summarize(class_counts[name]) for name in sorted(class_counts)}, + } + encoded = json.dumps(summary, indent=2, sort_keys=True) + "\n" + print(encoded, end="") + if args.output_json: + output_path = args.output_json.expanduser().resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(encoded, encoding="ascii") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/evaluate_reviewed_route_events.py b/scripts/speed_limit_vision/evaluate_reviewed_route_events.py index d35f5e621..e17268d9c 100644 --- a/scripts/speed_limit_vision/evaluate_reviewed_route_events.py +++ b/scripts/speed_limit_vision/evaluate_reviewed_route_events.py @@ -59,6 +59,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--track-max-age", type=float, help="Override the maximum proposal track lifetime.") parser.add_argument("--crop-ocr", action="store_true", help="Evaluate with crop OCR confirmation enabled.") parser.add_argument("--classifier-min-confidence", type=float, help="Override the value classifier confidence threshold.") + parser.add_argument("--classifier-speed-values", help="Comma-separated classifier classes for a candidate model.") + parser.add_argument("--extended-classifier-min-confidence", type=float, help="Override confidence for 5/10/80/90/100.") parser.add_argument("--trusted-model-min-confidence", type=float, help="Override tiny-box trusted model confidence.") parser.add_argument("--classifier-expansion-limit", type=int, help="Evaluate only the first N detector crop expansions.") parser.add_argument("--classifier-expansion-indices", help="Comma-separated detector crop expansion indices to evaluate.") @@ -232,6 +234,10 @@ def main() -> int: slv.DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = args.crop_ocr if args.classifier_min_confidence is not None: slv.US_CLASSIFIER_MIN_CONFIDENCE = args.classifier_min_confidence + if args.classifier_speed_values: + slv.US_CLASSIFIER_SPEED_VALUES = tuple(int(value) for value in args.classifier_speed_values.split(",")) + if args.extended_classifier_min_confidence is not None: + slv.EXTENDED_CLASSIFIER_MIN_CONFIDENCE = args.extended_classifier_min_confidence if args.trusted_model_min_confidence is not None: slv.DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE = args.trusted_model_min_confidence if args.classifier_expansion_indices: diff --git a/scripts/speed_limit_vision/merge_classifier_datasets.py b/scripts/speed_limit_vision/merge_classifier_datasets.py index e6ab0d685..4f09f0c3d 100644 --- a/scripts/speed_limit_vision/merge_classifier_datasets.py +++ b/scripts/speed_limit_vision/merge_classifier_datasets.py @@ -34,7 +34,9 @@ def link_or_copy(source: Path, destination: Path) -> None: try: os.link(source, destination) except OSError: - shutil.copy2(source, destination) + # copy2 preserves macOS metadata as AppleDouble `._` files on exFAT. Image + # loaders then mistake those sidecars for corrupt training images. + shutil.copyfile(source, destination) def main() -> int: diff --git a/scripts/speed_limit_vision/serve_manual_review_queue.py b/scripts/speed_limit_vision/serve_manual_review_queue.py index 20e19221a..d186052e4 100644 --- a/scripts/speed_limit_vision/serve_manual_review_queue.py +++ b/scripts/speed_limit_vision/serve_manual_review_queue.py @@ -130,7 +130,8 @@ HTML = r"""