diff --git a/scripts/speed_limit_vision/evaluate_runtime_manifest.py b/scripts/speed_limit_vision/evaluate_runtime_manifest.py index 2bf223774..431ec2141 100644 --- a/scripts/speed_limit_vision/evaluate_runtime_manifest.py +++ b/scripts/speed_limit_vision/evaluate_runtime_manifest.py @@ -28,6 +28,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--detector-min-confidence", type=float, help="Override runtime US detector confidence threshold.") parser.add_argument("--classifier-min-confidence", type=float, help="Override runtime US classifier confidence threshold.") parser.add_argument("--classifier-reject-min-confidence", type=float, help="Override runtime reject-class confidence threshold.") + parser.add_argument("--include-uncertain", action="store_true", help="Include uncertain_positive review rows in positive metrics.") parser.add_argument("--strict-positive-recall", type=float, help="Exit non-zero if positive exact recall is below this value.") parser.add_argument("--strict-negative-fpr", type=float, help="Exit non-zero if negative false-positive rate is above this value.") return parser.parse_args() @@ -89,6 +90,15 @@ def main() -> int: raise FileNotFoundError(classifier_path) rows = load_rows(args.manifest.expanduser().resolve(), set(args.split) if args.split else None) + uncertain_count = sum( + row.get("sample_type", "") == "uncertain_positive" or row.get("review_status", "") == "uncertain" + for row in rows + ) + if not args.include_uncertain: + rows = [ + row for row in rows + if row.get("sample_type", "") != "uncertain_positive" and row.get("review_status", "") != "uncertain" + ] if args.max_rows > 0 and len(rows) > args.max_rows: rng = random.Random(args.seed) rows = rng.sample(rows, args.max_rows) @@ -159,6 +169,8 @@ def main() -> int: negative_fpr = negative_false_positive / negative_count if negative_count else 0.0 print(f"Rows evaluated: {positive_count + negative_count}") + if uncertain_count and not args.include_uncertain: + print(f"Skipped uncertain rows: {uncertain_count}") print(f"Unreadable rows: {unreadable_count}") print( f"Positive exact: {positive_exact}/{positive_count} " diff --git a/scripts/speed_limit_vision/import_manifest_classifier_masks.py b/scripts/speed_limit_vision/import_manifest_classifier_masks.py index 32ce06b87..3e6a55937 100644 --- a/scripts/speed_limit_vision/import_manifest_classifier_masks.py +++ b/scripts/speed_limit_vision/import_manifest_classifier_masks.py @@ -44,6 +44,12 @@ def safe_stem(text: str) -> str: return cleaned.strip("._")[:180] or "sample" +def short_stem(text: str, max_prefix: int = 80) -> str: + prefix = safe_stem(text)[:max_prefix].strip("._") or "sample" + digest = hashlib.sha1(text.encode("utf-8")).hexdigest()[:12] + return f"{prefix}_{digest}" + + def read_rows(path: Path) -> list[dict[str, str]]: with path.open("r", encoding="utf-8", newline="") as csv_file: return list(csv.DictReader(csv_file)) @@ -163,9 +169,9 @@ def load_crop(row: dict[str, str], manifest_path: Path, default_padding: float): return crop_box(image, boxes[bbox_index], padding) -def write_mask(workspace: Path, split: str, speed_value: int, stem: str, image_bgr) -> None: +def write_mask(workspace: Path, split: str, speed_value: int, stem: str, image_bgr) -> bool: output_dir = ensure_dir(workspace / "classifier" / split / str(speed_value)) - cv2.imwrite(str(output_dir / f"{stem}.png"), image_bgr) + return cv2.imwrite(str(output_dir / f"{stem}.png"), image_bgr) def main() -> int: @@ -178,6 +184,7 @@ def main() -> int: skipped_no_speed = 0 skipped_no_crop = 0 skipped_no_mask = 0 + skipped_write_failed = 0 written = 0 for manifest_path in [path.expanduser().resolve() for path in args.manifest]: @@ -207,25 +214,32 @@ def main() -> int: skipped_no_mask += 1 continue - manifest_stem = safe_stem(manifest_path.stem) - source_stem = safe_stem(key_text) + manifest_stem = short_stem(manifest_path.stem, max_prefix=48) + source_stem = short_stem(key_text, max_prefix=72) base_stem = f"manifest_{manifest_stem}_{row_index:06d}_{source_stem}" base_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) - write_mask(workspace, split, speed_value, f"{base_stem}_base", base_mask) - written += 1 + row_written = 0 + if write_mask(workspace, split, speed_value, f"{base_stem}_base", base_mask): + written += 1 + row_written += 1 for variant_index in range(max(args.variants_per_example, 0)): augmented = augment_mask(mask, rng) - write_mask(workspace, split, speed_value, f"{base_stem}_var{variant_index:02d}", augmented) - written += 1 - imported += 1 + if write_mask(workspace, split, speed_value, f"{base_stem}_var{variant_index:02d}", augmented): + written += 1 + row_written += 1 + if row_written: + imported += 1 + else: + skipped_write_failed += 1 if args.max_rows > 0 and attempted >= args.max_rows: break print( "Imported manifest classifier masks: " f"attempted={attempted} imported={imported} written={written} " - f"skipped_no_speed={skipped_no_speed} skipped_no_crop={skipped_no_crop} skipped_no_mask={skipped_no_mask}" + f"skipped_no_speed={skipped_no_speed} skipped_no_crop={skipped_no_crop} skipped_no_mask={skipped_no_mask} " + f"skipped_write_failed={skipped_write_failed}" ) return 0 diff --git a/scripts/speed_limit_vision/import_manual_review_queue.py b/scripts/speed_limit_vision/import_manual_review_queue.py index 1781ac6f1..a2971a95a 100644 --- a/scripts/speed_limit_vision/import_manual_review_queue.py +++ b/scripts/speed_limit_vision/import_manual_review_queue.py @@ -63,6 +63,7 @@ DETECTOR_MANIFEST_FIELDNAMES = [ ] POSITIVE_STATUSES = {"accepted", "corrected"} +UNCERTAIN_STATUS = "uncertain" NEGATIVE_STATUS = "ignore" SIGN_TYPE_CLASS_IDS = { "regulatory": 0, @@ -226,6 +227,14 @@ def is_positive(row: dict[str, str]) -> bool: return Path(row.get("crop_path", "")).is_file() and Path(row.get("frame_path", "")).is_file() +def is_uncertain_positive(row: dict[str, str]) -> bool: + if row.get("review_status") != UNCERTAIN_STATUS: + return False + if not parse_speed(row.get("review_speed_limit_mph", "")): + return False + return Path(row.get("frame_path", "")).is_file() + + def is_true_negative(row: dict[str, str]) -> bool: if row.get("review_status") != NEGATIVE_STATUS: return False @@ -253,7 +262,7 @@ def positive_classifier_row(row: dict[str, str], split: str) -> dict[str, object def runtime_row(row: dict[str, str], split: str, sample_type: str) -> dict[str, object]: - speed = parse_speed(row.get("review_speed_limit_mph", "")) if sample_type == "positive" else 0 + speed = parse_speed(row.get("review_speed_limit_mph", "")) if sample_type in ("positive", "uncertain_positive") else 0 return { "record_key": row["record_key"], "split": split, @@ -331,6 +340,7 @@ def main() -> int: rows = merged_review_rows(queue_path, labels_path) positive_rows = [row for row in rows if is_positive(row)] + uncertain_positive_rows = [row for row in rows if is_uncertain_positive(row)] true_negative_rows = [row for row in rows if is_true_negative(row)] if args.max_detector_negatives > 0: true_negative_rows = true_negative_rows[:args.max_detector_negatives] @@ -347,6 +357,10 @@ def main() -> int: if detector_row is not None: detector_rows.append(detector_row) + for row in uncertain_positive_rows: + split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder) + runtime_rows.append(runtime_row(row, split, "uncertain_positive")) + for row in true_negative_rows: split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder) runtime_rows.append(runtime_row(row, split, "negative_empty")) @@ -363,6 +377,7 @@ def main() -> int: "labels": str(labels_path), "reviewed_rows": len(rows), "positive_rows": len(positive_rows), + "uncertain_positive_rows": len(uncertain_positive_rows), "true_negative_rows": len(true_negative_rows), "classifier_manifest": str(classifier_manifest), "runtime_manifest": str(runtime_manifest), @@ -374,7 +389,7 @@ def main() -> int: print( "Imported manual review queue: " - f"reviewed={len(rows)} positives={len(positive_rows)} true_negatives={len(true_negative_rows)} " + f"reviewed={len(rows)} positives={len(positive_rows)} uncertain_positives={len(uncertain_positive_rows)} true_negatives={len(true_negative_rows)} " f"detector_imported={len(detector_rows)}" ) print(f"Classifier manifest: {classifier_manifest}") diff --git a/scripts/speed_limit_vision/serve_manual_review_queue.py b/scripts/speed_limit_vision/serve_manual_review_queue.py index 9fbecec17..1f3e21d76 100644 --- a/scripts/speed_limit_vision/serve_manual_review_queue.py +++ b/scripts/speed_limit_vision/serve_manual_review_queue.py @@ -79,7 +79,7 @@ HTML = r""" - Keys: Space/p accept model, type speed to correct, i/x ignore, Enter save correction, j/k next/prev, s school, r regulatory, a advisory + Keys: Space/p accept model, type speed to correct, u uncertain, i/x ignore, Enter save correction, j/k next/prev, s school, r regulatory, a advisory
@@ -107,6 +107,7 @@ HTML = r"""

Action

+
@@ -356,7 +357,7 @@ function render() { } function manualReviewStatus() { - if (draft.review_status === "ignore" || draft.review_status === "needs_later") return draft.review_status; + if (draft.review_status === "ignore" || draft.review_status === "needs_later" || draft.review_status === "uncertain") return draft.review_status; return "corrected"; } @@ -440,6 +441,17 @@ document.addEventListener("keydown", ev => { save(true, "ignore"); return; } + if (key === "u") { + clearSpeedBuffer(); + draft.review_status = "uncertain"; + if (!draft.review_speed_limit_mph) draft.review_speed_limit_mph = current.candidate_speed_limit_mph || ""; + ensureSpeedSignType(); + setActive("#statusButtons button", "status", "uncertain"); + setActive("#speedButtons button", "speed", draft.review_speed_limit_mph); + setActive("#typeButtons button", "type", draft.review_sign_type); + save(true, "uncertain"); + return; + } if (key === "enter") { clearSpeedBuffer(); save(true); } }); loadQueue(); diff --git a/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx b/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx index 7985f71c8..6c2b97297 100755 Binary files a/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx and b/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx differ