diff --git a/common/libcommon.a b/common/libcommon.a index b9a6e64e..8d0fee0d 100644 Binary files a/common/libcommon.a and b/common/libcommon.a differ diff --git a/common/params_keys.h b/common/params_keys.h index a95750b4..6e6f1920 100644 --- a/common/params_keys.h +++ b/common/params_keys.h @@ -328,6 +328,13 @@ inline static std::unordered_map keys = { {"MapsSelected", {PERSISTENT, STRING, "", "", 0}}, {"MapSpeedLimit", {CLEAR_ON_MANAGER_START, FLOAT, "0.0", "0.0"}}, {"NextMapSpeedLimit", {CLEAR_ON_MANAGER_START, JSON, "{}", "{}"}}, + {"VisionSpeedLimit", {CLEAR_ON_MANAGER_START, FLOAT, "0.0", "0.0"}}, + {"VisionSpeedLimitConfidence", {CLEAR_ON_MANAGER_START, FLOAT, "0.0", "0.0"}}, + {"VisionSpeedLimitBookmarkCount", {CLEAR_ON_MANAGER_START, INT, "0", "0"}}, + {"VisionSpeedLimitDebugSession", {CLEAR_ON_MANAGER_START, STRING, "", ""}}, + {"VisionSpeedLimitLastEvent", {CLEAR_ON_MANAGER_START, STRING, "", ""}}, + {"VisionSpeedLimitStatus", {CLEAR_ON_MANAGER_START, STRING, "", ""}}, + {"VisionSpeedLimitStream", {CLEAR_ON_MANAGER_START, STRING, "", ""}}, {"MaxDesiredAcceleration", {PERSISTENT, FLOAT, "4.0", "2.0", 2}}, {"MinimumBackupSize", {PERSISTENT, INT, "0", "0"}}, {"MinimumLaneChangeSpeed", {PERSISTENT, FLOAT, "20.0", "20.0", 2}}, @@ -449,6 +456,9 @@ inline static std::unordered_map keys = { {"SpeedLimits", {PERSISTENT | DONT_LOG, JSON, "[]", "[]"}}, {"SpeedLimitsFiltered", {PERSISTENT | DONT_LOG, JSON, "[]", "[]"}}, {"SpeedLimitSources", {PERSISTENT, BOOL, "0", "0", 3}}, + {"VisionSpeedLimitAutoBookmark", {PERSISTENT, BOOL, "1", "0", 0}}, + {"VisionSpeedLimitAutoPreserveSegment", {PERSISTENT, BOOL, "0", "0", 0}}, + {"VisionSpeedLimitDetection", {PERSISTENT, BOOL, "0", "0", 0}}, {"StandardFollow", {PERSISTENT, FLOAT, "1.45", "1.45", 2}}, {"StandardFollowHigh", {PERSISTENT, FLOAT, "1.2", "1.2", 2}}, {"StandardJerkAcceleration", {PERSISTENT, FLOAT, "50.0", "50.0", 3}}, diff --git a/common/params_pyx.so b/common/params_pyx.so index 0ca74f24..36566229 100755 Binary files a/common/params_pyx.so and b/common/params_pyx.so differ diff --git a/docs/how-to/train-speed-limit-vision.md b/docs/how-to/train-speed-limit-vision.md new file mode 100644 index 00000000..9ba62c1a --- /dev/null +++ b/docs/how-to/train-speed-limit-vision.md @@ -0,0 +1,304 @@ +# Train Speed-Limit Vision + +This flow is for replacing the current imported `ayoubsa_best` checkpoint with a U.S.-focused detector plus a separate posted-speed classifier. + +The intended recipe is: + +1. bootstrap from public traffic-sign data +2. fine-tune on comma-specific bookmarked drives +3. export ONNX models +4. copy them onto the device for runtime testing + +## Why Two Models + +The detector and the value reader solve different problems. + +- detector: find the sign and decide whether it is a regulatory speed-limit sign +- classifier: read the posted value from the cropped sign + +This is a better fit for U.S. roads than a single detector with baked-in classes like `Speed Limit 10`, `Speed Limit 20`, `Speed Limit 30`, and so on. + +## Suggested Public Data + +- `LISA` for U.S. roadside sign geometry and sign style +- `ARTS` for U.S. MUTCD-style sign annotations and values +- `GLARE` for U.S. glare/lighting failures + +Use public data to get the model into the right regime, then fine-tune on comma bookmarks and replay clips. + +## Install Training Deps + +From the repo root: + +```bash +uv sync --extra speedvision +``` + +The runtime device does not need these packages. Only the training machine does. + +## Initialize a Workspace + +```bash +.venv/bin/python scripts/speed_limit_vision/init_workspace.py +``` + +This creates `.tmp/speed_limit_training` with: + +- detector image/label folders +- classifier crop folders +- review/bookmark manifests +- raw-source manifests +- export and run directories + +To keep the raw datasets off the internal disk, point `--workspace` at the SSD-backed workspace, for example: + +```bash +.venv/bin/python scripts/speed_limit_vision/init_workspace.py \ + --workspace /Volumes/T5/starpilot_speed_limit/workspace/speed_limit_training_clean +``` + +The workspace now also tracks: + +- `manifests/raw_sources.csv` +- `manifests/public_detector_samples.csv` +- `manifests/public_classifier_samples.csv` + +Those manifests are the provenance record for every imported public sample. + +## Import Public Datasets + +ARTS challenging subset: + +```bash +.venv/bin/python scripts/speed_limit_vision/import_arts_challenging.py \ + --workspace /Volumes/T5/starpilot_speed_limit/workspace/speed_limit_training_clean +``` + +This imports mapped speed-limit signs from the raw `challenging-dev.tar.gz` archive and appends detector/classifier provenance rows into the manifest CSVs. + +GLARE and LISA should be downloaded into the SSD raw tree first: + +- `/Volumes/T5/starpilot_speed_limit/raw/glare_official` +- `/Volumes/T5/starpilot_speed_limit/raw/lisa_official` + +Then import them into the same workspace so the detector/classifier datasets stay source-traceable. + +For GLARE, do not pull the whole Drive tree blindly. Use the filtered raw downloader so only the `Images/` and optional `Tracks/` files land on disk: + +```bash +.venv/bin/python scripts/speed_limit_vision/download_glare_raw.py \ + --workspace /Volumes/T5/starpilot_speed_limit/workspace/speed_limit_training_clean \ + --output-root /Volumes/T5/starpilot_speed_limit/raw/glare_raw \ + --prefix Images/ \ + --resume +``` + +Then import the completed image tree: + +```bash +.venv/bin/python scripts/speed_limit_vision/import_glare_images.py \ + --workspace /Volumes/T5/starpilot_speed_limit/workspace/speed_limit_training_clean \ + --overwrite +``` + +## Import Bookmarked Debug Sessions + +After a drive, copy or mount the debug session directory locally, then import it: + +```bash +.venv/bin/python scripts/speed_limit_vision/import_debug_sessions.py +``` + +Or point at specific sessions: + +```bash +.venv/bin/python scripts/speed_limit_vision/import_debug_sessions.py 20260330_220102 20260330_223355 +``` + +This writes: + +- `review/bookmarks.csv` +- snapshot images under `review/images` + +That manifest is the shortlist for labeling. + +If the route only lives on comma connect, fetch it directly into the same clip layout the bookmark tools expect: + +```bash +.venv/bin/python scripts/speed_limit_vision/download_connect_routes.py \ + / \ + --streams fcamera,qlog +``` + +This reads the JWT from `~/.comma/auth.json`, downloads the requested route files from comma connect, and updates: + +- `/Volumes/T5/starpilot_speed_limit/live_route_clips/bookmark_windows/data/media/0/realdata/...` when the SSD is mounted, otherwise the same `.tmp/...` paths under the repo +- `/Volumes/T5/starpilot_speed_limit/live_routes_meta/qlog_mtimes.txt` when the SSD is mounted, otherwise `.tmp/live_routes_meta/qlog_mtimes.txt` +- `/Volumes/T5/starpilot_speed_limit/live_routes_meta/files.txt` when the SSD is mounted, otherwise `.tmp/live_routes_meta/files.txt` + +For multiple routes, either pass several ids on the command line or use `--routes-file`. + +## Evaluate Real Sign Lead-Ins + +Bookmark stills are often too late. To score what matters, replay the real `fcamera.hevc` footage from the `5` seconds before each bookmark: + +```bash +.venv/bin/python scripts/speed_limit_vision/evaluate_bookmark_leadins.py \ + --json-out .tmp/live_route_clips/bookmark_windows_report.json +``` + +This produces a per-bookmark report of whether the current runtime saw anything in the usable sign approach window. + +The evaluator now reads a local session-to-route map from `session_route_map.json` under the same `live_routes_meta` root. Keep that file local or on the SSD so personal route ids never need to live in the repo. + +## Import Missed Lead-Ins for Labeling + +Turn those lead-in misses into review frames and contact sheets: + +```bash +.venv/bin/python scripts/speed_limit_vision/import_bookmark_leadins.py \ + --mode misses +``` + +This writes: + +- `review/bookmark_leadins.csv` +- sampled frames under `review/leadins/frames` +- contact sheets under `review/leadins/contact_sheets` + +That review set is the right source for labeling missed `55 mph`, night, and town-sequence failures. + +The bookmark/lead-in importers also accept source metadata fields such as region, device, and driver. Use those when importing debug sessions from multiple users so the comma-specific fine-tune can be sliced by contributor or geography instead of becoming one opaque pool. + +To shrink that review set to the most promising frames per missed sign window: + +```bash +.venv/bin/python scripts/speed_limit_vision/rank_bookmark_leadin_frames.py +``` + +This writes `review/bookmark_leadin_shortlist.csv` with the top-ranked sampled frames per bookmark. + +## Build the Detector Dataset + +Take the imported review images and move or copy the ones you want into: + +- `detector/images/train` +- `detector/images/val` + +Label them in YOLO detect format into: + +- `detector/labels/train` +- `detector/labels/val` + +Recommended classes: + +- `regulatory_speed_limit` +- `advisory_speed_limit` +- `school_zone_speed_limit` + +The dataset YAML is already generated at: + +- `detector/dataset.yaml` + +## Build the Value Classifier Dataset + +Fill out: + +- `classifier/value_labels.csv` + +Columns: + +- `image_path`: source image file +- `split`: `train` or `val` +- `speed_limit_mph`: posted value such as `25`, `35`, or `55` +- `bbox_index`: which YOLO box to crop if an image has multiple labeled signs +- `padding`: optional crop padding ratio +- `label_path`: optional explicit path to the YOLO label file + +Then generate the classifier crop folders: + +```bash +.venv/bin/python scripts/speed_limit_vision/build_value_dataset.py +``` + +This writes cropped sign images into: + +- `classifier/train//...` +- `classifier/val//...` + +## Train + +Detector: + +```bash +.venv/bin/python scripts/speed_limit_vision/train_detector.py --device mps +``` + +Classifier: + +```bash +.venv/bin/python scripts/speed_limit_vision/train_value_classifier.py --device mps +``` + +Use `--device cpu`, `--device mps`, or a CUDA device string depending on the training host. + +## Rebalance Toward Real Comma Data + +If the detector starts overfitting to synthetic/public data, build a lighter rebalanced dataset that keeps all `real_*` detector images and samples the rest: + +```bash +.venv/bin/python scripts/speed_limit_vision/rebalance_detector_dataset.py \ + --workspace .tmp/speed_limit_training \ + --max-other-train 3200 +``` + +Then point the detector trainer at the generated YAML: + +```bash +.venv/bin/python scripts/speed_limit_vision/train_detector.py \ + --workspace .tmp/speed_limit_training \ + --data .tmp/speed_limit_training/detector_rebalanced/dataset.yaml \ + --device mps +``` + +This keeps validation unchanged while making retrains faster and more comma-biased. + +## Export ONNX + +```bash +.venv/bin/python scripts/speed_limit_vision/export_models.py \ + --detector-weights .tmp/speed_limit_training/runs/detector/yolo11n-speed-limit-us/weights/best.pt \ + --classifier-weights .tmp/speed_limit_training/runs/classifier/yolo11n-cls-speed-limit-us/weights/best.pt \ + --install-repo-assets +``` + +That writes: + +- `.tmp/speed_limit_training/exports/speed_limit_us_detector.onnx` +- `.tmp/speed_limit_training/exports/speed_limit_us_value_classifier.onnx` + +And optionally copies them into: + +- `starpilot/assets/vision_models` + +## Copy to the Device + +```bash +.venv/bin/python scripts/speed_limit_vision/install_models.py --host comma@192.168.3.110 +``` + +The runtime already prefers `speed_limit_us_detector.onnx` plus `speed_limit_us_value_classifier.onnx` when both files exist in `starpilot/assets/vision_models`. + +## Evaluate the Runtime Path + +Run the real StarPilot runtime path, using the installed ONNX pair, against the known saved-frame cases: + +```bash +.venv/bin/python scripts/speed_limit_vision/evaluate_runtime_cases.py --strict +``` + +For temporal behavior on a saved frame directory or route extract, replay the runtime directly: + +```bash +.venv/bin/python scripts/replay_speed_limit_vision.py .tmp/vision_iter/seg10_5fps --frames-fps 5 +``` diff --git a/panda/board/obj/body_h7.bin.signed b/panda/board/obj/body_h7.bin.signed index 431e214a..c36f2b76 100644 Binary files a/panda/board/obj/body_h7.bin.signed and b/panda/board/obj/body_h7.bin.signed differ diff --git a/panda/board/obj/body_h7/bootstub.elf b/panda/board/obj/body_h7/bootstub.elf index 44f60add..384d6a70 100755 Binary files a/panda/board/obj/body_h7/bootstub.elf and b/panda/board/obj/body_h7/bootstub.elf differ diff --git a/panda/board/obj/body_h7/main.bin b/panda/board/obj/body_h7/main.bin index 0dc715cd..70ec5a5c 100755 Binary files a/panda/board/obj/body_h7/main.bin and b/panda/board/obj/body_h7/main.bin differ diff --git a/panda/board/obj/body_h7/main.elf b/panda/board/obj/body_h7/main.elf index 8b9daa9f..17ecac4a 100755 Binary files a/panda/board/obj/body_h7/main.elf and b/panda/board/obj/body_h7/main.elf differ diff --git a/panda/board/obj/bootstub.body_h7.bin b/panda/board/obj/bootstub.body_h7.bin index eddce4a9..2e770c1e 100755 Binary files a/panda/board/obj/bootstub.body_h7.bin and b/panda/board/obj/bootstub.body_h7.bin differ diff --git a/panda/board/obj/bootstub.panda.bin b/panda/board/obj/bootstub.panda.bin index e57e0fce..69ddeb37 100755 Binary files a/panda/board/obj/bootstub.panda.bin and b/panda/board/obj/bootstub.panda.bin differ diff --git a/panda/board/obj/bootstub.panda_h7.bin b/panda/board/obj/bootstub.panda_h7.bin index 81dfa741..01514729 100755 Binary files a/panda/board/obj/bootstub.panda_h7.bin and b/panda/board/obj/bootstub.panda_h7.bin differ diff --git a/panda/board/obj/bootstub.panda_h7_remote.bin b/panda/board/obj/bootstub.panda_h7_remote.bin index 81dfa741..01514729 100755 Binary files a/panda/board/obj/bootstub.panda_h7_remote.bin and b/panda/board/obj/bootstub.panda_h7_remote.bin differ diff --git a/panda/board/obj/bootstub.panda_jungle_h7.bin b/panda/board/obj/bootstub.panda_jungle_h7.bin index 236f50dd..ac7d0acf 100755 Binary files a/panda/board/obj/bootstub.panda_jungle_h7.bin and b/panda/board/obj/bootstub.panda_jungle_h7.bin differ diff --git a/panda/board/obj/bootstub.panda_remote.bin b/panda/board/obj/bootstub.panda_remote.bin index e57e0fce..69ddeb37 100755 Binary files a/panda/board/obj/bootstub.panda_remote.bin and b/panda/board/obj/bootstub.panda_remote.bin differ diff --git a/panda/board/obj/gitversion.h b/panda/board/obj/gitversion.h index f45a39e3..9c1eb1f8 100644 --- a/panda/board/obj/gitversion.h +++ b/panda/board/obj/gitversion.h @@ -1,2 +1,2 @@ extern const uint8_t gitversion[19]; -const uint8_t gitversion[19] = "DEV-523a26be-DEBUG"; +const uint8_t gitversion[19] = "DEV-f7ee2e65-DEBUG"; diff --git a/panda/board/obj/panda.bin.signed b/panda/board/obj/panda.bin.signed index 7c13fc96..c984e7d5 100644 Binary files a/panda/board/obj/panda.bin.signed and b/panda/board/obj/panda.bin.signed differ diff --git a/panda/board/obj/panda/bootstub.elf b/panda/board/obj/panda/bootstub.elf index 0f842cd7..75df98c8 100755 Binary files a/panda/board/obj/panda/bootstub.elf and b/panda/board/obj/panda/bootstub.elf differ diff --git a/panda/board/obj/panda/main.bin b/panda/board/obj/panda/main.bin index 36e4c3ad..42cf606a 100755 Binary files a/panda/board/obj/panda/main.bin and b/panda/board/obj/panda/main.bin differ diff --git a/panda/board/obj/panda/main.elf b/panda/board/obj/panda/main.elf index f30d12ac..76a51c70 100755 Binary files a/panda/board/obj/panda/main.elf and b/panda/board/obj/panda/main.elf differ diff --git a/panda/board/obj/panda_h7.bin.signed b/panda/board/obj/panda_h7.bin.signed index 402cdc04..e2bdff74 100644 Binary files a/panda/board/obj/panda_h7.bin.signed and b/panda/board/obj/panda_h7.bin.signed differ diff --git a/panda/board/obj/panda_h7/bootstub.elf b/panda/board/obj/panda_h7/bootstub.elf index 23921c8c..86244584 100755 Binary files a/panda/board/obj/panda_h7/bootstub.elf and b/panda/board/obj/panda_h7/bootstub.elf differ diff --git a/panda/board/obj/panda_h7/main.bin b/panda/board/obj/panda_h7/main.bin index dadda28a..c0ed1fd7 100755 Binary files a/panda/board/obj/panda_h7/main.bin and b/panda/board/obj/panda_h7/main.bin differ diff --git a/panda/board/obj/panda_h7/main.elf b/panda/board/obj/panda_h7/main.elf index 2d7fb140..85aa68ec 100755 Binary files a/panda/board/obj/panda_h7/main.elf and b/panda/board/obj/panda_h7/main.elf differ diff --git a/panda/board/obj/panda_h7_remote.bin.signed b/panda/board/obj/panda_h7_remote.bin.signed index 1343ed9f..64098118 100644 Binary files a/panda/board/obj/panda_h7_remote.bin.signed and b/panda/board/obj/panda_h7_remote.bin.signed differ diff --git a/panda/board/obj/panda_h7_remote/bootstub.elf b/panda/board/obj/panda_h7_remote/bootstub.elf index effea6e5..44220a17 100755 Binary files a/panda/board/obj/panda_h7_remote/bootstub.elf and b/panda/board/obj/panda_h7_remote/bootstub.elf differ diff --git a/panda/board/obj/panda_h7_remote/main.bin b/panda/board/obj/panda_h7_remote/main.bin index 3ec2e0a7..f5694fd2 100755 Binary files a/panda/board/obj/panda_h7_remote/main.bin and b/panda/board/obj/panda_h7_remote/main.bin differ diff --git a/panda/board/obj/panda_h7_remote/main.elf b/panda/board/obj/panda_h7_remote/main.elf index e3831b82..fa10fbf3 100755 Binary files a/panda/board/obj/panda_h7_remote/main.elf and b/panda/board/obj/panda_h7_remote/main.elf differ diff --git a/panda/board/obj/panda_jungle_h7.bin.signed b/panda/board/obj/panda_jungle_h7.bin.signed index b1351816..d899fac0 100644 Binary files a/panda/board/obj/panda_jungle_h7.bin.signed and b/panda/board/obj/panda_jungle_h7.bin.signed differ diff --git a/panda/board/obj/panda_jungle_h7/bootstub.elf b/panda/board/obj/panda_jungle_h7/bootstub.elf index f36b775a..e27b328a 100755 Binary files a/panda/board/obj/panda_jungle_h7/bootstub.elf and b/panda/board/obj/panda_jungle_h7/bootstub.elf differ diff --git a/panda/board/obj/panda_jungle_h7/main.bin b/panda/board/obj/panda_jungle_h7/main.bin index b5cf1e5b..5ad6a137 100755 Binary files a/panda/board/obj/panda_jungle_h7/main.bin and b/panda/board/obj/panda_jungle_h7/main.bin differ diff --git a/panda/board/obj/panda_jungle_h7/main.elf b/panda/board/obj/panda_jungle_h7/main.elf index dc77e30f..ec326f2e 100755 Binary files a/panda/board/obj/panda_jungle_h7/main.elf and b/panda/board/obj/panda_jungle_h7/main.elf differ diff --git a/panda/board/obj/panda_remote.bin.signed b/panda/board/obj/panda_remote.bin.signed index d21534e3..eee27d23 100644 Binary files a/panda/board/obj/panda_remote.bin.signed and b/panda/board/obj/panda_remote.bin.signed differ diff --git a/panda/board/obj/panda_remote/bootstub.elf b/panda/board/obj/panda_remote/bootstub.elf index e4a9873c..310632ff 100755 Binary files a/panda/board/obj/panda_remote/bootstub.elf and b/panda/board/obj/panda_remote/bootstub.elf differ diff --git a/panda/board/obj/panda_remote/main.bin b/panda/board/obj/panda_remote/main.bin index 0b9abfb3..f726da00 100755 Binary files a/panda/board/obj/panda_remote/main.bin and b/panda/board/obj/panda_remote/main.bin differ diff --git a/panda/board/obj/panda_remote/main.elf b/panda/board/obj/panda_remote/main.elf index 61992b67..f64aa710 100755 Binary files a/panda/board/obj/panda_remote/main.elf and b/panda/board/obj/panda_remote/main.elf differ diff --git a/panda/board/obj/version b/panda/board/obj/version index f727c6ec..c47a32fb 100644 --- a/panda/board/obj/version +++ b/panda/board/obj/version @@ -1 +1 @@ -DEV-523a26be-DEBUG \ No newline at end of file +DEV-f7ee2e65-DEBUG \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ecda9364..9f1acfb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ # modeld "onnx >= 1.14.0", + "opencv-python-headless", # logging "pyzmq", @@ -111,7 +112,6 @@ dev = [ "dbus-next", # TODO: remove once we moved everything to jeepney "dictdiffer", "matplotlib", - "opencv-python-headless", "parameterized >=0.8, <0.9", "pyautogui", "pygame", @@ -129,6 +129,12 @@ tools = [ "dearpygui>=2.1.0; (sys_platform != 'linux' or platform_machine != 'aarch64')", # not vended for linux aarch64 ] +speedvision = [ + "torch>=2.4", + "torchvision>=0.19", + "ultralytics>=8.3,<9", +] + [project.urls] Homepage = "https://github.com/commaai/openpilot" diff --git a/scripts/analyze_speed_limit_vision_debug.py b/scripts/analyze_speed_limit_vision_debug.py new file mode 100644 index 00000000..07949946 --- /dev/null +++ b/scripts/analyze_speed_limit_vision_debug.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +DEBUG_BASE_DIR = Path("/data/media/0/vision_speed_limit_debug") + + +def load_events(session_path: Path): + events_path = session_path / "events.jsonl" + if not events_path.is_file(): + raise FileNotFoundError(f"Missing events file: {events_path}") + + events = [] + with events_path.open("r", encoding="utf-8") as log_file: + for line in log_file: + line = line.strip() + if not line: + continue + events.append(json.loads(line)) + return events + + +def resolve_session(session_arg: str | None) -> Path | None: + if session_arg: + session_path = Path(session_arg) + if session_path.is_dir(): + return session_path + candidate = DEBUG_BASE_DIR / session_arg + if candidate.is_dir(): + return candidate + raise FileNotFoundError(f"Session not found: {session_arg}") + + if not DEBUG_BASE_DIR.is_dir(): + return None + + sessions = sorted((path for path in DEBUG_BASE_DIR.iterdir() if path.is_dir()), reverse=True) + return sessions[0] if sessions else None + + +def list_sessions(): + if not DEBUG_BASE_DIR.is_dir(): + print(f"No debug sessions found in {DEBUG_BASE_DIR}") + return + + sessions = sorted((path for path in DEBUG_BASE_DIR.iterdir() if path.is_dir()), reverse=True) + if not sessions: + print(f"No debug sessions found in {DEBUG_BASE_DIR}") + return + + for session_path in sessions: + events_path = session_path / "events.jsonl" + event_count = 0 + bookmark_count = 0 + if events_path.is_file(): + with events_path.open("r", encoding="utf-8") as log_file: + for line in log_file: + if not line.strip(): + continue + event_count += 1 + if '"event":"bookmark"' in line: + bookmark_count += 1 + print(f"{session_path.name}: {event_count} events, {bookmark_count} bookmarks") + + +def print_event(event: dict): + fields = [ + event.get("wallTime", ""), + event.get("event", ""), + ] + if event.get("sessionSeconds") is not None: + fields.append(f"t+{event['sessionSeconds']}s") + + if event.get("roadName"): + fields.append(f"road={event['roadName']}") + if event.get("speedLimitMph"): + fields.append(f"speed={event['speedLimitMph']} mph") + if event.get("candidateSpeedLimitMph"): + fields.append(f"candidate={event['candidateSpeedLimitMph']} mph") + if event.get("confidence"): + fields.append(f"conf={event['confidence']}") + if event.get("candidateConfidence"): + fields.append(f"candidateConf={event['candidateConfidence']}") + if event.get("statusText"): + fields.append(f"status={event['statusText']}") + elif event.get("status"): + fields.append(f"status={event['status']}") + if event.get("snapshot"): + fields.append(f"snapshot={event['snapshot']}") + print(" | ".join(str(field) for field in fields if field != "")) + + +def summarize_session(session_path: Path, window: int): + events = load_events(session_path) + print(f"Session: {session_path}") + print(f"Events: {len(events)}") + + bookmarks = [idx for idx, event in enumerate(events) if event.get("event") == "bookmark"] + if not bookmarks: + print("Bookmarks: none") + return + + print(f"Bookmarks: {len(bookmarks)}") + for bookmark_number, event_idx in enumerate(bookmarks, start=1): + print(f"\nBookmark {bookmark_number}") + start = max(event_idx - window, 0) + end = min(event_idx + window + 1, len(events)) + for idx in range(start, end): + prefix = "->" if idx == event_idx else " " + print(prefix, end="") + print_event(events[idx]) + + +def main(): + parser = argparse.ArgumentParser(description="Summarize StarPilot speed-limit vision debug sessions.") + parser.add_argument("session", nargs="?", help="Session id or full path. Defaults to the latest session.") + parser.add_argument("--list", action="store_true", help="List available sessions and exit.") + parser.add_argument("--window", type=int, default=5, help="How many events before/after each bookmark to print.") + args = parser.parse_args() + + if args.list: + list_sessions() + return + + session_path = resolve_session(args.session) + if session_path is None: + print(f"No debug sessions found in {DEBUG_BASE_DIR}") + return + + summarize_session(session_path, max(args.window, 0)) + + +if __name__ == "__main__": + main() diff --git a/scripts/replay_speed_limit_vision.py b/scripts/replay_speed_limit_vision.py new file mode 100644 index 00000000..f9fc29bd --- /dev/null +++ b/scripts/replay_speed_limit_vision.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +from pathlib import Path + +import cv2 + +import starpilot.system.speed_limit_vision as slv + + +class ReplayDaemon(slv.SpeedLimitVisionDaemon): + def __init__(self): + super().__init__(use_runtime=False) + self.now = 0.0 + + def _write_debug_event(self, event_type, frame_bgr=None, snapshot_prefix=None, **fields): + if event_type in ("candidate", "publish", "stale_clear"): + print(f"t={self.now:6.2f}s {event_type:12} {fields}") + + def _publish_status(self, status, clear_speed=False): + if clear_speed: + self._clear_detection() + + def _publish_detection(self, speed_limit_mph, confidence, status_prefix): + super()._publish_detection(speed_limit_mph, confidence, status_prefix) + + def process_frame(self, now, frame_bgr): + self.now = now + slv.time.monotonic = lambda now=now: now + self.current_frame_bgr = frame_bgr + + detection = self._detect_sign(frame_bgr) + if detection is not None: + self._update_detection(detection) + elif self.published_speed_limit_mph > 0 and self._published_detection_stale(now): + print(f"t={self.now:6.2f}s stale_clear {{'reason': 'no_detection'}}") + self._clear_detection() + + +def iter_directory_frames(path: Path, fps: float): + for index, frame_path in enumerate(sorted(path.glob("frame_*.png")), start=1): + frame = cv2.imread(str(frame_path)) + if frame is None: + continue + yield (index - 1) / fps, frame + + +def iter_video_frames(path: Path): + cap = cv2.VideoCapture(str(path)) + fps = cap.get(cv2.CAP_PROP_FPS) or 25.0 + frame_index = 0 + while True: + ok, frame = cap.read() + if not ok: + break + yield frame_index / fps, frame + frame_index += 1 + cap.release() + + +def main(): + parser = argparse.ArgumentParser(description="Replay StarPilot speed-limit vision on saved video or extracted frames.") + parser.add_argument("path", help="Path to an fcamera.hevc file or a directory of frame_XXX.png images.") + parser.add_argument("--frames-fps", type=float, default=5.0, help="FPS to assume when replaying an extracted frame directory.") + parser.add_argument("--start", type=float, default=0.0, help="Skip frames before this timestamp in seconds.") + parser.add_argument("--end", type=float, default=None, help="Stop once this timestamp in seconds is exceeded.") + args = parser.parse_args() + + path = Path(args.path) + if not path.exists(): + raise FileNotFoundError(path) + + daemon = ReplayDaemon() + frame_iter = iter_directory_frames(path, max(args.frames_fps, 0.1)) if path.is_dir() else iter_video_frames(path) + for now, frame_bgr in frame_iter: + if now < args.start: + continue + if args.end is not None and now > args.end: + break + daemon.process_frame(now, frame_bgr) + + +if __name__ == "__main__": + main() diff --git a/scripts/speed_limit_vision/augment_real_classifier_masks.py b/scripts/speed_limit_vision/augment_real_classifier_masks.py new file mode 100644 index 00000000..c13b1c6e --- /dev/null +++ b/scripts/speed_limit_vision/augment_real_classifier_masks.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import random + +from dataclasses import dataclass +from pathlib import Path + +import cv2 + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore + from generate_value_roi_classifier_dataset import augment_mask, extract_value_mask # type: ignore +else: + from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + from .generate_value_roi_classifier_dataset import augment_mask, extract_value_mask + + +@dataclass(frozen=True) +class ExampleSpec: + name: str + speed_limit_mph: int + image_path: str | None = None + frame_path: str | None = None + bbox: tuple[int, int, int, int] | None = None + + +DEFAULT_EXAMPLES = ( + ExampleSpec( + name="live15_runtime", + speed_limit_mph=15, + frame_path=".tmp/live_c4_capture/stopped_sign_road.jpg", + bbox=(725, 253, 768, 314), + ), + ExampleSpec( + name="school20_crop", + speed_limit_mph=20, + image_path=".tmp/route_vision/frame_041_sign_tight.jpg", + ), + ExampleSpec( + name="town30_crop", + speed_limit_mph=30, + image_path=".tmp/route_12c_seg9_10/seg10_real30_crop.png", + ), + ExampleSpec( + name="town30_late_runtime", + speed_limit_mph=30, + frame_path=".tmp/vision_iter/seg10_5fps/frame_054.png", + bbox=(887, 275, 931, 378), + ), + ExampleSpec( + name="town40_crop", + speed_limit_mph=40, + image_path=".tmp/route_12c_seg9_10/seg10_real40_crop.png", + ), + ExampleSpec( + name="highway40_crop", + speed_limit_mph=40, + image_path=".tmp/speed_route_frames_seg2_10_20/t12_sign_crop.png", + ), +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Inject curated real runtime-style masks into the classifier dataset.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--variants-per-example", type=int, default=80, help="Augmented mask variants to generate per example.") + parser.add_argument("--seed", type=int, default=20260330, help="Random seed.") + return parser.parse_args() + + +def load_crop(spec: ExampleSpec): + if spec.image_path: + image = cv2.imread(spec.image_path) + if image is None: + raise FileNotFoundError(spec.image_path) + return image + if spec.frame_path and spec.bbox: + frame = cv2.imread(spec.frame_path) + if frame is None: + raise FileNotFoundError(spec.frame_path) + x1, y1, x2, y2 = spec.bbox + crop = frame[y1:y2, x1:x2] + if crop.size == 0: + raise ValueError(f"{spec.name}: empty crop for bbox {spec.bbox}") + return crop + raise ValueError(f"{spec.name}: provide image_path or frame_path+bbox") + + +def save_mask(workspace: Path, split: str, speed_limit_mph: int, stem: str, mask_bgr) -> None: + output_dir = ensure_dir(workspace / "classifier" / split / str(speed_limit_mph)) + cv2.imwrite(str(output_dir / f"{stem}.png"), mask_bgr) + + +def remove_appledouble_files(root: Path) -> None: + for path in root.rglob("._*"): + if path.is_file() or path.is_symlink(): + path.unlink() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + rng = random.Random(args.seed) + written = 0 + + for spec in DEFAULT_EXAMPLES: + crop = load_crop(spec) + mask = extract_value_mask(crop) + if mask is None: + print(f"{spec.name}: skipped, no mask extracted") + continue + + base_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) + save_mask(workspace, "train", spec.speed_limit_mph, f"real_runtime_{spec.name}_base", base_mask) + written += 1 + + for variant_index in range(max(args.variants_per_example, 0)): + split = "val" if variant_index % 7 == 0 else "train" + augmented = augment_mask(mask, rng) + save_mask(workspace, split, spec.speed_limit_mph, f"real_runtime_{spec.name}_{variant_index:03d}", augmented) + written += 1 + + print(f"{spec.name}: added {1 + max(args.variants_per_example, 0)} mask(s) for {spec.speed_limit_mph} mph") + + remove_appledouble_files(workspace / "classifier" / "train") + remove_appledouble_files(workspace / "classifier" / "val") + print(f"Wrote {written} classifier mask image(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/augment_real_detector_examples.py b/scripts/speed_limit_vision/augment_real_detector_examples.py new file mode 100644 index 00000000..a1b9dc3f --- /dev/null +++ b/scripts/speed_limit_vision/augment_real_detector_examples.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import random + +from pathlib import Path + +import cv2 +import numpy as np + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore +else: + from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + + +def parse_label(label_path: Path, image_shape: tuple[int, int, int]): + lines = [line.strip() for line in label_path.read_text(encoding="utf-8").splitlines() if line.strip()] + if len(lines) != 1: + raise ValueError(f"Expected exactly one box in {label_path}") + + class_id, x_center, y_center, width, height = lines[0].split() + image_height, image_width = image_shape[:2] + x_center = float(x_center) * image_width + y_center = float(y_center) * image_height + width = float(width) * image_width + height = float(height) * image_height + x1 = x_center - width / 2 + y1 = y_center - height / 2 + x2 = x_center + width / 2 + y2 = y_center + height / 2 + return int(class_id), np.array([x1, y1, x2, y2], dtype=np.float32) + + +def write_label(label_path: Path, class_id: int, box: np.ndarray, image_shape: tuple[int, int, int]): + image_height, image_width = image_shape[:2] + x1, y1, x2, y2 = box.tolist() + x_center = ((x1 + x2) / 2) / image_width + y_center = ((y1 + y2) / 2) / image_height + width = (x2 - x1) / image_width + height = (y2 - y1) / image_height + label_path.write_text(f"{class_id} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}\n", encoding="utf-8") + + +def random_motion_blur(image: np.ndarray, rng: random.Random): + size = rng.choice((3, 5, 7)) + if rng.random() < 0.5: + kernel = np.zeros((size, size), dtype=np.float32) + kernel[size // 2, :] = 1.0 / size + else: + kernel = np.zeros((size, size), dtype=np.float32) + kernel[:, size // 2] = 1.0 / size + return cv2.filter2D(image, -1, kernel) + + +def augment_image(image: np.ndarray, box: np.ndarray, rng: random.Random): + image_height, image_width = image.shape[:2] + + scale = rng.uniform(0.92, 1.08) + translate_x = rng.uniform(-0.05, 0.05) * image_width + translate_y = rng.uniform(-0.04, 0.04) * image_height + center = (image_width / 2, image_height / 2) + matrix = cv2.getRotationMatrix2D(center, rng.uniform(-1.5, 1.5), scale) + matrix[0, 2] += translate_x + matrix[1, 2] += translate_y + + warped = cv2.warpAffine( + image, + matrix, + (image_width, image_height), + flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_REPLICATE, + ) + + corners = np.array([ + [box[0], box[1], 1.0], + [box[2], box[1], 1.0], + [box[2], box[3], 1.0], + [box[0], box[3], 1.0], + ], dtype=np.float32) + transformed = corners @ matrix.T + x_coords = transformed[:, 0] + y_coords = transformed[:, 1] + warped_box = np.array([ + np.clip(np.min(x_coords), 0, image_width - 1), + np.clip(np.min(y_coords), 0, image_height - 1), + np.clip(np.max(x_coords), 0, image_width - 1), + np.clip(np.max(y_coords), 0, image_height - 1), + ], dtype=np.float32) + + alpha = rng.uniform(0.85, 1.18) + beta = rng.uniform(-18.0, 16.0) + augmented = cv2.convertScaleAbs(warped, alpha=alpha, beta=beta) + + if rng.random() < 0.55: + augmented = cv2.GaussianBlur(augmented, (3, 3), rng.uniform(0.1, 1.0)) + if rng.random() < 0.35: + augmented = random_motion_blur(augmented, rng) + if rng.random() < 0.45: + noise = rng.uniform(4.0, 12.0) + augmented = np.clip(augmented.astype(np.float32) + np.random.normal(0.0, noise, augmented.shape), 0, 255).astype(np.uint8) + + return augmented, warped_box + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Oversample bootstrapped real detector frames with light augmentation.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--split", default="train", choices=("train", "val"), help="Detector split to augment.") + parser.add_argument("--variants-per-image", type=int, default=80, help="How many augmented variants to generate for each real_*.jpg frame.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + image_dir = workspace / "detector" / "images" / args.split + label_dir = workspace / "detector" / "labels" / args.split + ensure_dir(image_dir) + ensure_dir(label_dir) + + rng = random.Random(42) + base_images = sorted(image_dir.glob("real_*.jpg")) + created = 0 + for image_path in base_images: + label_path = label_dir / f"{image_path.stem}.txt" + image = cv2.imread(str(image_path)) + if image is None or not label_path.is_file(): + continue + + class_id, box = parse_label(label_path, image.shape) + for variant_index in range(args.variants_per_image): + augmented, warped_box = augment_image(image, box, rng) + if warped_box[2] - warped_box[0] < 10 or warped_box[3] - warped_box[1] < 12: + continue + + output_stem = f"{image_path.stem}_aug_{variant_index:03d}" + output_image = image_dir / f"{output_stem}.jpg" + output_label = label_dir / f"{output_stem}.txt" + cv2.imwrite(str(output_image), augmented, [cv2.IMWRITE_JPEG_QUALITY, 92]) + write_label(output_label, class_id, warped_box, augmented.shape) + created += 1 + + print(f"Augmented {len(base_images)} real detector images into {created} variants") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/bootstrap_real_detector_examples.py b/scripts/speed_limit_vision/bootstrap_real_detector_examples.py new file mode 100644 index 00000000..7b34e307 --- /dev/null +++ b/scripts/speed_limit_vision/bootstrap_real_detector_examples.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv + +from dataclasses import dataclass +from pathlib import Path + +import cv2 + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore +else: + from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + + +@dataclass(frozen=True) +class ExampleSpec: + name: str + detector_class: int + frame_path: str | None = None + frame_dir: str | None = None + template_path: str | None = None + bbox_override: tuple[int, int, int, int] | None = None + + +DEFAULT_EXAMPLES = ( + ExampleSpec( + name="route_seg2_t12_speed40", + detector_class=0, + frame_path=".tmp/speed_route_frames_seg2_10_20/t12.png", + template_path=".tmp/speed_route_frames_seg2_10_20/t12_sign_crop.png", + ), + ExampleSpec( + name="route_seg38_school20", + detector_class=2, + frame_path=".tmp/route_vision/seg38_frames/frame_041.jpg", + template_path=".tmp/route_vision/frame_041_sign_tight.jpg", + ), + ExampleSpec( + name="route_seg10_early_speed40", + detector_class=0, + frame_path=".tmp/route_12c_seg9_10/seg10_early/frame_005.png", + template_path=".tmp/route_12c_seg9_10/seg10_real40_crop.png", + ), + ExampleSpec( + name="route_seg10_early_speed30", + detector_class=0, + frame_path=".tmp/route_12c_seg9_10/seg10_early/frame_012.png", + template_path=".tmp/route_12c_seg9_10/seg10_real30_crop.png", + ), + ExampleSpec( + name="route_seg10_late_speed30", + detector_class=0, + frame_path=".tmp/vision_iter/seg10_5fps/frame_054.png", + template_path=".tmp/route_12c_seg9_10/seg10_real30_crop.png", + bbox_override=(885, 250, 941, 386), + ), + ExampleSpec( + name="route_seg10_late_speed30_clear", + detector_class=0, + frame_path=".tmp/vision_iter/seg10_5fps/frame_052.png", + template_path=".tmp/route_12c_seg9_10/seg10_real30_crop.png", + bbox_override=(829, 284, 870, 382), + ), + ExampleSpec( + name="live_capture_speed15", + detector_class=0, + frame_path=".tmp/live_c4_capture/stopped_sign_road.jpg", + template_path=".tmp/live_c4_capture/stopped_sign_crop_manual.png", + bbox_override=(724, 258, 763, 307), + ), +) + + +def detector_label_line(detector_class: int, x1: int, y1: int, x2: int, y2: int, image_shape: tuple[int, int, int]) -> str: + image_h, image_w = image_shape[:2] + x_center = ((x1 + x2) / 2) / image_w + y_center = ((y1 + y2) / 2) / image_h + width = (x2 - x1) / image_w + height = (y2 - y1) / image_h + return f"{detector_class} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}\n" + + +def match_template(frame: cv2.typing.MatLike, template: cv2.typing.MatLike): + result = cv2.matchTemplate(frame, template, cv2.TM_CCOEFF_NORMED) + _, max_value, _, max_location = cv2.minMaxLoc(result) + template_h, template_w = template.shape[:2] + x1, y1 = max_location + x2 = x1 + template_w + y2 = y1 + template_h + return float(max_value), (x1, y1, x2, y2) + + +def resolve_match(spec: ExampleSpec): + if spec.template_path is None: + raise FileNotFoundError(f"{spec.name}: missing template_path") + template = cv2.imread(spec.template_path) + if template is None: + raise FileNotFoundError(f"{spec.name}: failed to read template {spec.template_path}") + + if spec.frame_path: + frame = cv2.imread(spec.frame_path) + if frame is None: + raise FileNotFoundError(f"{spec.name}: failed to read frame {spec.frame_path}") + if spec.bbox_override is not None: + return Path(spec.frame_path), frame, 1.0, spec.bbox_override + confidence, bbox = match_template(frame, template) + return Path(spec.frame_path), frame, confidence, bbox + + if spec.frame_dir: + best = None + for frame_path in sorted(Path(spec.frame_dir).glob("*")): + frame = cv2.imread(str(frame_path)) + if frame is None: + continue + if frame.shape[0] < template.shape[0] or frame.shape[1] < template.shape[1]: + continue + confidence, bbox = match_template(frame, template) + if best is None or confidence > best[2]: + best = (frame_path, frame, confidence, bbox) + if best is None: + raise FileNotFoundError(f"{spec.name}: no readable frames found in {spec.frame_dir}") + return best + + raise ValueError(f"{spec.name}: one of frame_path or frame_dir must be provided") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import a few known real comma sign examples into the detector dataset.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--split", default="train", choices=("train", "val"), help="Detector dataset split to populate.") + parser.add_argument("--manifest", type=Path, help="Optional CSV manifest path. Defaults to /review/real_detector_examples.csv.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + split = args.split + image_dir = ensure_dir(workspace / "detector" / "images" / split) + label_dir = ensure_dir(workspace / "detector" / "labels" / split) + manifest_path = args.manifest.resolve() if args.manifest else (ensure_dir(workspace / "review") / "real_detector_examples.csv") + + records: list[dict[str, object]] = [] + for spec in DEFAULT_EXAMPLES: + frame_path, frame_bgr, confidence, (x1, y1, x2, y2) = resolve_match(spec) + stem = f"real_{spec.name}" + image_path = image_dir / f"{stem}.jpg" + label_path = label_dir / f"{stem}.txt" + + cv2.imwrite(str(image_path), frame_bgr, [cv2.IMWRITE_JPEG_QUALITY, 95]) + label_path.write_text(detector_label_line(spec.detector_class, x1, y1, x2, y2, frame_bgr.shape), encoding="utf-8") + + records.append({ + "name": spec.name, + "split": split, + "source_frame": str(frame_path), + "template_path": spec.template_path or "", + "template_match_confidence": round(confidence, 6), + "detector_class": spec.detector_class, + "bbox_x1": x1, + "bbox_y1": y1, + "bbox_x2": x2, + "bbox_y2": y2, + "dataset_image": str(image_path), + "dataset_label": str(label_path), + }) + + with manifest_path.open("w", encoding="utf-8", newline="") as manifest_file: + fieldnames = [ + "name", + "split", + "source_frame", + "template_path", + "template_match_confidence", + "detector_class", + "bbox_x1", + "bbox_y1", + "bbox_x2", + "bbox_y2", + "dataset_image", + "dataset_label", + ] + writer = csv.DictWriter(manifest_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(records) + + print(f"Imported {len(records)} real detector examples into {split} split") + print(f"Manifest: {manifest_path}") + for record in records: + print( + f"{record['name']}: conf={record['template_match_confidence']} " + f"bbox=({record['bbox_x1']},{record['bbox_y1']},{record['bbox_x2']},{record['bbox_y2']})" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/build_value_dataset.py b/scripts/speed_limit_vision/build_value_dataset.py new file mode 100644 index 00000000..db7b798d --- /dev/null +++ b/scripts/speed_limit_vision/build_value_dataset.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv + +from pathlib import Path + +import cv2 + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, VALUE_LABEL_FIELDS, ensure_dir, resolve_workspace # type: ignore +else: + from .common import DEFAULT_WORKSPACE, VALUE_LABEL_FIELDS, ensure_dir, resolve_workspace + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build a classifier crop dataset from detector labels and a value label manifest.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--labels-csv", type=Path, help="CSV manifest describing which labeled detector images map to which posted speed values.") + parser.add_argument("--default-padding", type=float, default=0.10, help="Default crop padding ratio when a row does not provide one.") + parser.add_argument("--overwrite", action="store_true", help="Overwrite existing classifier crops.") + return parser.parse_args() + + +def load_rows(csv_path: Path) -> list[dict[str, str]]: + with csv_path.open("r", encoding="utf-8", newline="") as csv_file: + reader = csv.DictReader(csv_file) + missing = [field for field in VALUE_LABEL_FIELDS if field not in (reader.fieldnames or [])] + if missing: + raise ValueError(f"Missing required CSV columns: {', '.join(missing)}") + return [row for row in reader if (row.get("image_path") or "").strip()] + + +def resolve_image_path(workspace: Path, image_path_text: str) -> Path: + image_path = Path(image_path_text).expanduser() + if image_path.is_file(): + return image_path.resolve() + + candidate = (workspace / image_path_text).resolve() + if candidate.is_file(): + return candidate + + basename = Path(image_path_text).name + for search_root in (workspace / "detector" / "images", workspace / "review" / "images"): + if not search_root.is_dir(): + continue + for found in search_root.rglob(basename): + if found.is_file(): + return found.resolve() + + raise FileNotFoundError(f"Image not found: {image_path_text}") + + +def resolve_label_path(workspace: Path, image_path: Path, label_path_text: str, split: str) -> Path: + if label_path_text: + label_path = Path(label_path_text).expanduser() + if label_path.is_file(): + return label_path.resolve() + candidate = (workspace / label_path_text).resolve() + if candidate.is_file(): + return candidate + raise FileNotFoundError(f"Label path not found: {label_path_text}") + + train_label = workspace / "detector" / "labels" / split / f"{image_path.stem}.txt" + if train_label.is_file(): + return train_label.resolve() + + for split_name in ("train", "val"): + candidate = workspace / "detector" / "labels" / split_name / f"{image_path.stem}.txt" + if candidate.is_file(): + return candidate.resolve() + + raise FileNotFoundError(f"Detector label not found for {image_path.name}") + + +def parse_yolo_labels(label_path: Path) -> list[tuple[int, float, float, float, float]]: + boxes = [] + with label_path.open("r", encoding="utf-8") as label_file: + for raw_line in label_file: + line = raw_line.strip() + if not line: + continue + class_id, x_center, y_center, width, height = line.split(maxsplit=4) + boxes.append((int(class_id), float(x_center), float(y_center), float(width), float(height))) + return boxes + + +def crop_box(image, yolo_box: tuple[int, float, float, float, float], padding: float): + _, x_center, y_center, width, height = yolo_box + image_height, image_width = image.shape[:2] + + box_width = width * image_width + box_height = height * image_height + pad_width = box_width * padding + pad_height = box_height * padding + + x1 = max(int(round((x_center * image_width) - box_width / 2 - pad_width)), 0) + y1 = max(int(round((y_center * image_height) - box_height / 2 - pad_height)), 0) + x2 = min(int(round((x_center * image_width) + box_width / 2 + pad_width)), image_width) + y2 = min(int(round((y_center * image_height) + box_height / 2 + pad_height)), image_height) + + if x2 <= x1 or y2 <= y1: + raise ValueError("Resolved crop has no area") + return image[y1:y2, x1:x2] + + +def remove_appledouble_files(root: Path) -> None: + for path in root.rglob("._*"): + if path.is_file() or path.is_symlink(): + path.unlink() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + labels_csv = args.labels_csv.resolve() if args.labels_csv else (workspace / "classifier" / "value_labels.csv") + rows = load_rows(labels_csv) + + built = 0 + for row in rows: + split = (row.get("split") or "train").strip().lower() + if split not in ("train", "val"): + raise ValueError(f"Unsupported split '{split}' in {labels_csv}") + + speed_limit = (row.get("speed_limit_mph") or "").strip() + if not speed_limit: + raise ValueError(f"Missing speed_limit_mph for image {row['image_path']}") + + bbox_index = int((row.get("bbox_index") or "0").strip()) + padding_text = (row.get("padding") or "").strip() + padding = float(padding_text) if padding_text else args.default_padding + + image_path = resolve_image_path(workspace, row["image_path"]) + label_path = resolve_label_path(workspace, image_path, (row.get("label_path") or "").strip(), split) + boxes = parse_yolo_labels(label_path) + if bbox_index >= len(boxes): + raise IndexError(f"bbox_index {bbox_index} out of range for {label_path}") + + image = cv2.imread(str(image_path)) + if image is None: + raise RuntimeError(f"Failed to read {image_path}") + + crop = crop_box(image, boxes[bbox_index], padding) + output_dir = ensure_dir(workspace / "classifier" / split / speed_limit) + output_path = output_dir / f"{image_path.stem}_bbox{bbox_index}.jpg" + if output_path.exists() and not args.overwrite: + continue + + cv2.imwrite(str(output_path), crop) + built += 1 + + remove_appledouble_files(workspace / "classifier") + print(f"Built {built} classifier crop(s) into {workspace / 'classifier'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/common.py b/scripts/speed_limit_vision/common.py new file mode 100644 index 00000000..5f4afb9d --- /dev/null +++ b/scripts/speed_limit_vision/common.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import csv +import json + +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_WORKSPACE = REPO_ROOT / ".tmp" / "speed_limit_training" +DEFAULT_DEBUG_BASE = Path("/data/media/0/vision_speed_limit_debug") +REPO_ASSET_DIR = REPO_ROOT / "starpilot" / "assets" / "vision_models" +DEFAULT_LOCAL_CLIP_ROOT = REPO_ROOT / ".tmp" / "live_route_clips" / "bookmark_windows" / "data" / "media" / "0" / "realdata" +DEFAULT_LOCAL_QLOG_MTIMES = REPO_ROOT / ".tmp" / "live_routes_meta" / "qlog_mtimes.txt" +DEFAULT_LOCAL_FILES_MANIFEST = REPO_ROOT / ".tmp" / "live_routes_meta" / "files.txt" +DEFAULT_LOCAL_SESSION_ROUTE_MAP = REPO_ROOT / ".tmp" / "live_routes_meta" / "session_route_map.json" +DEFAULT_EXTERNAL_ROOT = Path("/Volumes/T5/starpilot_speed_limit") + +DETECTOR_CLASS_NAMES = ( + "regulatory_speed_limit", + "advisory_speed_limit", + "school_zone_speed_limit", +) +DEFAULT_SPEED_VALUES = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75) + +DETECTOR_EXPORT_NAME = "speed_limit_us_detector.onnx" +CLASSIFIER_EXPORT_NAME = "speed_limit_us_value_classifier.onnx" + + +def resolve_workspace(path: str | Path | None) -> Path: + return Path(path).expanduser().resolve() if path else DEFAULT_WORKSPACE + + +def preferred_external_root() -> Path | None: + return DEFAULT_EXTERNAL_ROOT if DEFAULT_EXTERNAL_ROOT.is_dir() else None + + +def preferred_analysis_root() -> Path | None: + external_root = preferred_external_root() + if external_root is None: + return None + analysis_root = external_root / "analysis" + return analysis_root if analysis_root.is_dir() else external_root + + +def preferred_clip_root() -> Path: + external_root = preferred_analysis_root() + if external_root is not None: + return external_root / "live_route_clips" / "bookmark_windows" / "data" / "media" / "0" / "realdata" + return DEFAULT_LOCAL_CLIP_ROOT + + +def preferred_qlog_mtimes_path() -> Path: + external_root = preferred_analysis_root() + if external_root is not None: + return external_root / "live_routes_meta" / "qlog_mtimes.txt" + return DEFAULT_LOCAL_QLOG_MTIMES + + +def preferred_files_manifest_path() -> Path: + external_root = preferred_analysis_root() + if external_root is not None: + return external_root / "live_routes_meta" / "files.txt" + return DEFAULT_LOCAL_FILES_MANIFEST + + +def preferred_session_route_map_path() -> Path: + external_root = preferred_analysis_root() + if external_root is not None: + return external_root / "live_routes_meta" / "session_route_map.json" + return DEFAULT_LOCAL_SESSION_ROUTE_MAP + + +def load_session_route_map(path: str | Path | None = None) -> dict[str, str]: + route_map_path = Path(path).expanduser().resolve() if path else preferred_session_route_map_path() + if not route_map_path.is_file(): + return {} + data = json.loads(route_map_path.read_text(encoding="utf-8")) + return {str(key): str(value) for key, value in data.items() if key and value} + + +def default_raw_root(workspace: str | Path | None = None) -> Path: + resolved = resolve_workspace(workspace) + if resolved.parent.name == "workspace": + return resolved.parent.parent / "raw" + return resolved / "raw" + + +def ensure_dir(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + return path + + +def write_text(path: Path, text: str, force: bool = False) -> None: + if path.exists() and not force: + return + ensure_dir(path.parent) + path.write_text(text, encoding="utf-8") + + +def write_csv_header(path: Path, fieldnames: list[str], force: bool = False) -> None: + if path.exists() and not force: + return + ensure_dir(path.parent) + with path.open("w", encoding="utf-8", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + + +def read_jsonl(path: Path) -> list[dict]: + records: list[dict] = [] + with path.open("r", encoding="utf-8") as jsonl_file: + for line in jsonl_file: + line = line.strip() + if line: + records.append(json.loads(line)) + return records + + +def latest_debug_sessions(debug_base: Path, count: int = 1) -> list[Path]: + if not debug_base.is_dir(): + return [] + sessions = sorted((path for path in debug_base.iterdir() if path.is_dir()), reverse=True) + return sessions[:max(count, 0)] + + +def detector_dataset_yaml(workspace: Path) -> str: + detector_root = workspace / "detector" + lines = [ + f"path: {detector_root}", + "train: images/train", + "val: images/val", + "names:", + ] + for index, class_name in enumerate(DETECTOR_CLASS_NAMES): + lines.append(f" {index}: {class_name}") + return "\n".join(lines) + "\n" + + +def workspace_readme(speed_values: tuple[int, ...]) -> str: + supported_values = ", ".join(str(value) for value in speed_values) + return f"""# Speed Limit Vision Training Workspace + +This workspace is generated by `scripts/speed_limit_vision/init_workspace.py`. + +Directory layout: + +- `detector/images/train` and `detector/images/val`: detector training images +- `detector/labels/train` and `detector/labels/val`: YOLO detector labels +- `classifier/value_labels.csv`: value labels used to crop detector boxes into classifier folders +- `classifier/train` and `classifier/val`: classifier-ready crop folders +- `review/images`: imported snapshots from live debug sessions +- `review/bookmarks.csv`: bookmark/publish/candidate manifest built from debug sessions +- `review/leadins/frames` and `review/leadins/contact_sheets`: sampled frames from 5-second pre-bookmark route windows +- `review/bookmark_leadins.csv`: manifest describing sampled pre-bookmark review frames +- `exports`: exported ONNX models +- `runs`: training outputs + +Suggested detector classes: + +- `regulatory_speed_limit` +- `advisory_speed_limit` +- `school_zone_speed_limit` + +Suggested classifier values: + +- `{supported_values}` + +Generated manifests: + +- `manifests/raw_sources.csv`: raw public/comma source provenance +- `manifests/public_detector_samples.csv`: imported detector samples and sign metadata +- `manifests/public_classifier_samples.csv`: imported classifier-ready value samples +""" + + +BOOKMARK_MANIFEST_FIELDS = [ + "record_key", + "source_name", + "source_region", + "source_device", + "source_driver", + "session_id", + "event_index", + "event", + "session_seconds", + "wall_time", + "road_name", + "stream", + "status", + "candidate_speed_limit_mph", + "candidate_confidence", + "speed_limit_mph", + "confidence", + "published_speed_limit_mph", + "published_confidence", + "bookmark_count", + "snapshot_path", + "source_session_path", +] + + +BOOKMARK_LEADIN_MANIFEST_FIELDS = [ + "record_key", + "source_name", + "source_region", + "source_device", + "source_driver", + "session_id", + "bookmark_number", + "route", + "segment", + "segment_offset_s", + "leadin_start_s", + "sample_offset_s", + "window_result", + "published_values", + "candidate_values", + "frame_path", + "contact_sheet_path", + "source_video_path", +] + + +VALUE_LABEL_FIELDS = [ + "image_path", + "split", + "speed_limit_mph", + "bbox_index", + "padding", + "label_path", +] + + +RAW_SOURCE_FIELDS = [ + "source_name", + "source_version", + "source_license", + "source_type", + "raw_path", + "notes", +] + + +PUBLIC_DETECTOR_SAMPLE_FIELDS = [ + "record_key", + "source_name", + "split", + "image_path", + "label_path", + "annotation_path", + "source_image_id", + "class_name", + "speed_limit_mph", + "sign_code", + "bbox_left", + "bbox_top", + "bbox_right", + "bbox_bottom", +] + + +PUBLIC_CLASSIFIER_SAMPLE_FIELDS = [ + "record_key", + "source_name", + "split", + "image_path", + "speed_limit_mph", + "bbox_index", + "label_path", + "source_image_id", + "sign_code", +] diff --git a/scripts/speed_limit_vision/download_connect_routes.py b/scripts/speed_limit_vision/download_connect_routes.py new file mode 100644 index 00000000..6e0d3099 --- /dev/null +++ b/scripts/speed_limit_vision/download_connect_routes.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import re + +from dataclasses import dataclass +from email.utils import parsedate_to_datetime +from pathlib import Path +from urllib.parse import urlparse + +import requests + +from scripts.speed_limit_vision import common + + +API_HOST = os.getenv("COMMA_API_HOST", "https://api.commadotai.com") +DEFAULT_FILES_MANIFEST = common.preferred_files_manifest_path() +STREAM_FILE_NAMES = { + "fcamera": {"fcamera.hevc"}, + "qlog": {"qlog.zst", "qlog.bz2", "qlog"}, + "rlog": {"rlog.zst", "rlog.bz2", "rlog"}, + "qcamera": {"qcamera.ts"}, + "dcamera": {"dcamera.hevc"}, + "ecamera": {"ecamera.hevc"}, +} + + +@dataclass(frozen=True) +class RouteRequest: + dongle_id: str + log_id: str + segment_filter: set[int] | None + + @property + def canonical_name(self) -> str: + return f"{self.dongle_id}|{self.log_id}" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Download route files directly from comma connect into the speed-limit review layout.") + parser.add_argument("routes", nargs="*", help="Route ids like 'dongle/logid' or segment ids like 'dongle/logid--9'.") + parser.add_argument("--routes-file", type=Path, help="Optional file containing one route id per line.") + parser.add_argument("--clip-root", type=Path, default=common.preferred_clip_root(), help="Destination root for downloaded segment directories.") + parser.add_argument("--qlog-mtimes", type=Path, default=common.preferred_qlog_mtimes_path(), help="Path to qlog mtime manifest used by bookmark replay.") + parser.add_argument("--files-manifest", type=Path, default=DEFAULT_FILES_MANIFEST, help="Path to downloaded-files manifest.") + parser.add_argument("--streams", default="fcamera,qlog", help="Comma-separated stream set: fcamera,qlog,rlog,qcamera,dcamera,ecamera.") + parser.add_argument("--segments", help="Optional segment filter, e.g. '0,2,5-8'.") + parser.add_argument("--overwrite", action="store_true", help="Redownload files even if they already exist locally.") + parser.add_argument("--timeout", type=float, default=60.0, help="HTTP timeout in seconds.") + return parser.parse_args() + + +def read_token() -> str: + token = os.getenv("COMMA_JWT", "").strip() + if token: + return token + + auth_path = Path.home() / ".comma" / "auth.json" + if not auth_path.is_file(): + raise FileNotFoundError(f"Missing auth token at {auth_path}. Run python3 tools/lib/auth.py first or set COMMA_JWT.") + auth = json.loads(auth_path.read_text(encoding="utf-8")) + token = auth.get("access_token", "").strip() + if not token: + raise ValueError(f"No access_token in {auth_path}") + return token + + +def parse_segment_spec(spec: str | None) -> set[int] | None: + if not spec: + return None + + selected: set[int] = set() + for part in spec.split(","): + part = part.strip() + if not part: + continue + if "-" in part: + start_text, end_text = part.split("-", 1) + start = int(start_text) + end = int(end_text) + selected.update(range(min(start, end), max(start, end) + 1)) + else: + selected.add(int(part)) + return selected + + +def load_route_inputs(args: argparse.Namespace) -> list[str]: + raw_routes = list(args.routes) + if args.routes_file: + raw_routes.extend(line.strip() for line in args.routes_file.expanduser().resolve().read_text(encoding="utf-8").splitlines()) + if not raw_routes: + raise ValueError("No routes provided.") + return [route for route in raw_routes if route and not route.lstrip().startswith("#")] + + +def parse_route_request(raw: str, default_segments: set[int] | None) -> RouteRequest: + text = raw.strip().strip("'\"") + text = text.replace("|", "/") + match = re.fullmatch(r"([0-9a-f]{16})/([^/]+)", text) + if not match: + raise ValueError(f"Unrecognized route id: {raw}") + + dongle_id = match.group(1) + tail = match.group(2) + segment_filter = set(default_segments) if default_segments else None + + parts = tail.split("--") + if len(parts) == 3 and parts[-1].isdigit(): + log_id = "--".join(parts[:2]) + segment = int(parts[-1]) + if segment_filter is None: + segment_filter = {segment} + else: + segment_filter.add(segment) + else: + log_id = tail + + if len(log_id) != 20: + raise ValueError(f"Invalid log id in route: {raw}") + return RouteRequest(dongle_id=dongle_id, log_id=log_id, segment_filter=segment_filter) + + +def merge_route_requests(raw_routes: list[str], default_segments: set[int] | None) -> list[RouteRequest]: + merged: dict[tuple[str, str], set[int] | None] = {} + for raw in raw_routes: + request = parse_route_request(raw, default_segments) + key = (request.dongle_id, request.log_id) + if key not in merged: + merged[key] = None if request.segment_filter is None else set(request.segment_filter) + continue + if merged[key] is None or request.segment_filter is None: + merged[key] = None + else: + merged[key].update(request.segment_filter) + return [RouteRequest(dongle_id=dongle_id, log_id=log_id, segment_filter=segments) for (dongle_id, log_id), segments in sorted(merged.items())] + + +def selected_file_names(streams_csv: str) -> set[str]: + selected: set[str] = set() + for stream in streams_csv.split(","): + stream = stream.strip() + if not stream: + continue + if stream not in STREAM_FILE_NAMES: + raise ValueError(f"Unknown stream '{stream}'. Valid streams: {', '.join(sorted(STREAM_FILE_NAMES))}") + selected.update(STREAM_FILE_NAMES[stream]) + return selected + + +def api_get_json(session: requests.Session, endpoint: str, timeout: float): + response = session.get(f"{API_HOST}/{endpoint.lstrip('/')}", timeout=timeout) + response.raise_for_status() + return response.json() + + +def iter_route_file_urls(files_payload: dict, file_names: set[str]): + for value in files_payload.values(): + if not isinstance(value, list): + continue + for url in value: + parsed = urlparse(url) + parts = parsed.path.strip("/").split("/") + if len(parts) < 4: + continue + segment_text = parts[-2] + file_name = parts[-1] + if not segment_text.isdigit() or file_name not in file_names: + continue + yield int(segment_text), file_name, url + + +def download_to_path(session: requests.Session, url: str, dest_path: Path, overwrite: bool, timeout: float) -> int | None: + dest_path.parent.mkdir(parents=True, exist_ok=True) + if dest_path.exists() and not overwrite: + return int(dest_path.stat().st_mtime) + + response = session.get(url, stream=True, timeout=timeout) + response.raise_for_status() + temp_path = dest_path.with_suffix(dest_path.suffix + ".part") + with temp_path.open("wb") as handle: + for chunk in response.iter_content(chunk_size=1 << 20): + if chunk: + handle.write(chunk) + temp_path.replace(dest_path) + + last_modified = response.headers.get("Last-Modified") + if last_modified: + epoch = int(parsedate_to_datetime(last_modified).timestamp()) + os.utime(dest_path, (epoch, epoch)) + return epoch + return int(dest_path.stat().st_mtime) + + +def load_manifest_lines(path: Path) -> dict[str, str]: + if not path.is_file(): + return {} + lines: dict[str, str] = {} + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line: + continue + key = line.split(" ", 1)[0] + lines[key] = line + return lines + + +def write_manifest(path: Path, lines: dict[str, str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + for key in sorted(lines): + handle.write(lines[key] + "\n") + + +def main() -> int: + args = parse_args() + token = read_token() + route_requests = merge_route_requests(load_route_inputs(args), parse_segment_spec(args.segments)) + file_names = selected_file_names(args.streams) + clip_root = args.clip_root.expanduser().resolve() + qlog_mtimes_path = args.qlog_mtimes.expanduser().resolve() + files_manifest_path = args.files_manifest.expanduser().resolve() + + api_session = requests.Session() + api_session.headers.update({ + "Authorization": f"JWT {token}", + "User-Agent": "OpenpilotTools", + }) + download_session = requests.Session() + download_session.headers.update({ + "User-Agent": "OpenpilotTools", + }) + + qlog_lines = load_manifest_lines(qlog_mtimes_path) + file_lines = load_manifest_lines(files_manifest_path) + + for request in route_requests: + route_meta = api_get_json(api_session, f"v1/route/{request.canonical_name}", timeout=args.timeout) + files_payload = api_get_json(api_session, f"v1/route/{request.canonical_name}/files", timeout=args.timeout) + log_id = request.log_id + start_time = route_meta.get("start_time", "") + print(f"{request.canonical_name}: downloading streams={sorted(file_names)} segments={sorted(request.segment_filter) if request.segment_filter else 'all'}") + + for segment, file_name, url in iter_route_file_urls(files_payload, file_names): + if request.segment_filter is not None and segment not in request.segment_filter: + continue + + segment_name = f"{log_id}--{segment}" + segment_dir = clip_root / segment_name + dest_path = segment_dir / file_name + epoch = download_to_path(download_session, url, dest_path, args.overwrite, args.timeout) + print(f" {segment_name}/{file_name} <- {urlparse(url).netloc}") + file_lines[f"{segment_name} {dest_path}"] = f"{segment_name} {dest_path}" + if file_name.startswith("qlog") and epoch is not None: + qlog_lines[str(dest_path)] = f"{dest_path} {epoch}" + + if start_time: + print(f" route start_time={start_time}") + + write_manifest(qlog_mtimes_path, qlog_lines) + write_manifest(files_manifest_path, file_lines) + print(f"Updated {qlog_mtimes_path}") + print(f"Updated {files_manifest_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/download_glare_raw.py b/scripts/speed_limit_vision/download_glare_raw.py new file mode 100644 index 00000000..be263d79 --- /dev/null +++ b/scripts/speed_limit_vision/download_glare_raw.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import importlib + +from pathlib import Path + +import gdown + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, default_raw_root, ensure_dir, resolve_workspace # type: ignore +else: + from .common import DEFAULT_WORKSPACE, default_raw_root, ensure_dir, resolve_workspace + + +GLARE_ROOT_URL = "https://drive.google.com/drive/folders/1gmoOSgvjR4DP7jGfGS_xAmxcMShyeThx?usp=sharing" +DEFAULT_PREFIXES = ("Images/", "Tracks/") +MANIFEST_FIELDS = ["file_id", "relative_path", "local_path"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Download the raw GLARE image/track files without checkpoint artifacts.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--output-root", type=Path, help="Download destination. Defaults to /glare_raw.") + parser.add_argument("--prefix", dest="prefixes", nargs="+", default=list(DEFAULT_PREFIXES), help="Path prefixes to keep from the GLARE Drive tree.") + parser.add_argument("--manifest", type=Path, help="Optional CSV path for the filtered file manifest.") + parser.add_argument("--list-only", action="store_true", help="List matching files without downloading them.") + parser.add_argument("--resume", action="store_true", help="Resume partial downloads and skip completed files.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + output_root = args.output_root.resolve() if args.output_root else (default_raw_root(workspace) / "glare_raw") + manifest_path = args.manifest.resolve() if args.manifest else (output_root / "manifest.csv") + ensure_dir(output_root) + + download_folder_mod = importlib.import_module("gdown.download_folder") + entries = download_folder_mod.download_folder( + url=GLARE_ROOT_URL, + skip_download=True, + quiet=True, + remaining_ok=True, + ) + if entries is None: + raise RuntimeError("Failed to enumerate the GLARE Drive tree") + + selected = [entry for entry in entries if any(entry.path.startswith(prefix) for prefix in args.prefixes)] + print(f"Matched {len(selected)} GLARE file(s) under prefixes: {', '.join(args.prefixes)}") + + with manifest_path.open("w", encoding="utf-8", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=MANIFEST_FIELDS) + writer.writeheader() + for entry in selected: + local_path = output_root / entry.path + writer.writerow({ + "file_id": entry.id, + "relative_path": entry.path, + "local_path": str(local_path), + }) + if args.list_only: + continue + ensure_dir(local_path.parent) + gdown.download( + id=entry.id, + output=str(local_path), + quiet=False, + resume=args.resume, + ) + + print(f"GLARE manifest: {manifest_path}") + print(f"GLARE output: {output_root}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/evaluate_bookmark_leadins.py b/scripts/speed_limit_vision/evaluate_bookmark_leadins.py new file mode 100644 index 00000000..a5c9112b --- /dev/null +++ b/scripts/speed_limit_vision/evaluate_bookmark_leadins.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re + +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +import cv2 + +import starpilot.system.speed_limit_vision as slv + +from scripts.speed_limit_vision import common + + +DEFAULT_SESSION_ROOT = Path(".tmp/live_drive_debug") +DEFAULT_CLIP_ROOT = common.preferred_clip_root() +DEFAULT_QLOG_MTIMES = common.preferred_qlog_mtimes_path() +EVENT_TYPES = ("candidate", "publish", "stale_clear") + + +@dataclass(frozen=True) +class BookmarkWindow: + bookmark_number: int + route: str + segment: int + segment_offset_s: float + leadin_start_s: float + spans_previous_segment: bool + + +class LiveReplayDaemon(slv.SpeedLimitVisionDaemon): + def __init__(self): + super().__init__(use_runtime=False) + self.now = 0.0 + self.captured_events: list[dict] = [] + + def _write_debug_event(self, event_type, frame_bgr=None, snapshot_prefix=None, **fields): + if event_type not in EVENT_TYPES: + return + record = {"event": event_type, "t": round(self.now, 3)} + record.update(fields) + self.captured_events.append(record) + + def _publish_status(self, status, clear_speed=False): + if clear_speed: + self._clear_detection() + + def process_frame(self, now, frame_bgr): + self.now = now + slv.time.monotonic = lambda now=now: now + self.current_frame_bgr = frame_bgr + + inference_interval = slv.FOLLOWUP_INFERENCE_INTERVAL if now < self.followup_until else slv.INFERENCE_INTERVAL + if now - self.last_inference_at < inference_interval: + if self.published_speed_limit_mph > 0 and self._published_detection_stale(now): + self._write_debug_event("stale_clear", reason="hold_timeout") + self._publish_status("Scanning replay", clear_speed=True) + return + + self.last_inference_at = now + detection = self._detect_sign(frame_bgr) + if detection is not None: + self._update_detection(detection) + elif self.published_speed_limit_mph > 0 and self._published_detection_stale(now): + self._write_debug_event("stale_clear", reason="no_detection") + self._publish_status("Scanning replay", clear_speed=True) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Replay 5-second pre-bookmark sign windows through the live speed-limit vision path.") + parser.add_argument("--session-root", type=Path, default=DEFAULT_SESSION_ROOT, help="Directory containing debug session folders.") + parser.add_argument("--clip-root", type=Path, default=DEFAULT_CLIP_ROOT, help="Directory containing copied route clips under --/fcamera.hevc.") + parser.add_argument("--qlog-mtimes", type=Path, default=DEFAULT_QLOG_MTIMES, help="Text file with ' ' lines.") + parser.add_argument("--session-route-map", type=Path, default=common.preferred_session_route_map_path(), help="JSON file mapping debug session ids to route log ids.") + parser.add_argument("--models-dir", type=Path, help="Directory containing speed_limit_us_detector.onnx and speed_limit_us_value_classifier.onnx.") + parser.add_argument("--lead-in", type=float, default=5.0, help="Seconds before each bookmark to replay.") + parser.add_argument("--sample-fps", type=float, help="Optional decode sample rate. Use 5 for faster bookmark sweeps that still match the live inference cadence.") + parser.add_argument("--session", action="append", help="Optional session id filter. Repeat to run more than one.") + parser.add_argument("--bookmark", action="append", type=int, help="Optional bookmark number filter within the selected sessions.") + parser.add_argument("--json-out", type=Path, help="Optional path to write the summary JSON.") + return parser.parse_args() + + +def load_qlog_mtimes(path: Path): + route_mtimes: dict[str, dict[int, int]] = {} + pattern = re.compile(r"/([^/]+)--(\d+)/(qlog(?:\.(?:zst|bz2))?)$") + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + qlog_path, timestamp = line.rsplit(" ", 1) + match = pattern.search(qlog_path) + if match is None: + continue + route, segment = match.group(1), int(match.group(2)) + route_mtimes.setdefault(route, {})[segment] = int(timestamp) + return route_mtimes + + +def load_bookmarks(session_path: Path): + events = [] + with (session_path / "events.jsonl").open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + event = json.loads(line) + if event.get("event") in ("bookmark", "auto_bookmark"): + events.append(event) + return events + + +def locate_window(route: str, event: dict, route_mtimes: dict[str, dict[int, int]], lead_in: float): + event_wall_time = datetime.fromisoformat(event["wallTime"]).timestamp() + segment_mtimes = route_mtimes.get(route, {}) + for segment, start_epoch in sorted(segment_mtimes.items()): + if start_epoch <= event_wall_time < start_epoch + 60: + offset_s = event_wall_time - start_epoch + return BookmarkWindow( + bookmark_number=0, + route=route, + segment=segment, + segment_offset_s=offset_s, + leadin_start_s=offset_s - lead_in, + spans_previous_segment=offset_s - lead_in < 0.0, + ) + return None + + +def iter_video_window(path: Path, start_s: float, end_s: float, sample_fps: float | None = None): + capture = cv2.VideoCapture(str(path)) + fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + start_frame = max(int(start_s * fps), 0) + end_frame = max(int(end_s * fps), start_frame) + frame_step = 1 + if sample_fps is not None and sample_fps > 0.0 and sample_fps < fps: + frame_step = max(int(round(fps / sample_fps)), 1) + capture.set(cv2.CAP_PROP_POS_FRAMES, start_frame) + frame_index = start_frame + + while frame_index <= end_frame: + ok, frame_bgr = capture.read() + if not ok: + break + frame_time_s = frame_index / fps + if start_s <= frame_time_s <= end_s: + yield frame_time_s, frame_bgr + frame_index += 1 + + skipped = 1 + while skipped < frame_step and frame_index <= end_frame: + ok = capture.grab() + if not ok: + capture.release() + return + frame_index += 1 + skipped += 1 + + capture.release() + + +def replay_window(window: BookmarkWindow, clip_root: Path, sample_fps: float | None = None): + daemon = LiveReplayDaemon() + elapsed_base_s = 0.0 + replayed_frames = 0 + segments: list[tuple[Path, float, float]] = [] + + if window.spans_previous_segment: + previous_segment = window.segment - 1 + if previous_segment >= 0: + previous_clip = clip_root / f"{window.route}--{previous_segment}" / "fcamera.hevc" + previous_start_s = 60.0 + window.leadin_start_s + segments.append((previous_clip, max(previous_start_s, 0.0), 60.0)) + elapsed_base_s += max(60.0 - max(previous_start_s, 0.0), 0.0) + current_start_s = 0.0 + else: + current_start_s = window.leadin_start_s + + current_clip = clip_root / f"{window.route}--{window.segment}" / "fcamera.hevc" + segments.append((current_clip, max(current_start_s, 0.0), window.segment_offset_s)) + + cumulative_offset_s = 0.0 + for clip_path, start_s, end_s in segments: + if not clip_path.is_file(): + return { + "missingClip": str(clip_path), + "events": [], + "replayedFrames": replayed_frames, + } + + for frame_time_s, frame_bgr in iter_video_window(clip_path, start_s, end_s, sample_fps=sample_fps): + replay_time_s = cumulative_offset_s + (frame_time_s - start_s) + daemon.process_frame(replay_time_s, frame_bgr) + replayed_frames += 1 + + cumulative_offset_s += max(end_s - start_s, 0.0) + + candidate_values = [event["candidateSpeedLimitMph"] for event in daemon.captured_events if event["event"] == "candidate"] + published_values = [event["speedLimitMph"] for event in daemon.captured_events if event["event"] == "publish"] + return { + "events": daemon.captured_events, + "candidateValues": candidate_values, + "publishedValues": published_values, + "replayedFrames": replayed_frames, + "hit": bool(candidate_values or published_values), + } + + +def main(): + args = parse_args() + selected_sessions = set(args.session or []) + selected_bookmarks = set(args.bookmark or []) + route_mtimes = load_qlog_mtimes(args.qlog_mtimes.expanduser().resolve()) + session_route_map = common.load_session_route_map(args.session_route_map) + if not session_route_map: + raise FileNotFoundError(f"No session route map found at {args.session_route_map}") + + if args.models_dir: + models_dir = args.models_dir.expanduser().resolve() + detector_path = models_dir / "speed_limit_us_detector.onnx" + classifier_path = models_dir / "speed_limit_us_value_classifier.onnx" + if not detector_path.is_file(): + raise FileNotFoundError(detector_path) + if not classifier_path.is_file(): + raise FileNotFoundError(classifier_path) + slv.US_DETECTOR_MODEL_PATH = detector_path + slv.US_CLASSIFIER_MODEL_PATH = classifier_path + + summaries = [] + for session_id, route in session_route_map.items(): + if selected_sessions and session_id not in selected_sessions: + continue + + session_path = args.session_root.expanduser().resolve() / session_id + if not session_path.is_dir(): + continue + + bookmarks = load_bookmarks(session_path) + for bookmark_number, event in enumerate(bookmarks, start=1): + if selected_bookmarks and bookmark_number not in selected_bookmarks: + continue + + window = locate_window(route, event, route_mtimes, args.lead_in) + if window is None: + summary = { + "sessionId": session_id, + "bookmarkNumber": bookmark_number, + "route": route, + "status": "unmapped", + } + else: + window = BookmarkWindow( + bookmark_number=bookmark_number, + route=window.route, + segment=window.segment, + segment_offset_s=window.segment_offset_s, + leadin_start_s=window.leadin_start_s, + spans_previous_segment=window.spans_previous_segment, + ) + replay_summary = replay_window(window, args.clip_root.expanduser().resolve(), sample_fps=args.sample_fps) + summary = { + "sessionId": session_id, + "bookmarkNumber": bookmark_number, + "route": route, + "segment": window.segment, + "segmentOffsetS": round(window.segment_offset_s, 3), + "leadinStartS": round(window.leadin_start_s, 3), + "spansPreviousSegment": window.spans_previous_segment, + } + summary.update(replay_summary) + + summaries.append(summary) + + hit_count = sum(1 for summary in summaries if summary.get("hit")) + print(f"Bookmarks with detections in lead-in: {hit_count}/{len(summaries)}") + for summary in summaries: + if summary.get("status") == "unmapped": + print(f"{summary['sessionId']} bookmark {summary['bookmarkNumber']:02d}: unmapped") + continue + + result = "hit" if summary.get("hit") else "miss" + publish_values = ",".join(str(value) for value in summary.get("publishedValues", [])) or "-" + candidate_values = ",".join(str(value) for value in summary.get("candidateValues", [])) or "-" + note = "" + if summary.get("missingClip"): + note = f" missing={summary['missingClip']}" + print( + f"{summary['sessionId']} bookmark {summary['bookmarkNumber']:02d}: " + f"seg {summary['segment']} @ {summary['segmentOffsetS']:.2f}s " + f"lead-in [{summary['leadinStartS']:.2f}s, {summary['segmentOffsetS']:.2f}s] " + f"{result} publish={publish_values} candidate={candidate_values}{note}" + ) + + if args.json_out: + args.json_out.expanduser().resolve().write_text(json.dumps(summaries, indent=2), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/scripts/speed_limit_vision/evaluate_detector_classifier.py b/scripts/speed_limit_vision/evaluate_detector_classifier.py new file mode 100644 index 00000000..ccf02837 --- /dev/null +++ b/scripts/speed_limit_vision/evaluate_detector_classifier.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse + +from pathlib import Path + +import cv2 +import numpy as np + +from ultralytics import YOLO + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_SPEED_VALUES # type: ignore + from generate_value_roi_classifier_dataset import extract_value_mask # type: ignore +else: + from .common import DEFAULT_SPEED_VALUES + from .generate_value_roi_classifier_dataset import extract_value_mask + + +DETECTOR_SIGN_CLASSES = { + "regulatory_speed_limit", + "school_zone_speed_limit", + "speedLimit15", + "speedLimit20", + "speedLimit25", + "speedLimit30", + "speedLimit35", + "speedLimit40", + "speedLimit45", + "speedLimit50", + "speedLimit55", + "speedLimit60", + "speedLimit65", + "speedLimit70", + "speedLimit75", + "schoolSpeedLimit25", + "speedLimit55Ahead", +} + + +def iter_frames(path: Path): + if path.is_dir(): + for frame_path in sorted(path.glob("frame_*")): + frame = cv2.imread(str(frame_path)) + if frame is not None: + yield frame_path.name, frame + return + + cap = cv2.VideoCapture(str(path)) + fps = cap.get(cv2.CAP_PROP_FPS) or 20.0 + frame_index = 0 + while True: + ok, frame = cap.read() + if not ok: + break + yield f"t={frame_index / fps:06.2f}s", frame + frame_index += 1 + cap.release() + + +def crop_with_margin(frame: np.ndarray, xyxy: np.ndarray, margin_ratio: float = 0.16): + frame_h, frame_w = frame.shape[:2] + x1, y1, x2, y2 = xyxy.astype(int) + box_w = x2 - x1 + box_h = y2 - y1 + margin_x = int(box_w * margin_ratio) + margin_y = int(box_h * margin_ratio) + left = max(x1 - margin_x, 0) + top = max(y1 - margin_y, 0) + right = min(x2 + margin_x, frame_w) + bottom = min(y2 + margin_y, frame_h) + return frame[top:bottom, left:right] + + +def main(): + parser = argparse.ArgumentParser(description="Evaluate a detector + value classifier pair on saved frames or route video.") + parser.add_argument("path", help="Frame directory or video path.") + parser.add_argument("--detector", required=True, help="Ultralytics detector checkpoint (.pt).") + parser.add_argument("--classifier", required=True, help="Ultralytics classifier checkpoint (.pt).") + parser.add_argument("--conf", type=float, default=0.10, help="Detector confidence threshold.") + parser.add_argument("--imgsz", type=int, default=960, help="Detector image size.") + parser.add_argument("--device", default="mps", help="Inference device, such as mps or cpu.") + parser.add_argument("--max-frames", type=int, default=0, help="Optional frame cap.") + args = parser.parse_args() + + detector = YOLO(args.detector) + classifier = YOLO(args.classifier) + path = Path(args.path).expanduser().resolve() + speed_values = tuple(DEFAULT_SPEED_VALUES) + + for frame_index, (label, frame_bgr) in enumerate(iter_frames(path), start=1): + if args.max_frames > 0 and frame_index > args.max_frames: + break + + detector_result = detector.predict(source=frame_bgr, conf=args.conf, imgsz=args.imgsz, device=args.device, verbose=False)[0] + if detector_result.boxes is None or len(detector_result.boxes) == 0: + continue + + printed = False + for box, cls, conf in zip(detector_result.boxes.xyxy.cpu().numpy(), detector_result.boxes.cls.cpu().numpy(), detector_result.boxes.conf.cpu().numpy()): + class_name = detector.names[int(cls)] + if class_name not in DETECTOR_SIGN_CLASSES: + continue + + crop = crop_with_margin(frame_bgr, box) + if crop.size == 0: + continue + + mask = extract_value_mask(crop) + if mask is None: + continue + classifier_input = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) + classifier_result = classifier.predict(source=classifier_input, imgsz=128, device=args.device, verbose=False)[0] + probabilities = classifier_result.probs + if probabilities is None: + continue + + top_index = int(probabilities.top1) + if top_index >= len(speed_values): + continue + predicted_value = speed_values[top_index] + predicted_confidence = float(probabilities.top1conf) + + if not printed: + print(label) + printed = True + print( + f" detector={class_name} det_conf={float(conf):.3f} " + f"classifier={predicted_value} cls_conf={predicted_confidence:.3f} " + f"box={[round(float(v), 1) for v in box.tolist()]}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/speed_limit_vision/evaluate_runtime_cases.py b/scripts/speed_limit_vision/evaluate_runtime_cases.py new file mode 100644 index 00000000..a82d6cc5 --- /dev/null +++ b/scripts/speed_limit_vision/evaluate_runtime_cases.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse + +from dataclasses import dataclass +from pathlib import Path + +import cv2 + +import starpilot.system.speed_limit_vision as slv + + +@dataclass(frozen=True) +class RuntimeCase: + name: str + frame_path: str + expected_speed_limit_mph: int + + +DEFAULT_CASES = ( + RuntimeCase("live15", ".tmp/live_c4_capture/stopped_sign_road.jpg", 15), + RuntimeCase("school20", ".tmp/route_vision/seg38_frames/frame_041.jpg", 20), + RuntimeCase("highway40", ".tmp/speed_route_frames_seg2_10_20/t12.png", 40), + RuntimeCase("town40", ".tmp/route_12c_seg9_10/seg10_early/frame_005.png", 40), + RuntimeCase("town30", ".tmp/route_12c_seg9_10/seg10_early/frame_012.png", 30), + RuntimeCase("town30_late", ".tmp/vision_iter/seg10_5fps/frame_054.png", 30), + RuntimeCase("town30_late_earlier", ".tmp/vision_iter/seg10_5fps/frame_052.png", 30), +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate the StarPilot speed-limit runtime path on known saved-frame cases.") + parser.add_argument( + "--models-dir", + type=Path, + default=Path("starpilot/assets/vision_models"), + help="Directory containing speed_limit_us_detector.onnx and speed_limit_us_value_classifier.onnx.", + ) + parser.add_argument( + "--case", + action="append", + dest="selected_cases", + help="Optional case name filter. Repeat to run more than one case.", + ) + parser.add_argument("--strict", action="store_true", help="Exit non-zero if any evaluated case misses the expected value.") + return parser.parse_args() + + +def resolve_cases(selected_cases: list[str] | None): + if not selected_cases: + return DEFAULT_CASES + + selected = set(selected_cases) + cases = tuple(case for case in DEFAULT_CASES if case.name in selected) + missing = sorted(selected - {case.name for case in cases}) + if missing: + raise ValueError(f"Unknown case names: {', '.join(missing)}") + return cases + + +def main() -> int: + args = parse_args() + models_dir = args.models_dir.expanduser().resolve() + detector_path = models_dir / "speed_limit_us_detector.onnx" + classifier_path = models_dir / "speed_limit_us_value_classifier.onnx" + if not detector_path.is_file(): + raise FileNotFoundError(detector_path) + if not classifier_path.is_file(): + raise FileNotFoundError(classifier_path) + + slv.US_DETECTOR_MODEL_PATH = detector_path + slv.US_CLASSIFIER_MODEL_PATH = classifier_path + daemon = slv.SpeedLimitVisionDaemon(use_runtime=False) + + failures = 0 + for case in resolve_cases(args.selected_cases): + image_path = Path(case.frame_path).expanduser().resolve() + frame_bgr = cv2.imread(str(image_path)) + if frame_bgr is None: + raise FileNotFoundError(image_path) + + detection = daemon._detect_sign(frame_bgr) + predicted_speed = detection.speed_limit_mph if detection is not None else None + confidence = round(detection.confidence, 4) if detection is not None else None + passed = predicted_speed == case.expected_speed_limit_mph + if not passed: + failures += 1 + + print( + f"{case.name}: expected={case.expected_speed_limit_mph} " + f"predicted={predicted_speed} confidence={confidence} " + f"{'PASS' if passed else 'FAIL'}" + ) + + return 1 if args.strict and failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/export_models.py b/scripts/speed_limit_vision/export_models.py new file mode 100644 index 00000000..7c8972c3 --- /dev/null +++ b/scripts/speed_limit_vision/export_models.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import shutil + +from pathlib import Path + +import cv2 +import numpy as np + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import ( # type: ignore + CLASSIFIER_EXPORT_NAME, + DEFAULT_WORKSPACE, + DETECTOR_EXPORT_NAME, + REPO_ASSET_DIR, + ensure_dir, + resolve_workspace, + ) +else: + from .common import ( + CLASSIFIER_EXPORT_NAME, + DEFAULT_WORKSPACE, + DETECTOR_EXPORT_NAME, + REPO_ASSET_DIR, + ensure_dir, + resolve_workspace, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Export trained speed-limit detector/classifier checkpoints to ONNX.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--detector-weights", type=Path, help="Path to the trained detector .pt weights.") + parser.add_argument("--classifier-weights", type=Path, help="Path to the trained classifier .pt weights.") + parser.add_argument("--output-dir", type=Path, help="Where exported ONNX models should be written. Defaults to /exports.") + parser.add_argument("--detector-imgsz", type=int, default=640, help="Detector export image size.") + parser.add_argument("--classifier-imgsz", type=int, default=128, help="Classifier export image size.") + parser.add_argument("--opset", type=int, default=12, help="ONNX opset.") + parser.add_argument("--install-repo-assets", action="store_true", help="Copy exported ONNX files into starpilot/assets/vision_models.") + parser.add_argument("--skip-verify", action="store_true", help="Skip the OpenCV DNN load/forward smoke test after export.") + return parser.parse_args() + + +def export_yolo(weights_path: Path, output_path: Path, imgsz: int, opset: int, nms: bool) -> None: + from ultralytics import YOLO + + model = YOLO(str(weights_path)) + exported_path = Path(model.export(format="onnx", imgsz=imgsz, opset=opset, simplify=False, dynamic=False, nms=nms)) + ensure_dir(output_path.parent) + shutil.copy2(exported_path, output_path) + + +def verify_onnx_with_opencv(model_path: Path, input_size: int) -> None: + net = cv2.dnn.readNetFromONNX(str(model_path)) + blob = np.zeros((1, 3, input_size, input_size), dtype=np.float32) + net.setInput(blob) + _ = net.forward() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + output_dir = args.output_dir.resolve() if args.output_dir else (workspace / "exports") + ensure_dir(output_dir) + + try: + from ultralytics import YOLO # noqa: F401 + except Exception as exc: + raise SystemExit( + "Ultralytics is not installed. Run `uv sync --extra speedvision` in the repo root before exporting." + ) from exc + + exported_paths: list[Path] = [] + + if args.detector_weights: + detector_weights = args.detector_weights.resolve() + detector_output = output_dir / DETECTOR_EXPORT_NAME + export_yolo(detector_weights, detector_output, args.detector_imgsz, args.opset, nms=False) + if not args.skip_verify: + verify_onnx_with_opencv(detector_output, args.detector_imgsz) + exported_paths.append(detector_output) + + if args.classifier_weights: + classifier_weights = args.classifier_weights.resolve() + classifier_output = output_dir / CLASSIFIER_EXPORT_NAME + export_yolo(classifier_weights, classifier_output, args.classifier_imgsz, args.opset, nms=False) + if not args.skip_verify: + verify_onnx_with_opencv(classifier_output, args.classifier_imgsz) + exported_paths.append(classifier_output) + + if not exported_paths: + raise SystemExit("Pass at least one of --detector-weights or --classifier-weights") + + if args.install_repo_assets: + ensure_dir(REPO_ASSET_DIR) + for exported_path in exported_paths: + shutil.copy2(exported_path, REPO_ASSET_DIR / exported_path.name) + + for exported_path in exported_paths: + print(f"Exported {exported_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/generate_synthetic_us_speed_limits.py b/scripts/speed_limit_vision/generate_synthetic_us_speed_limits.py new file mode 100644 index 00000000..f11a18fb --- /dev/null +++ b/scripts/speed_limit_vision/generate_synthetic_us_speed_limits.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import math +import random + +from dataclasses import dataclass +from pathlib import Path + +import cv2 +import numpy as np + +from PIL import Image, ImageDraw, ImageEnhance, ImageFilter, ImageFont + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore +else: + from .common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + + +DEFAULT_BACKGROUND_DIR = DEFAULT_WORKSPACE / "backgrounds" +HEADER_FONT_CANDIDATES = ( + "/System/Library/Fonts/Supplemental/Arial Bold.ttf", + "/System/Library/Fonts/ArialHB.ttc", + "/System/Library/Fonts/Supplemental/Arial.ttf", +) +NUMBER_FONT_CANDIDATES = ( + "/System/Library/Fonts/Supplemental/DIN Condensed Bold.ttf", + "/System/Library/Fonts/Supplemental/Arial Black.ttf", + "/System/Library/Fonts/Supplemental/Arial Bold.ttf", +) +KNOWN_REAL_CROPS = ( + (".tmp/live_c4_capture/stopped_sign_crop_manual.png", 15), + (".tmp/route_vision/frame_041_sign_tight.jpg", 20), + (".tmp/route_vision/frame_041_sign_manual.jpg", 20), + (".tmp/route_12c_seg9_10/seg10_real30_crop.png", 30), + (".tmp/speed_route_frames_seg2_10_20/t12_sign_crop.png", 40), +) + + +@dataclass(frozen=True) +class SignSpec: + detector_class: int + style: str + speed_value: int | None + + +def load_font(candidates: tuple[str, ...], size: int) -> ImageFont.FreeTypeFont | ImageFont.ImageFont: + for candidate in candidates: + if Path(candidate).exists(): + return ImageFont.truetype(candidate, size=size) + return ImageFont.load_default() + + +def draw_centered(draw: ImageDraw.ImageDraw, box: tuple[int, int, int, int], text: str, font, fill: str): + left, top, right, bottom = box + bbox = draw.multiline_textbbox((0, 0), text, font=font, align="center", spacing=2) + width = bbox[2] - bbox[0] + height = bbox[3] - bbox[1] + x = left + (right - left - width) / 2 + y = top + (bottom - top - height) / 2 + draw.multiline_text((x, y), text, font=font, fill=fill, align="center", spacing=2) + + +def render_regulatory_sign(speed_value: int, school_zone: bool, seed: int) -> Image.Image: + rng = random.Random(seed) + sign_w = rng.randint(260, 320) + sign_h = rng.randint(390, 470) + image = Image.new("RGBA", (sign_w, sign_h), (255, 255, 255, 0)) + draw = ImageDraw.Draw(image) + + border_radius = max(int(sign_w * 0.08), 16) + draw.rounded_rectangle((0, 0, sign_w - 1, sign_h - 1), border_radius, fill="white", outline="black", width=max(sign_w // 38, 5)) + + header_font = load_font(HEADER_FONT_CANDIDATES, max(sign_w // 9, 26)) + number_font = load_font(NUMBER_FONT_CANDIDATES, max(sign_w // 3, 84)) + footer_font = load_font(HEADER_FONT_CANDIDATES, max(sign_w // 12, 18)) + + if school_zone: + draw_centered(draw, (int(sign_w * 0.10), int(sign_h * 0.06), int(sign_w * 0.90), int(sign_h * 0.24)), "SCHOOL", header_font, "black") + draw_centered(draw, (int(sign_w * 0.10), int(sign_h * 0.20), int(sign_w * 0.90), int(sign_h * 0.42)), "SPEED\nLIMIT", header_font, "black") + draw_centered(draw, (int(sign_w * 0.12), int(sign_h * 0.42), int(sign_w * 0.88), int(sign_h * 0.78)), str(speed_value), number_font, "black") + draw_centered(draw, (int(sign_w * 0.08), int(sign_h * 0.76), int(sign_w * 0.92), int(sign_h * 0.94)), "WHEN FLASHING", footer_font, "black") + else: + draw_centered(draw, (int(sign_w * 0.10), int(sign_h * 0.08), int(sign_w * 0.90), int(sign_h * 0.34)), "SPEED\nLIMIT", header_font, "black") + draw_centered(draw, (int(sign_w * 0.12), int(sign_h * 0.40), int(sign_w * 0.88), int(sign_h * 0.84)), str(speed_value), number_font, "black") + + if school_zone and rng.random() < 0.65: + lamp_y = int(sign_h * 0.12) + lamp_r = max(sign_w // 18, 12) + for lamp_x in (int(sign_w * 0.16), int(sign_w * 0.84)): + draw.ellipse((lamp_x - lamp_r, lamp_y - lamp_r, lamp_x + lamp_r, lamp_y + lamp_r), fill=(255, 192, 0), outline="black", width=3) + + return image + + +def render_advisory_sign(speed_value: int, seed: int) -> Image.Image: + rng = random.Random(seed) + size = rng.randint(240, 320) + image = Image.new("RGBA", (size, size), (255, 255, 255, 0)) + base = Image.new("RGBA", (size, size), (255, 255, 255, 0)) + draw = ImageDraw.Draw(base) + + draw.polygon(((size / 2, 0), (size, size / 2), (size / 2, size), (0, size / 2)), fill=(255, 214, 10), outline="black") + base = base.rotate(45, expand=True, resample=Image.Resampling.BICUBIC) + bbox = base.getbbox() + if bbox is not None: + base = base.crop(bbox) + + draw = ImageDraw.Draw(base) + number_font = load_font(NUMBER_FONT_CANDIDATES, max(base.size[0] // 3, 72)) + footer_font = load_font(HEADER_FONT_CANDIDATES, max(base.size[0] // 12, 18)) + draw_centered(draw, (0, int(base.size[1] * 0.18), base.size[0], int(base.size[1] * 0.68)), str(speed_value), number_font, "black") + if rng.random() < 0.7: + draw_centered(draw, (0, int(base.size[1] * 0.68), base.size[0], int(base.size[1] * 0.92)), "MPH", footer_font, "black") + return base + + +def add_motion_blur(image: Image.Image, radius: int) -> Image.Image: + if radius <= 1: + return image + kernel_size = radius * 2 + 1 + kernel = np.zeros((kernel_size, kernel_size), dtype=np.float32) + kernel[kernel_size // 2, :] = 1.0 / kernel_size + array = cv2.filter2D(np.array(image), -1, kernel) + return Image.fromarray(array) + + +def augment_sign(sign: Image.Image, rng: random.Random) -> Image.Image: + image = sign.copy() + if rng.random() < 0.7: + image = image.filter(ImageFilter.GaussianBlur(radius=rng.uniform(0.2, 1.8))) + if rng.random() < 0.35: + image = add_motion_blur(image, radius=rng.randint(2, 5)) + + brightness = ImageEnhance.Brightness(image) + image = brightness.enhance(rng.uniform(0.75, 1.18)) + contrast = ImageEnhance.Contrast(image) + image = contrast.enhance(rng.uniform(0.85, 1.25)) + return image + + +def choose_sign_spec(rng: random.Random, speed_values: tuple[int, ...]) -> SignSpec: + roll = rng.random() + if roll < 0.16: + return SignSpec(detector_class=1, style="advisory", speed_value=rng.choice((20, 25, 30, 35, 40, 45))) + if roll < 0.34: + school_choices = tuple(value for value in speed_values if value in (15, 20, 25)) + return SignSpec(detector_class=2, style="school_zone", speed_value=rng.choice(school_choices or speed_values)) + return SignSpec(detector_class=0, style="regulatory", speed_value=rng.choice(speed_values)) + + +def paste_transformed(background_bgr: np.ndarray, sign_rgba: Image.Image, rng: random.Random): + background = background_bgr.copy() + bg_h, bg_w = background.shape[:2] + sign = np.array(sign_rgba) + sign_h, sign_w = sign.shape[:2] + + target_h = int(rng.uniform(bg_h * 0.045, bg_h * 0.17)) + scale = target_h / max(sign_h, 1) + target_w = max(int(sign_w * scale), 12) + resized = cv2.resize(sign, (target_w, target_h), interpolation=cv2.INTER_LINEAR) + sign_h, sign_w = resized.shape[:2] + + center_x = int(rng.uniform(bg_w * 0.58, bg_w * 0.92)) + center_y = int(rng.uniform(bg_h * 0.10, bg_h * 0.58)) + + src = np.float32([[0, 0], [sign_w - 1, 0], [sign_w - 1, sign_h - 1], [0, sign_h - 1]]) + skew_x = sign_w * rng.uniform(0.04, 0.18) + skew_y = sign_h * rng.uniform(0.02, 0.12) + dst = np.float32([ + [center_x - sign_w * rng.uniform(0.35, 0.55), center_y - sign_h * rng.uniform(0.55, 0.70)], + [center_x + sign_w * rng.uniform(0.35, 0.55), center_y - sign_h * rng.uniform(0.45, 0.70)], + [center_x + sign_w * rng.uniform(0.28, 0.52), center_y + sign_h * rng.uniform(0.30, 0.58)], + [center_x - sign_w * rng.uniform(0.26, 0.48), center_y + sign_h * rng.uniform(0.34, 0.62)], + ]) + dst += np.float32([ + [rng.uniform(-skew_x, skew_x), rng.uniform(-skew_y, skew_y)], + [rng.uniform(-skew_x, skew_x), rng.uniform(-skew_y, skew_y)], + [rng.uniform(-skew_x, skew_x), rng.uniform(-skew_y, skew_y)], + [rng.uniform(-skew_x, skew_x), rng.uniform(-skew_y, skew_y)], + ]) + + matrix = cv2.getPerspectiveTransform(src, dst) + warped = cv2.warpPerspective(resized, matrix, (bg_w, bg_h), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0, 0)) + alpha = warped[:, :, 3:4].astype(np.float32) / 255.0 + if alpha.max() <= 0.01: + return background, None, None + + warped_rgb = warped[:, :, :3].astype(np.float32) + composite = background.astype(np.float32) * (1.0 - alpha) + warped_rgb * alpha + composite = composite.astype(np.uint8) + + ys, xs = np.where(alpha[:, :, 0] > 0.05) + if len(xs) == 0 or len(ys) == 0: + return background, None, None + + x1, x2 = int(xs.min()), int(xs.max()) + y1, y2 = int(ys.min()), int(ys.max()) + bbox = (x1, y1, x2, y2) + crop = composite[y1:y2 + 1, x1:x2 + 1] + return composite, bbox, crop + + +def detector_label_line(detector_class: int, bbox: tuple[int, int, int, int], image_shape: tuple[int, int, int]) -> str: + image_h, image_w = image_shape[:2] + x1, y1, x2, y2 = bbox + x_center = ((x1 + x2) / 2) / image_w + y_center = ((y1 + y2) / 2) / image_h + width = (x2 - x1) / image_w + height = (y2 - y1) / image_h + return f"{detector_class} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}\n" + + +def save_classifier_crop(base_dir: Path, split: str, speed_value: int, image_bgr: np.ndarray, stem: str): + output_dir = ensure_dir(base_dir / split / str(speed_value)) + output_path = output_dir / f"{stem}.jpg" + cv2.imwrite(str(output_path), image_bgr, [cv2.IMWRITE_JPEG_QUALITY, 92]) + + +def collect_backgrounds(background_dir: Path) -> list[Path]: + if not background_dir.is_dir(): + return [] + return sorted(path for path in background_dir.iterdir() if path.suffix.lower() in {".jpg", ".jpeg", ".png"}) + + +def main(): + parser = argparse.ArgumentParser(description="Generate a synthetic U.S. speed-limit detector/classifier dataset.") + parser.add_argument("--workspace", default=str(DEFAULT_WORKSPACE), help="Training workspace root.") + parser.add_argument("--background-dir", default=str(DEFAULT_BACKGROUND_DIR), help="Background image directory.") + parser.add_argument("--train-count", type=int, default=9000, help="Number of synthetic training detector images.") + parser.add_argument("--val-count", type=int, default=1200, help="Number of synthetic validation detector images.") + parser.add_argument("--negative-ratio", type=float, default=0.18, help="Share of detector images with no sign.") + parser.add_argument("--seed", type=int, default=20260330, help="Random seed.") + args = parser.parse_args() + + workspace = resolve_workspace(args.workspace) + background_dir = Path(args.background_dir).expanduser().resolve() + backgrounds = collect_backgrounds(background_dir) + if not backgrounds: + raise FileNotFoundError(f"No backgrounds found in {background_dir}") + + detector_image_dir = workspace / "detector" / "images" + detector_label_dir = workspace / "detector" / "labels" + classifier_dir = workspace / "classifier" + speed_values = tuple(DEFAULT_SPEED_VALUES) + rng = random.Random(args.seed) + + for split, count in (("train", max(args.train_count, 0)), ("val", max(args.val_count, 0))): + ensure_dir(detector_image_dir / split) + ensure_dir(detector_label_dir / split) + ensure_dir(classifier_dir / split) + + for index in range(count): + background_path = rng.choice(backgrounds) + background = cv2.imread(str(background_path)) + if background is None: + continue + + stem = f"{split}_{index:06d}" + image_path = detector_image_dir / split / f"{stem}.jpg" + label_path = detector_label_dir / split / f"{stem}.txt" + detector_lines: list[str] = [] + + if rng.random() >= args.negative_ratio: + sign_spec = choose_sign_spec(rng, speed_values) + if sign_spec.style == "advisory": + sign_image = render_advisory_sign(sign_spec.speed_value or 25, seed=rng.randint(0, 1_000_000)) + else: + sign_image = render_regulatory_sign(sign_spec.speed_value or 25, school_zone=sign_spec.style == "school_zone", seed=rng.randint(0, 1_000_000)) + sign_image = augment_sign(sign_image, rng) + composite, bbox, crop = paste_transformed(background, sign_image, rng) + if bbox is not None: + detector_lines.append(detector_label_line(sign_spec.detector_class, bbox, composite.shape)) + background = composite + if crop is not None and sign_spec.speed_value is not None and sign_spec.detector_class != 1: + save_classifier_crop(classifier_dir, split, sign_spec.speed_value, crop, stem) + + if rng.random() < 0.45: + alpha = rng.uniform(0.05, 0.20) + overlay = np.full_like(background, int(rng.uniform(180, 240))) + background = cv2.addWeighted(background, 1.0 - alpha, overlay, alpha, 0) + if rng.random() < 0.3: + background = cv2.GaussianBlur(background, (3, 3), rng.uniform(0.1, 0.9)) + if rng.random() < 0.25: + noise = rng.normalvariate(0, 6) + background = np.clip(background.astype(np.int16) + noise, 0, 255).astype(np.uint8) + + cv2.imwrite(str(image_path), background, [cv2.IMWRITE_JPEG_QUALITY, 92]) + label_path.write_text("".join(detector_lines), encoding="utf-8") + + repo_root = Path(__file__).resolve().parents[2] + imported_real = 0 + for relative_path, speed_value in KNOWN_REAL_CROPS: + crop_path = repo_root / relative_path + if not crop_path.is_file(): + continue + image = cv2.imread(str(crop_path)) + if image is None: + continue + split = "val" if imported_real % 4 == 0 else "train" + save_classifier_crop(classifier_dir, split, speed_value, image, f"real_{speed_value}_{imported_real:03d}") + imported_real += 1 + + print(f"Generated synthetic detector data in {workspace / 'detector'}") + print(f"Generated synthetic classifier data in {workspace / 'classifier'}") + print(f"Backgrounds used: {len(backgrounds)}") + print(f"Imported real crops: {imported_real}") + + +if __name__ == "__main__": + main() diff --git a/scripts/speed_limit_vision/generate_value_roi_classifier_dataset.py b/scripts/speed_limit_vision/generate_value_roi_classifier_dataset.py new file mode 100644 index 00000000..98bc26d3 --- /dev/null +++ b/scripts/speed_limit_vision/generate_value_roi_classifier_dataset.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import random +import shutil + +from pathlib import Path + +import cv2 +import numpy as np + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore + from generate_synthetic_us_speed_limits import KNOWN_REAL_CROPS, augment_sign, render_regulatory_sign # type: ignore +else: + from .common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + from .generate_synthetic_us_speed_limits import KNOWN_REAL_CROPS, augment_sign, render_regulatory_sign + + +VALUE_TEMPLATE_ROIS = ( + (0.35, 0.82, 0.15, 0.78), + (0.45, 0.85, 0.18, 0.78), + (0.40, 0.84, 0.18, 0.75), +) + + +def normalize_binary_mask(binary_mask: np.ndarray, size=(72, 96), padding=6): + points = cv2.findNonZero(binary_mask) + if points is None: + return None + + x, y, width, height = cv2.boundingRect(points) + digit = binary_mask[y:y + height, x:x + width] + target_w, target_h = size + scale = min((target_w - padding * 2) / max(width, 1), (target_h - padding * 2) / max(height, 1)) + resized_w = max(int(round(width * scale)), 1) + resized_h = max(int(round(height * scale)), 1) + resized = cv2.resize(digit, (resized_w, resized_h), interpolation=cv2.INTER_NEAREST) + + canvas = np.zeros((target_h, target_w), dtype=np.uint8) + offset_x = (target_w - resized_w) // 2 + offset_y = (target_h - resized_h) // 2 + canvas[offset_y:offset_y + resized_h, offset_x:offset_x + resized_w] = resized + return canvas + + +def extract_value_mask(sign_bgr: np.ndarray): + gray = cv2.cvtColor(sign_bgr, cv2.COLOR_BGR2GRAY) + height, width = gray.shape + best_mask = None + best_fill = 0.0 + + for top_ratio, bottom_ratio, left_ratio, right_ratio in VALUE_TEMPLATE_ROIS: + roi = gray[int(height * top_ratio):int(height * bottom_ratio), int(width * left_ratio):int(width * right_ratio)] + if roi.size == 0: + continue + + clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8)).apply(roi) + _, binary = cv2.threshold(clahe, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + binary = cv2.morphologyEx(binary, cv2.MORPH_OPEN, np.ones((2, 2), dtype=np.uint8)) + + num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(binary, 8) + mask = np.zeros_like(binary) + for label_idx in range(1, num_labels): + x, y, comp_w, comp_h, area = stats[label_idx] + if area < roi.shape[0] * roi.shape[1] * 0.01: + continue + if y < binary.shape[0] * 0.08: + continue + if comp_h < binary.shape[0] * 0.18: + continue + if comp_w > binary.shape[1] * 0.75: + continue + mask[labels == label_idx] = 255 + + normalized = normalize_binary_mask(mask, size=(72, 96)) + if normalized is None: + continue + + fill_ratio = float(np.count_nonzero(normalized)) / normalized.size + if fill_ratio > best_fill: + best_fill = fill_ratio + best_mask = normalized + + return best_mask + + +def perspective_jitter(sign_rgba, rng: random.Random): + sign = np.array(sign_rgba) + sign_h, sign_w = sign.shape[:2] + pad = max(sign_w, sign_h) // 5 + canvas = np.zeros((sign_h + pad * 2, sign_w + pad * 2, 4), dtype=np.uint8) + canvas[pad:pad + sign_h, pad:pad + sign_w] = sign + sign_h, sign_w = canvas.shape[:2] + + src = np.float32([[0, 0], [sign_w - 1, 0], [sign_w - 1, sign_h - 1], [0, sign_h - 1]]) + jitter_x = sign_w * 0.08 + jitter_y = sign_h * 0.08 + dst = src + np.float32([ + [rng.uniform(-jitter_x, jitter_x), rng.uniform(-jitter_y, jitter_y)], + [rng.uniform(-jitter_x, jitter_x), rng.uniform(-jitter_y, jitter_y)], + [rng.uniform(-jitter_x, jitter_x), rng.uniform(-jitter_y, jitter_y)], + [rng.uniform(-jitter_x, jitter_x), rng.uniform(-jitter_y, jitter_y)], + ]) + matrix = cv2.getPerspectiveTransform(src, dst) + warped = cv2.warpPerspective(canvas, matrix, (sign_w, sign_h), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0, 0)) + ys, xs = np.where(warped[:, :, 3] > 0) + if len(xs) == 0 or len(ys) == 0: + return canvas + return warped[ys.min():ys.max() + 1, xs.min():xs.max() + 1] + + +def augment_mask(mask: np.ndarray, rng: random.Random): + canvas = np.zeros((128, 128), dtype=np.uint8) + resized = cv2.resize(mask, None, fx=rng.uniform(0.85, 1.15), fy=rng.uniform(0.85, 1.15), interpolation=cv2.INTER_NEAREST) + offset_x = max((canvas.shape[1] - resized.shape[1]) // 2 + rng.randint(-8, 8), 0) + offset_y = max((canvas.shape[0] - resized.shape[0]) // 2 + rng.randint(-8, 8), 0) + end_x = min(offset_x + resized.shape[1], canvas.shape[1]) + end_y = min(offset_y + resized.shape[0], canvas.shape[0]) + canvas[offset_y:end_y, offset_x:end_x] = resized[:end_y - offset_y, :end_x - offset_x] + + if rng.random() < 0.45: + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (rng.choice((1, 2, 3)), rng.choice((1, 2, 3)))) + operation = cv2.MORPH_DILATE if rng.random() < 0.5 else cv2.MORPH_ERODE + canvas = cv2.morphologyEx(canvas, operation, kernel) + if rng.random() < 0.55: + canvas = cv2.GaussianBlur(canvas, (3, 3), rng.uniform(0.1, 1.0)) + if rng.random() < 0.35: + noise = np.random.normal(0.0, rng.uniform(2.0, 9.0), canvas.shape).astype(np.float32) + canvas = np.clip(canvas.astype(np.float32) + noise, 0, 255).astype(np.uint8) + + return cv2.cvtColor(canvas, cv2.COLOR_GRAY2BGR) + + +def save_mask(base_dir: Path, split: str, speed_value: int, image_bgr: np.ndarray, stem: str): + output_dir = ensure_dir(base_dir / split / str(speed_value)) + cv2.imwrite(str(output_dir / f"{stem}.png"), image_bgr) + + +def main(): + parser = argparse.ArgumentParser(description="Generate a value-ROI classifier dataset from synthetic U.S. speed-limit signs.") + parser.add_argument("--workspace", default=str(DEFAULT_WORKSPACE), help="Training workspace root.") + parser.add_argument("--train-per-class", type=int, default=1800, help="Synthetic training samples per value.") + parser.add_argument("--val-per-class", type=int, default=260, help="Synthetic validation samples per value.") + parser.add_argument("--real-augmentations", type=int, default=28, help="Augmented mask samples to create per known real crop.") + parser.add_argument("--seed", type=int, default=20260330, help="Random seed.") + args = parser.parse_args() + + workspace = resolve_workspace(args.workspace) + classifier_dir = workspace / "classifier" + if classifier_dir.exists(): + shutil.rmtree(classifier_dir) + ensure_dir(classifier_dir / "train") + ensure_dir(classifier_dir / "val") + + rng = random.Random(args.seed) + speed_values = tuple(DEFAULT_SPEED_VALUES) + + for split, per_class in (("train", max(args.train_per_class, 0)), ("val", max(args.val_per_class, 0))): + for speed_value in speed_values: + for index in range(per_class): + school_zone = speed_value in (15, 20, 25) and rng.random() < 0.45 + sign_rgba = render_regulatory_sign(speed_value, school_zone=school_zone, seed=rng.randint(0, 1_000_000)) + sign_rgba = augment_sign(sign_rgba, rng) + sign_rgba = perspective_jitter(sign_rgba, rng) + sign_bgr = cv2.cvtColor(sign_rgba[:, :, :3], cv2.COLOR_RGB2BGR) + mask = extract_value_mask(sign_bgr) + if mask is None: + continue + output = augment_mask(mask, rng) + save_mask(classifier_dir, split, speed_value, output, f"{split}_{speed_value}_{index:05d}") + + repo_root = Path(__file__).resolve().parents[2] + imported_real = 0 + for relative_path, speed_value in KNOWN_REAL_CROPS: + crop_path = repo_root / relative_path + if not crop_path.is_file(): + continue + crop_bgr = cv2.imread(str(crop_path)) + if crop_bgr is None: + continue + mask = extract_value_mask(crop_bgr) + if mask is None: + continue + for augmentation_index in range(max(args.real_augmentations, 1)): + split = "val" if augmentation_index % 5 == 0 else "train" + output = augment_mask(mask, rng) + save_mask(classifier_dir, split, speed_value, output, f"real_{speed_value}_{imported_real:03d}_{augmentation_index:03d}") + imported_real += 1 + + print(f"Generated ROI classifier dataset in {classifier_dir}") + print(f"Imported real crops: {imported_real}") + + +if __name__ == "__main__": + main() diff --git a/scripts/speed_limit_vision/import_arts_challenging.py b/scripts/speed_limit_vision/import_arts_challenging.py new file mode 100644 index 00000000..83323017 --- /dev/null +++ b/scripts/speed_limit_vision/import_arts_challenging.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import tarfile +import xml.etree.ElementTree as ET + +from collections import defaultdict +from pathlib import Path + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import ( # type: ignore + DEFAULT_WORKSPACE, + DETECTOR_CLASS_NAMES, + PUBLIC_CLASSIFIER_SAMPLE_FIELDS, + PUBLIC_DETECTOR_SAMPLE_FIELDS, + RAW_SOURCE_FIELDS, + VALUE_LABEL_FIELDS, + default_raw_root, + ensure_dir, + resolve_workspace, + ) +else: + from .common import ( + DEFAULT_WORKSPACE, + DETECTOR_CLASS_NAMES, + PUBLIC_CLASSIFIER_SAMPLE_FIELDS, + PUBLIC_DETECTOR_SAMPLE_FIELDS, + RAW_SOURCE_FIELDS, + VALUE_LABEL_FIELDS, + default_raw_root, + ensure_dir, + resolve_workspace, + ) + + +DEFAULT_ARCHIVE_RELATIVE = Path("external/arts_probe/Public/ARTS-V1/Challenging/challenging-dev.tar.gz") +SOURCE_NAME = "arts_challenging" +SOURCE_VERSION = "ARTS-V1 Challenging" +SOURCE_LICENSE = "Research dataset, see source distribution" + +ARTS_CODE_MAP: dict[str, tuple[str, int | None]] = { + "R2-1": ("regulatory_speed_limit", None), + "R2-125": ("regulatory_speed_limit", 25), + "R2-130": ("regulatory_speed_limit", 30), + "R2-135": ("regulatory_speed_limit", 35), + "R2-140": ("regulatory_speed_limit", 40), + "R2-145": ("regulatory_speed_limit", 45), + "R2-150": ("regulatory_speed_limit", 50), + "R2-155": ("regulatory_speed_limit", 55), + "R2-165": ("regulatory_speed_limit", 65), + "W13-1": ("advisory_speed_limit", None), + "W13-2": ("advisory_speed_limit", None), + "W13-3": ("advisory_speed_limit", None), +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import ARTS Challenging speed-limit samples into the training workspace.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--archive", type=Path, help="Path to challenging-dev.tar.gz. Defaults to the SSD raw-data layout.") + parser.add_argument("--train-split", type=float, default=0.85, help="Fallback train split ratio when ARTS split files are unavailable.") + parser.add_argument("--overwrite", action="store_true", help="Overwrite previously imported ARTS images/labels.") + return parser.parse_args() + + +def default_archive_path(workspace: Path) -> Path: + return default_raw_root(workspace) / DEFAULT_ARCHIVE_RELATIVE + + +def read_existing_rows(path: Path) -> list[dict[str, str]]: + if not path.is_file(): + return [] + with path.open("r", encoding="utf-8", newline="") as csv_file: + return list(csv.DictReader(csv_file)) + + +def write_rows(path: Path, fieldnames: list[str], rows: list[dict[str, str]]) -> None: + ensure_dir(path.parent) + with path.open("w", encoding="utf-8", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def build_fallback_split_lookup(stem_names: list[str], train_ratio: float) -> dict[str, str]: + cutoff = int(len(stem_names) * max(0.0, min(train_ratio, 1.0))) + fallback: dict[str, str] = {} + for index, stem in enumerate(sorted(stem_names)): + fallback[stem] = "train" if index < cutoff else "val" + return fallback + + +def yolo_box(size: tuple[int, int], bbox: tuple[int, int, int, int]) -> tuple[float, float, float, float]: + width, height = size + xmin, ymin, xmax, ymax = bbox + box_width = max(xmax - xmin, 1) + box_height = max(ymax - ymin, 1) + x_center = xmin + box_width / 2.0 + y_center = ymin + box_height / 2.0 + return ( + x_center / width, + y_center / height, + box_width / width, + box_height / height, + ) + + +def parse_annotation(xml_bytes: bytes) -> tuple[tuple[int, int], list[dict[str, object]]]: + root = ET.fromstring(xml_bytes) + size_node = root.find("size") + width = int(size_node.findtext("width", default="0")) + height = int(size_node.findtext("height", default="0")) + parsed: list[dict[str, object]] = [] + for obj in root.findall("object"): + sign_code = (obj.findtext("name") or "").strip() + mapped = ARTS_CODE_MAP.get(sign_code) + if mapped is None: + continue + bbox_node = obj.find("bndbox") + xmin = int(float(bbox_node.findtext("xmin", default="0"))) + ymin = int(float(bbox_node.findtext("ymin", default="0"))) + xmax = int(float(bbox_node.findtext("xmax", default="0"))) + ymax = int(float(bbox_node.findtext("ymax", default="0"))) + class_name, speed_value = mapped + parsed.append({ + "sign_code": sign_code, + "class_name": class_name, + "speed_limit_mph": speed_value, + "bbox": (xmin, ymin, xmax, ymax), + }) + return (width, height), parsed + + +def update_split_lookup(split_lookup: dict[str, str], split_name: str, text: str) -> None: + normalized_split = "val" if split_name == "test" else split_name + for line in text.splitlines(): + stem = line.strip() + if stem: + split_lookup[stem] = normalized_split + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + archive_path = args.archive.resolve() if args.archive else default_archive_path(workspace) + if not archive_path.is_file(): + raise FileNotFoundError(f"ARTS archive not found: {archive_path}") + + detector_manifest_path = workspace / "manifests" / "public_detector_samples.csv" + classifier_manifest_path = workspace / "manifests" / "public_classifier_samples.csv" + value_labels_path = workspace / "classifier" / "value_labels.csv" + raw_sources_path = workspace / "manifests" / "raw_sources.csv" + + existing_detector_rows = [row for row in read_existing_rows(detector_manifest_path) if row.get("source_name") != SOURCE_NAME] + existing_classifier_rows = [row for row in read_existing_rows(classifier_manifest_path) if row.get("source_name") != SOURCE_NAME] + existing_value_rows = [row for row in read_existing_rows(value_labels_path) if not (row.get("image_path") or "").startswith("detector/images/") or SOURCE_NAME not in (row.get("image_path") or "")] + existing_source_rows = [row for row in read_existing_rows(raw_sources_path) if row.get("source_name") != SOURCE_NAME] + + detector_rows: list[dict[str, str]] = [] + classifier_rows: list[dict[str, str]] = [] + value_rows: list[dict[str, str]] = [] + split_lookup: dict[str, str] = {} + annotations_by_stem: dict[str, dict[str, object]] = {} + + with tarfile.open(archive_path, "r:gz") as tar: + for member in tar: + if not member.isfile(): + continue + if member.name == "challenging/ImageSets/Main/train.txt": + update_split_lookup(split_lookup, "train", tar.extractfile(member).read().decode("utf-8", "ignore")) + continue + if member.name == "challenging/ImageSets/Main/val.txt": + update_split_lookup(split_lookup, "val", tar.extractfile(member).read().decode("utf-8", "ignore")) + continue + if member.name == "challenging/ImageSets/Main/test.txt": + update_split_lookup(split_lookup, "test", tar.extractfile(member).read().decode("utf-8", "ignore")) + continue + if not (member.name.startswith("challenging/Annotations/") and member.name.endswith(".xml")): + continue + stem = Path(member.name).stem + image_size, parsed_boxes = parse_annotation(tar.extractfile(member).read()) + if not parsed_boxes: + continue + annotations_by_stem[stem] = { + "image_size": image_size, + "boxes": parsed_boxes, + "annotation_name": member.name, + } + + if not split_lookup: + split_lookup = build_fallback_split_lookup(list(annotations_by_stem), args.train_split) + + imported_images = 0 + imported_boxes = 0 + class_counts: dict[str, int] = defaultdict(int) + + with tarfile.open(archive_path, "r:gz") as tar: + for member in tar: + if not member.isfile(): + continue + if not (member.name.startswith("challenging/JPEGImages/") and member.name.endswith(".jpg")): + continue + stem = Path(member.name).stem + annotation = annotations_by_stem.get(stem) + if annotation is None: + continue + + split = split_lookup.get(stem, "train") + image_bytes = tar.extractfile(member).read() + image_size = annotation["image_size"] # type: ignore[assignment] + parsed_boxes = annotation["boxes"] # type: ignore[assignment] + + image_out = workspace / "detector" / "images" / split / f"{SOURCE_NAME}_{stem}.jpg" + label_out = workspace / "detector" / "labels" / split / f"{SOURCE_NAME}_{stem}.txt" + if args.overwrite or not image_out.exists(): + ensure_dir(image_out.parent) + image_out.write_bytes(image_bytes) + yolo_lines: list[str] = [] + for bbox_index, box in enumerate(parsed_boxes): + class_name = str(box["class_name"]) + speed_value = box["speed_limit_mph"] + class_id = DETECTOR_CLASS_NAMES.index(class_name) + x_center, y_center, width, height = yolo_box(image_size, box["bbox"]) # type: ignore[arg-type] + yolo_lines.append(f"{class_id} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}") + xmin, ymin, xmax, ymax = box["bbox"] # type: ignore[misc] + record_key = f"{SOURCE_NAME}:{stem}:{bbox_index}" + detector_rows.append({ + "record_key": record_key, + "source_name": SOURCE_NAME, + "split": split, + "image_path": str(image_out.relative_to(workspace)), + "label_path": str(label_out.relative_to(workspace)), + "annotation_path": f"{archive_path}:{annotation['annotation_name']}", + "source_image_id": stem, + "class_name": class_name, + "speed_limit_mph": "" if speed_value is None else str(speed_value), + "sign_code": str(box["sign_code"]), + "bbox_left": str(xmin), + "bbox_top": str(ymin), + "bbox_right": str(xmax), + "bbox_bottom": str(ymax), + }) + class_counts[class_name] += 1 + imported_boxes += 1 + if speed_value is not None: + classifier_rows.append({ + "record_key": record_key, + "source_name": SOURCE_NAME, + "split": split, + "image_path": str(image_out.relative_to(workspace)), + "speed_limit_mph": str(speed_value), + "bbox_index": str(bbox_index), + "label_path": str(label_out.relative_to(workspace)), + "source_image_id": stem, + "sign_code": str(box["sign_code"]), + }) + value_rows.append({ + "image_path": str(image_out.relative_to(workspace)), + "split": split, + "speed_limit_mph": str(speed_value), + "bbox_index": str(bbox_index), + "padding": "0.10", + "label_path": str(label_out.relative_to(workspace)), + }) + + if yolo_lines and (args.overwrite or not label_out.exists()): + ensure_dir(label_out.parent) + label_out.write_text("\n".join(yolo_lines) + "\n", encoding="utf-8") + imported_images += 1 + + source_row = { + "source_name": SOURCE_NAME, + "source_version": SOURCE_VERSION, + "source_license": SOURCE_LICENSE, + "source_type": "public_detector_and_classifier_seed", + "raw_path": str(archive_path), + "notes": "ARTS challenging subset imported from VOC XML. Only mapped speed-limit classes were kept.", + } + + write_rows(raw_sources_path, RAW_SOURCE_FIELDS, existing_source_rows + [source_row]) + write_rows(detector_manifest_path, PUBLIC_DETECTOR_SAMPLE_FIELDS, existing_detector_rows + detector_rows) + write_rows(classifier_manifest_path, PUBLIC_CLASSIFIER_SAMPLE_FIELDS, existing_classifier_rows + classifier_rows) + write_rows(value_labels_path, VALUE_LABEL_FIELDS, existing_value_rows + value_rows) + + class_summary = ", ".join(f"{name}={count}" for name, count in sorted(class_counts.items())) + print(f"Imported {imported_images} ARTS image(s) and {imported_boxes} box(es) from {archive_path}") + print(f" detector manifest: {detector_manifest_path}") + print(f" classifier manifest: {classifier_manifest_path}") + print(f" class counts: {class_summary or 'none'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/import_bookmark_leadins.py b/scripts/speed_limit_vision/import_bookmark_leadins.py new file mode 100644 index 00000000..147d9016 --- /dev/null +++ b/scripts/speed_limit_vision/import_bookmark_leadins.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json +import math + +from pathlib import Path + +import cv2 +import numpy as np + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import ( # type: ignore + BOOKMARK_LEADIN_MANIFEST_FIELDS, + DEFAULT_WORKSPACE, + ensure_dir, + preferred_clip_root, + resolve_workspace, + write_csv_header, + ) +else: + from .common import ( + BOOKMARK_LEADIN_MANIFEST_FIELDS, + DEFAULT_WORKSPACE, + ensure_dir, + preferred_clip_root, + resolve_workspace, + write_csv_header, + ) + + +DEFAULT_REPORT_JSON = Path(".tmp/live_route_clips/bookmark_windows_report.json") +DEFAULT_CLIP_ROOT = preferred_clip_root() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import sampled 5-second pre-bookmark route windows into the speed-limit training workspace.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--report-json", type=Path, default=DEFAULT_REPORT_JSON, help="JSON output from evaluate_bookmark_leadins.py.") + parser.add_argument("--clip-root", type=Path, default=DEFAULT_CLIP_ROOT, help="Copied route clip root used by evaluate_bookmark_leadins.py.") + parser.add_argument("--source-name", default="comma_bookmark", help="Logical source name for imported bookmark windows.") + parser.add_argument("--source-region", default="", help="Optional region or market label, e.g. us_midwest.") + parser.add_argument("--source-device", default="", help="Optional device identifier or platform label.") + parser.add_argument("--source-driver", default="", help="Optional contributor/driver identifier.") + parser.add_argument("--mode", choices=("misses", "hits", "all"), default="misses", help="Which bookmark windows to sample.") + parser.add_argument("--sample-every", type=float, default=0.5, help="Seconds between sampled review frames.") + parser.add_argument("--max-samples", type=int, default=12, help="Optional cap on sampled frames per bookmark window.") + parser.add_argument("--overwrite", action="store_true", help="Overwrite existing extracted review frames and contact sheets.") + return parser.parse_args() + + +def load_existing_rows(manifest_path: Path) -> dict[str, dict[str, str]]: + if not manifest_path.exists(): + return {} + + with manifest_path.open("r", encoding="utf-8", newline="") as manifest_file: + reader = csv.DictReader(manifest_file) + return {row["record_key"]: row for row in reader if row.get("record_key")} + + +def read_frame_at(video_path: Path, target_time_s: float): + capture = cv2.VideoCapture(str(video_path)) + fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + frame_index = max(int(round(target_time_s * fps)), 0) + capture.set(cv2.CAP_PROP_POS_FRAMES, frame_index) + ok, frame_bgr = capture.read() + capture.release() + if not ok or frame_bgr is None: + return None + actual_time_s = frame_index / fps + return actual_time_s, frame_bgr + + +def sample_offsets(leadin_start_s: float, segment_offset_s: float, sample_every: float, max_samples: int): + if sample_every <= 0: + raise ValueError("sample_every must be positive") + + duration_s = max(segment_offset_s - leadin_start_s, 0.0) + sample_count = int(math.floor(duration_s / sample_every)) + 1 + if max_samples > 0: + sample_count = min(sample_count, max_samples) + if sample_count <= 1: + return [segment_offset_s] + + if max_samples > 0 and sample_count == max_samples: + return [leadin_start_s + duration_s * index / (sample_count - 1) for index in range(sample_count)] + + offsets = [] + sample_offset_s = leadin_start_s + while sample_offset_s <= segment_offset_s + 1e-6: + offsets.append(sample_offset_s) + sample_offset_s += sample_every + return offsets[:sample_count] + + +def extract_window_frames(row: dict, clip_root: Path, sample_every: float, max_samples: int): + route = row["route"] + segment = int(row["segment"]) + segment_offset_s = float(row["segmentOffsetS"]) + leadin_start_s = float(row["leadinStartS"]) + spans_previous_segment = bool(row.get("spansPreviousSegment")) + + sampled = [] + previous_clip = clip_root / f"{route}--{segment - 1}" / "fcamera.hevc" + current_clip = clip_root / f"{route}--{segment}" / "fcamera.hevc" + + for relative_offset_s in sample_offsets(leadin_start_s, segment_offset_s, sample_every, max_samples): + if relative_offset_s < 0.0 and spans_previous_segment and segment > 0: + source_video = previous_clip + source_time_s = 60.0 + relative_offset_s + else: + source_video = current_clip + source_time_s = max(relative_offset_s, 0.0) + + if not source_video.is_file(): + continue + + frame_info = read_frame_at(source_video, source_time_s) + if frame_info is None: + continue + + actual_time_s, frame_bgr = frame_info + sampled.append({ + "relative_offset_s": relative_offset_s, + "actual_time_s": actual_time_s, + "source_video": source_video, + "frame_bgr": frame_bgr, + }) + + return sampled + + +def write_contact_sheet(output_path: Path, frames: list[np.ndarray], labels: list[str], overwrite: bool): + if output_path.exists() and not overwrite: + return + ensure_dir(output_path.parent) + + columns = 4 + rows = max(int(math.ceil(len(frames) / columns)), 1) + tile_height, tile_width = 256, 456 + header_height = 26 + sheet = np.full((rows * (tile_height + header_height), columns * tile_width, 3), 24, dtype=np.uint8) + + for index, (frame_bgr, label) in enumerate(zip(frames, labels, strict=False)): + row_index = index // columns + column_index = index % columns + y = row_index * (tile_height + header_height) + x = column_index * tile_width + + resized = cv2.resize(frame_bgr, (tile_width, tile_height), interpolation=cv2.INTER_AREA) + sheet[y + header_height:y + header_height + tile_height, x:x + tile_width] = resized + cv2.putText(sheet, label, (x + 8, y + 18), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (235, 235, 235), 1, cv2.LINE_AA) + + cv2.imwrite(str(output_path), sheet, [cv2.IMWRITE_JPEG_QUALITY, 88]) + + +def include_row(row: dict, mode: str): + hit = bool(row.get("hit")) + if mode == "all": + return "segment" in row + if mode == "hits": + return "segment" in row and hit + return "segment" in row and not hit + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + report_rows = json.loads(args.report_json.expanduser().resolve().read_text(encoding="utf-8")) + clip_root = args.clip_root.expanduser().resolve() + + frame_dir = ensure_dir(workspace / "review" / "leadins" / "frames") + contact_sheet_dir = ensure_dir(workspace / "review" / "leadins" / "contact_sheets") + manifest_path = workspace / "review" / "bookmark_leadins.csv" + write_csv_header(manifest_path, BOOKMARK_LEADIN_MANIFEST_FIELDS) + manifest_rows = load_existing_rows(manifest_path) + + imported_windows = 0 + imported_frames = 0 + for row in report_rows: + if not include_row(row, args.mode): + continue + + sampled_frames = extract_window_frames(row, clip_root, args.sample_every, args.max_samples) + if not sampled_frames: + continue + + imported_windows += 1 + window_result = "hit" if row.get("hit") else "miss" + session_id = row["sessionId"] + bookmark_number = int(row["bookmarkNumber"]) + route = row["route"] + segment = int(row["segment"]) + published_values = ",".join(str(value) for value in row.get("publishedValues", [])) + candidate_values = ",".join(str(value) for value in row.get("candidateValues", [])) + + contact_sheet_name = f"{session_id}_bookmark_{bookmark_number:03d}_{window_result}.jpg" + contact_sheet_path = contact_sheet_dir / contact_sheet_name + contact_sheet_labels = [] + contact_sheet_frames = [] + + for sample_index, sample in enumerate(sampled_frames, start=1): + frame_name = f"{session_id}_bookmark_{bookmark_number:03d}_sample_{sample_index:02d}.jpg" + frame_path = frame_dir / frame_name + if args.overwrite or not frame_path.exists(): + cv2.imwrite(str(frame_path), sample["frame_bgr"], [cv2.IMWRITE_JPEG_QUALITY, 90]) + + imported_frames += 1 + contact_sheet_frames.append(sample["frame_bgr"]) + contact_sheet_labels.append(f"t={sample['relative_offset_s']:+.2f}s") + + record_key = f"{session_id}:bookmark:{bookmark_number}:sample:{sample_index}" + manifest_rows[record_key] = { + "record_key": record_key, + "source_name": args.source_name, + "source_region": args.source_region, + "source_device": args.source_device, + "source_driver": args.source_driver, + "session_id": session_id, + "bookmark_number": str(bookmark_number), + "route": route, + "segment": str(segment), + "segment_offset_s": str(row["segmentOffsetS"]), + "leadin_start_s": str(row["leadinStartS"]), + "sample_offset_s": f"{sample['relative_offset_s']:.3f}", + "window_result": window_result, + "published_values": published_values, + "candidate_values": candidate_values, + "frame_path": str(frame_path.relative_to(workspace)), + "contact_sheet_path": str(contact_sheet_path.relative_to(workspace)), + "source_video_path": str(sample["source_video"]), + } + + write_contact_sheet(contact_sheet_path, contact_sheet_frames, contact_sheet_labels, args.overwrite) + + with manifest_path.open("w", encoding="utf-8", newline="") as manifest_file: + writer = csv.DictWriter(manifest_file, fieldnames=BOOKMARK_LEADIN_MANIFEST_FIELDS) + writer.writeheader() + for row in sorted(manifest_rows.values(), key=lambda entry: entry["record_key"]): + writer.writerow(row) + + print(f"Imported {imported_windows} lead-in window(s) and {imported_frames} sampled frame(s) into {manifest_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/import_debug_sessions.py b/scripts/speed_limit_vision/import_debug_sessions.py new file mode 100644 index 00000000..b890d235 --- /dev/null +++ b/scripts/speed_limit_vision/import_debug_sessions.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import shutil + +from pathlib import Path + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import ( # type: ignore + BOOKMARK_MANIFEST_FIELDS, + DEFAULT_DEBUG_BASE, + DEFAULT_WORKSPACE, + ensure_dir, + latest_debug_sessions, + read_jsonl, + resolve_workspace, + write_csv_header, + ) +else: + from .common import ( + BOOKMARK_MANIFEST_FIELDS, + DEFAULT_DEBUG_BASE, + DEFAULT_WORKSPACE, + ensure_dir, + latest_debug_sessions, + read_jsonl, + resolve_workspace, + write_csv_header, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import speed-limit debug sessions into a training workspace manifest.") + parser.add_argument("sessions", nargs="*", help="Session ids or full session paths. Defaults to the latest session under --debug-base.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--debug-base", type=Path, default=DEFAULT_DEBUG_BASE, help="Root directory containing vision debug sessions.") + parser.add_argument("--latest", type=int, default=1, help="How many latest sessions to import when no session ids are provided.") + parser.add_argument("--mode", choices=("symlink", "copy"), default="symlink", help="How to place snapshots into the workspace review/images directory.") + parser.add_argument("--force", action="store_true", help="Overwrite snapshot links/files if they already exist.") + parser.add_argument("--events", nargs="+", default=["bookmark", "auto_bookmark", "publish", "candidate"], help="Event types to include in the manifest.") + return parser.parse_args() + + +def resolve_sessions(session_args: list[str], debug_base: Path, latest_count: int) -> list[Path]: + if session_args: + resolved = [] + for session_arg in session_args: + candidate = Path(session_arg).expanduser() + if candidate.is_dir(): + resolved.append(candidate.resolve()) + continue + candidate = (debug_base / session_arg).resolve() + if candidate.is_dir(): + resolved.append(candidate) + continue + raise FileNotFoundError(f"Debug session not found: {session_arg}") + return resolved + + latest = latest_debug_sessions(debug_base, latest_count) + if not latest: + raise FileNotFoundError(f"No debug sessions found in {debug_base}") + return latest + + +def load_existing_rows(manifest_path: Path) -> dict[str, dict[str, str]]: + if not manifest_path.exists(): + return {} + + with manifest_path.open("r", encoding="utf-8", newline="") as manifest_file: + reader = csv.DictReader(manifest_file) + return {row["record_key"]: row for row in reader if row.get("record_key")} + + +def stage_snapshot(source_path: Path, dest_path: Path, mode: str, force: bool) -> None: + ensure_dir(dest_path.parent) + if dest_path.exists() or dest_path.is_symlink(): + if not force: + return + if dest_path.is_dir(): + shutil.rmtree(dest_path) + else: + dest_path.unlink() + + if mode == "copy": + shutil.copy2(source_path, dest_path) + else: + dest_path.symlink_to(source_path) + + +def event_row(event: dict, session_id: str, session_path: Path, event_index: int, snapshot_path: str) -> dict[str, str]: + return { + "record_key": f"{session_id}:{event_index}", + "session_id": session_id, + "event_index": str(event_index), + "event": str(event.get("event") or ""), + "session_seconds": str(event.get("sessionSeconds") or ""), + "wall_time": str(event.get("wallTime") or ""), + "road_name": str(event.get("roadName") or ""), + "stream": str(event.get("stream") or ""), + "status": str(event.get("status") or ""), + "candidate_speed_limit_mph": str(event.get("candidateSpeedLimitMph") or ""), + "candidate_confidence": str(event.get("candidateConfidence") or ""), + "speed_limit_mph": str(event.get("speedLimitMph") or ""), + "confidence": str(event.get("confidence") or ""), + "published_speed_limit_mph": str(event.get("publishedSpeedLimitMph") or ""), + "published_confidence": str(event.get("publishedConfidence") or ""), + "bookmark_count": str(event.get("bookmarkCount") or ""), + "snapshot_path": snapshot_path, + "source_session_path": str(session_path), + } + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + review_image_dir = ensure_dir(workspace / "review" / "images") + manifest_path = workspace / "review" / "bookmarks.csv" + write_csv_header(manifest_path, BOOKMARK_MANIFEST_FIELDS) + + existing_rows = load_existing_rows(manifest_path) + sessions = resolve_sessions(args.sessions, args.debug_base.expanduser().resolve(), args.latest) + selected_events = set(args.events) + + for session_path in sessions: + events_path = session_path / "events.jsonl" + if not events_path.is_file(): + continue + + session_id = session_path.name + for event_index, event in enumerate(read_jsonl(events_path)): + event_type = str(event.get("event") or "") + if event_type not in selected_events: + continue + + snapshot_rel_path = "" + snapshot_name = event.get("snapshot") + if snapshot_name: + source_snapshot = session_path / str(snapshot_name) + if source_snapshot.is_file(): + dest_name = f"{session_id}_{event_index:04d}_{event_type}{source_snapshot.suffix.lower()}" + dest_snapshot = review_image_dir / dest_name + stage_snapshot(source_snapshot, dest_snapshot, args.mode, args.force) + snapshot_rel_path = str(dest_snapshot.relative_to(workspace)) + + row = event_row(event, session_id, session_path, event_index, snapshot_rel_path) + existing_rows[row["record_key"]] = row + + with manifest_path.open("w", encoding="utf-8", newline="") as manifest_file: + writer = csv.DictWriter(manifest_file, fieldnames=BOOKMARK_MANIFEST_FIELDS) + writer.writeheader() + for row in sorted(existing_rows.values(), key=lambda entry: (entry["session_id"], int(entry["event_index"]))): + writer.writerow(row) + + print(f"Imported {len(sessions)} debug session(s) into {manifest_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/import_glare_images.py b/scripts/speed_limit_vision/import_glare_images.py new file mode 100644 index 00000000..3af63342 --- /dev/null +++ b/scripts/speed_limit_vision/import_glare_images.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import hashlib +import re + +from collections import defaultdict +from pathlib import Path + +import cv2 + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import ( # type: ignore + DEFAULT_WORKSPACE, + DETECTOR_CLASS_NAMES, + PUBLIC_CLASSIFIER_SAMPLE_FIELDS, + PUBLIC_DETECTOR_SAMPLE_FIELDS, + RAW_SOURCE_FIELDS, + VALUE_LABEL_FIELDS, + default_raw_root, + ensure_dir, + resolve_workspace, + ) +else: + from .common import ( + DEFAULT_WORKSPACE, + DETECTOR_CLASS_NAMES, + PUBLIC_CLASSIFIER_SAMPLE_FIELDS, + PUBLIC_DETECTOR_SAMPLE_FIELDS, + RAW_SOURCE_FIELDS, + VALUE_LABEL_FIELDS, + default_raw_root, + ensure_dir, + resolve_workspace, + ) + + +SOURCE_NAME = "glare_images" +SOURCE_VERSION = "GLARE Images" +SOURCE_LICENSE = "CC BY 4.0" + +SPEED_TAG_PATTERN = re.compile(r"(speedLimit|exitSpeedAdvisory|rampSpeedAdvisory)(\d+)$") +GLARE_IGNORE_TAGS = {"speedLimit55Ahead"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import GLARE image annotations into the speed-limit training workspace.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--images-root", type=Path, help="Path to the downloaded GLARE Images directory. Defaults to /glare_raw/Images.") + parser.add_argument("--train-split", type=float, default=0.85, help="Train split ratio by origin-track hash.") + parser.add_argument("--overwrite", action="store_true", help="Overwrite previously imported GLARE samples.") + return parser.parse_args() + + +def default_images_root(workspace: Path) -> Path: + return default_raw_root(workspace) / "glare_raw" / "Images" + + +def read_existing_rows(path: Path) -> list[dict[str, str]]: + if not path.is_file(): + return [] + with path.open("r", encoding="utf-8", newline="") as csv_file: + return list(csv.DictReader(csv_file)) + + +def write_rows(path: Path, fieldnames: list[str], rows: list[dict[str, str]]) -> None: + ensure_dir(path.parent) + with path.open("w", encoding="utf-8", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def infer_label(tag: str) -> tuple[str, int] | None: + if tag in GLARE_IGNORE_TAGS: + return None + match = SPEED_TAG_PATTERN.fullmatch(tag) + if not match: + return None + tag_type, speed_value_text = match.groups() + speed_value = int(speed_value_text) + if tag_type == "speedLimit": + return ("regulatory_speed_limit", speed_value) + return ("advisory_speed_limit", speed_value) + + +def split_for_track(track_name: str, train_ratio: float) -> str: + digest = hashlib.md5(track_name.encode("utf-8")).hexdigest() + value = int(digest[:8], 16) / 0xFFFFFFFF + return "train" if value < train_ratio else "val" + + +def yolo_box(image_width: int, image_height: int, xmin: int, ymin: int, xmax: int, ymax: int) -> tuple[float, float, float, float]: + box_width = max(xmax - xmin, 1) + box_height = max(ymax - ymin, 1) + x_center = xmin + box_width / 2.0 + y_center = ymin + box_height / 2.0 + return ( + x_center / image_width, + y_center / image_height, + box_width / image_width, + box_height / image_height, + ) + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + images_root = args.images_root.resolve() if args.images_root else default_images_root(workspace) + annotations_csv = images_root / "allAnnotations.csv" + if not annotations_csv.is_file(): + raise FileNotFoundError(f"GLARE allAnnotations.csv not found: {annotations_csv}") + + detector_manifest_path = workspace / "manifests" / "public_detector_samples.csv" + classifier_manifest_path = workspace / "manifests" / "public_classifier_samples.csv" + value_labels_path = workspace / "classifier" / "value_labels.csv" + raw_sources_path = workspace / "manifests" / "raw_sources.csv" + + existing_detector_rows = [row for row in read_existing_rows(detector_manifest_path) if row.get("source_name") != SOURCE_NAME] + existing_classifier_rows = [row for row in read_existing_rows(classifier_manifest_path) if row.get("source_name") != SOURCE_NAME] + existing_value_rows = [row for row in read_existing_rows(value_labels_path) if SOURCE_NAME not in (row.get("image_path") or "")] + existing_source_rows = [row for row in read_existing_rows(raw_sources_path) if row.get("source_name") != SOURCE_NAME] + + grouped: dict[str, list[dict[str, str]]] = defaultdict(list) + with annotations_csv.open("r", encoding="utf-8", newline="") as csv_file: + reader = csv.DictReader(csv_file) + for row in reader: + tag = (row.get("Annotation tag") or "").strip() + if infer_label(tag) is None: + continue + filename = (row.get("Filename") or "").strip() + if filename: + grouped[filename].append(row) + + detector_rows: list[dict[str, str]] = [] + classifier_rows: list[dict[str, str]] = [] + value_rows: list[dict[str, str]] = [] + class_counts: dict[str, int] = defaultdict(int) + imported_images = 0 + imported_boxes = 0 + + for filename in sorted(grouped): + source_image = images_root / filename + if not source_image.is_file(): + continue + + box_rows = grouped[filename] + track_name = (box_rows[0].get("Origin track") or filename).strip() + split = split_for_track(track_name, args.train_split) + stem = Path(filename).stem + image_out = workspace / "detector" / "images" / split / f"{SOURCE_NAME}_{stem}.png" + label_out = workspace / "detector" / "labels" / split / f"{SOURCE_NAME}_{stem}.txt" + image_bgr = cv2.imread(str(source_image)) + if image_bgr is None: + continue + image_height, image_width = image_bgr.shape[:2] + + if args.overwrite or not image_out.exists(): + ensure_dir(image_out.parent) + image_out.write_bytes(source_image.read_bytes()) + + yolo_lines: list[str] = [] + for bbox_index, row in enumerate(box_rows): + tag = row["Annotation tag"].strip() + inferred = infer_label(tag) + if inferred is None: + continue + class_name, speed_value = inferred + class_id = DETECTOR_CLASS_NAMES.index(class_name) + xmin = int(float(row["Upper left corner X"])) + ymin = int(float(row["Upper left corner Y"])) + xmax = int(float(row["Lower right corner X"])) + ymax = int(float(row["Lower right corner Y"])) + x_center, y_center, width, height = yolo_box(image_width, image_height, xmin, ymin, xmax, ymax) + yolo_lines.append(f"{class_id} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}") + + record_key = f"{SOURCE_NAME}:{stem}:{bbox_index}" + detector_rows.append({ + "record_key": record_key, + "source_name": SOURCE_NAME, + "split": split, + "image_path": str(image_out.relative_to(workspace)), + "label_path": str(label_out.relative_to(workspace)), + "annotation_path": str(annotations_csv), + "source_image_id": filename, + "class_name": class_name, + "speed_limit_mph": str(speed_value), + "sign_code": tag, + "bbox_left": str(xmin), + "bbox_top": str(ymin), + "bbox_right": str(xmax), + "bbox_bottom": str(ymax), + }) + classifier_rows.append({ + "record_key": record_key, + "source_name": SOURCE_NAME, + "split": split, + "image_path": str(image_out.relative_to(workspace)), + "speed_limit_mph": str(speed_value), + "bbox_index": str(bbox_index), + "label_path": str(label_out.relative_to(workspace)), + "source_image_id": filename, + "sign_code": tag, + }) + value_rows.append({ + "image_path": str(image_out.relative_to(workspace)), + "split": split, + "speed_limit_mph": str(speed_value), + "bbox_index": str(bbox_index), + "padding": "0.10", + "label_path": str(label_out.relative_to(workspace)), + }) + class_counts[f"{class_name}:{speed_value}"] += 1 + imported_boxes += 1 + + if yolo_lines and (args.overwrite or not label_out.exists()): + ensure_dir(label_out.parent) + label_out.write_text("\n".join(yolo_lines) + "\n", encoding="utf-8") + imported_images += 1 + + source_row = { + "source_name": SOURCE_NAME, + "source_version": SOURCE_VERSION, + "source_license": SOURCE_LICENSE, + "source_type": "public_detector_and_classifier_seed", + "raw_path": str(images_root), + "notes": "Imported GLARE Images/allAnnotations.csv speed-limit and advisory-speed tags.", + } + + write_rows(raw_sources_path, RAW_SOURCE_FIELDS, existing_source_rows + [source_row]) + write_rows(detector_manifest_path, PUBLIC_DETECTOR_SAMPLE_FIELDS, existing_detector_rows + detector_rows) + write_rows(classifier_manifest_path, PUBLIC_CLASSIFIER_SAMPLE_FIELDS, existing_classifier_rows + classifier_rows) + write_rows(value_labels_path, VALUE_LABEL_FIELDS, existing_value_rows + value_rows) + + summary = ", ".join(f"{name}={count}" for name, count in sorted(class_counts.items())) + print(f"Imported {imported_images} GLARE image(s) and {imported_boxes} box(es) from {images_root}") + print(f" detector manifest: {detector_manifest_path}") + print(f" classifier manifest: {classifier_manifest_path}") + print(f" class counts: {summary or 'none'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/import_lisa_zip.py b/scripts/speed_limit_vision/import_lisa_zip.py new file mode 100644 index 00000000..4b8662c9 --- /dev/null +++ b/scripts/speed_limit_vision/import_lisa_zip.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import hashlib +import io + +from collections import defaultdict +from pathlib import Path +from zipfile import ZipFile + +import cv2 +import numpy as np + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import ( # type: ignore + DEFAULT_WORKSPACE, + DETECTOR_CLASS_NAMES, + PUBLIC_CLASSIFIER_SAMPLE_FIELDS, + PUBLIC_DETECTOR_SAMPLE_FIELDS, + RAW_SOURCE_FIELDS, + VALUE_LABEL_FIELDS, + default_raw_root, + ensure_dir, + resolve_workspace, + ) +else: + from .common import ( + DEFAULT_WORKSPACE, + DETECTOR_CLASS_NAMES, + PUBLIC_CLASSIFIER_SAMPLE_FIELDS, + PUBLIC_DETECTOR_SAMPLE_FIELDS, + RAW_SOURCE_FIELDS, + VALUE_LABEL_FIELDS, + default_raw_root, + ensure_dir, + resolve_workspace, + ) + + +SOURCE_NAME = "lisa_traffic_sign" +SOURCE_VERSION = "Kaggle omkarnadkarni/lisa-traffic-sign" +SOURCE_LICENSE = "See Kaggle dataset page" +DEFAULT_ZIP_RELATIVE = Path("lisa/lisa_traffic_sign.zip") + +LISA_LABEL_MAP: dict[str, tuple[str, int | None] | None] = { + "speedLimit15": ("regulatory_speed_limit", 15), + "speedLimit25": ("regulatory_speed_limit", 25), + "speedLimit30": ("regulatory_speed_limit", 30), + "speedLimit35": ("regulatory_speed_limit", 35), + "speedLimit40": ("regulatory_speed_limit", 40), + "speedLimit45": ("regulatory_speed_limit", 45), + "speedLimit50": ("regulatory_speed_limit", 50), + "speedLimit55": ("regulatory_speed_limit", 55), + "speedLimit65": ("regulatory_speed_limit", 65), + "speedLimitUrdbl": ("regulatory_speed_limit", None), + "schoolSpeedLimit25": ("school_zone_speed_limit", 25), + "rampSpeedAdvisory20": ("advisory_speed_limit", 20), + "rampSpeedAdvisory35": ("advisory_speed_limit", 35), + "rampSpeedAdvisory40": ("advisory_speed_limit", 40), + "rampSpeedAdvisory45": ("advisory_speed_limit", 45), + "rampSpeedAdvisory50": ("advisory_speed_limit", 50), + "rampSpeedAdvisoryUrdbl": ("advisory_speed_limit", None), + "truckSpeedLimit55": None, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import speed-related LISA samples from the Kaggle ZIP into the training workspace.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--zip-path", type=Path, help="Path to lisa_traffic_sign.zip. Defaults to the SSD raw-data layout.") + parser.add_argument("--train-split", type=float, default=0.85, help="Train split ratio by origin-track hash.") + parser.add_argument("--overwrite", action="store_true", help="Overwrite previously imported LISA samples.") + return parser.parse_args() + + +def default_zip_path(workspace: Path) -> Path: + return default_raw_root(workspace) / DEFAULT_ZIP_RELATIVE + + +def read_existing_rows(path: Path) -> list[dict[str, str]]: + if not path.is_file(): + return [] + with path.open("r", encoding="utf-8", newline="") as csv_file: + return list(csv.DictReader(csv_file)) + + +def write_rows(path: Path, fieldnames: list[str], rows: list[dict[str, str]]) -> None: + ensure_dir(path.parent) + with path.open("w", encoding="utf-8", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def split_for_track(track_name: str, train_ratio: float) -> str: + digest = hashlib.md5(track_name.encode("utf-8")).hexdigest() + value = int(digest[:8], 16) / 0xFFFFFFFF + return "train" if value < train_ratio else "val" + + +def yolo_box(image_width: int, image_height: int, xmin: int, ymin: int, xmax: int, ymax: int) -> tuple[float, float, float, float]: + box_width = max(xmax - xmin, 1) + box_height = max(ymax - ymin, 1) + x_center = xmin + box_width / 2.0 + y_center = ymin + box_height / 2.0 + return ( + x_center / image_width, + y_center / image_height, + box_width / image_width, + box_height / image_height, + ) + + +def parse_lisa_csv(csv_text: str) -> dict[str, list[dict[str, str]]]: + grouped: dict[str, list[dict[str, str]]] = defaultdict(list) + reader = csv.DictReader(io.StringIO(csv_text), delimiter=";") + for row in reader: + tag = (row.get("Annotation tag") or "").strip() + mapped = LISA_LABEL_MAP.get(tag) + if mapped is None: + continue + filename = (row.get("Filename") or "").strip() + if filename: + grouped[filename].append(row) + return grouped + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + zip_path = args.zip_path.resolve() if args.zip_path else default_zip_path(workspace) + if not zip_path.is_file(): + raise FileNotFoundError(f"LISA ZIP not found: {zip_path}") + + detector_manifest_path = workspace / "manifests" / "public_detector_samples.csv" + classifier_manifest_path = workspace / "manifests" / "public_classifier_samples.csv" + value_labels_path = workspace / "classifier" / "value_labels.csv" + raw_sources_path = workspace / "manifests" / "raw_sources.csv" + + existing_detector_rows = [row for row in read_existing_rows(detector_manifest_path) if row.get("source_name") != SOURCE_NAME] + existing_classifier_rows = [row for row in read_existing_rows(classifier_manifest_path) if row.get("source_name") != SOURCE_NAME] + existing_value_rows = [row for row in read_existing_rows(value_labels_path) if SOURCE_NAME not in (row.get("image_path") or "")] + existing_source_rows = [row for row in read_existing_rows(raw_sources_path) if row.get("source_name") != SOURCE_NAME] + + detector_rows: list[dict[str, str]] = [] + classifier_rows: list[dict[str, str]] = [] + value_rows: list[dict[str, str]] = [] + class_counts: dict[str, int] = defaultdict(int) + imported_images = 0 + imported_boxes = 0 + + with ZipFile(zip_path) as zip_file: + csv_members = sorted(name for name in zip_file.namelist() if name.endswith("frameAnnotations.csv")) + for csv_member in csv_members: + csv_text = zip_file.read(csv_member).decode("utf-8", "ignore") + grouped = parse_lisa_csv(csv_text) + csv_parent = Path(csv_member).parent + drive_slug = Path(csv_member).parts[0] + + for filename in sorted(grouped): + source_image_member = str(csv_parent / filename) + try: + image_bytes = zip_file.read(source_image_member) + except KeyError: + continue + + image_array = cv2.imdecode(np.frombuffer(image_bytes, dtype=np.uint8), cv2.IMREAD_COLOR) + if image_array is None: + continue + image_height, image_width = image_array.shape[:2] + box_rows = grouped[filename] + track_name = (box_rows[0].get("Origin track") or filename).strip() + split = split_for_track(f"{drive_slug}:{track_name}", args.train_split) + stem = Path(filename).stem + out_stem = f"{SOURCE_NAME}_{drive_slug}_{stem}" + image_out = workspace / "detector" / "images" / split / f"{out_stem}.png" + label_out = workspace / "detector" / "labels" / split / f"{out_stem}.txt" + + if args.overwrite or not image_out.exists(): + ensure_dir(image_out.parent) + image_out.write_bytes(image_bytes) + + yolo_lines: list[str] = [] + valid_boxes = 0 + for bbox_index, row in enumerate(box_rows): + tag = row["Annotation tag"].strip() + mapped = LISA_LABEL_MAP.get(tag) + if mapped is None: + continue + class_name, speed_value = mapped + class_id = DETECTOR_CLASS_NAMES.index(class_name) + xmin = int(float(row["Upper left corner X"])) + ymin = int(float(row["Upper left corner Y"])) + xmax = int(float(row["Lower right corner X"])) + ymax = int(float(row["Lower right corner Y"])) + x_center, y_center, width, height = yolo_box(image_width, image_height, xmin, ymin, xmax, ymax) + yolo_lines.append(f"{class_id} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}") + + record_key = f"{SOURCE_NAME}:{drive_slug}:{stem}:{bbox_index}" + detector_rows.append({ + "record_key": record_key, + "source_name": SOURCE_NAME, + "split": split, + "image_path": str(image_out.relative_to(workspace)), + "label_path": str(label_out.relative_to(workspace)), + "annotation_path": f"{zip_path}:{csv_member}", + "source_image_id": f"{drive_slug}/{filename}", + "class_name": class_name, + "speed_limit_mph": "" if speed_value is None else str(speed_value), + "sign_code": tag, + "bbox_left": str(xmin), + "bbox_top": str(ymin), + "bbox_right": str(xmax), + "bbox_bottom": str(ymax), + }) + if speed_value is not None: + classifier_rows.append({ + "record_key": record_key, + "source_name": SOURCE_NAME, + "split": split, + "image_path": str(image_out.relative_to(workspace)), + "speed_limit_mph": str(speed_value), + "bbox_index": str(bbox_index), + "label_path": str(label_out.relative_to(workspace)), + "source_image_id": f"{drive_slug}/{filename}", + "sign_code": tag, + }) + value_rows.append({ + "image_path": str(image_out.relative_to(workspace)), + "split": split, + "speed_limit_mph": str(speed_value), + "bbox_index": str(bbox_index), + "padding": "0.10", + "label_path": str(label_out.relative_to(workspace)), + }) + class_counts[f"{class_name}:{'' if speed_value is None else speed_value}"] += 1 + imported_boxes += 1 + valid_boxes += 1 + + if valid_boxes and (args.overwrite or not label_out.exists()): + ensure_dir(label_out.parent) + label_out.write_text("\n".join(yolo_lines) + "\n", encoding="utf-8") + imported_images += 1 + + source_row = { + "source_name": SOURCE_NAME, + "source_version": SOURCE_VERSION, + "source_license": SOURCE_LICENSE, + "source_type": "public_detector_and_classifier_seed", + "raw_path": str(zip_path), + "notes": "Imported speed-related LISA samples directly from the Kaggle ZIP. Ignores truckSpeedLimit55.", + } + + write_rows(raw_sources_path, RAW_SOURCE_FIELDS, existing_source_rows + [source_row]) + write_rows(detector_manifest_path, PUBLIC_DETECTOR_SAMPLE_FIELDS, existing_detector_rows + detector_rows) + write_rows(classifier_manifest_path, PUBLIC_CLASSIFIER_SAMPLE_FIELDS, existing_classifier_rows + classifier_rows) + write_rows(value_labels_path, VALUE_LABEL_FIELDS, existing_value_rows + value_rows) + + summary = ", ".join(f"{name}={count}" for name, count in sorted(class_counts.items())) + print(f"Imported {imported_images} LISA image(s) and {imported_boxes} box(es) from {zip_path}") + print(f" detector manifest: {detector_manifest_path}") + print(f" classifier manifest: {classifier_manifest_path}") + print(f" class counts: {summary or 'none'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/import_localized_bookmark_detector_examples.py b/scripts/speed_limit_vision/import_localized_bookmark_detector_examples.py new file mode 100644 index 00000000..c7be60c1 --- /dev/null +++ b/scripts/speed_limit_vision/import_localized_bookmark_detector_examples.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import random + +from pathlib import Path + +import cv2 + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore +else: + from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + + +LOCALIZED_MANIFEST = Path(".tmp/bookmark_sign_localization/localized_bookmarks.csv") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import high-confidence localized bookmark sign frames into the detector dataset.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--localized-manifest", type=Path, default=LOCALIZED_MANIFEST, help="CSV output from localize_bookmark_signs.py.") + parser.add_argument("--manifest-out", type=Path, help="Optional output CSV manifest path.") + parser.add_argument("--split", choices=("train", "val"), default="train", help="Target detector split.") + parser.add_argument("--source-name", default="comma_bookmark_localized", help="Logical source name for imported localized bookmarks.") + parser.add_argument("--source-region", default="", help="Optional region or market label, e.g. us_midwest.") + parser.add_argument("--source-device", default="", help="Optional device identifier or platform label.") + parser.add_argument("--source-driver", default="", help="Optional contributor/driver identifier.") + parser.add_argument("--session-id", action="append", default=[], help="Optional session_id filter. May be specified multiple times.") + parser.add_argument("--min-score", type=float, default=1.2, help="Minimum localization score to keep.") + parser.add_argument("--min-width", type=int, default=25, help="Minimum bbox width in pixels.") + parser.add_argument("--min-height", type=int, default=40, help="Minimum bbox height in pixels.") + parser.add_argument("--require-consistent", type=int, default=2, help="Require this many matching non-empty value reads across model/ocr/full_detection.") + parser.add_argument("--variants-per-example", type=int, default=6, help="Photometric variants to generate per localized frame.") + parser.add_argument("--temporal-radius-s", type=float, default=0.0, help="Optional radius in seconds to sample neighboring source-video frames around each localized hit.") + parser.add_argument("--temporal-step-s", type=float, default=0.15, help="Seconds between temporal neighbor samples when --temporal-radius-s is enabled.") + parser.add_argument("--temporal-box-expand", type=float, default=0.14, help="Fractional box expansion to apply to temporal neighbor labels.") + parser.add_argument("--seed", type=int, default=20260331, help="Random seed.") + parser.add_argument("--jpeg-quality", type=int, default=95, help="JPEG quality for written detector images.") + return parser.parse_args() + + +def detector_label_line(detector_class: int, x1: int, y1: int, x2: int, y2: int, image_shape: tuple[int, int, int]) -> str: + image_h, image_w = image_shape[:2] + x_center = ((x1 + x2) / 2) / image_w + y_center = ((y1 + y2) / 2) / image_h + width = (x2 - x1) / image_w + height = (y2 - y1) / image_h + return f"{detector_class} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}\n" + + +def remove_appledouble_files(root: Path) -> None: + for path in root.rglob("._*"): + if path.is_file() or path.is_symlink(): + path.unlink() + + +def dominant_read(row: dict[str, str]) -> tuple[str, int]: + counts: dict[str, int] = {} + for key in ("model_read", "ocr_read", "full_detection"): + raw = row.get(key, "") + value = raw.split("@", 1)[0].strip() if raw else "" + if value: + counts[value] = counts.get(value, 0) + 1 + if not counts: + return "", 0 + best_value, best_count = max(counts.items(), key=lambda item: (item[1], item[0])) + return best_value, best_count + + +def read_bbox(text: str) -> tuple[int, int, int, int]: + x1, y1, x2, y2 = (int(part) for part in text.split(",")) + return x1, y1, x2, y2 + + +def clamp_bbox(x1: int, y1: int, x2: int, y2: int, image_shape: tuple[int, int, int]) -> tuple[int, int, int, int]: + image_h, image_w = image_shape[:2] + x1 = max(0, min(x1, image_w - 2)) + y1 = max(0, min(y1, image_h - 2)) + x2 = max(x1 + 1, min(x2, image_w - 1)) + y2 = max(y1 + 1, min(y2, image_h - 1)) + return x1, y1, x2, y2 + + +def expand_bbox(x1: int, y1: int, x2: int, y2: int, image_shape: tuple[int, int, int], expand_ratio: float) -> tuple[int, int, int, int]: + width = x2 - x1 + height = y2 - y1 + x_pad = int(round(width * max(expand_ratio, 0.0) * 0.5)) + y_pad = int(round(height * max(expand_ratio, 0.0) * 0.5)) + return clamp_bbox(x1 - x_pad, y1 - y_pad, x2 + x_pad, y2 + y_pad, image_shape) + + +def read_frame_at(video_path: Path, target_time_s: float): + capture = cv2.VideoCapture(str(video_path)) + fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + frame_index = max(int(round(target_time_s * fps)), 0) + capture.set(cv2.CAP_PROP_POS_FRAMES, frame_index) + ok, frame_bgr = capture.read() + capture.release() + if not ok or frame_bgr is None: + return None + return frame_bgr + + +def apply_variant(frame_bgr, variant_index: int, rng: random.Random): + output = frame_bgr.copy() + mode = variant_index % 6 + if mode == 0: + alpha = rng.uniform(0.82, 0.94) + beta = rng.randint(-18, 8) + output = cv2.convertScaleAbs(output, alpha=alpha, beta=beta) + elif mode == 1: + alpha = rng.uniform(1.04, 1.14) + beta = rng.randint(-8, 18) + output = cv2.convertScaleAbs(output, alpha=alpha, beta=beta) + elif mode == 2: + output = cv2.GaussianBlur(output, (3, 3), rng.uniform(0.2, 0.8)) + elif mode == 3: + noise = rng.normalvariate(0.0, 8.0) + output = cv2.add(output, noise) + elif mode == 4: + hsv = cv2.cvtColor(output, cv2.COLOR_BGR2HSV) + hsv[..., 2] = cv2.convertScaleAbs(hsv[..., 2], alpha=rng.uniform(0.78, 0.9), beta=0) + output = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) + else: + encode_params = [cv2.IMWRITE_JPEG_QUALITY, rng.randint(72, 88)] + ok, encoded = cv2.imencode(".jpg", output, encode_params) + if ok: + output = cv2.imdecode(encoded, cv2.IMREAD_COLOR) + return output + + +def iter_temporal_offsets(radius_s: float, step_s: float) -> list[float]: + if radius_s <= 0.0 or step_s <= 0.0: + return [] + steps = int(radius_s / step_s) + offsets: list[float] = [] + for step in range(1, steps + 1): + delta = round(step * step_s, 3) + offsets.extend((-delta, delta)) + return sorted(offsets) + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + localized_manifest = args.localized_manifest.expanduser().resolve() + if not localized_manifest.is_file(): + raise FileNotFoundError(localized_manifest) + session_filter = set(args.session_id) + + image_dir = ensure_dir(workspace / "detector" / "images" / args.split) + label_dir = ensure_dir(workspace / "detector" / "labels" / args.split) + manifest_out = args.manifest_out.expanduser().resolve() if args.manifest_out else (ensure_dir(workspace / "review") / "localized_bookmark_detector_examples.csv") + rng = random.Random(args.seed) + temporal_offsets = iter_temporal_offsets(args.temporal_radius_s, args.temporal_step_s) + + records: list[dict[str, object]] = [] + kept = 0 + + with localized_manifest.open("r", encoding="utf-8", newline="") as csv_file: + reader = csv.DictReader(csv_file) + for row in reader: + if session_filter and row.get("session_id") not in session_filter: + continue + dominant_value, consistent_count = dominant_read(row) + if consistent_count < max(args.require_consistent, 0): + continue + if row.get("is_regulatory", "").lower() != "true": + continue + score = float(row["score"]) + if score < args.min_score: + continue + x1, y1, x2, y2 = read_bbox(row["box"]) + width = x2 - x1 + height = y2 - y1 + if width < args.min_width or height < args.min_height: + continue + + frame_path = Path(row["frame_path"]) + frame_bgr = cv2.imread(str(frame_path)) + if frame_bgr is None: + continue + + class_id = int(row["class_id"]) + stem_base = f"real_localized_{row['session_id']}_{int(row['bookmark_number']):03d}" + source_video_path = Path(row["source_video_path"]) if row.get("source_video_path") else None + relative_time_s = float(row["relative_time_s"]) if row.get("relative_time_s") else 0.0 + + source_frames = [("base", frame_bgr, (x1, y1, x2, y2))] + if source_video_path and source_video_path.is_file(): + for offset_s in temporal_offsets: + temporal_frame = read_frame_at(source_video_path, max(relative_time_s + offset_s, 0.0)) + if temporal_frame is None: + continue + temporal_bbox = expand_bbox(x1, y1, x2, y2, temporal_frame.shape, args.temporal_box_expand) + source_frames.append((f"t{offset_s:+.2f}s".replace(".", "p"), temporal_frame, temporal_bbox)) + + variants = [] + for source_suffix, source_bgr, source_bbox in source_frames: + variants.append((source_suffix, source_bgr, source_bbox)) + for variant_index in range(max(args.variants_per_example, 0)): + variants.append((f"{source_suffix}_var{variant_index:02d}", apply_variant(source_bgr, variant_index, rng), source_bbox)) + + for suffix, variant_bgr, variant_bbox in variants: + image_path = image_dir / f"{stem_base}_{suffix}.jpg" + label_path = label_dir / f"{stem_base}_{suffix}.txt" + cv2.imwrite(str(image_path), variant_bgr, [cv2.IMWRITE_JPEG_QUALITY, args.jpeg_quality]) + vx1, vy1, vx2, vy2 = variant_bbox + label_path.write_text(detector_label_line(class_id, vx1, vy1, vx2, vy2, variant_bgr.shape), encoding="utf-8") + records.append({ + "record_key": f"{row['session_id']}_{row['bookmark_number']}_{suffix}", + "split": args.split, + "source_name": row.get("source_name", args.source_name), + "source_region": row.get("source_region", args.source_region), + "source_device": row.get("source_device", args.source_device), + "source_driver": row.get("source_driver", args.source_driver), + "session_id": row["session_id"], + "bookmark_number": row["bookmark_number"], + "route": row["route"], + "segment": row["segment"], + "relative_time_s": row["relative_time_s"], + "score": row["score"], + "class_id": class_id, + "dominant_value": dominant_value, + "consistent_count": consistent_count, + "bbox_x1": vx1, + "bbox_y1": vy1, + "bbox_x2": vx2, + "bbox_y2": vy2, + "source_frame": str(frame_path), + "source_crop": row["crop_path"], + "source_video": str(source_video_path) if source_video_path else "", + "source_relative_time_s": relative_time_s, + "dataset_image": str(image_path), + "dataset_label": str(label_path), + }) + kept += 1 + + remove_appledouble_files(image_dir) + remove_appledouble_files(label_dir) + + fieldnames = [ + "record_key", + "split", + "source_name", + "source_region", + "source_device", + "source_driver", + "session_id", + "bookmark_number", + "route", + "segment", + "relative_time_s", + "score", + "class_id", + "dominant_value", + "consistent_count", + "bbox_x1", + "bbox_y1", + "bbox_x2", + "bbox_y2", + "source_frame", + "source_crop", + "source_video", + "source_relative_time_s", + "dataset_image", + "dataset_label", + ] + with manifest_out.open("w", encoding="utf-8", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(records) + + print(f"Imported {kept} localized bookmark example(s)") + print(f"Wrote {len(records)} detector image(s) into {args.split}") + print(f"Manifest: {manifest_out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/init_workspace.py b/scripts/speed_limit_vision/init_workspace.py new file mode 100644 index 00000000..33218cf3 --- /dev/null +++ b/scripts/speed_limit_vision/init_workspace.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse + +from pathlib import Path + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import ( # type: ignore + BOOKMARK_MANIFEST_FIELDS, + BOOKMARK_LEADIN_MANIFEST_FIELDS, + DEFAULT_SPEED_VALUES, + VALUE_LABEL_FIELDS, + DEFAULT_WORKSPACE, + PUBLIC_CLASSIFIER_SAMPLE_FIELDS, + PUBLIC_DETECTOR_SAMPLE_FIELDS, + RAW_SOURCE_FIELDS, + detector_dataset_yaml, + ensure_dir, + resolve_workspace, + workspace_readme, + write_csv_header, + write_text, + ) +else: + from .common import ( + BOOKMARK_MANIFEST_FIELDS, + BOOKMARK_LEADIN_MANIFEST_FIELDS, + DEFAULT_SPEED_VALUES, + VALUE_LABEL_FIELDS, + DEFAULT_WORKSPACE, + PUBLIC_CLASSIFIER_SAMPLE_FIELDS, + PUBLIC_DETECTOR_SAMPLE_FIELDS, + RAW_SOURCE_FIELDS, + detector_dataset_yaml, + ensure_dir, + resolve_workspace, + workspace_readme, + write_csv_header, + write_text, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Initialize a speed-limit detector/classifier training workspace.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Workspace root. Defaults to .tmp/speed_limit_training under the repo root.") + parser.add_argument("--force", action="store_true", help="Overwrite generated template files if they already exist.") + parser.add_argument("--speed-values", nargs="+", type=int, default=list(DEFAULT_SPEED_VALUES), help="Classifier speed values to document in the workspace README.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + + for relative_dir in ( + "detector/images/train", + "detector/images/val", + "detector/labels/train", + "detector/labels/val", + "classifier/train", + "classifier/val", + "review/images", + "review/leadins/frames", + "review/leadins/contact_sheets", + "manifests", + "raw", + "staging", + "exports", + "runs", + ): + ensure_dir(workspace / relative_dir) + + write_text(workspace / "detector" / "dataset.yaml", detector_dataset_yaml(workspace), force=args.force) + write_text(workspace / "README.md", workspace_readme(tuple(args.speed_values)), force=args.force) + write_csv_header(workspace / "review" / "bookmarks.csv", BOOKMARK_MANIFEST_FIELDS, force=args.force) + write_csv_header(workspace / "review" / "bookmark_leadins.csv", BOOKMARK_LEADIN_MANIFEST_FIELDS, force=args.force) + write_csv_header(workspace / "classifier" / "value_labels.csv", VALUE_LABEL_FIELDS, force=args.force) + write_csv_header(workspace / "manifests" / "raw_sources.csv", RAW_SOURCE_FIELDS, force=args.force) + write_csv_header(workspace / "manifests" / "public_detector_samples.csv", PUBLIC_DETECTOR_SAMPLE_FIELDS, force=args.force) + write_csv_header(workspace / "manifests" / "public_classifier_samples.csv", PUBLIC_CLASSIFIER_SAMPLE_FIELDS, force=args.force) + + print(f"Initialized speed-limit training workspace at {workspace}") + print(f" detector dataset: {workspace / 'detector/dataset.yaml'}") + print(f" review manifest: {workspace / 'review/bookmarks.csv'}") + print(f" lead-in manifest: {workspace / 'review/bookmark_leadins.csv'}") + print(f" value labels: {workspace / 'classifier/value_labels.csv'}") + print(f" raw sources: {workspace / 'manifests/raw_sources.csv'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/install_models.py b/scripts/speed_limit_vision/install_models.py new file mode 100644 index 00000000..1eb21289 --- /dev/null +++ b/scripts/speed_limit_vision/install_models.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import subprocess + +from pathlib import Path + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import ( # type: ignore + CLASSIFIER_EXPORT_NAME, + DEFAULT_WORKSPACE, + DETECTOR_EXPORT_NAME, + resolve_workspace, + ) +else: + from .common import ( + CLASSIFIER_EXPORT_NAME, + DEFAULT_WORKSPACE, + DETECTOR_EXPORT_NAME, + resolve_workspace, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Copy exported speed-limit ONNX models to a comma device.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--detector", type=Path, help="Detector ONNX path. Defaults to /exports/speed_limit_us_detector.onnx.") + parser.add_argument("--classifier", type=Path, help="Classifier ONNX path. Defaults to /exports/speed_limit_us_value_classifier.onnx.") + parser.add_argument("--host", default="comma@192.168.3.110", help="scp target host.") + parser.add_argument("--remote-dir", default="/data/openpilot/starpilot/assets/vision_models", help="Remote vision model directory.") + return parser.parse_args() + + +def scp_file(local_path: Path, host: str, remote_dir: str) -> None: + subprocess.run(["scp", str(local_path), f"{host}:{remote_dir}/{local_path.name}"], check=True) + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + + detector_path = args.detector.resolve() if args.detector else (workspace / "exports" / DETECTOR_EXPORT_NAME) + classifier_path = args.classifier.resolve() if args.classifier else (workspace / "exports" / CLASSIFIER_EXPORT_NAME) + + copied = 0 + for local_path in (detector_path, classifier_path): + if not local_path.is_file(): + continue + scp_file(local_path, args.host, args.remote_dir) + print(f"Copied {local_path.name} to {args.host}:{args.remote_dir}") + copied += 1 + + if copied == 0: + raise SystemExit("No ONNX models found to copy") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/localize_bookmark_signs.py b/scripts/speed_limit_vision/localize_bookmark_signs.py new file mode 100644 index 00000000..18cc79fb --- /dev/null +++ b/scripts/speed_limit_vision/localize_bookmark_signs.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv + +from pathlib import Path + +import cv2 + +import starpilot.system.speed_limit_vision as slv + +from scripts.speed_limit_vision import common +from scripts.speed_limit_vision import evaluate_bookmark_leadins as ebl + + +DEFAULT_OUTPUT_DIR = Path(".tmp/bookmark_sign_localization") + + +def parse_args(): + parser = argparse.ArgumentParser(description="Search around bookmarks for the most sign-like speed-limit frame and crop.") + parser.add_argument("--clip-root", type=Path, default=ebl.DEFAULT_CLIP_ROOT, help="Copied route clip root.") + parser.add_argument("--qlog-mtimes", type=Path, default=ebl.DEFAULT_QLOG_MTIMES, help="Text file with ' ' lines.") + parser.add_argument("--session-root", type=Path, default=ebl.DEFAULT_SESSION_ROOT, help="Directory containing debug session folders.") + parser.add_argument("--session-route-map", type=Path, default=common.preferred_session_route_map_path(), help="JSON file mapping debug session ids to route log ids.") + parser.add_argument("--models-dir", type=Path, help="Directory containing speed_limit_us_detector.onnx and speed_limit_us_value_classifier.onnx.") + parser.add_argument("--search-before", type=float, default=18.0, help="Seconds before the bookmark to scan.") + parser.add_argument("--search-after", type=float, default=2.0, help="Seconds after the bookmark to scan.") + parser.add_argument("--sample-every", type=float, default=0.5, help="Seconds between sampled frames.") + parser.add_argument("--top-k", type=int, default=1, help="How many candidate frames to save per bookmark.") + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR, help="Output directory for frames/crops/manifest.") + return parser.parse_args() + + +def configure_models(models_dir: Path | None): + if not models_dir: + return + models_dir = models_dir.expanduser().resolve() + detector_path = models_dir / "speed_limit_us_detector.onnx" + classifier_path = models_dir / "speed_limit_us_value_classifier.onnx" + if not detector_path.is_file(): + raise FileNotFoundError(detector_path) + if not classifier_path.is_file(): + raise FileNotFoundError(classifier_path) + slv.US_DETECTOR_MODEL_PATH = detector_path + slv.US_CLASSIFIER_MODEL_PATH = classifier_path + + +def iter_video_samples(clip_path: Path, start_s: float, end_s: float, sample_every: float): + capture = cv2.VideoCapture(str(clip_path)) + fps = capture.get(cv2.CAP_PROP_FPS) or 20.0 + start_frame = max(int(start_s * fps), 0) + end_frame = max(int(end_s * fps), start_frame) + + frame_index = 0 + while frame_index < start_frame: + ok, _ = capture.read() + if not ok: + capture.release() + return + frame_index += 1 + + next_sample_s = start_s + while frame_index <= end_frame: + ok, frame_bgr = capture.read() + if not ok or frame_bgr is None: + break + + frame_time_s = frame_index / fps + if frame_time_s + 1e-6 >= next_sample_s: + yield frame_time_s, frame_bgr + next_sample_s += sample_every + + frame_index += 1 + + capture.release() + + +def iter_context_frames(clip_root: Path, window: ebl.BookmarkWindow, search_before: float, search_after: float, sample_every: float): + ranges: list[tuple[Path, float, float]] = [] + start_s = window.segment_offset_s - search_before + end_s = window.segment_offset_s + search_after + + if start_s < 0.0 and window.segment > 0: + previous_clip = clip_root / f"{window.route}--{window.segment - 1}" / "fcamera.hevc" + if previous_clip.is_file(): + ranges.append((previous_clip, max(60.0 + start_s, 0.0), 60.0)) + start_s = 0.0 + + current_clip = clip_root / f"{window.route}--{window.segment}" / "fcamera.hevc" + if current_clip.is_file(): + ranges.append((current_clip, max(start_s, 0.0), min(end_s, 60.0))) + + for clip_path, range_start_s, range_end_s in ranges: + for source_time_s, frame_bgr in iter_video_samples(clip_path, range_start_s, range_end_s, sample_every): + if clip_path.parent.name.endswith(f"--{window.segment - 1}"): + relative_time_s = source_time_s - 60.0 + else: + relative_time_s = source_time_s + yield relative_time_s, clip_path, source_time_s, frame_bgr + + +def _score_expanded_candidate(daemon: slv.SpeedLimitVisionDaemon, frame_bgr, class_id: int, proposal_confidence: float, box, full_detection): + frame_height, frame_width = frame_bgr.shape[:2] + x1, y1, x2, y2 = box + box_width = x2 - x1 + box_height = y2 - y1 + if box_width <= 0 or box_height <= 0: + return None + + best = None + for expand_left, expand_top, expand_right, expand_bottom, expansion_weight in slv.DETECTOR_CLASSIFIER_EXPANSIONS: + expanded_x1 = max(int(x1 - box_width * expand_left), 0) + expanded_y1 = max(int(y1 - box_height * expand_top), 0) + expanded_x2 = min(int(x2 + box_width * expand_right), frame_width) + expanded_y2 = min(int(y2 + box_height * expand_bottom), frame_height) + if expanded_x2 <= expanded_x1 or expanded_y2 <= expanded_y1: + continue + + sign_crop = frame_bgr[expanded_y1:expanded_y2, expanded_x1:expanded_x2] + if sign_crop.size == 0: + continue + + is_regulatory = daemon._is_regulatory_speed_sign(sign_crop) or class_id == 2 + model_read = daemon._classify_speed_limit_from_model(sign_crop) + ocr_read = daemon._read_speed_limit_from_crop(sign_crop) + if model_read is None and ocr_read is None: + continue + + if class_id == 2: + read_result = model_read or ocr_read + if read_result is None or read_result[0] not in slv.SCHOOL_ZONE_SPEED_VALUES: + continue + elif not is_regulatory: + if model_read is None or ocr_read is None or model_read[0] != ocr_read[0]: + continue + read_result = (model_read[0], min(model_read[1], ocr_read[1])) + else: + if model_read is not None and ocr_read is not None and model_read[0] == ocr_read[0]: + read_result = (model_read[0], max(model_read[1], ocr_read[1])) + else: + read_result = model_read or ocr_read + + speed_limit_mph, read_confidence = read_result + area_ratio = ((expanded_x2 - expanded_x1) * (expanded_y2 - expanded_y1)) / max(frame_width * frame_height, 1) + score = read_confidence * 1.1 + proposal_confidence * 0.16 + expansion_weight * 0.08 + score += min(area_ratio * 6.0, 0.18) + if is_regulatory: + score += 0.14 + if model_read is not None and ocr_read is not None and model_read[0] == ocr_read[0]: + score += 0.24 + if full_detection is not None and full_detection.speed_limit_mph == speed_limit_mph: + score += 0.18 + full_detection.confidence * 0.08 + + candidate = { + "score": score, + "box": (expanded_x1, expanded_y1, expanded_x2, expanded_y2), + "proposal_confidence": proposal_confidence, + "class_id": class_id, + "is_regulatory": is_regulatory, + "model_read": model_read, + "ocr_read": ocr_read, + "full_detection": full_detection, + "read_result": read_result, + } + if best is None or candidate["score"] > best["score"]: + best = candidate + + return best + + +def score_frame(daemon: slv.SpeedLimitVisionDaemon, frame_bgr): + full_detection = daemon._detect_sign(frame_bgr) + best = None + + for proposal_confidence, class_id, (x1, y1, x2, y2) in daemon._collect_detector_classifier_proposals(frame_bgr): + if class_id == 1: + continue + + candidate = _score_expanded_candidate(daemon, frame_bgr, class_id, proposal_confidence, (x1, y1, x2, y2), full_detection) + if candidate is None: + continue + if best is None or candidate["score"] > best["score"]: + best = candidate + + return best + + +def write_manifest(rows: list[dict], path: Path): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=[ + "session_id", + "bookmark_number", + "route", + "segment", + "relative_time_s", + "source_video_path", + "score", + "proposal_confidence", + "class_id", + "is_regulatory", + "model_read", + "ocr_read", + "full_detection", + "frame_path", + "crop_path", + "box", + ]) + writer.writeheader() + writer.writerows(rows) + + +def main(): + args = parse_args() + configure_models(args.models_dir) + daemon = slv.SpeedLimitVisionDaemon(use_runtime=False) + route_mtimes = ebl.load_qlog_mtimes(args.qlog_mtimes.expanduser().resolve()) + session_route_map = common.load_session_route_map(args.session_route_map) + if not session_route_map: + raise FileNotFoundError(f"No session route map found at {args.session_route_map}") + clip_root = args.clip_root.expanduser().resolve() + session_root = args.session_root.expanduser().resolve() + output_dir = args.output_dir.expanduser().resolve() + frames_dir = output_dir / "frames" + crops_dir = output_dir / "crops" + frames_dir.mkdir(parents=True, exist_ok=True) + crops_dir.mkdir(parents=True, exist_ok=True) + + manifest_rows = [] + for session_id, route in session_route_map.items(): + bookmarks = ebl.load_bookmarks(session_root / session_id) + for bookmark_number, event in enumerate(bookmarks, start=1): + window = ebl.locate_window(route, event, route_mtimes, 5.0) + if window is None: + continue + + ranked = [] + for relative_time_s, source_video_path, source_time_s, frame_bgr in iter_context_frames( + clip_root, + window, + args.search_before, + args.search_after, + args.sample_every, + ): + scored = score_frame(daemon, frame_bgr) + if scored is None: + continue + ranked.append((scored["score"], relative_time_s, source_video_path, source_time_s, frame_bgr, scored)) + + ranked.sort(key=lambda item: item[0], reverse=True) + for rank_index, (_, relative_time_s, source_video_path, _, frame_bgr, scored) in enumerate(ranked[:max(args.top_k, 1)], start=1): + x1, y1, x2, y2 = scored["box"] + crop = frame_bgr[y1:y2, x1:x2] + + frame_name = f"{session_id}_bookmark_{bookmark_number:03d}_rank_{rank_index:02d}.jpg" + crop_name = f"{session_id}_bookmark_{bookmark_number:03d}_rank_{rank_index:02d}_crop.jpg" + frame_path = frames_dir / frame_name + crop_path = crops_dir / crop_name + cv2.imwrite(str(frame_path), frame_bgr, [cv2.IMWRITE_JPEG_QUALITY, 90]) + cv2.imwrite(str(crop_path), crop, [cv2.IMWRITE_JPEG_QUALITY, 90]) + + def fmt_detection(result): + if result is None: + return "" + return f"{result[0]}@{result[1]:.3f}" + + full_detection = scored["full_detection"] + manifest_rows.append({ + "session_id": session_id, + "bookmark_number": bookmark_number, + "route": route, + "segment": window.segment, + "relative_time_s": f"{relative_time_s:.3f}", + "source_video_path": str(source_video_path), + "score": f"{scored['score']:.4f}", + "proposal_confidence": f"{scored['proposal_confidence']:.4f}", + "class_id": str(scored["class_id"]), + "is_regulatory": str(bool(scored["is_regulatory"])), + "model_read": fmt_detection(scored["model_read"]), + "ocr_read": fmt_detection(scored["ocr_read"]), + "full_detection": "" if full_detection is None else f"{full_detection.speed_limit_mph}@{full_detection.confidence:.3f}", + "frame_path": str(frame_path), + "crop_path": str(crop_path), + "box": ",".join(str(value) for value in scored["box"]), + }) + + manifest_path = output_dir / "localized_bookmarks.csv" + write_manifest(manifest_rows, manifest_path) + print(f"Wrote {len(manifest_rows)} localized candidate(s) to {manifest_path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/speed_limit_vision/rank_bookmark_leadin_frames.py b/scripts/speed_limit_vision/rank_bookmark_leadin_frames.py new file mode 100644 index 00000000..589007c7 --- /dev/null +++ b/scripts/speed_limit_vision/rank_bookmark_leadin_frames.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv + +from collections import defaultdict +from pathlib import Path + +import cv2 + +import starpilot.system.speed_limit_vision as slv + + +DEFAULT_MANIFEST = Path(".tmp/speed_limit_training/review/bookmark_leadins.csv") +DEFAULT_OUTPUT = Path(".tmp/speed_limit_training/review/bookmark_leadin_shortlist.csv") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Rank sampled pre-bookmark lead-in frames by sign-likelihood for labeling.") + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST, help="CSV from import_bookmark_leadins.py.") + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help="Output CSV of top-ranked frames per bookmark.") + parser.add_argument("--top-k", type=int, default=3, help="How many frames to keep per bookmark window.") + return parser.parse_args() + + +def load_rows(path: Path): + with path.open("r", encoding="utf-8", newline="") as handle: + return [row for row in csv.DictReader(handle) if row.get("frame_path")] + + +def score_frame(daemon: slv.SpeedLimitVisionDaemon, frame_bgr): + detection = daemon._detect_sign_from_detector_classifier(frame_bgr) + ocr_detection = daemon._detect_sign_from_ocr_candidates(frame_bgr) + proposals = daemon._collect_detector_classifier_proposals(frame_bgr) + + best_proposal_score = 0.0 + best_box = None + frame_height, frame_width = frame_bgr.shape[:2] + for proposal_confidence, class_id, (x1, y1, x2, y2) in proposals[:8]: + box_width = x2 - x1 + box_height = y2 - y1 + if box_width <= 0 or box_height <= 0: + continue + crop = frame_bgr[y1:y2, x1:x2] + regulatory_bonus = 0.15 if daemon._is_regulatory_speed_sign(crop) else 0.0 + class_bonus = 0.10 if class_id in (0, 2) else 0.0 + area_bonus = min((box_width * box_height) / max(frame_width * frame_height, 1), 0.04) + score = proposal_confidence + regulatory_bonus + class_bonus + area_bonus + if score > best_proposal_score: + best_proposal_score = score + best_box = (x1, y1, x2, y2) + + score = best_proposal_score + reason = "proposal" + predicted_speed = "" + if ocr_detection is not None: + score = max(score, 0.55 + ocr_detection.confidence) + reason = "ocr" + predicted_speed = str(ocr_detection.speed_limit_mph) + if detection is not None: + score = max(score, 1.0 + detection.confidence) + reason = "detector" + predicted_speed = str(detection.speed_limit_mph) + + return { + "score": round(score, 4), + "reason": reason, + "predicted_speed": predicted_speed, + "best_box": "" if best_box is None else ",".join(str(v) for v in best_box), + } + + +def main() -> int: + args = parse_args() + manifest_path = args.manifest.expanduser().resolve() + output_path = args.output.expanduser().resolve() + rows = load_rows(manifest_path) + daemon = slv.SpeedLimitVisionDaemon(use_runtime=False) + + grouped_rows = defaultdict(list) + for row in rows: + grouped_rows[(row["session_id"], row["bookmark_number"])].append(row) + + shortlist_rows = [] + for (_, _), group_rows in grouped_rows.items(): + scored_rows = [] + for row in group_rows: + frame_path = (manifest_path.parents[1] / row["frame_path"]).resolve() + frame_bgr = cv2.imread(str(frame_path)) + if frame_bgr is None: + continue + scored_rows.append((score_frame(daemon, frame_bgr), row)) + + scored_rows.sort(key=lambda item: (-item[0]["score"], float(item[1]["sample_offset_s"]))) + for rank, (scored, row) in enumerate(scored_rows[:max(args.top_k, 1)], start=1): + shortlist_rows.append({ + "session_id": row["session_id"], + "bookmark_number": row["bookmark_number"], + "rank": rank, + "score": scored["score"], + "reason": scored["reason"], + "predicted_speed": scored["predicted_speed"], + "sample_offset_s": row["sample_offset_s"], + "frame_path": row["frame_path"], + "contact_sheet_path": row["contact_sheet_path"], + "window_result": row["window_result"], + "best_box": scored["best_box"], + }) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=[ + "session_id", + "bookmark_number", + "rank", + "score", + "reason", + "predicted_speed", + "sample_offset_s", + "frame_path", + "contact_sheet_path", + "window_result", + "best_box", + ]) + writer.writeheader() + writer.writerows(shortlist_rows) + + print(f"Wrote {len(shortlist_rows)} shortlisted frame(s) to {output_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/rebalance_detector_dataset.py b/scripts/speed_limit_vision/rebalance_detector_dataset.py new file mode 100644 index 00000000..3c5e99b2 --- /dev/null +++ b/scripts/speed_limit_vision/rebalance_detector_dataset.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import random + +from collections import defaultdict +from pathlib import Path + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, DETECTOR_CLASS_NAMES, ensure_dir, resolve_workspace # type: ignore +else: + from .common import DEFAULT_WORKSPACE, DETECTOR_CLASS_NAMES, ensure_dir, resolve_workspace + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Create a rebalanced detector dataset rooted in symlinks to the main workspace.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--output-root", type=Path, help="Output dataset root. Defaults to /detector_rebalanced.") + parser.add_argument("--max-other-train", type=int, default=4000, help="Maximum number of non-real train images to keep.") + parser.add_argument("--real-val-count", type=int, default=0, help="Hold out this many real train images as extra validation examples.") + parser.add_argument( + "--source-cap", + action="append", + default=[], + metavar="PREFIX=COUNT", + help="Cap train images for a filename prefix, for example arts_challenging=1200 or lisa_traffic_sign=1000.", + ) + parser.add_argument("--seed", type=int, default=0, help="Random seed for sampling non-real images.") + parser.add_argument("--copy", action="store_true", help="Copy files instead of creating symlinks.") + return parser.parse_args() + + +def link_or_copy(src: Path, dst: Path, copy_files: bool) -> None: + if dst.exists() or dst.is_symlink(): + dst.unlink() + ensure_dir(dst.parent) + if copy_files: + dst.write_bytes(src.read_bytes()) + else: + dst.symlink_to(src.resolve()) + + +def write_dataset_yaml(dataset_root: Path) -> Path: + yaml_lines = [ + f"path: {dataset_root}", + "train: images/train", + "val: images/val", + "names:", + ] + for index, class_name in enumerate(DETECTOR_CLASS_NAMES): + yaml_lines.append(f" {index}: {class_name}") + dataset_yaml = dataset_root / "dataset.yaml" + dataset_yaml.write_text("\n".join(yaml_lines) + "\n", encoding="utf-8") + return dataset_yaml + + +def remove_appledouble_files(root: Path) -> None: + for path in root.rglob("._*"): + if path.is_file() or path.is_symlink(): + path.unlink() + + +def visible_file_count(root: Path) -> int: + return sum(1 for path in root.glob("*") if path.is_file() and not path.name.startswith("._")) + + +def safe_unlink(path: Path) -> None: + try: + path.unlink() + except FileNotFoundError: + pass + + +def parse_source_caps(values: list[str]) -> dict[str, int]: + caps: dict[str, int] = {} + for value in values: + if "=" not in value: + raise ValueError(f"Invalid --source-cap '{value}', expected PREFIX=COUNT") + prefix, count_text = value.split("=", 1) + prefix = prefix.strip() + if not prefix: + raise ValueError(f"Invalid --source-cap '{value}', missing prefix") + caps[prefix] = max(int(count_text), 0) + return caps + + +def prefix_for_path(path: Path) -> str: + name = path.name + if name.startswith("real_"): + return "real" + if name.startswith("arts_challenging_"): + return "arts_challenging" + if name.startswith("lisa_traffic_sign_"): + return "lisa_traffic_sign" + if name.startswith("glare_images_"): + return "glare_images" + return name.split("_", 1)[0] + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + source_root = workspace / "detector" + output_root = args.output_root.resolve() if args.output_root else (workspace / "detector_rebalanced") + source_caps = parse_source_caps(args.source_cap) + rng = random.Random(args.seed) + + train_images = sorted((source_root / "images" / "train").glob("*")) + train_labels = source_root / "labels" / "train" + val_image_dir = source_root / "images" / "val" + val_label_dir = source_root / "labels" / "val" + + real_images = [path for path in train_images if path.name.startswith("real_")] + other_images = [path for path in train_images if not path.name.startswith("real_")] + rng.shuffle(real_images) + heldout_real_images = sorted(real_images[:max(args.real_val_count, 0)], key=lambda path: path.name) + target_real_train_images = sorted(real_images[max(args.real_val_count, 0):], key=lambda path: path.name) + + grouped_other_images: dict[str, list[Path]] = defaultdict(list) + for image_path in other_images: + grouped_other_images[prefix_for_path(image_path)].append(image_path) + + sampled_other_images: list[Path] = [] + uncapped_other_images: list[Path] = [] + for prefix, paths in grouped_other_images.items(): + rng.shuffle(paths) + if prefix in source_caps: + sampled_other_images.extend(paths[:source_caps[prefix]]) + else: + uncapped_other_images.extend(paths) + + rng.shuffle(uncapped_other_images) + remaining_slots = max(args.max_other_train - len(sampled_other_images), 0) + sampled_other_images.extend(uncapped_other_images[:remaining_slots]) + + target_train_images = target_real_train_images + sampled_other_images + target_train_images.sort(key=lambda path: path.name) + + output_train_image_dir = ensure_dir(output_root / "images" / "train") + output_train_label_dir = ensure_dir(output_root / "labels" / "train") + output_val_image_dir = ensure_dir(output_root / "images" / "val") + output_val_label_dir = ensure_dir(output_root / "labels" / "val") + + for existing in output_train_image_dir.glob("*"): + safe_unlink(existing) + for existing in output_train_label_dir.glob("*"): + safe_unlink(existing) + for existing in output_val_image_dir.glob("*"): + safe_unlink(existing) + for existing in output_val_label_dir.glob("*"): + safe_unlink(existing) + + for image_path in target_train_images: + label_path = train_labels / f"{image_path.stem}.txt" + if not label_path.is_file(): + continue + link_or_copy(image_path, output_train_image_dir / image_path.name, args.copy) + link_or_copy(label_path, output_train_label_dir / label_path.name, args.copy) + + for image_path in sorted(val_image_dir.glob("*")): + label_path = val_label_dir / f"{image_path.stem}.txt" + if not label_path.is_file(): + continue + link_or_copy(image_path, output_val_image_dir / image_path.name, args.copy) + link_or_copy(label_path, output_val_label_dir / label_path.name, args.copy) + + for image_path in heldout_real_images: + label_path = train_labels / f"{image_path.stem}.txt" + if not label_path.is_file(): + continue + link_or_copy(image_path, output_val_image_dir / image_path.name, args.copy) + link_or_copy(label_path, output_val_label_dir / label_path.name, args.copy) + + remove_appledouble_files(output_root) + dataset_yaml = write_dataset_yaml(output_root) + print(f"Created rebalanced detector dataset at {output_root}") + print(f"Dataset YAML: {dataset_yaml}") + print(f"Train images: {visible_file_count(output_train_image_dir)}") + print(f" real train: {len(target_real_train_images)}") + print(f" real held out to val: {len(heldout_real_images)}") + print(f" sampled other: {len(sampled_other_images)}") + if source_caps: + print(f" source caps: {source_caps}") + print(f"Val images: {visible_file_count(output_val_image_dir)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/run_overnight_clean_pipeline.sh b/scripts/speed_limit_vision/run_overnight_clean_pipeline.sh new file mode 100755 index 00000000..c6261174 --- /dev/null +++ b/scripts/speed_limit_vision/run_overnight_clean_pipeline.sh @@ -0,0 +1,95 @@ +#!/bin/zsh +set -euo pipefail + +REPO_ROOT="/Users/dominickthompson/starpilot" +WORKSPACE="/Volumes/T5/starpilot_speed_limit/workspace/speed_limit_training_clean" +LOG_DIR="$WORKSPACE/logs" +BASE_DETECTOR_NAME="yolo11n-comma-us-clean-v1" +GLARE_DETECTOR_NAME="yolo11n-comma-us-clean-glare-v1" +CLASSIFIER_NAME="yolo11n-cls-speed-limit-us-clean-v1" +EXPORT_DIR="$WORKSPACE/exports/overnight_latest" + +mkdir -p "$LOG_DIR" +cd "$REPO_ROOT" + +echo "[$(date)] waiting for GLARE raw download to finish" +while true; do + if pgrep -f "download_glare_raw.py --workspace $WORKSPACE" >/dev/null; then + sleep 30 + else + break + fi +done + +echo "[$(date)] importing completed GLARE images" +.venv/bin/python scripts/speed_limit_vision/import_glare_images.py --workspace "$WORKSPACE" --overwrite +.venv/bin/python scripts/speed_limit_vision/build_value_dataset.py --workspace "$WORKSPACE" --overwrite + +echo "[$(date)] waiting for base detector run to finish" +while true; do + if pgrep -f "train_detector.py --workspace $WORKSPACE .*--name $BASE_DETECTOR_NAME" >/dev/null; then + sleep 30 + else + break + fi +done + +BASE_DETECTOR_WEIGHTS="$WORKSPACE/runs/detector/$BASE_DETECTOR_NAME/weights/best.pt" +if [[ ! -f "$BASE_DETECTOR_WEIGHTS" ]]; then + echo "[$(date)] missing base detector weights: $BASE_DETECTOR_WEIGHTS" >&2 + exit 1 +fi + +echo "[$(date)] starting GLARE-augmented detector fine-tune" +.venv/bin/python scripts/speed_limit_vision/train_detector.py \ + --workspace "$WORKSPACE" \ + --device mps \ + --epochs 20 \ + --batch 24 \ + --workers 4 \ + --model "$BASE_DETECTOR_WEIGHTS" \ + --name "$GLARE_DETECTOR_NAME" \ + --exist-ok + +DETECTOR_WEIGHTS="$WORKSPACE/runs/detector/$GLARE_DETECTOR_NAME/weights/best.pt" +if [[ ! -f "$DETECTOR_WEIGHTS" ]]; then + DETECTOR_WEIGHTS="$BASE_DETECTOR_WEIGHTS" +fi + +echo "[$(date)] training value classifier" +.venv/bin/python scripts/speed_limit_vision/train_value_classifier.py \ + --workspace "$WORKSPACE" \ + --device mps \ + --epochs 40 \ + --batch 64 \ + --workers 4 \ + --name "$CLASSIFIER_NAME" \ + --exist-ok + +CLASSIFIER_WEIGHTS="$WORKSPACE/runs/classifier/$CLASSIFIER_NAME/weights/best.pt" +if [[ ! -f "$CLASSIFIER_WEIGHTS" ]]; then + echo "[$(date)] missing classifier weights: $CLASSIFIER_WEIGHTS" >&2 + exit 1 +fi + +echo "[$(date)] exporting ONNX models into repo assets" +.venv/bin/python scripts/speed_limit_vision/export_models.py \ + --workspace "$WORKSPACE" \ + --detector-weights "$DETECTOR_WEIGHTS" \ + --classifier-weights "$CLASSIFIER_WEIGHTS" \ + --output-dir "$EXPORT_DIR" \ + --install-repo-assets + +echo "[$(date)] evaluating runtime saved-frame cases" +.venv/bin/python scripts/speed_limit_vision/evaluate_runtime_cases.py \ + --models-dir "$EXPORT_DIR" \ + --strict | tee "$LOG_DIR/runtime_cases_overnight.txt" + +echo "[$(date)] evaluating bookmarked lead-ins" +.venv/bin/python scripts/speed_limit_vision/evaluate_bookmark_leadins.py \ + --models-dir "$EXPORT_DIR" \ + --lead-in 7 \ + --sample-fps 5 \ + --json-out "$LOG_DIR/bookmark_windows_overnight.json" | tee "$LOG_DIR/bookmark_windows_overnight.txt" + +echo "[$(date)] overnight clean pipeline complete" diff --git a/scripts/speed_limit_vision/sample_route_backgrounds.py b/scripts/speed_limit_vision/sample_route_backgrounds.py new file mode 100644 index 00000000..afdf3ca4 --- /dev/null +++ b/scripts/speed_limit_vision/sample_route_backgrounds.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse + +from pathlib import Path + +import cv2 + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore +else: + from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + + +IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp"} +VIDEO_SUFFIXES = {".hevc", ".mp4", ".mov", ".mkv", ".avi"} + + +def iter_source_files(paths: list[Path]): + for input_path in paths: + if input_path.is_file(): + yield input_path + continue + + if not input_path.exists(): + continue + + for path in sorted(input_path.rglob("*")): + if path.is_file() and path.suffix.lower() in IMAGE_SUFFIXES | VIDEO_SUFFIXES: + yield path + + +def sample_images(source_files: list[Path], output_dir: Path, max_per_file: int): + written = 0 + for file_index, image_path in enumerate(source_files, start=1): + frame = cv2.imread(str(image_path)) + if frame is None: + continue + stem = image_path.stem.replace(" ", "_") + output_path = output_dir / f"{file_index:05d}_{stem}.jpg" + cv2.imwrite(str(output_path), frame, [cv2.IMWRITE_JPEG_QUALITY, 90]) + written += 1 + if written >= max_per_file: + break + + +def sample_video(video_path: Path, output_dir: Path, seconds_between_frames: float, max_frames: int): + cap = cv2.VideoCapture(str(video_path)) + fps = cap.get(cv2.CAP_PROP_FPS) or 20.0 + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + frame_step = max(int(round(seconds_between_frames * fps)), 1) + frame_indices = range(0, total_frames if total_frames > 0 else frame_step * max_frames, frame_step) + written = 0 + + for frame_index in frame_indices: + if written >= max_frames: + break + cap.set(cv2.CAP_PROP_POS_FRAMES, frame_index) + ok, frame = cap.read() + if not ok or frame is None: + continue + + timestamp = frame_index / max(fps, 1.0) + safe_stem = video_path.parent.name.replace(" ", "_") + output_path = output_dir / f"{safe_stem}_{timestamp:06.2f}s.jpg" + cv2.imwrite(str(output_path), frame, [cv2.IMWRITE_JPEG_QUALITY, 90]) + written += 1 + + cap.release() + return written + + +def main(): + parser = argparse.ArgumentParser(description="Sample comma route frames into a reusable background pool.") + parser.add_argument("inputs", nargs="*", help="Route/video/image directories. Defaults to .tmp/comma_speed_training/realdata") + parser.add_argument("--workspace", default=str(DEFAULT_WORKSPACE), help="Training workspace root.") + parser.add_argument("--output-dir", default=None, help="Override background output directory.") + parser.add_argument("--sample-seconds", type=float, default=1.25, help="Seconds between sampled frames for each video.") + parser.add_argument("--max-per-video", type=int, default=6, help="Maximum frames to sample from each video.") + parser.add_argument("--limit-files", type=int, default=0, help="Optional cap on the number of source files to sample.") + args = parser.parse_args() + + workspace = resolve_workspace(args.workspace) + output_dir = Path(args.output_dir).expanduser().resolve() if args.output_dir else workspace / "backgrounds" + ensure_dir(output_dir) + + default_input = Path(".tmp/comma_speed_training/realdata") + input_paths = [Path(path).expanduser().resolve() for path in args.inputs] if args.inputs else [default_input.resolve()] + source_files = [path for path in iter_source_files(input_paths)] + if args.limit_files > 0: + source_files = source_files[:args.limit_files] + + image_files = [path for path in source_files if path.suffix.lower() in IMAGE_SUFFIXES] + video_files = [path for path in source_files if path.suffix.lower() in VIDEO_SUFFIXES] + + if not image_files and not video_files: + raise FileNotFoundError("No image or video sources found.") + + written = 0 + if image_files: + sample_images(image_files, output_dir, max_per_file=max(len(image_files), 1)) + written += len(image_files) + + for video_path in video_files: + written += sample_video(video_path, output_dir, max(args.sample_seconds, 0.1), max(args.max_per_video, 1)) + + print(f"Sampled {written} background frames into {output_dir}") + + +if __name__ == "__main__": + main() diff --git a/scripts/speed_limit_vision/train_detector.py b/scripts/speed_limit_vision/train_detector.py new file mode 100644 index 00000000..83fb4c43 --- /dev/null +++ b/scripts/speed_limit_vision/train_detector.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse + +from pathlib import Path + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, resolve_workspace # type: ignore +else: + from .common import DEFAULT_WORKSPACE, resolve_workspace + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Train the speed-limit detector using Ultralytics YOLO.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--data", type=Path, help="Detector dataset YAML. Defaults to /detector/dataset.yaml.") + parser.add_argument("--model", default="yolo11n.pt", help="Ultralytics detector checkpoint to fine-tune.") + parser.add_argument("--epochs", type=int, default=80, help="Training epochs.") + parser.add_argument("--imgsz", type=int, default=640, help="Training image size.") + parser.add_argument("--batch", type=int, default=16, help="Batch size.") + parser.add_argument("--workers", type=int, default=8, help="Data loader workers.") + parser.add_argument("--device", default="cpu", help="Ultralytics device string, for example cpu, mps, 0, or 0,1.") + parser.add_argument("--project", type=Path, help="Training output directory. Defaults to /runs/detector.") + parser.add_argument("--name", default="yolo11n-speed-limit-us", help="Run name under --project.") + parser.add_argument("--patience", type=int, default=20, help="Early stopping patience.") + parser.add_argument("--cache", action="store_true", help="Cache images in RAM if supported.") + parser.add_argument("--exist-ok", action="store_true", help="Allow overwriting an existing run directory.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + data_path = args.data.resolve() if args.data else (workspace / "detector" / "dataset.yaml") + project_path = args.project.resolve() if args.project else (workspace / "runs" / "detector") + + try: + from ultralytics import YOLO + except Exception as exc: + raise SystemExit( + "Ultralytics is not installed. Run `uv sync --extra speedvision` in the repo root before training." + ) from exc + + model = YOLO(args.model) + model.train( + data=str(data_path), + epochs=args.epochs, + imgsz=args.imgsz, + batch=args.batch, + workers=args.workers, + device=args.device, + project=str(project_path), + name=args.name, + patience=args.patience, + cache=args.cache, + exist_ok=args.exist_ok, + ) + print(f"Detector training complete under {project_path / args.name}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/train_value_classifier.py b/scripts/speed_limit_vision/train_value_classifier.py new file mode 100644 index 00000000..b5cac9b8 --- /dev/null +++ b/scripts/speed_limit_vision/train_value_classifier.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse + +from pathlib import Path + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, resolve_workspace # type: ignore +else: + from .common import DEFAULT_WORKSPACE, resolve_workspace + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Train the speed-limit value classifier using Ultralytics YOLO classification.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--data", type=Path, help="Classifier dataset root. Defaults to /classifier.") + parser.add_argument("--model", default="yolo11n-cls.pt", help="Ultralytics classification checkpoint to fine-tune.") + parser.add_argument("--epochs", type=int, default=60, help="Training epochs.") + parser.add_argument("--imgsz", type=int, default=128, help="Training image size.") + parser.add_argument("--batch", type=int, default=32, help="Batch size.") + parser.add_argument("--workers", type=int, default=8, help="Data loader workers.") + parser.add_argument("--device", default="cpu", help="Ultralytics device string, for example cpu, mps, 0, or 0,1.") + parser.add_argument("--project", type=Path, help="Training output directory. Defaults to /runs/classifier.") + parser.add_argument("--name", default="yolo11n-cls-speed-limit-us", help="Run name under --project.") + parser.add_argument("--patience", type=int, default=15, help="Early stopping patience.") + parser.add_argument("--cache", action="store_true", help="Cache images in RAM if supported.") + parser.add_argument("--exist-ok", action="store_true", help="Allow overwriting an existing run directory.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + data_path = args.data.resolve() if args.data else (workspace / "classifier") + project_path = args.project.resolve() if args.project else (workspace / "runs" / "classifier") + + try: + from ultralytics import YOLO + except Exception as exc: + raise SystemExit( + "Ultralytics is not installed. Run `uv sync --extra speedvision` in the repo root before training." + ) from exc + + model = YOLO(args.model) + model.train( + data=str(data_path), + epochs=args.epochs, + imgsz=args.imgsz, + batch=args.batch, + workers=args.workers, + device=args.device, + project=str(project_path), + name=args.name, + patience=args.patience, + cache=args.cache, + exist_ok=args.exist_ok, + ) + print(f"Classifier training complete under {project_path / args.name}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/watch_detector_eval.py b/scripts/speed_limit_vision/watch_detector_eval.py new file mode 100644 index 00000000..ea5e1472 --- /dev/null +++ b/scripts/speed_limit_vision/watch_detector_eval.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import time + +from datetime import datetime, timezone +from pathlib import Path + +import cv2 + +from ultralytics import YOLO + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore + from generate_value_roi_classifier_dataset import extract_value_mask # type: ignore +else: + from .common import DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + from .generate_value_roi_classifier_dataset import extract_value_mask + + +EVAL_WINDOWS = ( + ("full", (0.0, 0.0, 1.0, 1.0)), + ("roi3", (0.40, 0.12, 0.92, 0.86)), +) +SIGN_CLASSES = { + "regulatory_speed_limit", + "school_zone_speed_limit", + "speedLimit15", + "speedLimit20", + "speedLimit25", + "speedLimit30", + "speedLimit35", + "speedLimit40", + "speedLimit45", + "speedLimit50", + "speedLimit55", + "speedLimit60", + "speedLimit65", + "speedLimit70", + "speedLimit75", + "schoolSpeedLimit25", + "speedLimit55Ahead", +} +REAL_CASES = ( + ("live15", ".tmp/live_c4_capture/stopped_sign_road.jpg", 15), + ("school20", ".tmp/route_vision/seg38_frames/frame_041.jpg", 20), + ("highway40", ".tmp/speed_route_frames_seg2_10_20/t12.png", 40), + ("town40", ".tmp/route_12c_seg9_10/seg10_early/frame_005.png", 40), + ("town30", ".tmp/route_12c_seg9_10/seg10_early/frame_012.png", 30), + ("town30_late", ".tmp/vision_iter/seg10_5fps/frame_054.png", 30), +) + + +def detect_best(detector: YOLO, classifier: YOLO, frame_bgr): + frame_height, frame_width = frame_bgr.shape[:2] + best = None + for window_name, (left_ratio, top_ratio, right_ratio, bottom_ratio) in EVAL_WINDOWS: + left = int(frame_width * left_ratio) + top = int(frame_height * top_ratio) + right = int(frame_width * right_ratio) + bottom = int(frame_height * bottom_ratio) + roi = frame_bgr[top:bottom, left:right] + if roi.size == 0: + continue + + detector_result = detector.predict(source=roi, conf=0.03, imgsz=640, device="cpu", verbose=False)[0] + if detector_result.boxes is None: + continue + + for box, cls, det_conf in zip( + detector_result.boxes.xyxy.cpu().numpy(), + detector_result.boxes.cls.cpu().numpy(), + detector_result.boxes.conf.cpu().numpy(), + ): + class_name = detector.names[int(cls)] + if class_name not in SIGN_CLASSES: + continue + + x1, y1, x2, y2 = box.astype(int) + x1 += left + x2 += left + y1 += top + y2 += top + box_width = x2 - x1 + box_height = y2 - y1 + if box_width <= 0 or box_height <= 0: + continue + + expand = 0.18 + crop_left = max(x1 - int(box_width * expand), 0) + crop_top = max(y1 - int(box_height * expand), 0) + crop_right = min(x2 + int(box_width * expand), frame_width) + crop_bottom = min(y2 + int(box_height * expand), frame_height) + crop = frame_bgr[crop_top:crop_bottom, crop_left:crop_right] + if crop.size == 0: + continue + + mask = extract_value_mask(crop) + if mask is None: + continue + + classifier_result = classifier.predict(source=cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR), imgsz=128, device="cpu", verbose=False)[0] + probabilities = classifier_result.probs + if probabilities is None: + continue + + predicted_value = int(classifier_result.names[int(probabilities.top1)]) + classifier_confidence = float(probabilities.top1conf) + score = min(classifier_confidence * 0.82 + float(det_conf) * 0.26, 0.95) + candidate = { + "window": window_name, + "detectorClass": class_name, + "detectorConfidence": round(float(det_conf), 4), + "predictedValue": predicted_value, + "classifierConfidence": round(classifier_confidence, 4), + "score": round(score, 4), + "box": [int(x1), int(y1), int(x2), int(y2)], + } + if best is None or candidate["score"] > best["score"]: + best = candidate + return best + + +def evaluate_once(detector_weights: Path, classifier_weights: Path): + detector = YOLO(str(detector_weights)) + classifier = YOLO(str(classifier_weights)) + records = [] + for label, frame_path, expected in REAL_CASES: + frame = cv2.imread(frame_path) + best = detect_best(detector, classifier, frame) if frame is not None else None + correct = best is None if expected is None else bool(best is not None and best["predictedValue"] == expected) + records.append({ + "case": label, + "frame": frame_path, + "expected": expected, + "best": best, + "correct": correct, + }) + return records + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Watch a detector run and evaluate improved checkpoints on the saved comma examples.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--detector-weights", type=Path, required=True, help="Detector weights to watch, usually runs/.../weights/best.pt.") + parser.add_argument("--classifier-weights", type=Path, required=True, help="Classifier weights to use for evaluation.") + parser.add_argument("--interval", type=float, default=30.0, help="Polling interval in seconds.") + parser.add_argument("--output", type=Path, help="JSONL output log path. Defaults to /review/detector_watch.jsonl.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + detector_weights = args.detector_weights.resolve() + classifier_weights = args.classifier_weights.resolve() + output_path = args.output.resolve() if args.output else (ensure_dir(workspace / "review") / "detector_watch.jsonl") + ensure_dir(output_path.parent) + + last_mtime = None + while True: + if detector_weights.is_file(): + mtime = detector_weights.stat().st_mtime + if last_mtime is None or mtime > last_mtime: + last_mtime = mtime + records = evaluate_once(detector_weights, classifier_weights) + payload = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "detectorWeights": str(detector_weights), + "classifierWeights": str(classifier_weights), + "records": records, + } + with output_path.open("a", encoding="utf-8") as output_file: + output_file.write(json.dumps(payload, separators=(",", ":")) + "\n") + print(json.dumps(payload, indent=2)) + time.sleep(max(args.interval, 1.0)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/selfdrive/pandad/pandad b/selfdrive/pandad/pandad index 545fd74f..14106c61 100755 Binary files a/selfdrive/pandad/pandad and b/selfdrive/pandad/pandad differ diff --git a/selfdrive/selfdrived/events.py b/selfdrive/selfdrived/events.py index e9e5d1ed..90a8a0ce 100644 --- a/selfdrive/selfdrived/events.py +++ b/selfdrive/selfdrived/events.py @@ -265,6 +265,15 @@ def below_steer_speed_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.S Priority.LOW, VisualAlert.none, AudibleAlert.prompt, 1.5) +def speed_limit_changed_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMaster, metric: bool, soft_disable_time: int, personality, starpilot_toggles: SimpleNamespace) -> Alert: + speed_limit = sm["starpilotPlan"].unconfirmedSlcSpeedLimit or sm["starpilotPlan"].slcSpeedLimit + return Alert( + "Speed limit detected", + f"Confirm {get_display_speed(speed_limit, metric)}?", + StarPilotAlertStatus.starpilot, AlertSize.mid, + Priority.LOW, VisualAlert.none, AudibleAlert.prompt, 3.0) + + def calibration_incomplete_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMaster, metric: bool, soft_disable_time: int, personality, starpilot_toggles: SimpleNamespace) -> Alert: first_word = 'Recalibrating' if sm['liveCalibration'].calStatus == log.LiveCalibrationData.Status.recalibrating else 'Calibrating' return Alert( @@ -1176,11 +1185,7 @@ STARPILOT_EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = { }, StarPilotEventName.speedLimitChanged: { - ET.PERMANENT: Alert( - "Speed limit changed", - "", - StarPilotAlertStatus.starpilot, AlertSize.small, - Priority.LOW, VisualAlert.none, AudibleAlert.prompt, 3.), + ET.PERMANENT: speed_limit_changed_alert, }, StarPilotEventName.trafficModeActive: { diff --git a/selfdrive/ui/layouts/settings/starpilot/longitudinal.py b/selfdrive/ui/layouts/settings/starpilot/longitudinal.py index 2309e0ef..e1e95ce9 100644 --- a/selfdrive/ui/layouts/settings/starpilot/longitudinal.py +++ b/selfdrive/ui/layouts/settings/starpilot/longitudinal.py @@ -611,7 +611,7 @@ class StarPilotSpeedLimitControllerLayout(StarPilotPanel): return f"{primary}, {secondary}" def _on_priority_clicked(self): - primary_options = ["Dashboard", "Map Data", "Highest", "Lowest"] + primary_options = ["Dashboard", "Map Data", "Vision", "Highest", "Lowest"] current_primary = self._params.get("SLCPriority1", encoding='utf-8') or "Map Data" current_secondary = self._params.get("SLCPriority2", encoding='utf-8') or "None" @@ -622,7 +622,7 @@ class StarPilotSpeedLimitControllerLayout(StarPilotPanel): self._rebuild_grid() def show_secondary_dialog(primary): - secondary_options = ["None"] + [option for option in ("Dashboard", "Map Data") if option != primary] + secondary_options = ["None"] + [option for option in ("Dashboard", "Map Data", "Vision") if option != primary] selected_secondary = current_secondary if current_secondary in secondary_options else "None" gui_app.set_modal_overlay( SelectionDialog( diff --git a/selfdrive/ui/mici/onroad/augmented_road_view.py b/selfdrive/ui/mici/onroad/augmented_road_view.py index 9a827dac..da7324e2 100644 --- a/selfdrive/ui/mici/onroad/augmented_road_view.py +++ b/selfdrive/ui/mici/onroad/augmented_road_view.py @@ -377,6 +377,7 @@ class AugmentedRoadView(CameraView): return in_reverse = self._is_in_reverse() + self._hud_renderer.prepare(self._content_rect) # Draw all UI overlays if not in_reverse: @@ -384,6 +385,8 @@ class AugmentedRoadView(CameraView): # Fade out bottom of overlays for looks rl.draw_texture_ex(self._fade_texture, rl.Vector2(self._content_rect.x, self._content_rect.y), 0.0, 1.0, rl.WHITE) + if not in_reverse: + self._hud_renderer.render_background() alert_to_render, not_animating_out = self._alert_renderer.will_render() @@ -400,7 +403,7 @@ class AugmentedRoadView(CameraView): if ui_state.started: self._alert_renderer.render(self._content_rect) if not in_reverse: - self._hud_renderer.render(self._content_rect) + self._hud_renderer.render_foreground() if (not in_reverse) and alert_to_render is None: self._experimental_mode_banner.render(self._content_rect) if not in_reverse: diff --git a/selfdrive/ui/mici/onroad/hud_renderer.py b/selfdrive/ui/mici/onroad/hud_renderer.py index 7f489ccf..19744442 100644 --- a/selfdrive/ui/mici/onroad/hud_renderer.py +++ b/selfdrive/ui/mici/onroad/hud_renderer.py @@ -2,6 +2,7 @@ import pyray as rl from dataclasses import dataclass from openpilot.common.constants import CV from openpilot.selfdrive.ui.mici.onroad.torque_bar import TorqueBar +from openpilot.selfdrive.ui.mici.onroad.speed_limit_utils import resolve_display_speed_limit_ms from openpilot.selfdrive.ui.ui_state import ui_state, UIStatus from openpilot.system.ui.lib.application import gui_app, FontWeight from openpilot.system.ui.lib.multilang import tr @@ -19,6 +20,15 @@ CRUISE_DISABLED_CHAR = '–' SET_SPEED_PERSISTENCE = 2.5 # seconds +SPEED_LIMIT_PROMPT_CARD_WIDTH = 500 +SPEED_LIMIT_PROMPT_CARD_HEIGHT = 208 +SPEED_LIMIT_PROMPT_BUTTON_SIZE = 112 +SPEED_LIMIT_PROMPT_BUTTON_GAP = 28 +SPEED_LIMIT_PROMPT_CARD_PADDING = 34 +SPEED_LIMIT_PROMPT_US_SIGN_WIDTH = 132 +SPEED_LIMIT_PROMPT_US_SIGN_HEIGHT = 150 +SPEED_LIMIT_PROMPT_EU_SIGN_SIZE = 148 + @dataclass(frozen=True) class FontSizes: @@ -106,6 +116,15 @@ class HudRenderer(Widget): self.speed: float = 0.0 self.v_ego_cluster_seen: bool = False self._engaged: bool = False + self._show_speed_limit: bool = False + self._speed_limit: float = 0.0 + self._speed_limit_overridden: bool = False + self._pending_speed_limit: float = 0.0 + self._prompt_visible: bool = False + self._prompt_card_rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0) + self._prompt_sign_rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0) + self._prompt_deny_rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0) + self._prompt_accept_rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0) self._can_draw_top_icons = True self._show_wheel_critical = False @@ -169,16 +188,68 @@ class HudRenderer(Widget): speed_conversion = CV.MS_TO_KPH if ui_state.is_metric else CV.MS_TO_MPH self.speed = max(0.0, v_ego * speed_conversion) - def _render(self, rect: rl.Rectangle) -> None: - """Render HUD elements to the screen.""" + if sm.recv_frame["starpilotPlan"] >= ui_state.started_frame: + starpilot_plan = sm["starpilotPlan"] + self._show_speed_limit = ui_state.params.get_bool("ShowSpeedLimits") or ui_state.params.get_bool("SpeedLimitController") + if self._show_speed_limit: + dashboard_speed_limit = sm["starpilotCarState"].dashboardSpeedLimit if sm.valid.get("starpilotCarState", False) else 0.0 + vision_speed_limit = ui_state.params_memory.get_float("VisionSpeedLimit") if ui_state.params.get_bool("VisionSpeedLimitDetection") else 0.0 + primary_priority = ui_state.params.get("SLCPriority1", encoding='utf-8') or "Map Data" + secondary_priority = ui_state.params.get("SLCPriority2", encoding='utf-8') or "None" + source_limits = { + "Dashboard": dashboard_speed_limit, + "Map Data": starpilot_plan.slcMapSpeedLimit, + "Vision": vision_speed_limit, + "Mapbox": starpilot_plan.slcMapboxSpeedLimit if ui_state.params.get_bool("SLCMapboxFiller") else 0.0, + } + resolved_speed_limit = resolve_display_speed_limit_ms( + slc_speed_limit=starpilot_plan.slcSpeedLimit, + speed_limit_source=starpilot_plan.slcSpeedLimitSource, + source_limits=source_limits, + primary_priority=primary_priority, + secondary_priority=secondary_priority, + ) + self._speed_limit = max(0.0, resolved_speed_limit * speed_conversion) + self._speed_limit_overridden = bool(starpilot_plan.slcOverriddenSpeed > 0 and starpilot_plan.slcSpeedLimit > 0) + self._pending_speed_limit = max(0.0, starpilot_plan.unconfirmedSlcSpeedLimit * speed_conversion) + else: + self._speed_limit = 0.0 + self._speed_limit_overridden = False + self._pending_speed_limit = 0.0 + self._prompt_visible = self._pending_speed_limit > 0 + else: + self._show_speed_limit = False + self._speed_limit = 0.0 + self._speed_limit_overridden = False + self._pending_speed_limit = 0.0 + self._prompt_visible = False + def prepare(self, rect: rl.Rectangle) -> None: + """Update HUD state once before drawing background/foreground passes.""" + self.set_rect(rect) + self._update_state() + self._update_prompt_layout(rect) + + def render_background(self) -> None: + """Draw HUD elements that should sit behind alerts.""" + self._draw_speed_limit(self._rect) + self._draw_speed_limit_prompt(self._rect) + + def render_foreground(self) -> None: + """Draw HUD elements that should sit above alerts.""" if ui_state.sm['controlsState'].lateralControlState.which() != 'angleState': - self._torque_bar.render(rect) + self._torque_bar.render(self._rect) if self.is_cruise_set: - self._draw_set_speed(rect) + self._draw_set_speed(self._rect) - self._draw_steering_wheel(rect) + self._draw_steering_wheel(self._rect) + + def _render(self, rect: rl.Rectangle) -> None: + """Render HUD elements to the screen.""" + self.prepare(rect) + self.render_background() + self.render_foreground() def _draw_steering_wheel(self, rect: rl.Rectangle) -> None: wheel_txt = self._txt_wheel_critical if self._show_wheel_critical else self._txt_wheel @@ -264,6 +335,208 @@ class HudRenderer(Widget): max_color, ) + def _draw_speed_limit(self, rect: rl.Rectangle) -> None: + if not self._show_speed_limit: + return + + display_speed = self._speed_limit if self._speed_limit > 0 else self._pending_speed_limit + if display_speed <= 0: + return + + sign_alpha = 72 if self._speed_limit_overridden and self._pending_speed_limit <= 0 else 255 + use_vienna_speed_limit = ui_state.params.get_bool("UseVienna") + sign_width = 118 if use_vienna_speed_limit else 116 + sign_height = 118 if use_vienna_speed_limit else 132 + base_x = rect.x + rect.width - sign_width - 28 + sign_x = base_x + sign_y = rect.y + (28 if use_vienna_speed_limit else 20) + + speed_text = str(round(display_speed)) + if use_vienna_speed_limit: + center_x = sign_x + sign_width / 2 + center_y = sign_y + sign_height / 2 + radius = sign_width / 2 + + rl.draw_circle(int(center_x), int(center_y), radius, rl.Color(255, 255, 255, sign_alpha)) + rl.draw_ring( + rl.Vector2(center_x, center_y), + radius - 12, + radius, + 0, + 360, + 64, + rl.Color(201, 34, 49, sign_alpha), + ) + + font_size = 58 if len(speed_text) <= 2 else 48 + text_size = measure_text_cached(self._font_bold, speed_text, font_size) + text_pos = rl.Vector2(center_x - text_size.x / 2, center_y - text_size.y / 2 + 4) + rl.draw_text_ex(self._font_bold, speed_text, text_pos, font_size, 0, rl.Color(0, 0, 0, sign_alpha)) + else: + sign_rect = rl.Rectangle(sign_x, sign_y, sign_width, sign_height) + border_rect = rl.Rectangle(sign_x + 6, sign_y + 6, sign_width - 12, sign_height - 12) + rl.draw_rectangle_rounded(sign_rect, 0.18, 16, rl.Color(255, 255, 255, sign_alpha)) + rl.draw_rectangle_rounded_lines_ex(border_rect, 0.14, 16, 4, rl.Color(0, 0, 0, sign_alpha)) + + header_font_size = 20 + header_gap = 18 + speed_font_size = 50 if len(speed_text) <= 2 else 42 + + speed_label = tr("SPEED") + limit_label = tr("LIMIT") + speed_label_size = measure_text_cached(self._font_semi_bold, speed_label, header_font_size) + limit_label_size = measure_text_cached(self._font_semi_bold, limit_label, header_font_size) + speed_label_pos = rl.Vector2(sign_x + sign_width / 2 - speed_label_size.x / 2, sign_y + 18) + limit_label_pos = rl.Vector2(sign_x + sign_width / 2 - limit_label_size.x / 2, sign_y + 18 + header_gap) + rl.draw_text_ex(self._font_semi_bold, speed_label, speed_label_pos, header_font_size, 0, rl.Color(0, 0, 0, sign_alpha)) + rl.draw_text_ex(self._font_semi_bold, limit_label, limit_label_pos, header_font_size, 0, rl.Color(0, 0, 0, sign_alpha)) + + speed_text_size = measure_text_cached(self._font_bold, speed_text, speed_font_size) + speed_text_pos = rl.Vector2(sign_x + sign_width / 2 - speed_text_size.x / 2, sign_y + 66) + rl.draw_text_ex(self._font_bold, speed_text, speed_text_pos, speed_font_size, 0, rl.Color(0, 0, 0, sign_alpha)) + + def _update_prompt_layout(self, rect: rl.Rectangle) -> None: + if not self._prompt_visible: + self._prompt_card_rect = rl.Rectangle(0, 0, 0, 0) + self._prompt_sign_rect = rl.Rectangle(0, 0, 0, 0) + self._prompt_deny_rect = rl.Rectangle(0, 0, 0, 0) + self._prompt_accept_rect = rl.Rectangle(0, 0, 0, 0) + return + + use_vienna_speed_limit = ui_state.params.get_bool("UseVienna") + sign_width = SPEED_LIMIT_PROMPT_EU_SIGN_SIZE if use_vienna_speed_limit else SPEED_LIMIT_PROMPT_US_SIGN_WIDTH + sign_height = SPEED_LIMIT_PROMPT_EU_SIGN_SIZE if use_vienna_speed_limit else SPEED_LIMIT_PROMPT_US_SIGN_HEIGHT + button_size = SPEED_LIMIT_PROMPT_BUTTON_SIZE + card_width = max( + SPEED_LIMIT_PROMPT_CARD_WIDTH, + SPEED_LIMIT_PROMPT_CARD_PADDING * 2 + sign_width + button_size * 2 + SPEED_LIMIT_PROMPT_BUTTON_GAP * 2, + ) + card_x = rect.x + (rect.width - card_width) / 2 + card_y = rect.y + rect.height * 0.34 + card_rect = rl.Rectangle(card_x, card_y, card_width, SPEED_LIMIT_PROMPT_CARD_HEIGHT) + + controls_y = card_y + 68 + deny_x = card_x + SPEED_LIMIT_PROMPT_CARD_PADDING + sign_x = card_x + (card_width - sign_width) / 2 + accept_x = card_x + card_width - SPEED_LIMIT_PROMPT_CARD_PADDING - button_size + + self._prompt_card_rect = card_rect + self._prompt_sign_rect = rl.Rectangle(sign_x, controls_y - (sign_height - button_size) / 2, sign_width, sign_height) + self._prompt_deny_rect = rl.Rectangle(deny_x, controls_y, button_size, button_size) + self._prompt_accept_rect = rl.Rectangle(accept_x, controls_y, button_size, button_size) + + def _draw_prompt_button(self, rect: rl.Rectangle, symbol: str, fill: rl.Color, outline: rl.Color, pressed: bool) -> None: + scale = 0.95 if pressed else 1.0 + width = rect.width * scale + height = rect.height * scale + x = rect.x + (rect.width - width) / 2 + y = rect.y + (rect.height - height) / 2 + button_rect = rl.Rectangle(x, y, width, height) + center = rl.Vector2(button_rect.x + button_rect.width / 2, button_rect.y + button_rect.height / 2) + radius = min(button_rect.width, button_rect.height) / 2 + + rl.draw_circle_gradient(int(center.x), int(center.y), radius, rl.Color(0, 0, 0, 90), rl.BLANK) + rl.draw_circle(int(center.x), int(center.y), radius, fill) + rl.draw_ring(center, radius - 6, radius, 0, 360, 48, outline) + + symbol_size = 88 if symbol == "+" else 98 + symbol_text = tr(symbol) + symbol_measure = measure_text_cached(self._font_display, symbol_text, symbol_size) + symbol_pos = rl.Vector2(center.x - symbol_measure.x / 2, center.y - symbol_measure.y / 2 - (6 if symbol == "-" else 0)) + rl.draw_text_ex(self._font_display, symbol_text, symbol_pos, symbol_size, 0, rl.WHITE) + + def _draw_speed_limit_prompt(self, rect: rl.Rectangle) -> None: + if not self._prompt_visible: + return + + accent_color = rl.Color(255, 115, 0, 255) + card_rect = self._prompt_card_rect + rl.draw_rectangle_rounded(card_rect, 0.14, 18, rl.Color(10, 10, 10, 210)) + rl.draw_rectangle_rounded_lines_ex(card_rect, 0.14, 18, 4, accent_color) + + title_text = tr("NEW SPEED LIMIT") + title_size = measure_text_cached(self._font_semi_bold, title_text, 28) + title_pos = rl.Vector2(card_rect.x + card_rect.width / 2 - title_size.x / 2, card_rect.y + 18) + rl.draw_text_ex(self._font_semi_bold, title_text, title_pos, 28, 0, rl.Color(255, 255, 255, 235)) + + use_vienna_speed_limit = ui_state.params.get_bool("UseVienna") + speed_text = str(round(self._pending_speed_limit)) + sign_rect = self._prompt_sign_rect + + if use_vienna_speed_limit: + center_x = sign_rect.x + sign_rect.width / 2 + center_y = sign_rect.y + sign_rect.height / 2 + radius = sign_rect.width / 2 + + rl.draw_circle(int(center_x), int(center_y), radius, rl.WHITE) + rl.draw_ring( + rl.Vector2(center_x, center_y), + radius - 14, + radius, + 0, + 360, + 64, + rl.Color(201, 34, 49, 255), + ) + + font_size = 78 if len(speed_text) <= 2 else 66 + text_size = measure_text_cached(self._font_bold, speed_text, font_size) + text_pos = rl.Vector2(center_x - text_size.x / 2, center_y - text_size.y / 2 + 4) + rl.draw_text_ex(self._font_bold, speed_text, text_pos, font_size, 0, rl.BLACK) + else: + border_rect = rl.Rectangle(sign_rect.x + 8, sign_rect.y + 8, sign_rect.width - 16, sign_rect.height - 16) + rl.draw_rectangle_rounded(sign_rect, 0.18, 16, rl.WHITE) + rl.draw_rectangle_rounded_lines_ex(border_rect, 0.14, 16, 5, rl.BLACK) + + header_font_size = 24 + header_gap = 20 + speed_font_size = 72 if len(speed_text) <= 2 else 60 + + speed_label = tr("SPEED") + limit_label = tr("LIMIT") + speed_label_size = measure_text_cached(self._font_semi_bold, speed_label, header_font_size) + limit_label_size = measure_text_cached(self._font_semi_bold, limit_label, header_font_size) + rl.draw_text_ex( + self._font_semi_bold, + speed_label, + rl.Vector2(sign_rect.x + sign_rect.width / 2 - speed_label_size.x / 2, sign_rect.y + 20), + header_font_size, + 0, + rl.BLACK, + ) + rl.draw_text_ex( + self._font_semi_bold, + limit_label, + rl.Vector2(sign_rect.x + sign_rect.width / 2 - limit_label_size.x / 2, sign_rect.y + 20 + header_gap), + header_font_size, + 0, + rl.BLACK, + ) + + speed_size = measure_text_cached(self._font_bold, speed_text, speed_font_size) + speed_pos = rl.Vector2(sign_rect.x + sign_rect.width / 2 - speed_size.x / 2, sign_rect.y + 80) + rl.draw_text_ex(self._font_bold, speed_text, speed_pos, speed_font_size, 0, rl.BLACK) + + self._draw_prompt_button( + self._prompt_deny_rect, + "-", + rl.Color(104, 20, 20, 235), + rl.Color(255, 78, 78, 255), + False, + ) + self._draw_prompt_button( + self._prompt_accept_rect, + "+", + rl.Color(12, 110, 66, 235), + rl.Color(78, 255, 173, 255), + False, + ) + + hint_text = tr("USE WHEEL - / +") + hint_size = measure_text_cached(self._font_medium, hint_text, 24) + hint_pos = rl.Vector2(card_rect.x + card_rect.width / 2 - hint_size.x / 2, card_rect.y + card_rect.height - 34) + rl.draw_text_ex(self._font_medium, hint_text, hint_pos, 24, 0, rl.Color(255, 255, 255, 180)) + def _draw_current_speed(self, rect: rl.Rectangle) -> None: """Draw the current vehicle speed and unit.""" speed_text = str(round(self.speed)) diff --git a/selfdrive/ui/mici/onroad/speed_limit_utils.py b/selfdrive/ui/mici/onroad/speed_limit_utils.py new file mode 100644 index 00000000..77786564 --- /dev/null +++ b/selfdrive/ui/mici/onroad/speed_limit_utils.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from collections.abc import Mapping + + +def _normalize_limit(limit: float) -> float: + return limit if limit >= 1.0 else 0.0 + + +def resolve_display_speed_limit_ms( + slc_speed_limit: float, + speed_limit_source: str, + source_limits: Mapping[str, float], + primary_priority: str, + secondary_priority: str, +) -> float: + slc_speed_limit = _normalize_limit(slc_speed_limit) + if slc_speed_limit > 0.0: + return slc_speed_limit + + normalized_limits = {source: _normalize_limit(limit) for source, limit in source_limits.items()} + + active_source_limit = normalized_limits.get(speed_limit_source, 0.0) + if active_source_limit > 0.0: + return active_source_limit + + available_limits = {source: limit for source, limit in normalized_limits.items() if limit > 0.0} + if not available_limits: + return 0.0 + + if primary_priority == "Highest": + return max(available_limits.values()) + if primary_priority == "Lowest": + return min(available_limits.values()) + + for priority in (primary_priority, secondary_priority, "Mapbox"): + limit = available_limits.get(priority, 0.0) + if limit > 0.0: + return limit + + return next(iter(available_limits.values())) diff --git a/selfdrive/ui/tests/test_speed_limit_utils.py b/selfdrive/ui/tests/test_speed_limit_utils.py new file mode 100644 index 00000000..69788890 --- /dev/null +++ b/selfdrive/ui/tests/test_speed_limit_utils.py @@ -0,0 +1,77 @@ +import importlib.util +import unittest +from pathlib import Path + + +MODULE_PATH = Path(__file__).resolve().parents[1] / "mici" / "onroad" / "speed_limit_utils.py" +SPEC = importlib.util.spec_from_file_location("speed_limit_utils_under_test", MODULE_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC is not None and SPEC.loader is not None +SPEC.loader.exec_module(MODULE) +resolve_display_speed_limit_ms = MODULE.resolve_display_speed_limit_ms + + +class TestSpeedLimitUtils(unittest.TestCase): + def test_resolve_display_speed_limit_prefers_slc_target(self): + speed_limit = resolve_display_speed_limit_ms( + slc_speed_limit=24.6, + speed_limit_source="Dashboard", + source_limits={ + "Dashboard": 22.4, + "Map Data": 20.1, + "Vision": 0.0, + "Mapbox": 0.0, + }, + primary_priority="Map Data", + secondary_priority="Dashboard", + ) + + self.assertEqual(speed_limit, 24.6) + + def test_resolve_display_speed_limit_uses_active_source_when_display_only(self): + speed_limit = resolve_display_speed_limit_ms( + slc_speed_limit=0.0, + speed_limit_source="Dashboard", + source_limits={ + "Dashboard": 22.4, + "Map Data": 20.1, + "Vision": 0.0, + "Mapbox": 0.0, + }, + primary_priority="Map Data", + secondary_priority="Vision", + ) + + self.assertEqual(speed_limit, 22.4) + + def test_resolve_display_speed_limit_honors_priority_order(self): + speed_limit = resolve_display_speed_limit_ms( + slc_speed_limit=0.0, + speed_limit_source="None", + source_limits={ + "Dashboard": 22.4, + "Map Data": 20.1, + "Vision": 24.6, + "Mapbox": 0.0, + }, + primary_priority="Map Data", + secondary_priority="Vision", + ) + + self.assertEqual(speed_limit, 20.1) + + def test_resolve_display_speed_limit_falls_back_to_mapbox(self): + speed_limit = resolve_display_speed_limit_ms( + slc_speed_limit=0.0, + speed_limit_source="None", + source_limits={ + "Dashboard": 0.0, + "Map Data": 0.0, + "Vision": 0.0, + "Mapbox": 17.9, + }, + primary_priority="Map Data", + secondary_priority="Dashboard", + ) + + self.assertEqual(speed_limit, 17.9) diff --git a/selfdrive/ui/ui b/selfdrive/ui/ui index a50fe22e..220434f7 100755 Binary files a/selfdrive/ui/ui and b/selfdrive/ui/ui differ diff --git a/selfdrive/ui/ui_state.py b/selfdrive/ui/ui_state.py index dbf816f0..bbbb909b 100644 --- a/selfdrive/ui/ui_state.py +++ b/selfdrive/ui/ui_state.py @@ -57,6 +57,7 @@ class UIState: "liveParameters", "rawAudioData", "starpilotCarState", + "starpilotPlan", ] ) diff --git a/starpilot/assets/vision_models/speed_limit_us_detector.onnx b/starpilot/assets/vision_models/speed_limit_us_detector.onnx new file mode 100755 index 00000000..b8711154 Binary files /dev/null and b/starpilot/assets/vision_models/speed_limit_us_detector.onnx differ diff --git a/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx b/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx new file mode 100755 index 00000000..c0c7e5b6 Binary files /dev/null and b/starpilot/assets/vision_models/speed_limit_us_value_classifier.onnx differ diff --git a/starpilot/assets/vision_models/speed_limit_vision.onnx b/starpilot/assets/vision_models/speed_limit_vision.onnx new file mode 100644 index 00000000..eadeda70 Binary files /dev/null and b/starpilot/assets/vision_models/speed_limit_vision.onnx differ diff --git a/starpilot/common/starpilot_variables.py b/starpilot/common/starpilot_variables.py index ea4f3419..f95e382a 100644 --- a/starpilot/common/starpilot_variables.py +++ b/starpilot/common/starpilot_variables.py @@ -883,8 +883,9 @@ class StarPilotVariables: toggle.sng_hack = self.get_value("SNGHack", condition=toggle.openpilot_longitudinal and toggle.car_make == "toyota" and not toggle.has_pedal and not has_sng) toggle.speed_limit_controller = toggle.openpilot_longitudinal and self.get_value("SpeedLimitController") - toggle.map_speed_lookahead_higher = self.get_value("SLCLookaheadHigher", cast=float, condition=toggle.speed_limit_controller) - toggle.map_speed_lookahead_lower = self.get_value("SLCLookaheadLower", cast=float, condition=toggle.speed_limit_controller) + speed_limit_display = toggle.show_speed_limits or toggle.speed_limit_controller + toggle.map_speed_lookahead_higher = self.get_value("SLCLookaheadHigher", cast=float, condition=speed_limit_display) + toggle.map_speed_lookahead_lower = self.get_value("SLCLookaheadLower", cast=float, condition=speed_limit_display) toggle.set_speed_limit = self.get_value("SetSpeedLimit", condition=toggle.speed_limit_controller) toggle.show_speed_limit_offset = self.get_value("ShowSLCOffset", condition=toggle.speed_limit_controller) or toggle.debug_mode slc_fallback_method = self.get_value("SLCFallback", cast=float, condition=toggle.speed_limit_controller) @@ -905,13 +906,14 @@ class StarPilotVariables: toggle.speed_limit_offset5 = self.get_value("Offset5", cast=float, condition=toggle.speed_limit_controller, conversion=speed_conversion) toggle.speed_limit_offset6 = self.get_value("Offset6", cast=float, condition=toggle.speed_limit_controller, conversion=speed_conversion) toggle.speed_limit_offset7 = self.get_value("Offset7", cast=float, condition=toggle.speed_limit_controller, conversion=speed_conversion) - toggle.speed_limit_priority1 = self.get_value("SLCPriority1", cast=None, condition=toggle.speed_limit_controller) - toggle.speed_limit_priority2 = self.get_value("SLCPriority2", cast=None, condition=toggle.speed_limit_controller) + toggle.speed_limit_priority1 = self.get_value("SLCPriority1", cast=None, condition=speed_limit_display) + toggle.speed_limit_priority2 = self.get_value("SLCPriority2", cast=None, condition=speed_limit_display) toggle.speed_limit_priority_highest = toggle.speed_limit_priority1 == "Highest" toggle.speed_limit_priority_lowest = toggle.speed_limit_priority1 == "Lowest" - toggle.speed_limit_sources = self.get_value("SpeedLimitSources", condition=toggle.speed_limit_controller) or toggle.debug_mode + toggle.speed_limit_sources = self.get_value("SpeedLimitSources", condition=speed_limit_display) or toggle.debug_mode toggle.speed_limit_filler = self.get_value("SpeedLimitFiller") + toggle.vision_speed_limit_detection = self.get_value("VisionSpeedLimitDetection") toggle.startup_alert_top = self.get_value("StartupMessageTop", cast=str, default="") toggle.startup_alert_bottom = self.get_value("StartupMessageBottom", cast=str, default="") diff --git a/starpilot/controls/lib/speed_limit_controller.py b/starpilot/controls/lib/speed_limit_controller.py index b61f9201..bb2d0029 100644 --- a/starpilot/controls/lib/speed_limit_controller.py +++ b/starpilot/controls/lib/speed_limit_controller.py @@ -51,6 +51,7 @@ class SpeedLimitController: self.speed_limit_changed_timer = 0 self.target = 0 self.unconfirmed_speed_limit = 0 + self.vision_limit = 0 self.previous_source = "None" self.source = "None" @@ -255,12 +256,14 @@ class SpeedLimitController: self.starpilot_planner.params.put_nonblocking("PreviousSpeedLimit", self.target) - def update_limits(self, dashboard_speed_limit, now, time_validated, v_cruise, v_ego, sm): + def update_limits(self, dashboard_speed_limit, now, time_validated, v_cruise, v_ego, sm, display_only=False): self.update_map_speed_limit(v_ego, sm) + self.vision_limit = self.starpilot_planner.params_memory.get_float("VisionSpeedLimit") if getattr(self.starpilot_toggles, "vision_speed_limit_detection", False) else 0 limits = { "Dashboard": dashboard_speed_limit, - "Map Data": self.map_speed_limit + "Map Data": self.map_speed_limit, + "Vision": self.vision_limit, } filtered_limits = {source: limit for source, limit in limits.items() if limit >= 1} @@ -297,7 +300,7 @@ class SpeedLimitController: desired_source = "Mapbox" desired_target = self.mapbox_limit - if desired_target == 0 or self.target == 0: + if not display_only and (desired_target == 0 or self.target == 0): if self.denied_target != self.previous_target > 0 and self.starpilot_toggles.slc_fallback_previous_speed_limit: desired_source = self.previous_source desired_target = self.previous_target @@ -311,6 +314,20 @@ class SpeedLimitController: self.mapbox_limit = 0 self.segment_distance = 0 + if display_only: + self.speed_limit_changed_timer = 0 + self.unconfirmed_speed_limit = 0 + self.overridden_speed = 0 + + if desired_target >= 1: + self.source = desired_source + self.target = desired_target + else: + self.source = "None" + self.target = 0 + + return + if abs(desired_target - self.previous_target) >= 1: self.handle_limit_change(desired_source, desired_target, sm) elif desired_source != self.source and abs(desired_target - self.target) < 1: diff --git a/starpilot/controls/lib/starpilot_vcruise.py b/starpilot/controls/lib/starpilot_vcruise.py index 2d9920b4..39e4c474 100644 --- a/starpilot/controls/lib/starpilot_vcruise.py +++ b/starpilot/controls/lib/starpilot_vcruise.py @@ -73,7 +73,7 @@ class StarPilotVCruise: self.slc_offset = self.slc.offset self.slc_target = self.slc.target elif starpilot_toggles.show_speed_limits: - self.slc.update_limits(sm["starpilotCarState"].dashboardSpeedLimit, now, time_validated, v_cruise, v_ego, sm) + self.slc.update_limits(sm["starpilotCarState"].dashboardSpeedLimit, now, time_validated, v_cruise, v_ego, sm, display_only=True) self.slc_offset = 0 self.slc_target = self.slc.target diff --git a/starpilot/system/speed_limit_filler.py b/starpilot/system/speed_limit_filler.py index 3a9f470c..dea91ffb 100644 --- a/starpilot/system/speed_limit_filler.py +++ b/starpilot/system/speed_limit_filler.py @@ -127,7 +127,9 @@ class MapSpeedLogger: self.filtered_dataset = list(self.cleanup_dataset(fallback_dataset)) def get_speed_limit_source(self): + vision_speed_limit = self.params_memory.get_float("VisionSpeedLimit") if self.params.get_bool("VisionSpeedLimitDetection") else 0 sources = [ + (vision_speed_limit, "Vision"), (self.sm["starpilotPlan"].slcMapboxSpeedLimit, "Mapbox"), (self.sm["starpilotCarState"].dashboardSpeedLimit, "Dashboard") ] diff --git a/starpilot/system/speed_limit_vision.py b/starpilot/system/speed_limit_vision.py new file mode 100644 index 00000000..ea37f3a4 --- /dev/null +++ b/starpilot/system/speed_limit_vision.py @@ -0,0 +1,1530 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import time + +from collections import Counter, deque +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +import cv2 +import numpy as np + +from openpilot.common.constants import CV + +INFERENCE_INTERVAL = 0.4 +FOLLOWUP_INFERENCE_INTERVAL = 0.2 +FOLLOWUP_WINDOW_SECONDS = 1.5 +MIN_DETECTION_CONFIDENCE = 0.2 +STRONG_DETECTION_CONFIDENCE = 0.72 +OCR_MIN_CONFIDENCE = 0.35 +VALUE_TEMPLATE_MIN_CONFIDENCE = 0.62 +HISTORY_SECONDS = 2.0 +CONSISTENT_DETECTIONS = 2 +MODEL_DETECTION_SHORT_CIRCUIT_CONFIDENCE = 0.65 +PUBLISHED_HOLD_SECONDS = 12.0 +PUBLISHED_CHANGE_COOLDOWN_SECONDS = 1.4 +PUBLISHED_REVERT_CONFIDENCE = 0.97 +AUTO_BOOKMARK_CONFIRM_DELAY_SECONDS = 0.9 +AUTO_BOOKMARK_COOLDOWN_SECONDS = 8.0 +AUTO_BOOKMARK_MIN_CONFIDENCE = 0.62 +MODEL_PROPOSAL_MIN_CONFIDENCE = 0.0001 +MODEL_PROPOSAL_MAX_COUNT = 16 +MODEL_PROPOSAL_MAX_AREA_RATIO = 0.18 +MODEL_PROPOSAL_MIN_WIDTH = 10 +MODEL_PROPOSAL_MIN_HEIGHT = 18 +MODEL_PROPOSAL_MIN_X_RATIO = 0.35 +MODEL_PROPOSAL_MAX_Y_RATIO = 0.82 +MODEL_PROPOSAL_EXPANSIONS = ( + (1.2, 1.6, 1.2, 1.8), +) +MODEL_PROPOSAL_TRIM_BOTTOM_RATIOS = (1.0, 0.78) + +ROI_WINDOWS = ( + {"bounds": (0.48, 0.00, 0.98, 0.42), "min_confidence": MIN_DETECTION_CONFIDENCE}, + {"bounds": (0.52, 0.02, 0.97, 0.58), "min_confidence": 0.22}, + {"bounds": (0.62, 0.02, 0.99, 0.68), "min_confidence": 0.18}, + {"bounds": (0.72, 0.05, 1.00, 0.82), "min_confidence": 0.15}, +) +EDGE_MARGIN_RATIO = 0.03 +MAX_BOX_AREA_RATIO = 0.22 + +OCR_SEARCH_WINDOWS = ( + (0.45, 0.05, 0.92, 0.86), + (0.65, 0.08, 0.98, 0.76), +) +OCR_SEARCH_THRESHOLDS = (130,) +OCR_SEARCH_CROP_VARIANTS = ( + (0.08, 0.06, 0.10), +) +OCR_SEARCH_UPSCALE = 3 +OCR_FALLBACK_MIN_CONFIDENCE = 0.55 +VALUE_TEMPLATE_ROIS = ( + (0.35, 0.82, 0.15, 0.78), + (0.45, 0.85, 0.18, 0.78), + (0.40, 0.84, 0.18, 0.75), +) +REGULATORY_WHITE_VALUE_MIN = 135 +REGULATORY_WHITE_SAT_MAX = 70 +REGULATORY_DARK_VALUE_MAX = 115 +REGULATORY_DARK_SAT_MAX = 110 +REGULATORY_YELLOW_HUE_MIN = 12 +REGULATORY_YELLOW_HUE_MAX = 45 +REGULATORY_YELLOW_SAT_MIN = 70 +REGULATORY_YELLOW_VALUE_MIN = 85 +REGULATORY_RED_LOW_HUE_MAX = 12 +REGULATORY_RED_HIGH_HUE_MIN = 168 +REGULATORY_RED_SAT_MIN = 80 +REGULATORY_RED_VALUE_MIN = 60 +REGULATORY_MIN_WHITE_RATIO = 0.08 +REGULATORY_MIN_DARK_RATIO = 0.01 +REGULATORY_MAX_YELLOW_RATIO = 0.12 +REGULATORY_MAX_RED_RATIO = 0.10 +REGULATORY_MIN_WHITE_COMPONENT_RATIO = 0.012 +REGULATORY_MIN_COMPONENT_FILL = 0.36 +REGULATORY_MIN_COMPONENT_HEIGHT_RATIO = 0.2 +REGULATORY_MIN_COMPONENT_WIDTH_RATIO = 0.12 +REGULATORY_MIN_ASPECT_RATIO = 0.28 +REGULATORY_MAX_ASPECT_RATIO = 1.25 + +SPEED_LIMIT_CLASSES = { + 2: 10, + 3: 100, + 4: 110, + 5: 120, + 6: 20, + 7: 30, + 8: 40, + 9: 50, + 10: 60, + 11: 70, + 12: 80, + 13: 90, +} + +VALID_SPEED_LIMITS_MPH = set(range(10, 125, 5)) +LEGACY_MODEL_PATH = Path(__file__).resolve().parents[1] / "assets" / "vision_models" / "speed_limit_vision.onnx" +US_DETECTOR_MODEL_PATH = Path(__file__).resolve().parents[1] / "assets" / "vision_models" / "speed_limit_us_detector.onnx" +US_CLASSIFIER_MODEL_PATH = Path(__file__).resolve().parents[1] / "assets" / "vision_models" / "speed_limit_us_value_classifier.onnx" +US_DETECTOR_CLASSES = { + 0: "regulatory_speed_limit", + 1: "advisory_speed_limit", + 2: "school_zone_speed_limit", +} +US_CLASSIFIER_SPEED_VALUES = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75) +SCHOOL_ZONE_SPEED_VALUES = frozenset((15, 20, 25)) +US_DETECTOR_MIN_CONFIDENCE = 0.10 +US_CLASSIFIER_MIN_CONFIDENCE = 0.38 +DETECTOR_CLASSIFIER_EXPANSIONS = ( + (0.00, 0.00, 0.00, 0.00, 1.10), + (0.10, 0.06, 0.10, 0.12, 1.00), + (0.06, 0.00, 0.10, 0.10, 0.75), + (0.00, 0.00, 0.18, 0.18, 0.55), +) +SCHOOL_ZONE_DIRECT_EXPANSIONS = ( + (0.00, 0.00, 0.18, 0.18), + (0.00, 0.00, 0.22, 0.18), +) +SCHOOL_ZONE_READ_VARIANTS = ( + (0.00, 0.00, 1.00, 1.00, 0.80), + (0.00, 0.35, 1.00, 1.00, 0.88), + (0.00, 0.45, 1.00, 1.00, 0.96), + (0.12, 0.38, 0.88, 1.00, 1.00), +) +DETECTOR_CLASSIFIER_SUPPORT_BONUS = 0.06 +DETECTOR_CLASSIFIER_REGULATORY_BONUS = 0.05 +DETECTOR_CLASSIFIER_NON_REGULATORY_PENALTY = 0.03 +DETECTOR_CLASSIFIER_SMALL_BOX_AREA_RATIO = 0.004 +DETECTOR_CLASSIFIER_MIN_ACCEPT_WIDTH = 28 +DETECTOR_CLASSIFIER_MIN_ACCEPT_HEIGHT = 40 +DETECTOR_CLASSIFIER_RESCUE_MIN_WIDTH = 14 +DETECTOR_CLASSIFIER_RESCUE_MIN_HEIGHT = 24 +DETECTOR_CLASSIFIER_RESCUE_MIN_X_RATIO = 0.52 +DETECTOR_CLASSIFIER_RESCUE_MIN_SUPPORT = 2 +DETECTOR_CLASSIFIER_RESCUE_MIN_CONFIDENCE = 0.90 +DETECTOR_CLASSIFIER_RESCUE_MAX_SCORE = 0.64 +SCHOOL_ZONE_SPEED_PRIOR = 0.12 +SCHOOL_ZONE_SUPPORT_BONUS = 0.08 +SCHOOL_ZONE_MIN_SUPPORT = 2 +SCHOOL_ZONE_MIN_CONFIDENCE = 0.70 +SCHOOL_ZONE_SINGLE_READ_CONFIDENCE = 0.975 +SCHOOL_ZONE_SHORT_CIRCUIT_CONFIDENCE = 0.78 +DEBUG_BASE_DIR = Path("/data/media/0/vision_speed_limit_debug") +DEBUG_CAPTURE_DIRNAME = "captures" +SNAPSHOT_JPEG_QUALITY = 85 + + +@dataclass +class Detection: + speed_limit_mph: int + confidence: float + + +@dataclass +class HistoryEntry: + speed_limit_mph: int + confidence: float + created_at: float + + +class SpeedLimitVisionDaemon: + def __init__(self, use_runtime=True): + self.use_runtime = use_runtime + self.params = None + self.params_memory = None + self.Ratekeeper = None + self.messaging = None + self.pm = None + self.VisionIpcClient = None + self.VisionStreamType = None + self.sm = None + + if self.use_runtime: + from cereal import messaging + from msgq.visionipc import VisionIpcClient, VisionStreamType + from openpilot.common.params import Params + from openpilot.common.realtime import Ratekeeper + + self.messaging = messaging + self.pm = messaging.PubMaster(["userBookmark"]) + self.params = Params(return_defaults=True) + self.params_memory = Params(memory=True) + self.Ratekeeper = Ratekeeper + self.VisionIpcClient = VisionIpcClient + self.VisionStreamType = VisionStreamType + self.sm = messaging.SubMaster(["deviceState", "mapdOut", "userBookmark"]) + + self.client = None + self.stream_name = "" + self.stream_type = None + + self.net = None + self.classifier_net = None + self.model_mode = "legacy" + self.last_error = "" + self.last_inference_at = 0.0 + self.last_detection_at = 0.0 + self.last_road_name = "" + self.followup_until = 0.0 + self.started_prev = False + + self.history: deque[HistoryEntry] = deque() + self.published_speed_limit_mph = 0 + self.published_confidence = 0.0 + self.previous_published_speed_limit_mph = 0 + self.last_publish_change_at = 0.0 + self.last_candidate_speed_limit_mph = 0 + self.last_candidate_confidence = 0.0 + self.last_candidate_at = 0.0 + self.last_auto_bookmark_at = 0.0 + self.last_auto_bookmark_speed_limit_mph = 0 + self.last_auto_bookmark_publish_at = 0.0 + self.pending_auto_bookmark = None + self.ignore_next_user_bookmark = False + self.current_frame_bgr = None + + self.debug_session_id = "" + self.debug_dir = None + self.debug_capture_dir = None + self.debug_log_path = None + self.debug_bookmark_count = 0 + self.debug_session_started_at = 0.0 + self.last_logged_status = "" + self.last_logged_candidate = None + + self.digit_templates = self._build_digit_templates() + self.speed_value_templates = self._build_speed_value_templates() + self._load_model() + + def _start_debug_session(self): + if not self.use_runtime or self.params_memory is None or self.debug_session_id: + return + + timestamp = datetime.now(timezone.utc) + session_id = timestamp.strftime("%Y%m%d_%H%M%S") + debug_dir = DEBUG_BASE_DIR / session_id + suffix = 1 + while debug_dir.exists(): + suffix += 1 + session_id = f"{timestamp.strftime('%Y%m%d_%H%M%S')}_{suffix}" + debug_dir = DEBUG_BASE_DIR / session_id + + debug_dir.mkdir(parents=True, exist_ok=True) + capture_dir = debug_dir / DEBUG_CAPTURE_DIRNAME + capture_dir.mkdir(parents=True, exist_ok=True) + + self.debug_session_id = session_id + self.debug_dir = debug_dir + self.debug_capture_dir = capture_dir + self.debug_log_path = debug_dir / "events.jsonl" + self.debug_bookmark_count = 0 + self.debug_session_started_at = time.monotonic() + self.last_logged_status = "" + self.last_logged_candidate = None + + self.params_memory.put("VisionSpeedLimitDebugSession", self.debug_session_id) + self.params_memory.put_int("VisionSpeedLimitBookmarkCount", self.debug_bookmark_count) + self.params_memory.put("VisionSpeedLimitLastEvent", "") + self._write_debug_event("session_start", reason="onroad") + + def _close_debug_session(self): + self.debug_session_id = "" + self.debug_dir = None + self.debug_capture_dir = None + self.debug_log_path = None + self.debug_bookmark_count = 0 + self.debug_session_started_at = 0.0 + self.last_logged_status = "" + self.last_logged_candidate = None + + def _write_debug_event(self, event_type, frame_bgr=None, snapshot_prefix=None, **fields): + if not self.use_runtime or self.params_memory is None: + return + if not self.debug_log_path: + return + + wall_time_ns = time.time_ns() + event = { + "event": event_type, + "wallTimeNs": wall_time_ns, + "wallTime": datetime.fromtimestamp(wall_time_ns / 1e9, timezone.utc).isoformat(), + "monoTimeNs": time.monotonic_ns(), + "roadName": self.last_road_name, + "stream": self.stream_name, + "publishedSpeedLimitMph": self.published_speed_limit_mph, + "publishedConfidence": round(self.published_confidence, 4), + "bookmarkCount": self.debug_bookmark_count, + "status": self.params_memory.get("VisionSpeedLimitStatus", encoding="utf-8") or "", + } + if self.debug_session_started_at > 0.0: + event["sessionSeconds"] = round(max(time.monotonic() - self.debug_session_started_at, 0.0), 3) + event.update(fields) + + if frame_bgr is not None and self.debug_capture_dir is not None and snapshot_prefix: + snapshot_name = f"{wall_time_ns}_{snapshot_prefix}.jpg" + snapshot_path = self.debug_capture_dir / snapshot_name + try: + cv2.imwrite(str(snapshot_path), frame_bgr, [cv2.IMWRITE_JPEG_QUALITY, SNAPSHOT_JPEG_QUALITY]) + event["snapshot"] = f"{DEBUG_CAPTURE_DIRNAME}/{snapshot_name}" + except Exception: + pass + + try: + with self.debug_log_path.open("a", encoding="utf-8") as log_file: + log_file.write(json.dumps(event, separators=(",", ":")) + "\n") + except Exception: + pass + + summary_parts = [event_type.replace("_", " ")] + if "speedLimitMph" in fields: + summary_parts.append(f"{fields['speedLimitMph']} mph") + elif "candidateSpeedLimitMph" in fields: + summary_parts.append(f"{fields['candidateSpeedLimitMph']} mph") + summary = " ".join(summary_parts) + self.params_memory.put("VisionSpeedLimitLastEvent", summary[:160]) + + def _record_bookmark(self): + if not self.use_runtime or self.params_memory is None or not self.debug_log_path: + return + + self.debug_bookmark_count += 1 + self.params_memory.put_int("VisionSpeedLimitBookmarkCount", self.debug_bookmark_count) + + fields = { + "candidateSpeedLimitMph": self.last_candidate_speed_limit_mph, + "candidateConfidence": round(self.last_candidate_confidence, 4), + "publishedSpeedLimitMph": self.published_speed_limit_mph, + "publishedConfidence": round(self.published_confidence, 4), + } + self._write_debug_event( + "bookmark", + frame_bgr=self.current_frame_bgr, + snapshot_prefix=f"bookmark_{self.debug_bookmark_count:03d}", + **fields, + ) + + def _record_auto_bookmark(self, speed_limit_mph, confidence): + if not self.use_runtime or self.params_memory is None or not self.debug_log_path: + return + + self.debug_bookmark_count += 1 + self.params_memory.put_int("VisionSpeedLimitBookmarkCount", self.debug_bookmark_count) + + self._write_debug_event( + "auto_bookmark", + frame_bgr=self.current_frame_bgr, + snapshot_prefix=f"auto_bookmark_{self.debug_bookmark_count:03d}", + candidateSpeedLimitMph=self.last_candidate_speed_limit_mph, + candidateConfidence=round(self.last_candidate_confidence, 4), + publishedSpeedLimitMph=self.published_speed_limit_mph, + publishedConfidence=round(self.published_confidence, 4), + speedLimitMph=speed_limit_mph, + confidence=round(confidence, 4), + bookmarkCount=self.debug_bookmark_count, + ) + + def _schedule_auto_bookmark(self, speed_limit_mph, confidence, published_at): + if not self.use_runtime or self.params is None: + return + if not self.params.get_bool("VisionSpeedLimitAutoBookmark"): + return + if confidence < AUTO_BOOKMARK_MIN_CONFIDENCE: + return + if published_at - self.last_auto_bookmark_at < AUTO_BOOKMARK_COOLDOWN_SECONDS and speed_limit_mph == self.last_auto_bookmark_speed_limit_mph: + return + + self.pending_auto_bookmark = { + "due_at": published_at + AUTO_BOOKMARK_CONFIRM_DELAY_SECONDS, + "published_at": published_at, + "speed_limit_mph": speed_limit_mph, + "confidence": confidence, + } + + def _emit_preserve_bookmark(self): + if not self.use_runtime or self.pm is None or self.messaging is None: + return + self.ignore_next_user_bookmark = True + msg = self.messaging.new_message("userBookmark", valid=True) + self.pm.send("userBookmark", msg) + + def _maybe_commit_auto_bookmark(self, now): + pending = self.pending_auto_bookmark + if pending is None or now < pending["due_at"]: + return + if self.params is not None and not self.params.get_bool("VisionSpeedLimitAutoBookmark"): + self.pending_auto_bookmark = None + return + + self.pending_auto_bookmark = None + speed_limit_mph = int(pending["speed_limit_mph"]) + confidence = float(pending["confidence"]) + published_at = float(pending["published_at"]) + + if self.published_speed_limit_mph != speed_limit_mph: + return + if self.published_confidence + 0.02 < confidence: + return + if self.current_frame_bgr is None: + return + + self.last_auto_bookmark_at = now + self.last_auto_bookmark_speed_limit_mph = speed_limit_mph + self.last_auto_bookmark_publish_at = published_at + self._record_auto_bookmark(speed_limit_mph, max(confidence, self.published_confidence)) + + if self.params is not None and self.params.get_bool("VisionSpeedLimitAutoPreserveSegment"): + self._emit_preserve_bookmark() + + def _published_detection_stale(self, now): + return self.published_speed_limit_mph > 0 and now - self.last_detection_at > PUBLISHED_HOLD_SECONDS + + def _load_model(self): + self.net = None + self.classifier_net = None + self.model_mode = "legacy" + + if US_DETECTOR_MODEL_PATH.is_file() and US_CLASSIFIER_MODEL_PATH.is_file(): + try: + self.net = cv2.dnn.readNetFromONNX(str(US_DETECTOR_MODEL_PATH)) + self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) + self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) + self.classifier_net = cv2.dnn.readNetFromONNX(str(US_CLASSIFIER_MODEL_PATH)) + self.classifier_net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) + self.classifier_net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) + self.model_mode = "detector_classifier" + self.last_error = "" + return + except Exception as exc: + self.net = None + self.classifier_net = None + self.model_mode = "legacy" + self.last_error = f"Failed to load U.S. vision models: {exc}" + + if not LEGACY_MODEL_PATH.is_file(): + if not self.last_error: + self.last_error = f"Missing vision model: {LEGACY_MODEL_PATH.name}" + self._publish_status(self.last_error, clear_speed=True) + return + + try: + self.net = cv2.dnn.readNetFromONNX(str(LEGACY_MODEL_PATH)) + self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) + self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) + self.last_error = "" + except Exception as exc: + self.last_error = f"Failed to load vision model: {exc}" + self._publish_status(self.last_error, clear_speed=True) + + def _build_digit_templates(self): + templates = {str(digit): [] for digit in range(10)} + fonts = [ + cv2.FONT_HERSHEY_SIMPLEX, + cv2.FONT_HERSHEY_DUPLEX, + cv2.FONT_HERSHEY_COMPLEX, + cv2.FONT_HERSHEY_TRIPLEX, + ] + + for digit in templates: + for font in fonts: + for scale, thickness in ((1.5, 2), (1.7, 3), (1.9, 3), (2.1, 4)): + canvas = np.full((120, 80), 255, dtype=np.uint8) + text_size, baseline = cv2.getTextSize(digit, font, scale, thickness) + x = max((canvas.shape[1] - text_size[0]) // 2, 0) + y = max((canvas.shape[0] + text_size[1]) // 2 - baseline, text_size[1]) + cv2.putText(canvas, digit, (x, y), font, scale, 0, thickness, cv2.LINE_AA) + _, binary = cv2.threshold(canvas, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + normalized = self._normalize_binary_digit(binary) + if normalized is not None: + templates[digit].append(normalized) + + return templates + + def _build_speed_value_templates(self): + templates = {str(speed_limit): [] for speed_limit in VALID_SPEED_LIMITS_MPH} + fonts = [ + cv2.FONT_HERSHEY_SIMPLEX, + cv2.FONT_HERSHEY_DUPLEX, + cv2.FONT_HERSHEY_COMPLEX, + cv2.FONT_HERSHEY_TRIPLEX, + ] + + for speed_limit in templates: + for font in fonts: + for scale, thickness in ((1.2, 2), (1.4, 2), (1.6, 3), (1.8, 3), (2.0, 4)): + canvas = np.full((120, 100), 255, dtype=np.uint8) + text_size, baseline = cv2.getTextSize(speed_limit, font, scale, thickness) + x = max((canvas.shape[1] - text_size[0]) // 2, 0) + y = max((canvas.shape[0] + text_size[1]) // 2 - baseline, text_size[1]) + cv2.putText(canvas, speed_limit, (x, y), font, scale, 0, thickness, cv2.LINE_AA) + _, binary = cv2.threshold(canvas, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + normalized = self._normalize_binary_digit(binary, size=(72, 96)) + if normalized is not None: + templates[speed_limit].append(normalized) + + return templates + + def _normalize_binary_digit(self, binary, size=(48, 72), padding=6): + points = cv2.findNonZero(binary) + if points is None: + return None + + x, y, w, h = cv2.boundingRect(points) + digit = binary[y:y + h, x:x + w] + target_w, target_h = size + scale = min((target_w - padding * 2) / max(w, 1), (target_h - padding * 2) / max(h, 1)) + resized_w = max(int(round(w * scale)), 1) + resized_h = max(int(round(h * scale)), 1) + resized = cv2.resize(digit, (resized_w, resized_h), interpolation=cv2.INTER_NEAREST) + + canvas = np.zeros((target_h, target_w), dtype=np.uint8) + offset_x = (target_w - resized_w) // 2 + offset_y = (target_h - resized_h) // 2 + canvas[offset_y:offset_y + resized_h, offset_x:offset_x + resized_w] = resized + return canvas + + def _connect_camera(self): + if not self.use_runtime or self.VisionIpcClient is None or self.VisionStreamType is None: + return False + + try: + available_streams = self.VisionIpcClient.available_streams("camerad", block=False) + except Exception: + available_streams = [] + + desired_stream = None + stream_name = "" + if self.VisionStreamType.VISION_STREAM_ROAD in available_streams: + desired_stream = self.VisionStreamType.VISION_STREAM_ROAD + stream_name = "road camera" + elif self.VisionStreamType.VISION_STREAM_WIDE_ROAD in available_streams: + desired_stream = self.VisionStreamType.VISION_STREAM_WIDE_ROAD + stream_name = "wide camera" + + if desired_stream is None: + self.client = None + self.stream_type = None + self.stream_name = "" + return False + + if self.client is None or self.stream_type != desired_stream: + self.client = self.VisionIpcClient("camerad", desired_stream, True) + self.stream_type = desired_stream + self.stream_name = stream_name + + if not self.client.is_connected(): + self.client.connect(True) + + return self.client.is_connected() + + @staticmethod + def _letterbox(image, shape=(640, 640), color=(114, 114, 114)): + image_height, image_width = image.shape[:2] + ratio = min(shape[0] / image_height, shape[1] / image_width) + resized_width = int(round(image_width * ratio)) + resized_height = int(round(image_height * ratio)) + pad_width = (shape[1] - resized_width) / 2 + pad_height = (shape[0] - resized_height) / 2 + + if (image_width, image_height) != (resized_width, resized_height): + image = cv2.resize(image, (resized_width, resized_height), interpolation=cv2.INTER_LINEAR) + + top = int(round(pad_height - 0.1)) + bottom = int(round(pad_height + 0.1)) + left = int(round(pad_width - 0.1)) + right = int(round(pad_width + 0.1)) + image = cv2.copyMakeBorder(image, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) + return image, ratio, pad_width, pad_height + + def _detect_sign(self, frame_bgr): + if self.net is None: + return self._detect_sign_from_ocr_candidates(frame_bgr) + + if self.model_mode == "detector_classifier" and self.classifier_net is not None: + detector_detection = self._detect_sign_from_detector_classifier(frame_bgr) + if detector_detection is not None: + return detector_detection + return self._detect_sign_from_ocr_candidates(frame_bgr) + + model_detection = self._detect_sign_from_model_proposals(frame_bgr) + if model_detection is not None and model_detection.confidence >= MODEL_DETECTION_SHORT_CIRCUIT_CONFIDENCE: + return model_detection + + ocr_detection = self._detect_sign_from_ocr_candidates(frame_bgr) + if ocr_detection is not None and (model_detection is None or ocr_detection.confidence > model_detection.confidence): + return ocr_detection + + return model_detection + + def _is_regulatory_speed_sign(self, sign_crop): + if sign_crop.size == 0: + return False + + crop_height, crop_width = sign_crop.shape[:2] + crop_area = crop_height * crop_width + if crop_area <= 0: + return False + + hsv = cv2.cvtColor(sign_crop, cv2.COLOR_BGR2HSV) + hue = hsv[:, :, 0] + saturation = hsv[:, :, 1] + value = hsv[:, :, 2] + + white_mask = ((value >= REGULATORY_WHITE_VALUE_MIN) & (saturation <= REGULATORY_WHITE_SAT_MAX)).astype(np.uint8) + dark_mask = ((value <= REGULATORY_DARK_VALUE_MAX) & (saturation <= REGULATORY_DARK_SAT_MAX)).astype(np.uint8) + yellow_mask = ( + (hue >= REGULATORY_YELLOW_HUE_MIN) & + (hue <= REGULATORY_YELLOW_HUE_MAX) & + (saturation >= REGULATORY_YELLOW_SAT_MIN) & + (value >= REGULATORY_YELLOW_VALUE_MIN) + ).astype(np.uint8) + red_mask = ( + (((hue <= REGULATORY_RED_LOW_HUE_MAX) | (hue >= REGULATORY_RED_HIGH_HUE_MIN))) & + (saturation >= REGULATORY_RED_SAT_MIN) & + (value >= REGULATORY_RED_VALUE_MIN) + ).astype(np.uint8) + + white_ratio = float(white_mask.mean()) + dark_ratio = float(dark_mask.mean()) + yellow_ratio = float(yellow_mask.mean()) + red_ratio = float(red_mask.mean()) + + if white_ratio < REGULATORY_MIN_WHITE_RATIO or dark_ratio < REGULATORY_MIN_DARK_RATIO: + return False + if yellow_ratio > REGULATORY_MAX_YELLOW_RATIO and yellow_ratio > white_ratio * 0.45: + return False + if red_ratio > REGULATORY_MAX_RED_RATIO and red_ratio > white_ratio * 0.35: + return False + + white_binary = (white_mask * 255).astype(np.uint8) + contours, _ = cv2.findContours(white_binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + min_component_area = crop_area * REGULATORY_MIN_WHITE_COMPONENT_RATIO + + for contour in contours: + area = cv2.contourArea(contour) + if area < min_component_area: + continue + + x, y, width, height = cv2.boundingRect(contour) + if height < crop_height * REGULATORY_MIN_COMPONENT_HEIGHT_RATIO: + continue + if width < crop_width * REGULATORY_MIN_COMPONENT_WIDTH_RATIO: + continue + + aspect_ratio = width / max(height, 1) + if aspect_ratio < REGULATORY_MIN_ASPECT_RATIO or aspect_ratio > REGULATORY_MAX_ASPECT_RATIO: + continue + + fill_ratio = area / max(width * height, 1) + if fill_ratio < REGULATORY_MIN_COMPONENT_FILL: + continue + + return True + + return False + + @staticmethod + def _softmax(scores): + scores = scores.astype(np.float32) + scores = scores - np.max(scores) + exp_scores = np.exp(scores) + denominator = np.sum(exp_scores) + if denominator <= 0: + return exp_scores + return exp_scores / denominator + + @staticmethod + def _normalize_classifier_output(scores): + scores = scores.astype(np.float32) + if scores.size == 0: + return scores + + # Ultralytics classifier ONNX exports may already emit normalized probabilities. + if np.all(scores >= 0.0) and np.all(scores <= 1.0): + total = float(np.sum(scores)) + if 0.99 <= total <= 1.01: + return scores + + return SpeedLimitVisionDaemon._softmax(scores) + + @staticmethod + def _square_resize(image, size=128, color=(114, 114, 114)): + image_height, image_width = image.shape[:2] + ratio = min(size / max(image_height, 1), size / max(image_width, 1)) + resized_width = max(int(round(image_width * ratio)), 1) + resized_height = max(int(round(image_height * ratio)), 1) + image = cv2.resize(image, (resized_width, resized_height), interpolation=cv2.INTER_LINEAR) + + canvas = np.full((size, size, image.shape[2]), color, dtype=image.dtype) + offset_x = (size - resized_width) // 2 + offset_y = (size - resized_height) // 2 + canvas[offset_y:offset_y + resized_height, offset_x:offset_x + resized_width] = image + return canvas + + @staticmethod + def _crop_by_ratio(image, left_ratio, top_ratio, right_ratio, bottom_ratio): + image_height, image_width = image.shape[:2] + x1 = max(int(image_width * left_ratio), 0) + y1 = max(int(image_height * top_ratio), 0) + x2 = min(int(image_width * right_ratio), image_width) + y2 = min(int(image_height * bottom_ratio), image_height) + if x2 <= x1 or y2 <= y1: + return None + crop = image[y1:y2, x1:x2] + return crop if crop.size > 0 else None + + def _iter_school_zone_read_crops(self, sign_crop): + yielded = set() + for left_ratio, top_ratio, right_ratio, bottom_ratio, weight in SCHOOL_ZONE_READ_VARIANTS: + crop = self._crop_by_ratio(sign_crop, left_ratio, top_ratio, right_ratio, bottom_ratio) + if crop is None: + continue + key = (crop.shape[1], crop.shape[0], left_ratio, top_ratio, right_ratio, bottom_ratio) + if key in yielded: + continue + yielded.add(key) + yield crop, weight + + def _collect_model_proposals(self, frame_bgr): + if self.net is None: + return [] + + frame_height, frame_width = frame_bgr.shape[:2] + letterboxed, ratio, pad_width, pad_height = self._letterbox(frame_bgr) + blob = cv2.dnn.blobFromImage(letterboxed, scalefactor=1 / 255.0, size=(640, 640), swapRB=True, crop=False) + self.net.setInput(blob) + + predictions = np.squeeze(self.net.forward()) + if predictions.ndim != 2: + return [] + if predictions.shape[0] < predictions.shape[1]: + predictions = predictions.T + + max_box_area = frame_width * frame_height * MODEL_PROPOSAL_MAX_AREA_RATIO + candidates = [] + for prediction in predictions: + class_scores = prediction[4:] + class_id = int(np.argmax(class_scores)) + confidence = float(class_scores[class_id]) + if class_id not in SPEED_LIMIT_CLASSES: + continue + if confidence < MODEL_PROPOSAL_MIN_CONFIDENCE: + continue + + center_x, center_y, width, height = prediction[:4] + x1 = max(int((center_x - width / 2 - pad_width) / ratio), 0) + y1 = max(int((center_y - height / 2 - pad_height) / ratio), 0) + x2 = min(int((center_x + width / 2 - pad_width) / ratio), frame_width) + y2 = min(int((center_y + height / 2 - pad_height) / ratio), frame_height) + if x2 <= x1 or y2 <= y1: + continue + + box_width = x2 - x1 + box_height = y2 - y1 + if box_width < MODEL_PROPOSAL_MIN_WIDTH or box_height < MODEL_PROPOSAL_MIN_HEIGHT: + continue + if box_width * box_height > max_box_area: + continue + if (x1 + x2) / 2 < frame_width * MODEL_PROPOSAL_MIN_X_RATIO: + continue + if y1 > frame_height * MODEL_PROPOSAL_MAX_Y_RATIO: + continue + + candidates.append((confidence, class_id, (x1, y1, x2, y2))) + + return sorted(candidates, reverse=True)[:MODEL_PROPOSAL_MAX_COUNT] + + def _collect_detector_classifier_proposals_from_region(self, frame_bgr, origin_x, origin_y, full_frame_width, full_frame_height, min_confidence): + if self.net is None: + return [] + + region_height, region_width = frame_bgr.shape[:2] + letterboxed, ratio, pad_width, pad_height = self._letterbox(frame_bgr) + blob = cv2.dnn.blobFromImage(letterboxed, scalefactor=1 / 255.0, size=(640, 640), swapRB=True, crop=False) + self.net.setInput(blob) + + predictions = np.squeeze(self.net.forward()) + if predictions.ndim != 2: + return [] + if predictions.shape[0] < predictions.shape[1]: + predictions = predictions.T + + candidates = [] + for prediction in predictions: + class_scores = prediction[4:] + class_id = int(np.argmax(class_scores)) + confidence = float(class_scores[class_id]) + if class_id not in US_DETECTOR_CLASSES: + continue + if confidence < min_confidence: + continue + + center_x, center_y, width, height = prediction[:4] + x1 = max(int((center_x - width / 2 - pad_width) / ratio), 0) + y1 = max(int((center_y - height / 2 - pad_height) / ratio), 0) + x2 = min(int((center_x + width / 2 - pad_width) / ratio), region_width) + y2 = min(int((center_y + height / 2 - pad_height) / ratio), region_height) + if x2 <= x1 or y2 <= y1: + continue + + x1 += origin_x + y1 += origin_y + x2 += origin_x + y2 += origin_y + + box_width = x2 - x1 + box_height = y2 - y1 + if box_width < MODEL_PROPOSAL_MIN_WIDTH or box_height < MODEL_PROPOSAL_MIN_HEIGHT: + continue + if box_width * box_height > full_frame_width * full_frame_height * MODEL_PROPOSAL_MAX_AREA_RATIO: + continue + if (x1 + x2) / 2 < full_frame_width * MODEL_PROPOSAL_MIN_X_RATIO: + continue + if y1 > full_frame_height * MODEL_PROPOSAL_MAX_Y_RATIO: + continue + + candidates.append((confidence, class_id, (x1, y1, x2, y2))) + + return candidates + + def _collect_detector_classifier_proposals(self, frame_bgr): + if self.net is None: + return [] + + frame_height, frame_width = frame_bgr.shape[:2] + candidates = self._collect_detector_classifier_proposals_from_region( + frame_bgr, + 0, + 0, + frame_width, + frame_height, + US_DETECTOR_MIN_CONFIDENCE, + ) + + # A second pass on a focused right-side ROI materially improves small U.S. sign reads. + if ROI_WINDOWS: + left_ratio, top_ratio, right_ratio, bottom_ratio = ROI_WINDOWS[-1]["bounds"] + left = int(frame_width * left_ratio) + top = int(frame_height * top_ratio) + right = int(frame_width * right_ratio) + bottom = int(frame_height * bottom_ratio) + roi = frame_bgr[top:bottom, left:right] + if roi.size > 0: + candidates.extend(self._collect_detector_classifier_proposals_from_region( + roi, + left, + top, + frame_width, + frame_height, + max(float(ROI_WINDOWS[-1]["min_confidence"]), US_DETECTOR_MIN_CONFIDENCE), + )) + + return sorted(candidates, reverse=True)[:MODEL_PROPOSAL_MAX_COUNT] + + def _classify_speed_limit_from_model(self, sign_crop): + if self.classifier_net is None or sign_crop.size == 0: + return None + + normalized_mask = self._extract_value_template_mask(sign_crop) + if normalized_mask is None: + return None + + classifier_input = cv2.cvtColor(normalized_mask, cv2.COLOR_GRAY2BGR) + padded_crop = self._square_resize(classifier_input, size=128) + blob = cv2.dnn.blobFromImage(padded_crop, scalefactor=1 / 255.0, size=(128, 128), swapRB=True, crop=False) + self.classifier_net.setInput(blob) + + scores = np.array(self.classifier_net.forward()).reshape(-1) + if scores.size != len(US_CLASSIFIER_SPEED_VALUES): + return None + + probabilities = self._normalize_classifier_output(scores) + class_index = int(np.argmax(probabilities)) + confidence = float(probabilities[class_index]) + if confidence < US_CLASSIFIER_MIN_CONFIDENCE: + return None + + return US_CLASSIFIER_SPEED_VALUES[class_index], confidence + + def _detect_sign_from_detector_classifier(self, frame_bgr): + frame_height, frame_width = frame_bgr.shape[:2] + best_detection = None + best_score = 0.0 + + for proposal_confidence, class_id, (x1, y1, x2, y2) in self._collect_detector_classifier_proposals(frame_bgr): + if class_id == 1: + continue + + box_width = x2 - x1 + box_height = y2 - y1 + if box_width <= 0 or box_height <= 0: + continue + is_small_box = box_width < DETECTOR_CLASSIFIER_MIN_ACCEPT_WIDTH or box_height < DETECTOR_CLASSIFIER_MIN_ACCEPT_HEIGHT + # Tiny far-away proposals are the main nighttime false-positive source. + # Keep a narrow rescue path for right-side proposals with strong repeated reads, + # then let temporal confirmation decide whether they are real. + if is_small_box and ( + box_width < DETECTOR_CLASSIFIER_RESCUE_MIN_WIDTH or + box_height < DETECTOR_CLASSIFIER_RESCUE_MIN_HEIGHT or + x1 < frame_width * DETECTOR_CLASSIFIER_RESCUE_MIN_X_RATIO + ): + continue + + if class_id == 2: + school_scores: dict[int, float] = {} + school_best_confidences: dict[int, float] = {} + school_support_counts: dict[int, int] = {} + for expand_left, expand_top, expand_right, expand_bottom in SCHOOL_ZONE_DIRECT_EXPANSIONS: + expanded_x1 = max(int(x1 - box_width * expand_left), 0) + expanded_y1 = max(int(y1 - box_height * expand_top), 0) + expanded_x2 = min(int(x2 + box_width * expand_right), frame_width) + expanded_y2 = min(int(y2 + box_height * expand_bottom), frame_height) + sign_crop = frame_bgr[expanded_y1:expanded_y2, expanded_x1:expanded_x2] + if sign_crop.size == 0: + continue + + for school_crop, crop_weight in self._iter_school_zone_read_crops(sign_crop): + read_result = self._classify_speed_limit_from_model(school_crop) + if read_result is None: + read_result = self._read_speed_limit_from_crop(school_crop) + if read_result is None: + continue + + speed_limit_mph, read_confidence = read_result + if speed_limit_mph not in SCHOOL_ZONE_SPEED_VALUES: + continue + + school_scores[speed_limit_mph] = school_scores.get(speed_limit_mph, 0.0) + read_confidence * crop_weight + school_best_confidences[speed_limit_mph] = max(school_best_confidences.get(speed_limit_mph, 0.0), read_confidence) + school_support_counts[speed_limit_mph] = school_support_counts.get(speed_limit_mph, 0) + 1 + + if school_scores: + speed_limit_mph = max( + school_scores, + key=lambda speed: ( + school_scores[speed] + max(school_support_counts[speed] - 1, 0) * SCHOOL_ZONE_SUPPORT_BONUS, + school_best_confidences[speed], + ), + ) + read_confidence = school_best_confidences[speed_limit_mph] + support_count = school_support_counts[speed_limit_mph] + if ( + (support_count >= SCHOOL_ZONE_MIN_SUPPORT and read_confidence >= SCHOOL_ZONE_MIN_CONFIDENCE) or + read_confidence >= SCHOOL_ZONE_SINGLE_READ_CONFIDENCE + ): + score = min( + read_confidence * 0.72 + + proposal_confidence * 0.22 + + max(support_count - 1, 0) * SCHOOL_ZONE_SUPPORT_BONUS + + 0.04, + 0.95, + ) + if score >= SCHOOL_ZONE_SHORT_CIRCUIT_CONFIDENCE: + return Detection(speed_limit_mph, score) + + proposal_area_ratio = (box_width * box_height) / max(frame_width * frame_height, 1) + speed_scores: dict[int, float] = {} + speed_best_confidences: dict[int, float] = {} + speed_support_counts: dict[int, int] = {} + speed_regulatory_support: dict[int, int] = {} + + for expand_left, expand_top, expand_right, expand_bottom, expansion_weight in DETECTOR_CLASSIFIER_EXPANSIONS: + expanded_x1 = max(int(x1 - box_width * expand_left), 0) + expanded_y1 = max(int(y1 - box_height * expand_top), 0) + expanded_x2 = min(int(x2 + box_width * expand_right), frame_width) + expanded_y2 = min(int(y2 + box_height * expand_bottom), frame_height) + sign_crop = frame_bgr[expanded_y1:expanded_y2, expanded_x1:expanded_x2] + if sign_crop.size == 0: + continue + + is_regulatory = self._is_regulatory_speed_sign(sign_crop) + if class_id == 2: + is_regulatory = True + + model_read = self._classify_speed_limit_from_model(sign_crop) + ocr_read = self._read_speed_limit_from_crop(sign_crop) + read_result = model_read or ocr_read + if read_result is None: + continue + + if class_id != 2 and not is_regulatory: + if model_read is None or ocr_read is None or model_read[0] != ocr_read[0]: + continue + read_result = (model_read[0], min(model_read[1], ocr_read[1])) + + speed_limit_mph, read_confidence = read_result + score = read_confidence * expansion_weight + if is_regulatory: + score += DETECTOR_CLASSIFIER_REGULATORY_BONUS + elif proposal_area_ratio >= DETECTOR_CLASSIFIER_SMALL_BOX_AREA_RATIO: + score -= DETECTOR_CLASSIFIER_NON_REGULATORY_PENALTY + if class_id == 2: + score += SCHOOL_ZONE_SPEED_PRIOR if speed_limit_mph in SCHOOL_ZONE_SPEED_VALUES else -SCHOOL_ZONE_SPEED_PRIOR + + speed_scores[speed_limit_mph] = speed_scores.get(speed_limit_mph, 0.0) + score + speed_best_confidences[speed_limit_mph] = max(speed_best_confidences.get(speed_limit_mph, 0.0), read_confidence) + speed_support_counts[speed_limit_mph] = speed_support_counts.get(speed_limit_mph, 0) + 1 + if is_regulatory or class_id == 2: + speed_regulatory_support[speed_limit_mph] = speed_regulatory_support.get(speed_limit_mph, 0) + 1 + + if not speed_scores: + continue + + speed_limit_mph = max( + speed_scores, + key=lambda speed: ( + speed_scores[speed] + max(speed_support_counts[speed] - 1, 0) * DETECTOR_CLASSIFIER_SUPPORT_BONUS, + speed_best_confidences[speed], + ), + ) + if class_id == 2 and speed_limit_mph not in SCHOOL_ZONE_SPEED_VALUES: + continue + read_confidence = speed_best_confidences[speed_limit_mph] + support_count = speed_support_counts[speed_limit_mph] + score = min( + read_confidence * 0.72 + + proposal_confidence * 0.24 + + max(support_count - 1, 0) * DETECTOR_CLASSIFIER_SUPPORT_BONUS, + 0.95, + ) + if class_id == 2: + if speed_limit_mph in SCHOOL_ZONE_SPEED_VALUES: + score = min(score + 0.06, 0.95) + else: + score = max(score - 0.06, 0.0) + elif is_small_box: + if speed_regulatory_support.get(speed_limit_mph, 0) < 1: + continue + if support_count < DETECTOR_CLASSIFIER_RESCUE_MIN_SUPPORT: + continue + if read_confidence < DETECTOR_CLASSIFIER_RESCUE_MIN_CONFIDENCE: + continue + score = min(score, DETECTOR_CLASSIFIER_RESCUE_MAX_SCORE) + if score > best_score: + best_score = score + best_detection = Detection(speed_limit_mph, score) + if best_score >= MODEL_DETECTION_SHORT_CIRCUIT_CONFIDENCE: + return best_detection + + return best_detection + + def _detect_sign_from_model_proposals(self, frame_bgr): + frame_height, frame_width = frame_bgr.shape[:2] + best_detection = None + best_score = 0.0 + + for proposal_confidence, class_id, (x1, y1, x2, y2) in self._collect_model_proposals(frame_bgr): + proposal_crop = frame_bgr[y1:y2, x1:x2] + if proposal_crop.size == 0 or not self._is_regulatory_speed_sign(proposal_crop): + continue + + box_width = x2 - x1 + box_height = y2 - y1 + + for expand_left, expand_top, expand_right, expand_bottom in MODEL_PROPOSAL_EXPANSIONS: + expanded_x1 = max(int(x1 - box_width * expand_left), 0) + expanded_y1 = max(int(y1 - box_height * expand_top), 0) + expanded_x2 = min(int(x2 + box_width * expand_right), frame_width) + expanded_y2 = min(int(y2 + box_height * expand_bottom), frame_height) + if expanded_x2 <= expanded_x1 or expanded_y2 <= expanded_y1: + continue + + for trim_bottom_ratio in MODEL_PROPOSAL_TRIM_BOTTOM_RATIOS: + trimmed_y2 = expanded_y1 + int((expanded_y2 - expanded_y1) * trim_bottom_ratio) + if trimmed_y2 <= expanded_y1: + continue + + sign_crop = frame_bgr[expanded_y1:trimmed_y2, expanded_x1:expanded_x2] + if sign_crop.size == 0: + continue + + read_result = self._read_speed_limit_from_crop(sign_crop) + if read_result is None: + continue + + speed_limit_mph, read_confidence = read_result + score = read_confidence + min(proposal_confidence * 8.0, 0.08) + if class_id in SPEED_LIMIT_CLASSES and SPEED_LIMIT_CLASSES[class_id] == speed_limit_mph: + score += 0.03 + + if score > best_score: + best_score = score + best_detection = Detection(speed_limit_mph, min(score, 0.85)) + if best_score >= MODEL_DETECTION_SHORT_CIRCUIT_CONFIDENCE: + return best_detection + + return best_detection + + def _detect_sign_from_ocr_candidates(self, frame_bgr): + frame_height, frame_width = frame_bgr.shape[:2] + best_detection = None + best_score = 0.0 + + for left_ratio, top_ratio, right_ratio, bottom_ratio in OCR_SEARCH_WINDOWS: + left = int(frame_width * left_ratio) + top = int(frame_height * top_ratio) + right = int(frame_width * right_ratio) + bottom = int(frame_height * bottom_ratio) + roi = frame_bgr[top:bottom, left:right] + if roi.size == 0: + continue + + gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY) + gray = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8)).apply(gray) + upscaled = cv2.resize(gray, None, fx=OCR_SEARCH_UPSCALE, fy=OCR_SEARCH_UPSCALE, interpolation=cv2.INTER_CUBIC) + roi_area = upscaled.shape[0] * upscaled.shape[1] + if roi_area <= 0: + continue + + for threshold in OCR_SEARCH_THRESHOLDS: + _, binary = cv2.threshold(upscaled, threshold, 255, cv2.THRESH_BINARY) + binary = cv2.morphologyEx(binary, cv2.MORPH_CLOSE, np.ones((5, 5), dtype=np.uint8)) + contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + for contour in contours: + x, y, w, h = cv2.boundingRect(contour) + area = w * h + aspect = w / max(h, 1) + if area < roi_area * 0.002 or area > roi_area * 0.08: + continue + if aspect < 0.45 or aspect > 1.4: + continue + if y > upscaled.shape[0] * 0.82: + continue + + for inset_left_ratio, inset_right_ratio, pad_ratio in OCR_SEARCH_CROP_VARIANTS: + inset_x = x + int(w * inset_left_ratio) + inset_width = max(int(w * (1 - inset_left_ratio - inset_right_ratio)), 20) + pad_x = int(inset_width * pad_ratio) + pad_y = int(h * pad_ratio) + + x1 = max(inset_x - pad_x, 0) + y1 = max(y - pad_y, 0) + x2 = min(inset_x + inset_width + pad_x, upscaled.shape[1]) + y2 = min(y + h + pad_y, upscaled.shape[0]) + candidate_crop = upscaled[y1:y2, x1:x2] + if candidate_crop.size == 0: + continue + candidate_crop_bgr = cv2.cvtColor(candidate_crop, cv2.COLOR_GRAY2BGR) + if not self._is_regulatory_speed_sign(candidate_crop_bgr): + continue + + ocr_result = self._read_speed_limit_from_crop(candidate_crop_bgr) + if ocr_result is None: + continue + + speed_limit_mph, confidence = ocr_result + if confidence < OCR_FALLBACK_MIN_CONFIDENCE: + continue + + score = confidence + min(area / roi_area, 0.03) + if score > best_score: + best_score = score + best_detection = Detection(speed_limit_mph, confidence) + + return best_detection + + def _read_speed_limit_from_crop(self, sign_crop): + gray = cv2.cvtColor(sign_crop, cv2.COLOR_BGR2GRAY) + height, width = gray.shape + roi = gray[int(height * 0.22):int(height * 0.92), int(width * 0.1):int(width * 0.9)] + if roi.size == 0: + return None + + blurred = cv2.GaussianBlur(roi, (5, 5), 0) + _, binary = cv2.threshold(blurred, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + binary = cv2.morphologyEx(binary, cv2.MORPH_OPEN, np.ones((2, 2), dtype=np.uint8)) + + contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + digit_boxes = [] + min_area = roi.shape[0] * roi.shape[1] * 0.01 + min_height = roi.shape[0] * 0.35 + min_width = roi.shape[1] * 0.05 + + for contour in contours: + x, y, w, h = cv2.boundingRect(contour) + area = cv2.contourArea(contour) + if area < min_area or h < min_height or w < min_width: + continue + if w / max(h, 1) > 1.1: + continue + if y + h < roi.shape[0] * 0.35: + continue + digit_boxes.append((x, y, w, h)) + + digit_boxes.sort(key=lambda box: box[0]) + if len(digit_boxes) < 2: + return self._read_speed_limit_from_value_template(sign_crop) + if len(digit_boxes) > 3: + digit_boxes = sorted(digit_boxes, key=lambda box: box[3], reverse=True)[:3] + digit_boxes.sort(key=lambda box: box[0]) + + digits = [] + confidences = [] + for x, y, w, h in digit_boxes: + digit_mask = binary[y:y + h, x:x + w] + normalized = self._normalize_binary_digit(digit_mask) + if normalized is None: + return self._read_speed_limit_from_value_template(sign_crop) + digit, confidence = self._classify_digit(normalized) + if digit is None or confidence < OCR_MIN_CONFIDENCE: + return self._read_speed_limit_from_value_template(sign_crop) + digits.append(digit) + confidences.append(confidence) + + if not 2 <= len(digits) <= 3: + return self._read_speed_limit_from_value_template(sign_crop) + + speed_limit_mph = int("".join(digits)) + if speed_limit_mph not in VALID_SPEED_LIMITS_MPH: + return self._read_speed_limit_from_value_template(sign_crop) + + return speed_limit_mph, float(sum(confidences) / len(confidences)) + + def _read_speed_limit_from_value_template(self, sign_crop): + normalized = self._extract_value_template_mask(sign_crop) + if normalized is None: + return None + + candidate = normalized > 0 + best_match = None + for speed_limit, templates in self.speed_value_templates.items(): + for template in templates: + template_mask = template > 0 + union = np.logical_or(candidate, template_mask).sum() + if union == 0: + continue + intersection = np.logical_and(candidate, template_mask).sum() + iou = intersection / union + if best_match is None or iou > best_match[1]: + best_match = (int(speed_limit), iou) + + if best_match is None or best_match[1] < VALUE_TEMPLATE_MIN_CONFIDENCE: + return None + return best_match + + def _extract_value_template_mask(self, sign_crop): + gray = cv2.cvtColor(sign_crop, cv2.COLOR_BGR2GRAY) + height, width = gray.shape + best_mask = None + best_fill = 0.0 + + for top_ratio, bottom_ratio, left_ratio, right_ratio in VALUE_TEMPLATE_ROIS: + roi = gray[int(height * top_ratio):int(height * bottom_ratio), int(width * left_ratio):int(width * right_ratio)] + if roi.size == 0: + continue + + clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8)).apply(roi) + _, binary = cv2.threshold(clahe, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + binary = cv2.morphologyEx(binary, cv2.MORPH_OPEN, np.ones((2, 2), dtype=np.uint8)) + + num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(binary, 8) + mask = np.zeros_like(binary) + for label_idx in range(1, num_labels): + x, y, w, h, area = stats[label_idx] + if area < roi.shape[0] * roi.shape[1] * 0.01: + continue + if y < binary.shape[0] * 0.08: + continue + if h < binary.shape[0] * 0.18: + continue + if w > binary.shape[1] * 0.75: + continue + mask[labels == label_idx] = 255 + + normalized = self._normalize_binary_digit(mask, size=(72, 96)) + if normalized is None: + continue + + fill_ratio = float(np.count_nonzero(normalized)) / normalized.size + if fill_ratio <= 0.02: + continue + if fill_ratio > best_fill: + best_fill = fill_ratio + best_mask = normalized + + return best_mask + + def _classify_digit(self, normalized_digit): + best_digit = None + best_iou = 0.0 + + candidate = normalized_digit > 0 + for digit, templates in self.digit_templates.items(): + for template in templates: + template_mask = template > 0 + union = np.logical_or(candidate, template_mask).sum() + if union == 0: + continue + intersection = np.logical_and(candidate, template_mask).sum() + iou = intersection / union + if iou > best_iou: + best_iou = iou + best_digit = digit + + return best_digit, best_iou + + def _prune_history(self, now): + while self.history and now - self.history[0].created_at > HISTORY_SECONDS: + self.history.popleft() + + def _confirm_detection(self): + if not self.history: + return None + + counts = Counter(entry.speed_limit_mph for entry in self.history) + candidate_speed_limit, candidate_count = counts.most_common(1)[0] + matching_entries = [entry for entry in self.history if entry.speed_limit_mph == candidate_speed_limit] + best_confidence = max(entry.confidence for entry in matching_entries) + + if best_confidence >= STRONG_DETECTION_CONFIDENCE or candidate_count >= CONSISTENT_DETECTIONS: + return candidate_speed_limit, best_confidence + return None + + def _clear_detection(self): + self.history.clear() + self.followup_until = 0.0 + self.pending_auto_bookmark = None + self.previous_published_speed_limit_mph = self.published_speed_limit_mph + self.published_speed_limit_mph = 0 + self.published_confidence = 0.0 + self.last_publish_change_at = 0.0 + self.last_candidate_speed_limit_mph = 0 + self.last_candidate_confidence = 0.0 + self.last_candidate_at = 0.0 + self.last_logged_candidate = None + if self.params_memory is not None: + self.params_memory.remove("VisionSpeedLimit") + self.params_memory.remove("VisionSpeedLimitConfidence") + + def _publish_status(self, status, clear_speed=False): + if clear_speed: + self._clear_detection() + if self.params_memory is None: + return + self.params_memory.put("VisionSpeedLimitStatus", status) + if self.stream_name: + self.params_memory.put("VisionSpeedLimitStream", self.stream_name) + else: + self.params_memory.remove("VisionSpeedLimitStream") + if status != self.last_logged_status: + self.last_logged_status = status + self._write_debug_event("status", statusText=status) + + def _publish_detection(self, speed_limit_mph, confidence, status_prefix): + if speed_limit_mph != self.published_speed_limit_mph or abs(confidence - self.published_confidence) >= 0.05: + published_changed = speed_limit_mph != self.published_speed_limit_mph + if speed_limit_mph != self.published_speed_limit_mph: + self.previous_published_speed_limit_mph = self.published_speed_limit_mph + self.last_publish_change_at = time.monotonic() + self.published_speed_limit_mph = speed_limit_mph + self.published_confidence = confidence + self._write_debug_event( + "publish", + frame_bgr=self.current_frame_bgr, + snapshot_prefix=f"publish_{speed_limit_mph:03d}", + speedLimitMph=speed_limit_mph, + confidence=round(confidence, 4), + ) + if self.params_memory is not None: + self.params_memory.put_float("VisionSpeedLimit", speed_limit_mph * CV.MPH_TO_MS) + self.params_memory.put_float("VisionSpeedLimitConfidence", confidence) + if published_changed: + self._schedule_auto_bookmark(speed_limit_mph, confidence, self.last_publish_change_at) + + status = f"{status_prefix} {speed_limit_mph} mph ({confidence * 100:.0f}%)" + self._publish_status(status, clear_speed=False) + + def _should_hold_current_publish(self, speed_limit_mph, confidence, now): + if self.published_speed_limit_mph <= 0: + return False + if speed_limit_mph == self.published_speed_limit_mph: + return False + if speed_limit_mph != self.previous_published_speed_limit_mph: + return False + if now - self.last_publish_change_at >= PUBLISHED_CHANGE_COOLDOWN_SECONDS: + return False + return confidence < PUBLISHED_REVERT_CONFIDENCE + + def _update_detection(self, detection): + now = time.monotonic() + self.last_detection_at = now + self.followup_until = max(self.followup_until, now + FOLLOWUP_WINDOW_SECONDS) + self.last_candidate_speed_limit_mph = detection.speed_limit_mph + self.last_candidate_confidence = detection.confidence + self.last_candidate_at = now + + candidate_signature = (detection.speed_limit_mph, round(detection.confidence, 2)) + if candidate_signature != self.last_logged_candidate: + self.last_logged_candidate = candidate_signature + self._write_debug_event( + "candidate", + candidateSpeedLimitMph=detection.speed_limit_mph, + candidateConfidence=round(detection.confidence, 4), + ) + + self.history.append(HistoryEntry(detection.speed_limit_mph, detection.confidence, now)) + self._prune_history(now) + + confirmed = self._confirm_detection() + if confirmed is not None: + speed_limit_mph, confidence = confirmed + if self._should_hold_current_publish(speed_limit_mph, confidence, now): + self._publish_status( + f"Candidate {speed_limit_mph} mph ({confidence * 100:.0f}%)", + clear_speed=False, + ) + else: + self._publish_detection(speed_limit_mph, confidence, "Holding") + else: + self._publish_status(f"Candidate {detection.speed_limit_mph} mph ({detection.confidence * 100:.0f}%)", clear_speed=False) + + def run(self): + if not self.use_runtime or self.sm is None: + raise RuntimeError("SpeedLimitVisionDaemon runtime loop requires use_runtime=True") + + ratekeeper = self.Ratekeeper(5, None) + + while True: + self.sm.update(0) + + if self.sm.updated["userBookmark"]: + if self.ignore_next_user_bookmark: + self.ignore_next_user_bookmark = False + else: + self._record_bookmark() + + if self.net is None: + self._publish_status(self.last_error or "Vision model unavailable", clear_speed=True) + ratekeeper.keep_time() + continue + + if not self.sm["deviceState"].started: + if self.started_prev: + self._write_debug_event("session_end", reason="offroad") + self._close_debug_session() + self.last_road_name = "" + self.started_prev = False + self.current_frame_bgr = None + self.pending_auto_bookmark = None + self._publish_status("Idle - offroad", clear_speed=True) + ratekeeper.keep_time() + continue + elif not self.started_prev: + self.started_prev = True + self._start_debug_session() + + road_name = self.sm["mapdOut"].roadName + if self.last_road_name and road_name and road_name != self.last_road_name: + self._write_debug_event("road_change", previousRoadName=self.last_road_name, roadName=road_name) + self._clear_detection() + self.last_road_name = road_name or self.last_road_name + + if not self._connect_camera(): + status = "Waiting for camera stream" + if self.published_speed_limit_mph > 0: + now = time.monotonic() + if self._published_detection_stale(now): + self._write_debug_event("stale_clear", reason="stream_unavailable") + self._publish_status("Waiting for camera stream", clear_speed=True) + ratekeeper.keep_time() + continue + status = f"{status}, holding {self.published_speed_limit_mph} mph" + self._publish_status(status, clear_speed=False) + ratekeeper.keep_time() + continue + + now = time.monotonic() + inference_interval = FOLLOWUP_INFERENCE_INTERVAL if now < self.followup_until else INFERENCE_INTERVAL + if now - self.last_inference_at < inference_interval: + if self.published_speed_limit_mph > 0: + if self._published_detection_stale(now): + self._write_debug_event("stale_clear", reason="hold_timeout") + self._publish_status(f"Scanning {self.stream_name}", clear_speed=True) + else: + self._publish_detection(self.published_speed_limit_mph, self.published_confidence, "Holding") + else: + self._publish_status(f"Scanning {self.stream_name}", clear_speed=False) + ratekeeper.keep_time() + continue + + buffer = self.client.recv() if self.client is not None else None + self.last_inference_at = now + if buffer is None or not buffer.data.any(): + if self._published_detection_stale(now): + self._write_debug_event("stale_clear", reason="empty_frame") + self._publish_status(f"Waiting for {self.stream_name}", clear_speed=True) + else: + self._publish_status(f"Waiting for {self.stream_name}", clear_speed=False) + ratekeeper.keep_time() + continue + + image = np.frombuffer(buffer.data, dtype=np.uint8).reshape((len(buffer.data) // self.client.stride, self.client.stride)) + frame_bgr = cv2.cvtColor(image[:self.client.height * 3 // 2, :self.client.width], cv2.COLOR_YUV2BGR_NV12) + self.current_frame_bgr = frame_bgr + + detection = self._detect_sign(frame_bgr) + if detection is not None: + self._update_detection(detection) + elif self.published_speed_limit_mph > 0: + if self._published_detection_stale(now): + self._write_debug_event("stale_clear", reason="no_detection") + self._publish_status(f"Scanning {self.stream_name}", clear_speed=True) + else: + self._publish_detection(self.published_speed_limit_mph, self.published_confidence, "Holding") + else: + self._publish_status(f"Scanning {self.stream_name}", clear_speed=False) + + self._maybe_commit_auto_bookmark(now) + + ratekeeper.keep_time() + + +def main(): + daemon = SpeedLimitVisionDaemon() + daemon.run() + + +if __name__ == "__main__": + main() diff --git a/starpilot/system/the_pond/assets/components/tools/device_settings_layout.json b/starpilot/system/the_pond/assets/components/tools/device_settings_layout.json index c3b39606..6116b5b1 100644 --- a/starpilot/system/the_pond/assets/components/tools/device_settings_layout.json +++ b/starpilot/system/the_pond/assets/components/tools/device_settings_layout.json @@ -1136,6 +1136,29 @@ "ui_type": "toggle", "is_parent_toggle": true }, + { + "key": "VisionSpeedLimitDetection", + "label": "Vision Speed Limit Detection", + "description": "Use the road camera to detect speed limit signs for SLC and speed limit filling.", + "data_type": "bool", + "ui_type": "toggle" + }, + { + "key": "VisionSpeedLimitAutoBookmark", + "label": "Auto-Bookmark Vision Signs", + "description": "Automatically save confirmed vision-detected speed limit signs into the speed-limit debug session so they can be imported into the training set later.", + "data_type": "bool", + "ui_type": "toggle", + "parent_key": "VisionSpeedLimitDetection" + }, + { + "key": "VisionSpeedLimitAutoPreserveSegment", + "label": "Preserve Auto-Bookmarked Segments", + "description": "Also send a real bookmark for confirmed auto-bookmarks so loggerd preserves the route segment. Leave this off unless you specifically want the extra storage usage.", + "data_type": "bool", + "ui_type": "toggle", + "parent_key": "VisionSpeedLimitAutoBookmark" + }, { "key": "SLCConfirmation", "label": "Confirm New Speed Limits", @@ -1180,6 +1203,62 @@ "ui_type": "toggle", "parent_key": "SpeedLimitController" }, + { + "key": "SLCPriority1", + "label": "Primary Speed Limit Source", + "description": "Choose the first speed-limit source to trust when more than one is available.", + "data_type": "string", + "ui_type": "dropdown", + "options": [ + { + "value": "Dashboard", + "label": "Dashboard" + }, + { + "value": "Map Data", + "label": "Map Data" + }, + { + "value": "Vision", + "label": "Vision" + }, + { + "value": "Highest", + "label": "Highest" + }, + { + "value": "Lowest", + "label": "Lowest" + } + ], + "parent_key": "SpeedLimitController" + }, + { + "key": "SLCPriority2", + "label": "Secondary Speed Limit Source", + "description": "Choose the backup speed-limit source when the primary one is unavailable.", + "data_type": "string", + "ui_type": "dropdown", + "options": [ + { + "value": "None", + "label": "None" + }, + { + "value": "Dashboard", + "label": "Dashboard" + }, + { + "value": "Map Data", + "label": "Map Data" + }, + { + "value": "Vision", + "label": "Vision" + } + ], + "parent_key": "SpeedLimitController" + }, { "key": "Offset1", "label": "Speed Offset (0\u201324 mph)", diff --git a/starpilot/system/the_pond/assets/components/tools/speed_limits.js b/starpilot/system/the_pond/assets/components/tools/speed_limits.js index 490a41f4..84d3aa29 100644 --- a/starpilot/system/the_pond/assets/components/tools/speed_limits.js +++ b/starpilot/system/the_pond/assets/components/tools/speed_limits.js @@ -8,6 +8,15 @@ const state = reactive({ reason: "", status: "Checking...", submitting: false, + visionConfidence: 0, + visionBookmarkCount: 0, + visionDebugSession: "", + visionDisplaySpeed: 0, + visionEnabled: false, + visionLastEvent: "", + visionSpeedUnit: "mph", + visionStatus: "Checking...", + visionStream: "", }) let pollTimer = null @@ -21,11 +30,24 @@ async function fetchStatus() { state.processing = Boolean(result.processing) state.reason = result.reason || "" state.status = result.status || "Idle" + state.visionBookmarkCount = Number(result.visionBookmarkCount || 0) + state.visionConfidence = Number(result.visionConfidence || 0) + state.visionDebugSession = result.visionDebugSession || "" + state.visionDisplaySpeed = Number(result.visionDisplaySpeed || 0) + state.visionEnabled = Boolean(result.visionEnabled) + state.visionLastEvent = result.visionLastEvent || "" + state.visionSpeedUnit = result.visionSpeedUnit || "mph" + state.visionStatus = result.visionStatus || (state.visionEnabled ? "Idle" : "Disabled") + state.visionStream = result.visionStream || "" } catch (error) { state.canProcessNow = false state.processing = false state.reason = "Failed to load processor status." state.status = "Unavailable" + state.visionBookmarkCount = 0 + state.visionDebugSession = "" + state.visionLastEvent = "" + state.visionStatus = "Unavailable" } state.loading = false @@ -83,6 +105,27 @@ export function SpeedLimits() {

${() => state.loading ? "Checking processor status..." : `Processor Status: ${state.status}`}

+

+ ${() => { + if (state.loading) { + return "Checking vision detector..." + } + + const suffix = state.visionDisplaySpeed > 0 + ? ` (${state.visionDisplaySpeed} ${state.visionSpeedUnit}${state.visionConfidence > 0 ? `, ${Math.round(state.visionConfidence * 100)}%` : ""})` + : "" + const stream = state.visionStream ? ` on ${state.visionStream}` : "" + return `Vision Detector: ${state.visionStatus}${stream}${suffix}` + }} +

+ ${() => !state.loading && state.visionEnabled ? html` +

+ ${`Vision Debug: ${state.visionDebugSession || "No active session"}${state.visionBookmarkCount ? `, ${state.visionBookmarkCount} bookmark${state.visionBookmarkCount === 1 ? "" : "s"}` : ""}`} +

+ ` : ""} + ${() => !state.loading && state.visionEnabled && state.visionLastEvent ? html` +

${`Latest Vision Event: ${state.visionLastEvent}`}

+ ` : ""} ${() => !state.loading && state.reason && state.reason !== state.status ? html`

${state.reason}

` : ""} diff --git a/starpilot/system/the_pond/the_pond.py b/starpilot/system/the_pond/the_pond.py index b6a23867..c044395d 100644 --- a/starpilot/system/the_pond/the_pond.py +++ b/starpilot/system/the_pond/the_pond.py @@ -30,6 +30,7 @@ from cereal import car, log, messaging from opendbc.can.parser import CANParser from opendbc.car.gm.values import GMFlags from opendbc.car.toyota.carcontroller import LOCK_CMD, UNLOCK_CMD +from openpilot.common.constants import CV from openpilot.common.params import ParamKeyType, Params from openpilot.common.realtime import DT_HW from openpilot.common.time_helpers import system_time_valid @@ -3845,6 +3846,18 @@ def setup(app): enabled = params.get_bool("SpeedLimitFiller") is_onroad = params.get_bool("IsOnroad") time_valid = system_time_valid() + is_metric = params.get_bool("IsMetric") + + vision_enabled = params.get_bool("VisionSpeedLimitDetection") + vision_speed_limit = params_memory.get_float("VisionSpeedLimit") if vision_enabled else 0 + vision_confidence = params_memory.get_float("VisionSpeedLimitConfidence") if vision_enabled else 0 + vision_bookmark_count = params_memory.get_int("VisionSpeedLimitBookmarkCount") if vision_enabled else 0 + vision_debug_session = params_memory.get("VisionSpeedLimitDebugSession", encoding="utf-8") if vision_enabled else "" + vision_last_event = params_memory.get("VisionSpeedLimitLastEvent", encoding="utf-8") if vision_enabled else "" + vision_status = params_memory.get("VisionSpeedLimitStatus", encoding="utf-8") or ("Idle" if vision_enabled else "Disabled") + vision_stream = params_memory.get("VisionSpeedLimitStream", encoding="utf-8") if vision_enabled else "" + vision_speed_unit = "km/h" if is_metric else "mph" + vision_display_speed = round(vision_speed_limit * (CV.MS_TO_KPH if is_metric else CV.MS_TO_MPH)) if vision_speed_limit > 0 else 0 network_connected = True try: @@ -3894,6 +3907,15 @@ def setup(app): "timeValid": time_valid, "totalRequests": total_requests, "maxRequests": max_requests, + "visionConfidence": vision_confidence, + "visionBookmarkCount": vision_bookmark_count, + "visionDebugSession": vision_debug_session, + "visionDisplaySpeed": vision_display_speed, + "visionEnabled": vision_enabled, + "visionLastEvent": vision_last_event, + "visionSpeedUnit": vision_speed_unit, + "visionStatus": vision_status, + "visionStream": vision_stream, } @app.route("/api/speed_limits/status", methods=["GET"]) diff --git a/starpilot/ui/qt/offroad/longitudinal_settings.cc b/starpilot/ui/qt/offroad/longitudinal_settings.cc index feb5534a..5da13aa3 100644 --- a/starpilot/ui/qt/offroad/longitudinal_settings.cc +++ b/starpilot/ui/qt/offroad/longitudinal_settings.cc @@ -186,7 +186,7 @@ StarPilotLongitudinalPanel::StarPilotLongitudinalPanel(StarPilotSettingsWindow * {"ReduceLateralAccelerationSnow", tr("Reduce Speed in Curves by:"), tr("Lower the desired speed while driving through curves in snow. Increase for safer, gentler turns; decrease for more aggressive driving in curves."), ""}, - {"SpeedLimitController", tr("Speed Limit Controller"), tr("Limit openpilot's maximum driving speed to the current speed limit obtained from downloaded maps, Mapbox, or the dashboard for supported vehicles (Ford, Genesis, Hyundai, Kia, Lexus, Toyota)."), "../../starpilot/assets/toggle_icons/icon_speed_limit.png"}, + {"SpeedLimitController", tr("Speed Limit Controller"), tr("Limit openpilot's maximum driving speed to the current speed limit obtained from downloaded maps, Mapbox, the dashboard, or vision-detected signs."), "../../starpilot/assets/toggle_icons/icon_speed_limit.png"}, {"SLCFallback", tr("Fallback Speed"), tr("The speed used by \"Speed Limit Controller\" when no speed limit is found.

- Set Speed: Use the cruise set speed
- Experimental Mode: Estimate the limit using the driving model
- Previous Limit: Keep using the last confirmed limit"), ""}, {"SLCOverride", tr("Override Speed"), tr("The speed used by \"Speed Limit Controller\" after you manually drive faster than the posted limit.

- Set with Gas Pedal: Use the highest speed reached while pressing the gas
- Max Set Speed: Use the cruise set speed

Overrides clear when openpilot disengages."), ""}, {"SLCQOL", tr("Quality of Life"), tr("Miscellaneous \"Speed Limit Controller\" changes to fine-tune how openpilot drives."), ""}, @@ -195,6 +195,7 @@ StarPilotLongitudinalPanel::StarPilotLongitudinalPanel(StarPilotSettingsWindow * {"SLCLookaheadLower", tr("Lower Limit Lookahead Time"), tr("How far ahead openpilot anticipates upcoming lower speed limits from downloaded map data."), ""}, {"SetSpeedLimit", tr("Match Speed Limit on Engage"), tr("When openpilot is first enabled, automatically set the max speed to the current posted limit."), ""}, {"SLCMapboxFiller", tr("Use Mapbox as Fallback"), tr("Use Mapbox speed-limit data when no other source is available."), ""}, + {"VisionSpeedLimitDetection", tr("Vision Speed Limit Detection"), tr("Use the road camera to detect speed limit signs for SLC and speed limit filling."), ""}, {"SLCPriority", tr("Speed Limit Source Priority"), tr("The source order for speed limits when more than one is available."), ""}, {"SLCOffsets", tr("Speed Limit Offsets"), tr("Add an offset to the posted speed limit to better match your driving style."), ""}, {"Offset1", tr("Speed Offset (0–24 mph)"), tr("How much to offset posted speed-limits between 0 and 24 mph."), ""}, @@ -474,8 +475,8 @@ StarPilotLongitudinalPanel::StarPilotLongitudinalPanel(StarPilotSettingsWindow * longitudinalToggle = overrideSelection; } else if (param == "SLCPriority") { ButtonControl *slcPriorityButton = new ButtonControl(title, tr("SELECT"), desc); - QStringList primaryPriorities = {tr("Dashboard"), tr("Map Data"), tr("Highest"), tr("Lowest")}; - QStringList otherPriorities = {tr("None"), tr("Dashboard"), tr("Map Data")}; + QStringList primaryPriorities = {tr("Dashboard"), tr("Map Data"), tr("Vision"), tr("Highest"), tr("Lowest")}; + QStringList otherPriorities = {tr("None"), tr("Dashboard"), tr("Map Data"), tr("Vision")}; QStringList priorityPrompts = {tr("Select your primary priority"), tr("Select your secondary priority")}; QObject::connect(slcPriorityButton, &ButtonControl::clicked, [=]() { diff --git a/starpilot/ui/qt/offroad/longitudinal_settings.h b/starpilot/ui/qt/offroad/longitudinal_settings.h index 18ab3ae9..720cb8af 100644 --- a/starpilot/ui/qt/offroad/longitudinal_settings.h +++ b/starpilot/ui/qt/offroad/longitudinal_settings.h @@ -38,7 +38,7 @@ private: QSet relaxedPersonalityKeys = {"RelaxedFollow", "RelaxedFollowHigh", "RelaxedJerkAcceleration", "RelaxedJerkDeceleration", "RelaxedJerkDanger", "RelaxedJerkSpeed", "RelaxedJerkSpeedDecrease", "ResetRelaxedPersonality"}; QSet speedLimitControllerKeys = {"SLCOffsets", "SLCFallback", "SLCOverride", "SLCPriority", "SLCQOL", "SLCVisuals"}; QSet speedLimitControllerOffsetsKeys = {"Offset1", "Offset2", "Offset3", "Offset4", "Offset5", "Offset6", "Offset7"}; - QSet speedLimitControllerQOLKeys = {"SetSpeedLimit", "SLCConfirmation", "SLCLookaheadHigher", "SLCLookaheadLower", "SLCMapboxFiller"}; + QSet speedLimitControllerQOLKeys = {"SetSpeedLimit", "SLCConfirmation", "SLCLookaheadHigher", "SLCLookaheadLower", "SLCMapboxFiller", "VisionSpeedLimitDetection"}; QSet speedLimitControllerVisualKeys = {"ShowSLCOffset", "SpeedLimitSources"}; QSet standardPersonalityKeys = {"StandardFollow", "StandardFollowHigh", "StandardJerkAcceleration", "StandardJerkDeceleration", "StandardJerkDanger", "StandardJerkSpeed", "StandardJerkSpeedDecrease", "ResetStandardPersonality"}; QSet trafficPersonalityKeys = {"TrafficFollow", "TrafficJerkAcceleration", "TrafficJerkDeceleration", "TrafficJerkDanger", "TrafficJerkSpeed", "TrafficJerkSpeedDecrease", "ResetTrafficPersonality"}; diff --git a/starpilot/ui/qt/onroad/starpilot_annotated_camera.cc b/starpilot/ui/qt/onroad/starpilot_annotated_camera.cc index c47e3b9f..126bcaaf 100644 --- a/starpilot/ui/qt/onroad/starpilot_annotated_camera.cc +++ b/starpilot/ui/qt/onroad/starpilot_annotated_camera.cc @@ -19,6 +19,7 @@ StarPilotAnnotatedCameraWidget::StarPilotAnnotatedCameraWidget(QWidget *parent) speedIcon = loadPixmap("../../starpilot/assets/other_images/speed_icon.png", {widget_size, widget_size}); stopSignImg = loadPixmap("../../starpilot/assets/other_images/stop_sign.png", {btn_size, btn_size}); turnIcon = loadPixmap("../../starpilot/assets/other_images/turn_icon.png", {widget_size, widget_size}); + visionIcon = loadPixmap("../../starpilot/assets/other_images/speed_icon.png", {btn_size / 2, btn_size / 2}).scaled(iconSize, Qt::KeepAspectRatio, Qt::SmoothTransformation); loadGif("../../starpilot/assets/other_images/curve_icon.gif", cemCurveIcon, QSize(widget_size, widget_size), this); loadGif("../../starpilot/assets/other_images/lead_icon.gif", cemLeadIcon, QSize(widget_size, widget_size), this); @@ -182,11 +183,12 @@ void StarPilotAnnotatedCameraWidget::updateState(const UIState &s, const StarPil roadCurvature = starpilotPlan.getRoadCurvature(); roadName = QString::fromStdString(mapdOut.getRoadName()); slcOverriddenSpeed = starpilotPlan.getSlcOverriddenSpeed(); - speedLimit = slcOverriddenSpeed != 0 ? slcOverriddenSpeed : starpilotPlan.getSlcSpeedLimit(); + speedLimit = starpilotPlan.getSlcSpeedLimit(); speedLimitChanged = starpilotPlan.getSpeedLimitChanged(); speedLimitSource = starpilotPlan.getSlcSpeedLimitSource(); stoppingDistance = modelV2.getPosition().getX().size() > 33 - 1 ? modelV2.getPosition().getX()[33 - 1] : 0.0; unconfirmedSpeedLimit = starpilotPlan.getUnconfirmedSlcSpeedLimit(); + visionSpeedLimit = params.getBool("VisionSpeedLimitDetection") ? params_memory.getFloat("VisionSpeedLimit") : 0.0; weatherDaytime = starpilotPlan.getWeatherDaytime(); weatherId = starpilotPlan.getWeatherId(); @@ -1028,11 +1030,13 @@ void StarPilotAnnotatedCameraWidget::paintSpeedLimitSources(QPainter &p) { QRect dashboardRect(speedLimitRect.x() - signMargin, speedLimitRect.y() + speedLimitRect.height() + UI_BORDER_SIZE, 450, 60); QRect mapDataRect(dashboardRect.x(), dashboardRect.y() + dashboardRect.height() + UI_BORDER_SIZE / 2, 450, 60); - QRect mapboxRect(mapDataRect.x(), mapDataRect.y() + mapDataRect.height() + UI_BORDER_SIZE / 2, 450, 60); + QRect visionRect(mapDataRect.x(), mapDataRect.y() + mapDataRect.height() + UI_BORDER_SIZE / 2, 450, 60); + QRect mapboxRect(visionRect.x(), visionRect.y() + visionRect.height() + UI_BORDER_SIZE / 2, 450, 60); QRect nextLimitRect(mapboxRect.x(), mapboxRect.y() + mapboxRect.height() + UI_BORDER_SIZE / 2, 450, 60); drawSource(dashboardRect, dashboardIcon, "Dashboard", dashboardSpeedLimit * speedConversion); drawSource(mapDataRect, mapDataIcon, "Map Data", mapSpeedLimit * speedConversion); + drawSource(visionRect, visionIcon, "Vision", visionSpeedLimit * speedConversion); drawSource(mapboxRect, mapboxIcon, "Mapbox", mapboxSpeedLimit * speedConversion); drawSource(nextLimitRect, nextMapsIcon, "Upcoming", nextSpeedLimit * speedConversion); diff --git a/starpilot/ui/qt/onroad/starpilot_annotated_camera.h b/starpilot/ui/qt/onroad/starpilot_annotated_camera.h index 77ace88e..2999f69f 100644 --- a/starpilot/ui/qt/onroad/starpilot_annotated_camera.h +++ b/starpilot/ui/qt/onroad/starpilot_annotated_camera.h @@ -119,6 +119,7 @@ private: float speedLimit; float stoppingDistance; float unconfirmedSpeedLimit; + float visionSpeedLimit; std::string speedLimitSource; @@ -144,6 +145,7 @@ private: QPixmap speedIcon; QPixmap stopSignImg; QPixmap turnIcon; + QPixmap visionIcon; QPoint cemStatusPosition; QPoint compassPosition; diff --git a/system/camerad/camerad b/system/camerad/camerad index d8eb9e33..7964ee7e 100755 Binary files a/system/camerad/camerad and b/system/camerad/camerad differ diff --git a/system/loggerd/bootlog b/system/loggerd/bootlog index 575858e1..53fed283 100755 Binary files a/system/loggerd/bootlog and b/system/loggerd/bootlog differ diff --git a/system/loggerd/encoderd b/system/loggerd/encoderd index 42cbb65b..cfe14809 100755 Binary files a/system/loggerd/encoderd and b/system/loggerd/encoderd differ diff --git a/system/loggerd/loggerd b/system/loggerd/loggerd index 0f3b34c0..7fb06d52 100755 Binary files a/system/loggerd/loggerd and b/system/loggerd/loggerd differ diff --git a/system/manager/process_config.py b/system/manager/process_config.py index 17684893..3462a645 100644 --- a/system/manager/process_config.py +++ b/system/manager/process_config.py @@ -73,6 +73,9 @@ def allow_uploads(started: bool, params: Params, CP: car.CarParams, starpilot_to def run_speed_limit_filler(started: bool, params: Params, CP: car.CarParams, starpilot_toggles: SimpleNamespace) -> bool: return starpilot_toggles.speed_limit_filler +def run_speed_limit_vision(started: bool, params: Params, CP: car.CarParams, starpilot_toggles: SimpleNamespace) -> bool: + return starpilot_toggles.vision_speed_limit_detection + procs = [ DaemonProcess("manage_athenad", "system.athena.manage_athenad", "AthenadPid"), @@ -140,6 +143,7 @@ procs += [ PythonProcess("the_pond", "starpilot.system.the_pond.the_pond", always_run, nice=19), PythonProcess("galaxy", "starpilot.system.galaxy.galaxy", always_run, nice=19), PythonProcess("speed_limit_filler", "starpilot.system.speed_limit_filler", run_speed_limit_filler), + PythonProcess("speed_limit_vision", "starpilot.system.speed_limit_vision", run_speed_limit_vision), ] managed_processes = {p.name: p for p in procs} diff --git a/uv.lock b/uv.lock index 641da25e..c8094b72 100644 --- a/uv.lock +++ b/uv.lock @@ -4,10 +4,12 @@ requires-python = ">=3.11, <3.13" resolution-markers = [ "python_full_version >= '3.12' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.12' and sys_platform == 'win32'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.12' and sys_platform == 'darwin'", "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.12' and sys_platform == 'win32'", + "(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", ] [[package]] @@ -441,6 +443,71 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/5c/3dab83cc4aba1f4b0e733e3f0c3e7d4386440d660ba5b1e3ff995feb734d/cryptography-43.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362", size = 3068026, upload-time = "2024-10-18T15:58:11.916Z" }, ] +[[package]] +name = "cuda-bindings" +version = "13.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/3a8241c6e19483ac1f1dcf5c10238205dcb8a6e9d0d4d4709240dff28ff4/cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d", size = 5730273, upload-time = "2026-03-11T00:12:37.18Z" }, + { url = "https://files.pythonhosted.org/packages/e9/94/2748597f47bb1600cd466b20cab4159f1530a3a33fe7f70fee199b3abb9e/cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1", size = 6313924, upload-time = "2026-03-11T00:12:39.462Z" }, + { url = "https://files.pythonhosted.org/packages/52/c8/b2589d68acf7e3d63e2be330b84bc25712e97ed799affbca7edd7eae25d6/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788", size = 5722404, upload-time = "2026-03-11T00:12:44.041Z" }, + { url = "https://files.pythonhosted.org/packages/1f/92/f899f7bbb5617bb65ec52a6eac1e9a1447a86b916c4194f8a5001b8cde0c/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955", size = 6320619, upload-time = "2026-03-11T00:12:45.939Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/66/0c02bd330e7d976f83fa68583d6198d76f23581bcbb5c0e98a6148f326e5/cuda_pathfinder-1.5.0-py3-none-any.whl", hash = "sha256:498f90a9e9de36044a7924742aecce11c50c49f735f1bc53e05aa46de9ea4110", size = 49739, upload-time = "2026-03-24T21:14:30.869Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +curand = [ + { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cusolver = [ + { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -628,6 +695,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, ] +[[package]] +name = "fsspec" +version = "2026.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/cf/b50ddf667c15276a9ab15a70ef5f257564de271957933ffea49d2cdbcdfb/fsspec-2026.3.0.tar.gz", hash = "sha256:1ee6a0e28677557f8c2f994e3eea77db6392b4de9cd1f5d7a9e87a0ae9d01b41", size = 313547, upload-time = "2026-03-27T19:11:14.892Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/1f/5f4a3cd9e4440e9d9bc78ad0a91a1c8d46b4d429d5239ebe6793c9fe5c41/fsspec-2026.3.0-py3-none-any.whl", hash = "sha256:d2ceafaad1b3457968ed14efa28798162f1638dbb5d2a6868a2db002a5ee39a4", size = 202595, upload-time = "2026-03-27T19:11:13.595Z" }, +] + [[package]] name = "future-fstrings" version = "1.2.0" @@ -1232,6 +1308,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl", hash = "sha256:4732914fb471f56b5cce04d7bae6f164a592c7712e1c85f9ef585e197299521c", size = 38268, upload-time = "2023-06-20T04:17:17.522Z" }, ] +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + [[package]] name = "numpy" version = "2.3.3" @@ -1269,6 +1354,155 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/11/0cc63f9f321ccf63886ac203336777140011fb669e739da36d8db3c53b98/numpy-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2e267c7da5bf7309670523896df97f93f6e469fb931161f483cd6882b3b1a5dc", size = 12971844, upload-time = "2025-09-09T15:58:57.359Z" }, ] +[[package]] +name = "nvidia-cublas" +version = "13.1.0.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, + { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.19.0.56" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, + { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas", marker = "sys_platform != 'darwin'" }, + { name = "nvidia-cusparse", marker = "sys_platform != 'darwin'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277, upload-time = "2025-08-13T19:22:40.982Z" }, + { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119, upload-time = "2025-08-13T19:23:41.967Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.28.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/55/1920646a2e43ffd4fc958536b276197ed740e9e0c54105b4bb3521591fc7/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:01c873ba1626b54caa12272ed228dc5b2781545e0ae8ba3f432a8ef1c6d78643", size = 196561677, upload-time = "2025-11-18T05:49:03.45Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b4/878fefaad5b2bcc6fcf8d474a25e3e3774bc5133e4b58adff4d0bca238bc/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:e4553a30f34195f3fa1da02a6da3d6337d28f2003943aa0a3d247bbc25fefc42", size = 196493177, upload-time = "2025-11-18T05:49:17.677Z" }, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, +] + [[package]] name = "onnx" version = "1.19.0" @@ -1295,6 +1529,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/37/ad500945b1b5c154fe9d7b826b30816ebd629d10211ea82071b5bcc30aa4/onnx-1.19.0-cp312-cp312-win_arm64.whl", hash = "sha256:efb768299580b786e21abe504e1652ae6189f0beed02ab087cd841cb4bb37e43", size = 16426022, upload-time = "2025-08-27T02:33:33.515Z" }, ] +[[package]] +name = "opencv-python" +version = "4.13.0.92" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/6f/5a28fef4c4a382be06afe3938c64cc168223016fa520c5abaf37e8862aa5/opencv_python-4.13.0.92-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:caf60c071ec391ba51ed00a4a920f996d0b64e3e46068aac1f646b5de0326a19", size = 46247052, upload-time = "2026-02-05T07:01:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/08/ac/6c98c44c650b8114a0fb901691351cfb3956d502e8e9b5cd27f4ee7fbf2f/opencv_python-4.13.0.92-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:5868a8c028a0b37561579bfb8ac1875babdc69546d236249fff296a8c010ccf9", size = 32568781, upload-time = "2026-02-05T07:01:41.379Z" }, + { url = "https://files.pythonhosted.org/packages/3e/51/82fed528b45173bf629fa44effb76dff8bc9f4eeaee759038362dfa60237/opencv_python-4.13.0.92-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0bc2596e68f972ca452d80f444bc404e08807d021fbba40df26b61b18e01838a", size = 47685527, upload-time = "2026-02-05T06:59:11.24Z" }, + { url = "https://files.pythonhosted.org/packages/db/07/90b34a8e2cf9c50fe8ed25cac9011cde0676b4d9d9c973751ac7616223a2/opencv_python-4.13.0.92-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:402033cddf9d294693094de5ef532339f14ce821da3ad7df7c9f6e8316da32cf", size = 70460872, upload-time = "2026-02-05T06:59:19.162Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/7a9cc719b3eaf4377b9c2e3edeb7ed3a81de41f96421510c0a169ca3cfd4/opencv_python-4.13.0.92-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bccaabf9eb7f897ca61880ce2869dcd9b25b72129c28478e7f2a5e8dee945616", size = 46708208, upload-time = "2026-02-05T06:59:15.419Z" }, + { url = "https://files.pythonhosted.org/packages/fd/55/b3b49a1b97aabcfbbd6c7326df9cb0b6fa0c0aefa8e89d500939e04aa229/opencv_python-4.13.0.92-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:620d602b8f7d8b8dab5f4b99c6eb353e78d3fb8b0f53db1bd258bb1aa001c1d5", size = 72927042, upload-time = "2026-02-05T06:59:23.389Z" }, + { url = "https://files.pythonhosted.org/packages/fb/17/de5458312bcb07ddf434d7bfcb24bb52c59635ad58c6e7c751b48949b009/opencv_python-4.13.0.92-cp37-abi3-win32.whl", hash = "sha256:372fe164a3148ac1ca51e5f3ad0541a4a276452273f503441d718fab9c5e5f59", size = 30932638, upload-time = "2026-02-05T07:02:14.98Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a5/1be1516390333ff9be3a9cb648c9f33df79d5096e5884b5df71a588af463/opencv_python-4.13.0.92-cp37-abi3-win_amd64.whl", hash = "sha256:423d934c9fafb91aad38edf26efb46da91ffbc05f3f59c4b0c72e699720706f5", size = 40212062, upload-time = "2026-02-05T07:02:12.724Z" }, +] + [[package]] name = "opencv-python-headless" version = "4.11.0.86" @@ -1332,6 +1584,7 @@ dependencies = [ { name = "mapbox-earcut" }, { name = "numpy" }, { name = "onnx" }, + { name = "opencv-python-headless" }, { name = "psutil" }, { name = "pyaudio" }, { name = "pycapnp" }, @@ -1365,7 +1618,6 @@ dev = [ { name = "dbus-next" }, { name = "dictdiffer" }, { name = "matplotlib" }, - { name = "opencv-python-headless" }, { name = "parameterized" }, { name = "pyautogui" }, { name = "pygame" }, @@ -1382,6 +1634,11 @@ docs = [ { name = "mkdocs" }, { name = "natsort" }, ] +speedvision = [ + { name = "torch" }, + { name = "torchvision" }, + { name = "ultralytics" }, +] testing = [ { name = "codespell" }, { name = "coverage" }, @@ -1436,7 +1693,7 @@ requires-dist = [ { name = "natsort", marker = "extra == 'docs'" }, { name = "numpy", specifier = ">=2.0" }, { name = "onnx", specifier = ">=1.14.0" }, - { name = "opencv-python-headless", marker = "extra == 'dev'" }, + { name = "opencv-python-headless" }, { name = "parameterized", marker = "extra == 'dev'", specifier = ">=0.8,<0.9" }, { name = "pre-commit-hooks", marker = "extra == 'testing'" }, { name = "psutil" }, @@ -1475,14 +1732,17 @@ requires-dist = [ { name = "spidev", marker = "sys_platform == 'linux'" }, { name = "sympy" }, { name = "tabulate", marker = "extra == 'dev'" }, + { name = "torch", marker = "extra == 'speedvision'", specifier = ">=2.4" }, + { name = "torchvision", marker = "extra == 'speedvision'", specifier = ">=0.19" }, { name = "tqdm" }, { name = "types-requests", marker = "extra == 'dev'" }, { name = "types-tabulate", marker = "extra == 'dev'" }, + { name = "ultralytics", marker = "extra == 'speedvision'", specifier = ">=8.3,<9" }, { name = "websocket-client" }, { name = "xattr" }, { name = "zstandard" }, ] -provides-extras = ["docs", "testing", "dev", "tools"] +provides-extras = ["docs", "testing", "dev", "tools", "speedvision"] [[package]] name = "packaging" @@ -1610,6 +1870,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "polars" +version = "1.39.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "polars-runtime-32" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/ab/f19e592fce9e000da49c96bf35e77cef67f9cb4b040bfa538a2764c0263e/polars-1.39.3.tar.gz", hash = "sha256:2e016c7f3e8d14fa777ef86fe0477cec6c67023a20ba4c94d6e8431eefe4a63c", size = 728987, upload-time = "2026-03-20T11:16:24.836Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/db/08f4ca10c5018813e7e0b59e4472302328b3d2ab1512f5a2157a814540e0/polars-1.39.3-py3-none-any.whl", hash = "sha256:c2b955ccc0a08a2bc9259785decf3d5c007b489b523bf2390cf21cec2bb82a56", size = 823985, upload-time = "2026-03-20T11:14:23.619Z" }, +] + +[[package]] +name = "polars-runtime-32" +version = "1.39.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/17/39/c8688696bc22b6c501e3b82ef3be10e543c07a785af5660f30997cd22dd2/polars_runtime_32-1.39.3.tar.gz", hash = "sha256:c728e4f469cafab501947585f36311b8fb222d3e934c6209e83791e0df20b29d", size = 2872335, upload-time = "2026-03-20T11:16:26.581Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/74/1b41205f7368c9375ab1dea91178eaa20435fe3eff036390a53a7660b416/polars_runtime_32-1.39.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:425c0b220b573fa097b4042edff73114cc6d23432a21dfd2dc41adf329d7d2e9", size = 45273243, upload-time = "2026-03-20T11:14:26.691Z" }, + { url = "https://files.pythonhosted.org/packages/90/bf/297716b3095fe719be20fcf7af1d2b6ab069c38199bbace2469608a69b3a/polars_runtime_32-1.39.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ef5884711e3c617d7dc93519a7d038e242f5741cfe5fe9afd32d58845d86c562", size = 40842924, upload-time = "2026-03-20T11:14:31.154Z" }, + { url = "https://files.pythonhosted.org/packages/3d/3e/e65236d9d0d9babfa0ecba593413c06530fca60a8feb8f66243aa5dba92e/polars_runtime_32-1.39.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06b47f535eb1f97a9a1e5b0053ef50db3a4276e241178e37bbb1a38b1fa53b14", size = 43220650, upload-time = "2026-03-20T11:14:35.458Z" }, + { url = "https://files.pythonhosted.org/packages/b0/15/fc3e43f3fdf3f20b7dfb5abe871ab6162cf8fb4aeabf4cfad822d5dc4c79/polars_runtime_32-1.39.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bc9e13dc1d2e828331f2fe8ccbc9757554dc4933a8d3e85e906b988178f95ed", size = 46877498, upload-time = "2026-03-20T11:14:40.14Z" }, + { url = "https://files.pythonhosted.org/packages/3c/81/bd5f895919e32c6ab0a7786cd0c0ca961cb03152c47c3645808b54383f31/polars_runtime_32-1.39.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:363d49e3a3e638fc943e2b9887940300a7d06789930855a178a4727949259dc2", size = 43380176, upload-time = "2026-03-20T11:14:45.566Z" }, + { url = "https://files.pythonhosted.org/packages/7a/3e/c86433c3b5ec0315bdfc7640d0c15d41f1216c0103a0eab9a9b5147d6c4c/polars_runtime_32-1.39.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7c206bdcc7bc62ea038d6adea8e44b02f0e675e0191a54c810703b4895208ea4", size = 46485933, upload-time = "2026-03-20T11:14:51.155Z" }, + { url = "https://files.pythonhosted.org/packages/54/ce/200b310cf91f98e652eb6ea09fdb3a9718aa0293ebf113dce325797c8572/polars_runtime_32-1.39.3-cp310-abi3-win_amd64.whl", hash = "sha256:d66ca522517554a883446957539c40dc7b75eb0c2220357fb28bc8940d305339", size = 46995458, upload-time = "2026-03-20T11:14:56.074Z" }, + { url = "https://files.pythonhosted.org/packages/da/76/2d48927e0aa2abbdde08cbf4a2536883b73277d47fbeca95e952de86df34/polars_runtime_32-1.39.3-cp310-abi3-win_arm64.whl", hash = "sha256:f49f51461de63f13e5dd4eb080421c8f23f856945f3f8bd5b2b1f59da52c2860", size = 41857648, upload-time = "2026-03-20T11:15:01.142Z" }, +] + [[package]] name = "pre-commit-hooks" version = "6.0.0" @@ -4749,6 +5037,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/04/afc078a12cf68592345b1e2d6ecdff837d286bac023d7a22c54c7a698c5b/ruff-0.13.1-py3-none-win_arm64.whl", hash = "sha256:c0bae9ffd92d54e03c2bf266f466da0a65e145f298ee5b5846ed435f6a00518a", size = 12437893, upload-time = "2025-09-18T19:52:41.283Z" }, ] +[[package]] +name = "scipy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/75/b4ce781849931fef6fd529afa6b63711d5a733065722d0c3e2724af9e40a/scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec", size = 31613675, upload-time = "2026-02-23T00:16:00.13Z" }, + { url = "https://files.pythonhosted.org/packages/f7/58/bccc2861b305abdd1b8663d6130c0b3d7cc22e8d86663edbc8401bfd40d4/scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696", size = 28162057, upload-time = "2026-02-23T00:16:09.456Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ee/18146b7757ed4976276b9c9819108adbc73c5aad636e5353e20746b73069/scipy-1.17.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee", size = 20334032, upload-time = "2026-02-23T00:16:17.358Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e6/cef1cf3557f0c54954198554a10016b6a03b2ec9e22a4e1df734936bd99c/scipy-1.17.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd", size = 22709533, upload-time = "2026-02-23T00:16:25.791Z" }, + { url = "https://files.pythonhosted.org/packages/4d/60/8804678875fc59362b0fb759ab3ecce1f09c10a735680318ac30da8cd76b/scipy-1.17.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c", size = 33062057, upload-time = "2026-02-23T00:16:36.931Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4", size = 35349300, upload-time = "2026-02-23T00:16:49.108Z" }, + { url = "https://files.pythonhosted.org/packages/b4/3d/7ccbbdcbb54c8fdc20d3b6930137c782a163fa626f0aef920349873421ba/scipy-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444", size = 35127333, upload-time = "2026-02-23T00:17:01.293Z" }, + { url = "https://files.pythonhosted.org/packages/e8/19/f926cb11c42b15ba08e3a71e376d816ac08614f769b4f47e06c3580c836a/scipy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082", size = 37741314, upload-time = "2026-02-23T00:17:12.576Z" }, + { url = "https://files.pythonhosted.org/packages/95/da/0d1df507cf574b3f224ccc3d45244c9a1d732c81dcb26b1e8a766ae271a8/scipy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff", size = 36607512, upload-time = "2026-02-23T00:17:23.424Z" }, + { url = "https://files.pythonhosted.org/packages/68/7f/bdd79ceaad24b671543ffe0ef61ed8e659440eb683b66f033454dcee90eb/scipy-1.17.1-cp311-cp311-win_arm64.whl", hash = "sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d", size = 24599248, upload-time = "2026-02-23T00:17:34.561Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/b992b488d6f299dbe3f11a20b24d3dda3d46f1a635ede1c46b5b17a7b163/scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8", size = 31610954, upload-time = "2026-02-23T00:17:49.855Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/cf107b01494c19dc100f1d0b7ac3cc08666e96ba2d64db7626066cee895e/scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76", size = 28172662, upload-time = "2026-02-23T00:18:01.64Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a9/599c28631bad314d219cf9ffd40e985b24d603fc8a2f4ccc5ae8419a535b/scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086", size = 20344366, upload-time = "2026-02-23T00:18:12.015Z" }, + { url = "https://files.pythonhosted.org/packages/35/f5/906eda513271c8deb5af284e5ef0206d17a96239af79f9fa0aebfe0e36b4/scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b", size = 22704017, upload-time = "2026-02-23T00:18:21.502Z" }, + { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" }, + { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/dc08d77fbf3d87d3ee27f6a0c6dcce1de5829a64f2eae85a0ecc1f0daa73/scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87", size = 36549682, upload-time = "2026-02-23T00:19:07.67Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3", size = 24547340, upload-time = "2026-02-23T00:19:12.024Z" }, +] + [[package]] name = "scons" version = "4.9.1" @@ -4927,6 +5246,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, ] +[[package]] +name = "torch" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/0d/98b410492609e34a155fa8b121b55c7dca229f39636851c3a9ec20edea21/torch-2.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7b6a60d48062809f58595509c524b88e6ddec3ebe25833d6462eeab81e5f2ce4", size = 80529712, upload-time = "2026-03-23T18:12:02.608Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/acea680005f098f79fd70c1d9d5ccc0cb4296ec2af539a0450108232fc0c/torch-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d91aac77f24082809d2c5a93f52a5f085032740a1ebc9252a7b052ef5a4fddc6", size = 419718178, upload-time = "2026-03-23T18:10:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/8b/d7be22fbec9ffee6cff31a39f8750d4b3a65d349a286cf4aec74c2375662/torch-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7aa2f9bbc6d4595ba72138026b2074be1233186150e9292865e04b7a63b8c67a", size = 530604548, upload-time = "2026-03-23T18:10:03.569Z" }, + { url = "https://files.pythonhosted.org/packages/d1/bd/9912d30b68845256aabbb4a40aeefeef3c3b20db5211ccda653544ada4b6/torch-2.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:73e24aaf8f36ab90d95cd1761208b2eb70841c2a9ca1a3f9061b39fc5331b708", size = 114519675, upload-time = "2026-03-23T18:11:52.995Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8b/69e3008d78e5cee2b30183340cc425081b78afc5eff3d080daab0adda9aa/torch-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b5866312ee6e52ea625cd211dcb97d6a2cdc1131a5f15cc0d87eec948f6dd34", size = 80606338, upload-time = "2026-03-23T18:11:34.781Z" }, + { url = "https://files.pythonhosted.org/packages/13/16/42e5915ebe4868caa6bac83a8ed59db57f12e9a61b7d749d584776ed53d5/torch-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f99924682ef0aa6a4ab3b1b76f40dc6e273fca09f367d15a524266db100a723f", size = 419731115, upload-time = "2026-03-23T18:11:06.944Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c9/82638ef24d7877510f83baf821f5619a61b45568ce21c0a87a91576510aa/torch-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0f68f4ac6d95d12e896c3b7a912b5871619542ec54d3649cf48cc1edd4dd2756", size = 530712279, upload-time = "2026-03-23T18:10:31.481Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ff/6756f1c7ee302f6d202120e0f4f05b432b839908f9071157302cedfc5232/torch-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:fbf39280699d1b869f55eac536deceaa1b60bd6788ba74f399cc67e60a5fab10", size = 114556047, upload-time = "2026-03-23T18:10:55.931Z" }, +] + +[[package]] +name = "torchvision" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, + { name = "torch" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/bd/d552a2521bade3295b2c6e7a4a0d1022261cab7ca7011f4e2a330dbb3caa/torchvision-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55bd6ad4ae77be01ba67a410b05b51f53b0d0ee45f146eb6a0dfb9007e70ab3c", size = 1863499, upload-time = "2026-03-23T18:12:58.696Z" }, + { url = "https://files.pythonhosted.org/packages/33/bf/21b899792b08cae7a298551c68398a79e333697479ed311b3b067aab4bdc/torchvision-0.26.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1c55dc8affbcc0eb2060fbabbe996ae9e5839b24bb6419777f17848945a411b1", size = 7767527, upload-time = "2026-03-23T18:12:44.348Z" }, + { url = "https://files.pythonhosted.org/packages/9a/45/57bbf9e216850d065e66dd31a50f57424b607f1d878ab8956e56a1f4e36b/torchvision-0.26.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fd10b5f994c210f4f6d6761cf686f82d748554adf486cb0979770c3252868c8f", size = 7519925, upload-time = "2026-03-23T18:12:53.283Z" }, + { url = "https://files.pythonhosted.org/packages/10/58/ed8f7754299f3e91d6414b6dc09f62b3fa7c6e5d63dfe48d69ab81498a37/torchvision-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:de6424b12887ad884f39a0ee446994ae3cd3b6a00a9cafe1bead85a031132af0", size = 3983834, upload-time = "2026-03-23T18:13:00.224Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e7/56b47cc3b132aea90ccce22bcb8975dec688b002150012acc842846039d0/torchvision-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c409e1c3fdebec7a3834465086dbda8bf7680eff79abf7fd2f10c6b59520a7a4", size = 1863502, upload-time = "2026-03-23T18:12:57.326Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/5c31c92c08b65662fe9604a4067ae8232582805949f11ddc042cebe818ed/torchvision-0.26.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:406557718e62fdf10f5706e88d8a5ec000f872da913bf629aab9297622585547", size = 7767944, upload-time = "2026-03-23T18:12:42.805Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d8/cb6ccda1a1f35a6597645818641701207b3e8e13553e75fce5d86bac74b2/torchvision-0.26.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d61a5abb6b42a0c0c311996c2ac4b83a94418a97182c83b055a2a4ae985e05aa", size = 7522205, upload-time = "2026-03-23T18:12:54.654Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a9/c272623a0f735c35f0f6cd6dc74784d4f970e800cf063bb76687895a2ab9/torchvision-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:7993c01648e7c61d191b018e84d38fe0825c8fcb2720cd0f37caf7ba14404aa1", size = 4255155, upload-time = "2026-03-23T18:12:32.652Z" }, +] + [[package]] name = "tqdm" version = "4.67.1" @@ -4939,6 +5309,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, ] +[[package]] +name = "triton" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/2c/96f92f3c60387e14cc45aed49487f3486f89ea27106c1b1376913c62abe4/triton-3.6.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651", size = 176081190, upload-time = "2026-01-20T16:16:00.523Z" }, + { url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" }, + { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, +] + [[package]] name = "types-requests" version = "2.32.4.20250913" @@ -4969,6 +5350,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] +[[package]] +name = "ultralytics" +version = "8.4.32" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy" }, + { name = "opencv-python" }, + { name = "pillow" }, + { name = "polars" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "scipy" }, + { name = "torch" }, + { name = "torchvision" }, + { name = "ultralytics-thop" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/80/b17c01884a5c3997775d40786efc81d94b2204298a6d8ef3f6a6e308fced/ultralytics-8.4.32.tar.gz", hash = "sha256:ae9d3c3fa2930248a4a93e65fb47803cf58b440e36f541d103b308f3774d574e", size = 1028243, upload-time = "2026-03-30T16:11:28.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/ee/7ba24da543b47971bf7e64c01e2f2b03ec25aec25cd3af0007922db35fe5/ultralytics-8.4.32-py3-none-any.whl", hash = "sha256:88093e6c311a1ea3dd3369481249fe202d4e8e946d24902b22c00850e2e29fae", size = 1219034, upload-time = "2026-03-30T16:11:24.343Z" }, +] + +[[package]] +name = "ultralytics-thop" +version = "2.0.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "torch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/63/21a32e1facfeee245dbdfb7b4669faf7a36ff7c00b50987932bdab126f4b/ultralytics_thop-2.0.18.tar.gz", hash = "sha256:21103bcd39cc9928477dc3d9374561749b66a1781b35f46256c8d8c4ac01d9cf", size = 34557, upload-time = "2025-10-29T16:58:13.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/c7/fb42228bb05473d248c110218ffb8b1ad2f76728ed8699856e5af21112ad/ultralytics_thop-2.0.18-py3-none-any.whl", hash = "sha256:2bb44851ad224b116c3995b02dd5e474a5ccf00acf237fe0edb9e1506ede04ec", size = 28941, upload-time = "2025-10-29T16:58:12.093Z" }, +] + [[package]] name = "urllib3" version = "2.5.0" diff --git a/yolo11n-cls.pt b/yolo11n-cls.pt new file mode 100644 index 00000000..8651b718 Binary files /dev/null and b/yolo11n-cls.pt differ