mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-07-16 06:42:12 +08:00
2561 lines
102 KiB
Python
2561 lines
102 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import time
|
|
|
|
from collections import Counter, deque
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import cv2
|
|
import numpy as np
|
|
|
|
from openpilot.common.constants import CV
|
|
from openpilot.common.realtime import set_core_affinity
|
|
from openpilot.system.hardware import PC
|
|
|
|
RUNTIME_LOOP_HZ = 20
|
|
INFERENCE_INTERVAL = 0.15
|
|
FOLLOWUP_INFERENCE_INTERVAL = 0.10
|
|
FOLLOWUP_WINDOW_SECONDS = 2.0
|
|
TEMPORAL_TRACKING_ENABLED = False
|
|
TRACK_CONFIRMED_PROPOSALS_ENABLED = False
|
|
TRACK_CLASSIFICATION_INTERVAL = 0.12
|
|
TRACK_BUSY_CLASSIFICATION_INTERVAL = 0.35
|
|
TRACK_DETECTOR_INTERVAL = 0.55
|
|
TRACK_MAX_AGE_SECONDS = 2.0
|
|
TRACK_MIN_PROPOSAL_CONFIDENCE = 0.10
|
|
TRACK_UNREADABLE_MIN_PROPOSAL_CONFIDENCE = 0.22
|
|
TRACK_MAX_CONSECUTIVE_FAILED_READS = 2
|
|
TRACK_MIN_FEATURE_COUNT = 4
|
|
TRACK_MAX_AREA_RATIO = 0.18
|
|
TRACK_CROP_PADDING_RATIO = 0.06
|
|
TRACK_REPEAT_CONFIDENCE_BONUS = 0.12
|
|
BUSY_INFERENCE_INTERVAL = 1.0
|
|
LIVE_POSE_RECOVERY_THROTTLE_SECONDS = 2.0
|
|
LIVE_POSE_RECOVERY_INFERENCE_INTERVAL = 1.0
|
|
RUNTIME_TELEMETRY_INTERVAL_SECONDS = 2.0
|
|
DEBUG_HEARTBEAT_INTERVAL_SECONDS = 30.0
|
|
DEFAULT_DETECTOR_INPUT_SIZE = 640
|
|
DETECTOR_INPUT_SIZE_CANDIDATES = (640, 512, 448, 416, 384, 320, 288, 256, 224, 192)
|
|
DEFAULT_CLASSIFIER_INPUT_SIZE = 128
|
|
CLASSIFIER_INPUT_SIZE_CANDIDATES = (128, 112, 96, 80, 64)
|
|
FULL_FRAME_OCR_FALLBACK_ENABLED = False
|
|
DETECTOR_CLASSIFIER_CROP_OCR_ENABLED = False
|
|
DETECTOR_CLASSIFIER_REGION_MODE = "right_roi" # full, right_roi, full_and_right_roi
|
|
DEVICE_BUSY_AVG_CPU_USAGE_PERCENT = 78.0
|
|
DEVICE_BUSY_MAX_CPU_USAGE_PERCENT = 92.0
|
|
DEVICE_BUSY_HOT_CORE_COUNT = 4
|
|
MIN_DETECTION_CONFIDENCE = 0.2
|
|
STRONG_DETECTION_CONFIDENCE = 0.72
|
|
OCR_MIN_CONFIDENCE = 0.35
|
|
VALUE_TEMPLATE_MIN_CONFIDENCE = 0.55
|
|
HISTORY_SECONDS = 2.0
|
|
CONSISTENT_DETECTIONS = 2
|
|
# These counts must remain achievable at the measured 1.5 Hz onroad cadence.
|
|
CHANGE_CONSISTENT_DETECTIONS = 2
|
|
CHANGE_SINGLE_READ_MIN_CONFIDENCE = 0.83
|
|
CHANGE_REPEAT_MIN_CONFIDENCE = 0.70
|
|
LOW_SPEED_CHANGE_CONSISTENT_DETECTIONS = 2
|
|
LOW_SPEED_CHANGE_MIN_CONFIDENCE = 0.90
|
|
LOW_SPEED_CHANGE_ALLOW_STRONG_CONSENSUS = True
|
|
MODEL_DETECTION_SHORT_CIRCUIT_CONFIDENCE = 0.65
|
|
PUBLISHED_HOLD_SECONDS = 300.0
|
|
PUBLISHED_CHANGE_COOLDOWN_SECONDS = 1.4
|
|
PUBLISHED_REVERT_CONFIDENCE = 0.97
|
|
AUTO_BOOKMARK_CONFIRM_DELAY_SECONDS = 0.9
|
|
AUTO_BOOKMARK_COOLDOWN_SECONDS = 8.0
|
|
AUTO_BOOKMARK_MIN_CONFIDENCE = 0.62
|
|
TRAINING_COLLECTOR_CONFIRM_DELAY_SECONDS = 0.7
|
|
TRAINING_COLLECTOR_COOLDOWN_SECONDS = 2.5
|
|
TRAINING_COLLECTOR_MIN_CONFIDENCE = 0.40
|
|
MAP_NEXT_REVIEW_DISTANCE_METERS = 120.0
|
|
MAP_TRANSITION_MISS_CAPTURE_COOLDOWN_SECONDS = 8.0
|
|
MAP_VISION_MATCH_WINDOW_SECONDS = 2.5
|
|
MODEL_PROPOSAL_MIN_CONFIDENCE = 0.0001
|
|
MODEL_PROPOSAL_MAX_COUNT = 4
|
|
MODEL_PROPOSAL_MAX_AREA_RATIO = 0.18
|
|
MODEL_PROPOSAL_MIN_WIDTH = 10
|
|
MODEL_PROPOSAL_MIN_HEIGHT = 18
|
|
MODEL_PROPOSAL_MIN_X_RATIO = 0.35
|
|
MODEL_PROPOSAL_MAX_Y_RATIO = 0.82
|
|
MODEL_PROPOSAL_EXPANSIONS = (
|
|
(1.2, 1.6, 1.2, 1.8),
|
|
)
|
|
MODEL_PROPOSAL_TRIM_BOTTOM_RATIOS = (1.0, 0.78)
|
|
|
|
ROI_WINDOWS = (
|
|
{"bounds": (0.48, 0.00, 0.98, 0.42), "min_confidence": MIN_DETECTION_CONFIDENCE},
|
|
{"bounds": (0.52, 0.02, 0.97, 0.58), "min_confidence": 0.22},
|
|
{"bounds": (0.62, 0.02, 0.99, 0.68), "min_confidence": 0.18},
|
|
{"bounds": (0.45, 0.00, 1.00, 0.82), "min_confidence": 0.06},
|
|
)
|
|
EDGE_MARGIN_RATIO = 0.03
|
|
MAX_BOX_AREA_RATIO = 0.22
|
|
|
|
OCR_SEARCH_WINDOWS = (
|
|
(0.45, 0.05, 0.92, 0.86),
|
|
(0.65, 0.08, 0.98, 0.76),
|
|
)
|
|
OCR_SEARCH_THRESHOLDS = (130,)
|
|
OCR_SEARCH_CROP_VARIANTS = (
|
|
(0.08, 0.06, 0.10),
|
|
)
|
|
OCR_SEARCH_UPSCALE = 3
|
|
OCR_FALLBACK_MIN_CONFIDENCE = 0.55
|
|
VALUE_TEMPLATE_ROIS = (
|
|
(0.35, 0.82, 0.15, 0.78),
|
|
(0.45, 0.85, 0.18, 0.78),
|
|
(0.40, 0.84, 0.18, 0.75),
|
|
)
|
|
REGULATORY_WHITE_VALUE_MIN = 135
|
|
REGULATORY_WHITE_SAT_MAX = 70
|
|
REGULATORY_DARK_VALUE_MAX = 115
|
|
REGULATORY_DARK_SAT_MAX = 110
|
|
REGULATORY_YELLOW_HUE_MIN = 12
|
|
REGULATORY_YELLOW_HUE_MAX = 45
|
|
REGULATORY_YELLOW_SAT_MIN = 70
|
|
REGULATORY_YELLOW_VALUE_MIN = 85
|
|
REGULATORY_RED_LOW_HUE_MAX = 12
|
|
REGULATORY_RED_HIGH_HUE_MIN = 168
|
|
REGULATORY_RED_SAT_MIN = 80
|
|
REGULATORY_RED_VALUE_MIN = 60
|
|
REGULATORY_GREEN_HUE_MIN = 45
|
|
REGULATORY_GREEN_HUE_MAX = 90
|
|
REGULATORY_BLUE_HUE_MIN = 90
|
|
REGULATORY_BLUE_HUE_MAX = 135
|
|
REGULATORY_COLORED_SAT_MIN = 70
|
|
REGULATORY_COLORED_VALUE_MIN = 70
|
|
REGULATORY_MIN_WHITE_RATIO = 0.08
|
|
REGULATORY_MIN_DARK_RATIO = 0.01
|
|
REGULATORY_MAX_YELLOW_RATIO = 0.12
|
|
REGULATORY_MAX_RED_RATIO = 0.10
|
|
REGULATORY_MAX_GREEN_RATIO = 0.35
|
|
REGULATORY_MAX_BLUE_RATIO = 0.35
|
|
REGULATORY_MIN_WHITE_COMPONENT_RATIO = 0.012
|
|
REGULATORY_MIN_COMPONENT_FILL = 0.36
|
|
REGULATORY_MIN_COMPONENT_HEIGHT_RATIO = 0.2
|
|
REGULATORY_MIN_COMPONENT_WIDTH_RATIO = 0.12
|
|
REGULATORY_MIN_ASPECT_RATIO = 0.28
|
|
REGULATORY_MAX_ASPECT_RATIO = 1.25
|
|
|
|
SPEED_LIMIT_CLASSES = {
|
|
2: 10,
|
|
3: 100,
|
|
4: 110,
|
|
5: 120,
|
|
6: 20,
|
|
7: 30,
|
|
8: 40,
|
|
9: 50,
|
|
10: 60,
|
|
11: 70,
|
|
12: 80,
|
|
13: 90,
|
|
}
|
|
|
|
VALID_SPEED_LIMITS_MPH = set(range(10, 125, 5))
|
|
MIN_PUBLISHABLE_SPEED_LIMIT_MPH = 20
|
|
LEGACY_MODEL_PATH = Path(__file__).resolve().parents[1] / "assets" / "vision_models" / "speed_limit_vision.onnx"
|
|
US_DETECTOR_MODEL_PATH = Path(__file__).resolve().parents[1] / "assets" / "vision_models" / "speed_limit_us_detector.onnx"
|
|
US_CLASSIFIER_MODEL_PATH = Path(__file__).resolve().parents[1] / "assets" / "vision_models" / "speed_limit_us_value_classifier.onnx"
|
|
US_REJECT_CLASSIFIER_MODEL_PATH = Path(__file__).resolve().parents[1] / "assets" / "vision_models" / "speed_limit_us_reject_classifier.onnx"
|
|
US_DETECTOR_CLASSES = {
|
|
0: "regulatory_speed_limit",
|
|
1: "advisory_speed_limit",
|
|
2: "school_zone_speed_limit",
|
|
}
|
|
US_CLASSIFIER_SPEED_VALUES = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75)
|
|
SCHOOL_ZONE_SPEED_VALUES = frozenset((15, 20, 25))
|
|
US_DETECTOR_MIN_CONFIDENCE = 0.06
|
|
US_CLASSIFIER_MIN_CONFIDENCE = 0.60
|
|
US_CLASSIFIER_REJECT_MIN_CONFIDENCE = 0.85
|
|
SEPARATE_REJECT_CLASSIFIER_ENABLED = False
|
|
US_REJECT_CLASSIFIER_MIN_CONFIDENCE = 0.85
|
|
DETECTOR_CLASSIFIER_EXPANSIONS = (
|
|
(0.00, 0.00, 0.00, 0.00, 1.10),
|
|
(0.10, 0.06, 0.10, 0.12, 1.00),
|
|
(0.00, 0.00, 0.18, 0.18, 0.55),
|
|
)
|
|
SCHOOL_ZONE_DIRECT_EXPANSIONS = (
|
|
(0.00, 0.00, 0.18, 0.18),
|
|
(0.00, 0.00, 0.22, 0.18),
|
|
)
|
|
SCHOOL_ZONE_READ_VARIANTS = (
|
|
(0.00, 0.00, 1.00, 1.00, 0.80),
|
|
(0.00, 0.35, 1.00, 1.00, 0.88),
|
|
(0.00, 0.45, 1.00, 1.00, 0.96),
|
|
(0.12, 0.38, 0.88, 1.00, 1.00),
|
|
)
|
|
DETECTOR_CLASSIFIER_SUPPORT_BONUS = 0.06
|
|
DETECTOR_CLASSIFIER_REGULATORY_BONUS = 0.05
|
|
DETECTOR_CLASSIFIER_NON_REGULATORY_PENALTY = 0.03
|
|
DETECTOR_CLASSIFIER_SMALL_BOX_AREA_RATIO = 0.004
|
|
DETECTOR_CLASSIFIER_TINY_LOW_CONF_AREA_RATIO = 0.002
|
|
DETECTOR_CLASSIFIER_TINY_LOW_CONF_MIN_CONFIDENCE = 0.16
|
|
DETECTOR_CLASSIFIER_MIN_ACCEPT_WIDTH = 28
|
|
DETECTOR_CLASSIFIER_MIN_ACCEPT_HEIGHT = 40
|
|
DETECTOR_CLASSIFIER_RESCUE_MIN_WIDTH = 14
|
|
DETECTOR_CLASSIFIER_RESCUE_MIN_HEIGHT = 18
|
|
DETECTOR_CLASSIFIER_RESCUE_MIN_X_RATIO = 0.52
|
|
DETECTOR_CLASSIFIER_RESCUE_MIN_SUPPORT = 2
|
|
DETECTOR_CLASSIFIER_RESCUE_MIN_CONFIDENCE = 0.90
|
|
DETECTOR_CLASSIFIER_RESCUE_MAX_SCORE = 0.64
|
|
DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_SUPPORT = 3
|
|
DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_PROPOSAL_CONFIDENCE = 0.60
|
|
DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_READ_CONFIDENCE = 0.995
|
|
DETECTOR_CLASSIFIER_STRONG_RESCUE_MAX_SCORE = 0.74
|
|
DETECTOR_CLASSIFIER_TRUSTED_MODEL_MAX_HEIGHT = 55
|
|
DETECTOR_CLASSIFIER_TRUSTED_MODEL_MAX_AREA_RATIO = 0.002
|
|
DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_PROPOSAL_CONFIDENCE = 0.18
|
|
DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_X_RATIO = 0.52
|
|
DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE = 0.65
|
|
DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_SUPPORT = 2
|
|
DETECTOR_CLASSIFIER_STRONG_MODEL_MIN_PROPOSAL_CONFIDENCE = 0.60
|
|
DETECTOR_CLASSIFIER_STRONG_MODEL_MIN_READ_CONFIDENCE = 0.995
|
|
DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_MIN_READ_CONFIDENCE = 0.95
|
|
DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_ENABLED = True
|
|
DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_MIN_SUPPORT = 2
|
|
DETECTOR_CLASSIFIER_MODEL_ONLY_CONSENSUS_MIN_CONFIDENCE = 0.90
|
|
DETECTOR_CLASSIFIER_MODEL_ONLY_CONSENSUS_MIN_SUPPORT = 2
|
|
SCHOOL_ZONE_SPEED_PRIOR = 0.12
|
|
SCHOOL_ZONE_SUPPORT_BONUS = 0.08
|
|
SCHOOL_ZONE_MIN_SUPPORT = 2
|
|
SCHOOL_ZONE_MIN_CONFIDENCE = 0.70
|
|
SCHOOL_ZONE_SINGLE_READ_CONFIDENCE = 0.975
|
|
SCHOOL_ZONE_SHORT_CIRCUIT_CONFIDENCE = 0.78
|
|
SCHOOL_ZONE_FALLBACK_MIN_CONFIDENCE = 0.35
|
|
NON_SCHOOL_LOW_SPEED_COMPETING_MIN_CONFIDENCE = 0.95
|
|
DEBUG_BASE_DIR = Path("/data/media/0/vision_speed_limit_debug")
|
|
DEBUG_RUNTIME_STATUS_PATH = DEBUG_BASE_DIR / "runtime_status.json"
|
|
DEBUG_CAPTURE_DIRNAME = "captures"
|
|
SNAPSHOT_JPEG_QUALITY = 85
|
|
SPEED_LIMIT_VISION_AFFINITY_CORES = [0, 1, 2]
|
|
|
|
|
|
def device_cpu_usage_busy(cpu_usage):
|
|
usage = list(cpu_usage)
|
|
if not usage:
|
|
return False
|
|
return (
|
|
sum(usage) / len(usage) >= DEVICE_BUSY_AVG_CPU_USAGE_PERCENT or
|
|
sum(core_usage >= DEVICE_BUSY_MAX_CPU_USAGE_PERCENT for core_usage in usage) >= DEVICE_BUSY_HOT_CORE_COUNT
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class Detection:
|
|
speed_limit_mph: int
|
|
confidence: float
|
|
strong_consensus: bool = False
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DetectorProposal:
|
|
confidence: float
|
|
class_id: int
|
|
bbox: tuple[int, int, int, int]
|
|
speed_limit_mph: int = 0
|
|
|
|
|
|
@dataclass
|
|
class ProposalTrack:
|
|
proposal: DetectorProposal
|
|
bbox: tuple[int, int, int, int]
|
|
previous_gray: np.ndarray
|
|
points: np.ndarray
|
|
started_at: float
|
|
last_classified_at: float
|
|
last_speed_limit_mph: int = 0
|
|
consistent_reads: int = 0
|
|
consecutive_failed_reads: int = 0
|
|
|
|
|
|
@dataclass
|
|
class HistoryEntry:
|
|
speed_limit_mph: int
|
|
confidence: float
|
|
created_at: float
|
|
strong_consensus: bool = False
|
|
|
|
|
|
class SpeedLimitVisionDaemon:
|
|
def __init__(self, use_runtime=True):
|
|
self.use_runtime = use_runtime
|
|
self.params = None
|
|
self.params_memory = None
|
|
self.Ratekeeper = None
|
|
self.messaging = None
|
|
self.pm = None
|
|
self.VisionIpcClient = None
|
|
self.VisionStreamType = None
|
|
self.sm = None
|
|
|
|
if self.use_runtime:
|
|
from cereal import messaging
|
|
from msgq.visionipc import VisionIpcClient, VisionStreamType
|
|
from openpilot.common.params import Params
|
|
from openpilot.common.realtime import Ratekeeper
|
|
|
|
self.messaging = messaging
|
|
self.pm = messaging.PubMaster(["userBookmark"])
|
|
self.params = Params(return_defaults=True)
|
|
self.params_memory = Params(memory=True)
|
|
self.Ratekeeper = Ratekeeper
|
|
self.VisionIpcClient = VisionIpcClient
|
|
self.VisionStreamType = VisionStreamType
|
|
self.sm = messaging.SubMaster(["deviceState", "mapdOut", "userBookmark", "livePose"])
|
|
|
|
self.client = None
|
|
self.stream_name = ""
|
|
self.stream_type = None
|
|
|
|
self.net = None
|
|
self.classifier_net = None
|
|
self.model_mode = "legacy"
|
|
self.detector_input_size = DEFAULT_DETECTOR_INPUT_SIZE
|
|
self.classifier_input_size = DEFAULT_CLASSIFIER_INPUT_SIZE
|
|
self.last_error = ""
|
|
self.last_inference_at = -float("inf")
|
|
self.last_detection_at = 0.0
|
|
self.last_live_pose_inputs_not_ok_at = -float("inf")
|
|
self.last_road_name = ""
|
|
self.followup_until = 0.0
|
|
self.latest_detector_proposal = None
|
|
self.proposal_track = None
|
|
self.track_inference_count = 0
|
|
self.track_failure_count = 0
|
|
self.track_start_count = 0
|
|
self.max_track_proposal_confidence = 0.0
|
|
self.started_prev = False
|
|
|
|
self.history: deque[HistoryEntry] = deque()
|
|
self.published_speed_limit_mph = 0
|
|
self.published_confidence = 0.0
|
|
self.previous_published_speed_limit_mph = 0
|
|
self.last_publish_change_at = 0.0
|
|
self.last_published_support_at = 0.0
|
|
self.last_candidate_speed_limit_mph = 0
|
|
self.last_candidate_confidence = 0.0
|
|
self.last_candidate_at = 0.0
|
|
self.last_auto_bookmark_at = 0.0
|
|
self.last_auto_bookmark_speed_limit_mph = 0
|
|
self.last_auto_bookmark_publish_at = 0.0
|
|
self.pending_auto_bookmark = None
|
|
self.last_training_capture_at = 0.0
|
|
self.last_training_capture_speed_limit_mph = 0
|
|
self.pending_training_capture = None
|
|
self.last_map_speed_limit_mph = 0
|
|
self.last_map_transition_miss_at = 0.0
|
|
self.last_map_transition_miss_speed_limit_mph = 0
|
|
self.ignore_next_user_bookmark = False
|
|
self.current_frame_bgr = None
|
|
|
|
self.debug_session_id = ""
|
|
self.debug_dir = None
|
|
self.debug_capture_dir = None
|
|
self.debug_log_path = None
|
|
self.debug_bookmark_count = 0
|
|
self.debug_session_started_at = 0.0
|
|
self.last_logged_status = ""
|
|
self.last_logged_candidate = None
|
|
self.last_runtime_telemetry_at = 0.0
|
|
self.last_debug_heartbeat_at = 0.0
|
|
self.loop_count = 0
|
|
self.inference_count = 0
|
|
self.detector_inference_count = 0
|
|
self.interval_skip_count = 0
|
|
self.busy_skip_count = 0
|
|
self.camera_unavailable_count = 0
|
|
self.empty_frame_count = 0
|
|
self.detection_count = 0
|
|
self.last_inference_interval = INFERENCE_INTERVAL
|
|
self.last_inference_interval_reason = "steady"
|
|
self.last_cpu_busy = False
|
|
self.last_frame_process_duration_s = 0.0
|
|
self.last_detector_forward_count = 0
|
|
self.last_detector_forward_duration_s = 0.0
|
|
self.last_classifier_forward_count = 0
|
|
self.last_classifier_forward_duration_s = 0.0
|
|
|
|
self.digit_templates = self._build_digit_templates()
|
|
self.speed_value_templates = self._build_speed_value_templates()
|
|
self._load_model()
|
|
|
|
def _start_debug_session(self):
|
|
if not self.use_runtime or self.params_memory is None or self.debug_session_id:
|
|
return
|
|
|
|
timestamp = datetime.now(UTC)
|
|
session_id = timestamp.strftime("%Y%m%d_%H%M%S")
|
|
debug_dir = DEBUG_BASE_DIR / session_id
|
|
suffix = 1
|
|
while debug_dir.exists():
|
|
suffix += 1
|
|
session_id = f"{timestamp.strftime('%Y%m%d_%H%M%S')}_{suffix}"
|
|
debug_dir = DEBUG_BASE_DIR / session_id
|
|
|
|
debug_dir.mkdir(parents=True, exist_ok=True)
|
|
capture_dir = debug_dir / DEBUG_CAPTURE_DIRNAME
|
|
capture_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
self.debug_session_id = session_id
|
|
self.debug_dir = debug_dir
|
|
self.debug_capture_dir = capture_dir
|
|
self.debug_log_path = debug_dir / "events.jsonl"
|
|
self.debug_bookmark_count = 0
|
|
self.debug_session_started_at = time.monotonic()
|
|
self.last_logged_status = ""
|
|
self.last_logged_candidate = None
|
|
|
|
self.params_memory.put("VisionSpeedLimitDebugSession", self.debug_session_id)
|
|
self.params_memory.put_int("VisionSpeedLimitBookmarkCount", self.debug_bookmark_count)
|
|
self.params_memory.put("VisionSpeedLimitLastEvent", "")
|
|
self._write_debug_event("session_start", reason="onroad")
|
|
|
|
def _close_debug_session(self):
|
|
self.debug_session_id = ""
|
|
self.debug_dir = None
|
|
self.debug_capture_dir = None
|
|
self.debug_log_path = None
|
|
self.debug_bookmark_count = 0
|
|
self.debug_session_started_at = 0.0
|
|
self.last_logged_status = ""
|
|
self.last_logged_candidate = None
|
|
self.last_debug_heartbeat_at = 0.0
|
|
|
|
def _read_next_map_speed_limit(self):
|
|
if self.params_memory is None:
|
|
return {}
|
|
|
|
next_map_speed_limit = self.params_memory.get("NextMapSpeedLimit") or {}
|
|
if isinstance(next_map_speed_limit, (bytes, str)):
|
|
try:
|
|
next_map_speed_limit = json.loads(next_map_speed_limit)
|
|
except Exception:
|
|
next_map_speed_limit = {}
|
|
return next_map_speed_limit if isinstance(next_map_speed_limit, dict) else {}
|
|
|
|
def _get_map_context(self):
|
|
current_limit_ms = 0.0
|
|
next_limit_ms = 0.0
|
|
next_distance_m = 0.0
|
|
source = "none"
|
|
|
|
if self.sm is not None:
|
|
try:
|
|
current_limit_ms = float(self.sm["mapdOut"].speedLimit or 0.0)
|
|
next_limit_ms = float(self.sm["mapdOut"].nextSpeedLimit or 0.0)
|
|
next_distance_m = float(self.sm["mapdOut"].nextSpeedLimitDistance or 0.0)
|
|
if current_limit_ms > 0.0 or next_limit_ms > 0.0:
|
|
source = "mapd"
|
|
except Exception:
|
|
current_limit_ms = 0.0
|
|
next_limit_ms = 0.0
|
|
next_distance_m = 0.0
|
|
|
|
if self.params_memory is not None:
|
|
filler_current_limit_ms = float(self.params_memory.get_float("MapSpeedLimit") or 0.0)
|
|
next_map_speed_limit = self._read_next_map_speed_limit()
|
|
filler_next_limit_ms = float(next_map_speed_limit.get("speedlimit") or 0.0)
|
|
filler_next_distance_m = float(next_map_speed_limit.get("distance") or 0.0)
|
|
if filler_current_limit_ms > 0.0 or filler_next_limit_ms > 0.0:
|
|
current_limit_ms = filler_current_limit_ms if filler_current_limit_ms > 0.0 else current_limit_ms
|
|
next_limit_ms = filler_next_limit_ms if filler_next_limit_ms > 0.0 else next_limit_ms
|
|
next_distance_m = filler_next_distance_m if filler_next_distance_m > 0.0 else next_distance_m
|
|
source = "filler"
|
|
|
|
current_limit_mph = int(round(current_limit_ms * CV.MS_TO_MPH)) if current_limit_ms > 0.0 else 0
|
|
next_limit_mph = int(round(next_limit_ms * CV.MS_TO_MPH)) if next_limit_ms > 0.0 else 0
|
|
next_distance_m = round(next_distance_m, 1) if next_distance_m > 0.0 else 0.0
|
|
|
|
return {
|
|
"source": source,
|
|
"current_speed_limit_mph": current_limit_mph,
|
|
"next_speed_limit_mph": next_limit_mph,
|
|
"next_speed_limit_distance_m": next_distance_m,
|
|
}
|
|
|
|
def _map_fields(self, speed_limit_mph=0):
|
|
map_context = self._get_map_context()
|
|
current_limit_mph = int(map_context["current_speed_limit_mph"])
|
|
next_limit_mph = int(map_context["next_speed_limit_mph"])
|
|
next_distance_m = float(map_context["next_speed_limit_distance_m"])
|
|
map_source = str(map_context["source"])
|
|
|
|
expected_speed_limit_mph = current_limit_mph if current_limit_mph > 0 else 0
|
|
map_relation = "no_map"
|
|
review_bucket = "vision_only"
|
|
|
|
next_is_relevant = next_limit_mph > 0 and next_limit_mph != current_limit_mph and 0.0 < next_distance_m <= MAP_NEXT_REVIEW_DISTANCE_METERS
|
|
|
|
if current_limit_mph > 0:
|
|
if speed_limit_mph > 0 and speed_limit_mph == current_limit_mph:
|
|
map_relation = "agree_current"
|
|
review_bucket = "map_agreement"
|
|
else:
|
|
map_relation = "disagree_current"
|
|
review_bucket = "map_disagreement"
|
|
|
|
if next_is_relevant:
|
|
if speed_limit_mph > 0 and speed_limit_mph == next_limit_mph:
|
|
expected_speed_limit_mph = next_limit_mph
|
|
map_relation = "agree_next"
|
|
review_bucket = "map_agreement"
|
|
elif current_limit_mph <= 0:
|
|
expected_speed_limit_mph = next_limit_mph
|
|
map_relation = "disagree_next"
|
|
review_bucket = "map_disagreement"
|
|
|
|
if speed_limit_mph <= 0:
|
|
if next_is_relevant:
|
|
expected_speed_limit_mph = next_limit_mph
|
|
map_relation = "map_transition_pending"
|
|
review_bucket = "map_transition_review"
|
|
elif current_limit_mph > 0:
|
|
map_relation = "map_present"
|
|
review_bucket = "map_context_only"
|
|
|
|
return {
|
|
"mapSource": map_source,
|
|
"mapCurrentSpeedLimitMph": current_limit_mph,
|
|
"mapNextSpeedLimitMph": next_limit_mph,
|
|
"mapNextSpeedLimitDistanceM": next_distance_m,
|
|
"mapExpectedSpeedLimitMph": expected_speed_limit_mph,
|
|
"mapRelation": map_relation,
|
|
"reviewBucket": review_bucket,
|
|
}
|
|
|
|
def _vision_recently_supported(self, speed_limit_mph, now):
|
|
if speed_limit_mph <= 0:
|
|
return False
|
|
if self.last_candidate_speed_limit_mph == speed_limit_mph and now - self.last_candidate_at <= MAP_VISION_MATCH_WINDOW_SECONDS:
|
|
return True
|
|
if self.published_speed_limit_mph == speed_limit_mph and now - self.last_detection_at <= MAP_VISION_MATCH_WINDOW_SECONDS:
|
|
return True
|
|
return any(
|
|
entry.speed_limit_mph == speed_limit_mph and now - entry.created_at <= MAP_VISION_MATCH_WINDOW_SECONDS
|
|
for entry in self.history
|
|
)
|
|
|
|
def _write_debug_event(self, event_type, frame_bgr=None, snapshot_prefix=None, **fields):
|
|
if not self.use_runtime or self.params_memory is None:
|
|
return
|
|
if not self.debug_log_path:
|
|
return
|
|
|
|
wall_time_ns = time.time_ns()
|
|
event = {
|
|
"event": event_type,
|
|
"wallTimeNs": wall_time_ns,
|
|
"wallTime": datetime.fromtimestamp(wall_time_ns / 1e9, UTC).isoformat(),
|
|
"monoTimeNs": time.monotonic_ns(),
|
|
"roadName": self.last_road_name,
|
|
"stream": self.stream_name,
|
|
"publishedSpeedLimitMph": self.published_speed_limit_mph,
|
|
"publishedConfidence": round(self.published_confidence, 4),
|
|
"bookmarkCount": self.debug_bookmark_count,
|
|
"status": self.params_memory.get("VisionSpeedLimitStatus", encoding="utf-8") or "",
|
|
}
|
|
if self.debug_session_started_at > 0.0:
|
|
event["sessionSeconds"] = round(max(time.monotonic() - self.debug_session_started_at, 0.0), 3)
|
|
event.update(self._map_fields(int(fields.get("speedLimitMph") or fields.get("candidateSpeedLimitMph") or 0)))
|
|
event.update(fields)
|
|
|
|
if frame_bgr is not None and self.debug_capture_dir is not None and snapshot_prefix:
|
|
snapshot_name = f"{wall_time_ns}_{snapshot_prefix}.jpg"
|
|
snapshot_path = self.debug_capture_dir / snapshot_name
|
|
try:
|
|
cv2.imwrite(str(snapshot_path), frame_bgr, [cv2.IMWRITE_JPEG_QUALITY, SNAPSHOT_JPEG_QUALITY])
|
|
event["snapshot"] = f"{DEBUG_CAPTURE_DIRNAME}/{snapshot_name}"
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
with self.debug_log_path.open("a", encoding="utf-8") as log_file:
|
|
log_file.write(json.dumps(event, separators=(",", ":")) + "\n")
|
|
except Exception:
|
|
pass
|
|
|
|
summary_parts = [event_type.replace("_", " ")]
|
|
if "speedLimitMph" in fields:
|
|
summary_parts.append(f"{fields['speedLimitMph']} mph")
|
|
elif "candidateSpeedLimitMph" in fields:
|
|
summary_parts.append(f"{fields['candidateSpeedLimitMph']} mph")
|
|
summary = " ".join(summary_parts)
|
|
self.params_memory.put("VisionSpeedLimitLastEvent", summary[:160])
|
|
|
|
def _record_bookmark(self):
|
|
if not self.use_runtime or self.params_memory is None or not self.debug_log_path:
|
|
return
|
|
|
|
self.debug_bookmark_count += 1
|
|
self.params_memory.put_int("VisionSpeedLimitBookmarkCount", self.debug_bookmark_count)
|
|
|
|
fields = {
|
|
"candidateSpeedLimitMph": self.last_candidate_speed_limit_mph,
|
|
"candidateConfidence": round(self.last_candidate_confidence, 4),
|
|
"publishedSpeedLimitMph": self.published_speed_limit_mph,
|
|
"publishedConfidence": round(self.published_confidence, 4),
|
|
}
|
|
self._write_debug_event(
|
|
"bookmark",
|
|
frame_bgr=self.current_frame_bgr,
|
|
snapshot_prefix=f"bookmark_{self.debug_bookmark_count:03d}",
|
|
**fields,
|
|
)
|
|
|
|
def _record_auto_bookmark(self, speed_limit_mph, confidence):
|
|
if not self.use_runtime or self.params_memory is None or not self.debug_log_path:
|
|
return
|
|
|
|
self.debug_bookmark_count += 1
|
|
self.params_memory.put_int("VisionSpeedLimitBookmarkCount", self.debug_bookmark_count)
|
|
|
|
self._write_debug_event(
|
|
"auto_bookmark",
|
|
frame_bgr=self.current_frame_bgr,
|
|
snapshot_prefix=f"auto_bookmark_{self.debug_bookmark_count:03d}",
|
|
candidateSpeedLimitMph=self.last_candidate_speed_limit_mph,
|
|
candidateConfidence=round(self.last_candidate_confidence, 4),
|
|
publishedSpeedLimitMph=self.published_speed_limit_mph,
|
|
publishedConfidence=round(self.published_confidence, 4),
|
|
speedLimitMph=speed_limit_mph,
|
|
confidence=round(confidence, 4),
|
|
bookmarkCount=self.debug_bookmark_count,
|
|
)
|
|
|
|
def _record_training_candidate(self, speed_limit_mph, confidence, source_confidence, source_event):
|
|
if not self.use_runtime or self.params_memory is None or not self.debug_log_path:
|
|
return
|
|
|
|
self._write_debug_event(
|
|
"training_candidate",
|
|
frame_bgr=self.current_frame_bgr,
|
|
snapshot_prefix=f"training_candidate_{speed_limit_mph:03d}",
|
|
candidateSpeedLimitMph=self.last_candidate_speed_limit_mph,
|
|
candidateConfidence=round(self.last_candidate_confidence, 4),
|
|
publishedSpeedLimitMph=self.published_speed_limit_mph,
|
|
publishedConfidence=round(self.published_confidence, 4),
|
|
speedLimitMph=speed_limit_mph,
|
|
confidence=round(confidence, 4),
|
|
sourceConfidence=round(source_confidence, 4),
|
|
sourceEvent=source_event,
|
|
)
|
|
|
|
def _record_map_transition_miss(self, speed_limit_mph, previous_speed_limit_mph):
|
|
if not self.use_runtime or self.params_memory is None or not self.debug_log_path:
|
|
return
|
|
|
|
self._write_debug_event(
|
|
"map_transition_miss",
|
|
frame_bgr=self.current_frame_bgr,
|
|
snapshot_prefix=f"map_transition_miss_{speed_limit_mph:03d}",
|
|
speedLimitMph=speed_limit_mph,
|
|
previousMapSpeedLimitMph=previous_speed_limit_mph,
|
|
candidateSpeedLimitMph=self.last_candidate_speed_limit_mph,
|
|
candidateConfidence=round(self.last_candidate_confidence, 4),
|
|
publishedSpeedLimitMph=self.published_speed_limit_mph,
|
|
publishedConfidence=round(self.published_confidence, 4),
|
|
reason="map_change_without_vision_support",
|
|
reviewBucket="map_transition_miss",
|
|
)
|
|
|
|
def _schedule_auto_bookmark(self, speed_limit_mph, confidence, published_at):
|
|
if not self.use_runtime or self.params is None:
|
|
return
|
|
if not self.params.get_bool("VisionSpeedLimitAutoBookmark"):
|
|
return
|
|
if confidence < AUTO_BOOKMARK_MIN_CONFIDENCE:
|
|
return
|
|
if published_at - self.last_auto_bookmark_at < AUTO_BOOKMARK_COOLDOWN_SECONDS and speed_limit_mph == self.last_auto_bookmark_speed_limit_mph:
|
|
return
|
|
|
|
self.pending_auto_bookmark = {
|
|
"due_at": published_at + AUTO_BOOKMARK_CONFIRM_DELAY_SECONDS,
|
|
"published_at": published_at,
|
|
"speed_limit_mph": speed_limit_mph,
|
|
"confidence": confidence,
|
|
}
|
|
|
|
def _schedule_training_capture(self, speed_limit_mph, confidence, detected_at):
|
|
if not self.use_runtime or self.params is None:
|
|
return
|
|
if not self.params.get_bool("VisionSpeedLimitTrainingCollector", default=True):
|
|
self.pending_training_capture = None
|
|
return
|
|
if confidence < TRAINING_COLLECTOR_MIN_CONFIDENCE:
|
|
return
|
|
if detected_at - self.last_training_capture_at < TRAINING_COLLECTOR_COOLDOWN_SECONDS and speed_limit_mph == self.last_training_capture_speed_limit_mph:
|
|
return
|
|
|
|
pending = self.pending_training_capture
|
|
if pending is not None:
|
|
if pending["speed_limit_mph"] == speed_limit_mph:
|
|
pending["confidence"] = max(float(pending["confidence"]), confidence)
|
|
pending["last_seen_at"] = detected_at
|
|
return
|
|
if detected_at < pending["due_at"] and confidence <= float(pending["confidence"]) + 0.08:
|
|
return
|
|
|
|
self.pending_training_capture = {
|
|
"due_at": detected_at + TRAINING_COLLECTOR_CONFIRM_DELAY_SECONDS,
|
|
"detected_at": detected_at,
|
|
"last_seen_at": detected_at,
|
|
"speed_limit_mph": speed_limit_mph,
|
|
"confidence": confidence,
|
|
}
|
|
|
|
def _emit_preserve_bookmark(self):
|
|
if not self.use_runtime or self.pm is None or self.messaging is None:
|
|
return
|
|
self.ignore_next_user_bookmark = True
|
|
msg = self.messaging.new_message("userBookmark", valid=True)
|
|
self.pm.send("userBookmark", msg)
|
|
|
|
def _maybe_commit_auto_bookmark(self, now):
|
|
pending = self.pending_auto_bookmark
|
|
if pending is None or now < pending["due_at"]:
|
|
return
|
|
if self.params is not None and not self.params.get_bool("VisionSpeedLimitAutoBookmark"):
|
|
self.pending_auto_bookmark = None
|
|
return
|
|
|
|
self.pending_auto_bookmark = None
|
|
speed_limit_mph = int(pending["speed_limit_mph"])
|
|
confidence = float(pending["confidence"])
|
|
published_at = float(pending["published_at"])
|
|
|
|
if self.published_speed_limit_mph != speed_limit_mph:
|
|
return
|
|
if self.published_confidence + 0.02 < confidence:
|
|
return
|
|
if self.current_frame_bgr is None:
|
|
return
|
|
|
|
self.last_auto_bookmark_at = now
|
|
self.last_auto_bookmark_speed_limit_mph = speed_limit_mph
|
|
self.last_auto_bookmark_publish_at = published_at
|
|
self._record_auto_bookmark(speed_limit_mph, max(confidence, self.published_confidence))
|
|
|
|
if self.params is not None and self.params.get_bool("VisionSpeedLimitAutoPreserveSegment"):
|
|
self._emit_preserve_bookmark()
|
|
|
|
def _maybe_commit_training_capture(self, now):
|
|
pending = self.pending_training_capture
|
|
if pending is None or now < pending["due_at"]:
|
|
return
|
|
if self.params is not None and not self.params.get_bool("VisionSpeedLimitTrainingCollector", default=True):
|
|
self.pending_training_capture = None
|
|
return
|
|
|
|
self.pending_training_capture = None
|
|
if self.current_frame_bgr is None:
|
|
return
|
|
|
|
speed_limit_mph = int(pending["speed_limit_mph"])
|
|
source_confidence = float(pending["confidence"])
|
|
if now - self.last_candidate_at > FOLLOWUP_WINDOW_SECONDS and self.published_speed_limit_mph != speed_limit_mph:
|
|
return
|
|
|
|
confidence = source_confidence
|
|
source_event = "candidate"
|
|
if self.last_candidate_speed_limit_mph == speed_limit_mph:
|
|
confidence = max(confidence, self.last_candidate_confidence)
|
|
if self.published_speed_limit_mph == speed_limit_mph:
|
|
confidence = max(confidence, self.published_confidence)
|
|
source_event = "publish"
|
|
if confidence < TRAINING_COLLECTOR_MIN_CONFIDENCE:
|
|
return
|
|
if now - self.last_training_capture_at < TRAINING_COLLECTOR_COOLDOWN_SECONDS and speed_limit_mph == self.last_training_capture_speed_limit_mph:
|
|
return
|
|
|
|
self.last_training_capture_at = now
|
|
self.last_training_capture_speed_limit_mph = speed_limit_mph
|
|
self._record_training_candidate(speed_limit_mph, confidence, source_confidence, source_event)
|
|
|
|
def _maybe_capture_map_transition_miss(self, now):
|
|
map_context = self._get_map_context()
|
|
current_speed_limit_mph = int(map_context["current_speed_limit_mph"])
|
|
previous_speed_limit_mph = self.last_map_speed_limit_mph
|
|
|
|
if current_speed_limit_mph != previous_speed_limit_mph:
|
|
self.last_map_speed_limit_mph = current_speed_limit_mph
|
|
|
|
if current_speed_limit_mph <= 0 or current_speed_limit_mph == previous_speed_limit_mph or previous_speed_limit_mph <= 0:
|
|
return
|
|
if self.current_frame_bgr is None:
|
|
return
|
|
capture_in_cooldown = now - self.last_map_transition_miss_at < MAP_TRANSITION_MISS_CAPTURE_COOLDOWN_SECONDS
|
|
if capture_in_cooldown and current_speed_limit_mph == self.last_map_transition_miss_speed_limit_mph:
|
|
return
|
|
if self._vision_recently_supported(current_speed_limit_mph, now):
|
|
return
|
|
|
|
self.last_map_transition_miss_at = now
|
|
self.last_map_transition_miss_speed_limit_mph = current_speed_limit_mph
|
|
self._record_map_transition_miss(current_speed_limit_mph, previous_speed_limit_mph)
|
|
|
|
def _published_detection_stale(self, now):
|
|
support_at = self.last_published_support_at or self.last_detection_at
|
|
return self.published_speed_limit_mph > 0 and now - support_at > PUBLISHED_HOLD_SECONDS
|
|
|
|
def _clear_published_detection_if_stale(self, now, reason):
|
|
if not self._published_detection_stale(now):
|
|
return False
|
|
stale_speed_limit_mph = self.published_speed_limit_mph
|
|
self._write_debug_event("stale_clear", reason=reason, speedLimitMph=stale_speed_limit_mph)
|
|
self._clear_detection()
|
|
return True
|
|
|
|
def _device_cpu_busy(self):
|
|
if self.sm is None:
|
|
return False
|
|
return device_cpu_usage_busy(self.sm["deviceState"].cpuUsagePercent)
|
|
|
|
def _inference_interval(self, now):
|
|
in_followup = now < self.followup_until
|
|
interval = FOLLOWUP_INFERENCE_INTERVAL if in_followup else INFERENCE_INTERVAL
|
|
reason = "followup" if in_followup else "steady"
|
|
self.last_cpu_busy = False
|
|
if now - self.last_live_pose_inputs_not_ok_at < LIVE_POSE_RECOVERY_THROTTLE_SECONDS:
|
|
interval = max(interval, LIVE_POSE_RECOVERY_INFERENCE_INTERVAL)
|
|
reason = "live_pose_recovery"
|
|
elif self._device_cpu_busy():
|
|
self.last_cpu_busy = True
|
|
interval = max(interval, BUSY_INFERENCE_INTERVAL)
|
|
reason = "cpu_busy"
|
|
self.last_inference_interval = interval
|
|
self.last_inference_interval_reason = reason
|
|
return interval
|
|
|
|
@staticmethod
|
|
def _read_onnx_square_input_size(model_path, default_size=DEFAULT_DETECTOR_INPUT_SIZE, candidates=DETECTOR_INPUT_SIZE_CANDIDATES):
|
|
try:
|
|
import onnx
|
|
|
|
model = onnx.load(str(model_path), load_external_data=False)
|
|
if not model.graph.input:
|
|
return default_size
|
|
|
|
shape = model.graph.input[0].type.tensor_type.shape.dim
|
|
if len(shape) < 4:
|
|
return default_size
|
|
|
|
height = int(shape[2].dim_value)
|
|
width = int(shape[3].dim_value)
|
|
if height == width and height in candidates:
|
|
return height
|
|
except Exception:
|
|
pass
|
|
return default_size
|
|
|
|
def _load_model(self):
|
|
self.net = None
|
|
self.classifier_net = None
|
|
self.reject_classifier_net = None
|
|
self.model_mode = "legacy"
|
|
self.detector_input_size = DEFAULT_DETECTOR_INPUT_SIZE
|
|
self.classifier_input_size = DEFAULT_CLASSIFIER_INPUT_SIZE
|
|
|
|
if US_DETECTOR_MODEL_PATH.is_file() and US_CLASSIFIER_MODEL_PATH.is_file():
|
|
try:
|
|
self.detector_input_size = self._read_onnx_square_input_size(US_DETECTOR_MODEL_PATH)
|
|
self.net = cv2.dnn.readNetFromONNX(str(US_DETECTOR_MODEL_PATH))
|
|
self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV)
|
|
self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)
|
|
self.classifier_input_size = self._read_onnx_square_input_size(
|
|
US_CLASSIFIER_MODEL_PATH,
|
|
DEFAULT_CLASSIFIER_INPUT_SIZE,
|
|
CLASSIFIER_INPUT_SIZE_CANDIDATES,
|
|
)
|
|
self.classifier_net = cv2.dnn.readNetFromONNX(str(US_CLASSIFIER_MODEL_PATH))
|
|
self.classifier_net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV)
|
|
self.classifier_net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)
|
|
if SEPARATE_REJECT_CLASSIFIER_ENABLED and US_REJECT_CLASSIFIER_MODEL_PATH.is_file():
|
|
try:
|
|
self.reject_classifier_net = cv2.dnn.readNetFromONNX(str(US_REJECT_CLASSIFIER_MODEL_PATH))
|
|
self.reject_classifier_net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV)
|
|
self.reject_classifier_net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)
|
|
except Exception:
|
|
self.reject_classifier_net = None
|
|
self.model_mode = "detector_classifier"
|
|
self.last_error = ""
|
|
return
|
|
except Exception as exc:
|
|
self.net = None
|
|
self.classifier_net = None
|
|
self.reject_classifier_net = None
|
|
self.model_mode = "legacy"
|
|
self.last_error = f"Failed to load U.S. vision models: {exc}"
|
|
|
|
if not LEGACY_MODEL_PATH.is_file():
|
|
if not self.last_error:
|
|
self.last_error = f"Missing vision model: {LEGACY_MODEL_PATH.name}"
|
|
self._publish_status(self.last_error, clear_speed=True)
|
|
return
|
|
|
|
try:
|
|
self.detector_input_size = self._read_onnx_square_input_size(LEGACY_MODEL_PATH)
|
|
self.net = cv2.dnn.readNetFromONNX(str(LEGACY_MODEL_PATH))
|
|
self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV)
|
|
self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)
|
|
self.last_error = ""
|
|
except Exception as exc:
|
|
self.last_error = f"Failed to load vision model: {exc}"
|
|
self._publish_status(self.last_error, clear_speed=True)
|
|
|
|
def _build_digit_templates(self):
|
|
templates = {str(digit): [] for digit in range(10)}
|
|
fonts = [
|
|
cv2.FONT_HERSHEY_SIMPLEX,
|
|
cv2.FONT_HERSHEY_DUPLEX,
|
|
cv2.FONT_HERSHEY_COMPLEX,
|
|
cv2.FONT_HERSHEY_TRIPLEX,
|
|
]
|
|
|
|
for digit in templates:
|
|
for font in fonts:
|
|
for scale, thickness in ((1.5, 2), (1.7, 3), (1.9, 3), (2.1, 4)):
|
|
canvas = np.full((120, 80), 255, dtype=np.uint8)
|
|
text_size, baseline = cv2.getTextSize(digit, font, scale, thickness)
|
|
x = max((canvas.shape[1] - text_size[0]) // 2, 0)
|
|
y = max((canvas.shape[0] + text_size[1]) // 2 - baseline, text_size[1])
|
|
cv2.putText(canvas, digit, (x, y), font, scale, 0, thickness, cv2.LINE_AA)
|
|
_, binary = cv2.threshold(canvas, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
|
|
normalized = self._normalize_binary_digit(binary)
|
|
if normalized is not None:
|
|
templates[digit].append(normalized)
|
|
|
|
return templates
|
|
|
|
def _build_speed_value_templates(self):
|
|
templates = {str(speed_limit): [] for speed_limit in VALID_SPEED_LIMITS_MPH}
|
|
fonts = [
|
|
cv2.FONT_HERSHEY_SIMPLEX,
|
|
cv2.FONT_HERSHEY_DUPLEX,
|
|
cv2.FONT_HERSHEY_COMPLEX,
|
|
cv2.FONT_HERSHEY_TRIPLEX,
|
|
]
|
|
|
|
for speed_limit in templates:
|
|
for font in fonts:
|
|
for scale, thickness in ((1.2, 2), (1.4, 2), (1.6, 3), (1.8, 3), (2.0, 4)):
|
|
canvas = np.full((120, 100), 255, dtype=np.uint8)
|
|
text_size, baseline = cv2.getTextSize(speed_limit, font, scale, thickness)
|
|
x = max((canvas.shape[1] - text_size[0]) // 2, 0)
|
|
y = max((canvas.shape[0] + text_size[1]) // 2 - baseline, text_size[1])
|
|
cv2.putText(canvas, speed_limit, (x, y), font, scale, 0, thickness, cv2.LINE_AA)
|
|
_, binary = cv2.threshold(canvas, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
|
|
normalized = self._normalize_binary_digit(binary, size=(72, 96))
|
|
if normalized is not None:
|
|
templates[speed_limit].append(normalized)
|
|
|
|
return templates
|
|
|
|
def _normalize_binary_digit(self, binary, size=(48, 72), padding=6):
|
|
points = cv2.findNonZero(binary)
|
|
if points is None:
|
|
return None
|
|
|
|
x, y, w, h = cv2.boundingRect(points)
|
|
digit = binary[y:y + h, x:x + w]
|
|
target_w, target_h = size
|
|
scale = min((target_w - padding * 2) / max(w, 1), (target_h - padding * 2) / max(h, 1))
|
|
resized_w = max(int(round(w * scale)), 1)
|
|
resized_h = max(int(round(h * scale)), 1)
|
|
resized = cv2.resize(digit, (resized_w, resized_h), interpolation=cv2.INTER_NEAREST)
|
|
|
|
canvas = np.zeros((target_h, target_w), dtype=np.uint8)
|
|
offset_x = (target_w - resized_w) // 2
|
|
offset_y = (target_h - resized_h) // 2
|
|
canvas[offset_y:offset_y + resized_h, offset_x:offset_x + resized_w] = resized
|
|
return canvas
|
|
|
|
def _connect_camera(self):
|
|
if not self.use_runtime or self.VisionIpcClient is None or self.VisionStreamType is None:
|
|
return False
|
|
|
|
try:
|
|
available_streams = self.VisionIpcClient.available_streams("camerad", block=False)
|
|
except Exception:
|
|
available_streams = []
|
|
|
|
desired_stream = None
|
|
stream_name = ""
|
|
if self.VisionStreamType.VISION_STREAM_ROAD in available_streams:
|
|
desired_stream = self.VisionStreamType.VISION_STREAM_ROAD
|
|
stream_name = "road camera"
|
|
elif self.VisionStreamType.VISION_STREAM_WIDE_ROAD in available_streams:
|
|
desired_stream = self.VisionStreamType.VISION_STREAM_WIDE_ROAD
|
|
stream_name = "wide camera"
|
|
|
|
if desired_stream is None:
|
|
self.client = None
|
|
self.stream_type = None
|
|
self.stream_name = ""
|
|
return False
|
|
|
|
if self.client is None or self.stream_type != desired_stream:
|
|
self.client = self.VisionIpcClient("camerad", desired_stream, True)
|
|
self.stream_type = desired_stream
|
|
self.stream_name = stream_name
|
|
|
|
if not self.client.is_connected():
|
|
self.client.connect(True)
|
|
|
|
return self.client.is_connected()
|
|
|
|
@staticmethod
|
|
def _letterbox(image, shape=(640, 640), color=(114, 114, 114)):
|
|
image_height, image_width = image.shape[:2]
|
|
ratio = min(shape[0] / image_height, shape[1] / image_width)
|
|
resized_width = int(round(image_width * ratio))
|
|
resized_height = int(round(image_height * ratio))
|
|
pad_width = (shape[1] - resized_width) / 2
|
|
pad_height = (shape[0] - resized_height) / 2
|
|
|
|
if (image_width, image_height) != (resized_width, resized_height):
|
|
image = cv2.resize(image, (resized_width, resized_height), interpolation=cv2.INTER_LINEAR)
|
|
|
|
top = int(round(pad_height - 0.1))
|
|
bottom = int(round(pad_height + 0.1))
|
|
left = int(round(pad_width - 0.1))
|
|
right = int(round(pad_width + 0.1))
|
|
image = cv2.copyMakeBorder(image, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)
|
|
return image, ratio, pad_width, pad_height
|
|
|
|
@staticmethod
|
|
def _clamp_track_bbox(bbox, width, height):
|
|
x1, y1, x2, y2 = bbox
|
|
result = (
|
|
max(int(round(x1)), 0),
|
|
max(int(round(y1)), 0),
|
|
min(int(round(x2)), width),
|
|
min(int(round(y2)), height),
|
|
)
|
|
return result if result[2] > result[0] and result[3] > result[1] else None
|
|
|
|
@staticmethod
|
|
def _track_feature_points(gray, bbox):
|
|
height, width = gray.shape[:2]
|
|
x1, y1, x2, y2 = bbox
|
|
box_width = x2 - x1
|
|
box_height = y2 - y1
|
|
pad_x = max(int(box_width * 0.20), 2)
|
|
pad_y = max(int(box_height * 0.20), 2)
|
|
mask = np.zeros_like(gray)
|
|
mask[max(y1 - pad_y, 0):min(y2 + pad_y, height), max(x1 - pad_x, 0):min(x2 + pad_x, width)] = 255
|
|
return cv2.goodFeaturesToTrack(gray, mask=mask, maxCorners=40, qualityLevel=0.005, minDistance=3, blockSize=5)
|
|
|
|
@classmethod
|
|
def _flow_track_bbox(cls, previous_gray, current_gray, bbox, points):
|
|
if points is None or len(points) < TRACK_MIN_FEATURE_COUNT:
|
|
points = cls._track_feature_points(previous_gray, bbox)
|
|
if points is None or len(points) < TRACK_MIN_FEATURE_COUNT:
|
|
return None, None
|
|
|
|
next_points, status, errors = cv2.calcOpticalFlowPyrLK(
|
|
previous_gray,
|
|
current_gray,
|
|
points,
|
|
None,
|
|
winSize=(25, 25),
|
|
maxLevel=3,
|
|
criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 20, 0.03),
|
|
)
|
|
if next_points is None or status is None:
|
|
return None, None
|
|
good = status.reshape(-1).astype(bool)
|
|
if errors is not None:
|
|
good &= errors.reshape(-1) < 35.0
|
|
old = points.reshape(-1, 2)[good]
|
|
new = next_points.reshape(-1, 2)[good]
|
|
if len(old) < TRACK_MIN_FEATURE_COUNT:
|
|
return None, None
|
|
|
|
transform, inliers = cv2.estimateAffinePartial2D(old, new, method=cv2.RANSAC, ransacReprojThreshold=3.0)
|
|
if transform is None or inliers is None or int(inliers.sum()) < TRACK_MIN_FEATURE_COUNT:
|
|
return None, None
|
|
scale = math.hypot(float(transform[0, 0]), float(transform[0, 1]))
|
|
if not 0.84 <= scale <= 1.24:
|
|
return None, None
|
|
|
|
x1, y1, x2, y2 = bbox
|
|
corners = np.float32(((x1, y1), (x2, y1), (x2, y2), (x1, y2))).reshape(-1, 1, 2)
|
|
moved = cv2.transform(corners, transform).reshape(-1, 2)
|
|
tracked = cls._clamp_track_bbox(
|
|
(moved[:, 0].min(), moved[:, 1].min(), moved[:, 0].max(), moved[:, 1].max()),
|
|
current_gray.shape[1],
|
|
current_gray.shape[0],
|
|
)
|
|
if tracked is None:
|
|
return None, None
|
|
inlier_points = new[inliers.reshape(-1).astype(bool)].reshape(-1, 1, 2)
|
|
return tracked, inlier_points
|
|
|
|
def _remember_detector_proposal(self, confidence, class_id, bbox, speed_limit_mph=0, preferred=False):
|
|
min_confidence = TRACK_MIN_PROPOSAL_CONFIDENCE if speed_limit_mph else TRACK_UNREADABLE_MIN_PROPOSAL_CONFIDENCE
|
|
if not TEMPORAL_TRACKING_ENABLED or class_id == 1 or confidence < min_confidence:
|
|
return
|
|
proposal = DetectorProposal(float(confidence), int(class_id), bbox, int(speed_limit_mph))
|
|
latest_proposal = getattr(self, "latest_detector_proposal", None)
|
|
if preferred or latest_proposal is None or proposal.confidence > latest_proposal.confidence:
|
|
self.latest_detector_proposal = proposal
|
|
|
|
def _start_latest_detector_track(self, frame_bgr, now):
|
|
proposal = self.latest_detector_proposal
|
|
self.latest_detector_proposal = None
|
|
if (
|
|
not TEMPORAL_TRACKING_ENABLED or
|
|
proposal is None or
|
|
(proposal.speed_limit_mph and not TRACK_CONFIRMED_PROPOSALS_ENABLED)
|
|
):
|
|
return False
|
|
gray = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2GRAY)
|
|
points = self._track_feature_points(gray, proposal.bbox)
|
|
if points is None or len(points) < TRACK_MIN_FEATURE_COUNT:
|
|
self.track_failure_count += 1
|
|
return False
|
|
self.proposal_track = ProposalTrack(
|
|
proposal=proposal,
|
|
bbox=proposal.bbox,
|
|
previous_gray=gray,
|
|
points=points,
|
|
started_at=now,
|
|
last_classified_at=now,
|
|
)
|
|
self.track_start_count += 1
|
|
self.max_track_proposal_confidence = max(self.max_track_proposal_confidence, proposal.confidence)
|
|
return True
|
|
|
|
def _clear_proposal_track(self, failed=False):
|
|
if failed and self.proposal_track is not None:
|
|
self.track_failure_count += 1
|
|
self.proposal_track = None
|
|
|
|
def _track_classification_interval(self, now):
|
|
interval = TRACK_CLASSIFICATION_INTERVAL
|
|
if now - self.last_live_pose_inputs_not_ok_at < LIVE_POSE_RECOVERY_THROTTLE_SECONDS:
|
|
return max(interval, LIVE_POSE_RECOVERY_INFERENCE_INTERVAL)
|
|
if self._device_cpu_busy():
|
|
return max(interval, TRACK_BUSY_CLASSIFICATION_INTERVAL)
|
|
return interval
|
|
|
|
def _track_classification_due(self, now):
|
|
track = self.proposal_track
|
|
if track is None:
|
|
return False
|
|
if now - track.started_at > TRACK_MAX_AGE_SECONDS:
|
|
self._clear_proposal_track()
|
|
return False
|
|
return now - track.last_classified_at >= self._track_classification_interval(now)
|
|
|
|
def _classify_proposal_track(self, frame_bgr, now):
|
|
track = self.proposal_track
|
|
if track is None:
|
|
return None
|
|
current_gray = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2GRAY)
|
|
bbox, points = self._flow_track_bbox(track.previous_gray, current_gray, track.bbox, track.points)
|
|
if bbox is None or points is None:
|
|
self._clear_proposal_track(failed=True)
|
|
return None
|
|
|
|
frame_height, frame_width = frame_bgr.shape[:2]
|
|
x1, y1, x2, y2 = bbox
|
|
box_width = x2 - x1
|
|
box_height = y2 - y1
|
|
area_ratio = box_width * box_height / max(frame_width * frame_height, 1)
|
|
if (
|
|
box_width < MODEL_PROPOSAL_MIN_WIDTH or
|
|
box_height < MODEL_PROPOSAL_MIN_HEIGHT or
|
|
area_ratio > TRACK_MAX_AREA_RATIO or
|
|
(x1 + x2) / 2 < frame_width * MODEL_PROPOSAL_MIN_X_RATIO
|
|
):
|
|
self._clear_proposal_track(failed=True)
|
|
return None
|
|
|
|
track.bbox = bbox
|
|
track.previous_gray = current_gray
|
|
track.points = points
|
|
track.last_classified_at = now
|
|
self.track_inference_count += 1
|
|
|
|
pad_x = int(box_width * TRACK_CROP_PADDING_RATIO)
|
|
pad_y = int(box_height * TRACK_CROP_PADDING_RATIO)
|
|
crop_x1 = max(x1 - pad_x, 0)
|
|
crop_y1 = max(y1 - pad_y, 0)
|
|
crop_x2 = min(x2 + pad_x, frame_width)
|
|
crop_y2 = min(y2 + pad_y, frame_height)
|
|
sign_crop = frame_bgr[crop_y1:crop_y2, crop_x1:crop_x2]
|
|
if sign_crop.size == 0:
|
|
self._clear_proposal_track(failed=True)
|
|
return None
|
|
|
|
read_result = self._classify_speed_limit_from_model(sign_crop)
|
|
if read_result is None:
|
|
track.consecutive_failed_reads += 1
|
|
track.last_speed_limit_mph = 0
|
|
track.consistent_reads = 0
|
|
if track.consecutive_failed_reads >= TRACK_MAX_CONSECUTIVE_FAILED_READS:
|
|
self._clear_proposal_track()
|
|
return None
|
|
speed_limit_mph, read_confidence = read_result
|
|
if track.proposal.speed_limit_mph and speed_limit_mph != track.proposal.speed_limit_mph:
|
|
self._clear_proposal_track()
|
|
return None
|
|
if track.proposal.class_id == 2 and speed_limit_mph not in SCHOOL_ZONE_SPEED_VALUES:
|
|
track.last_speed_limit_mph = 0
|
|
track.consistent_reads = 0
|
|
return None
|
|
|
|
track.consecutive_failed_reads = 0
|
|
|
|
if speed_limit_mph == track.last_speed_limit_mph:
|
|
track.consistent_reads += 1
|
|
else:
|
|
track.last_speed_limit_mph = speed_limit_mph
|
|
track.consistent_reads = 1
|
|
|
|
regulatory_bonus = 0.04 if self._is_regulatory_speed_sign(sign_crop) or track.proposal.class_id == 2 else 0.0
|
|
repeat_bonus = TRACK_REPEAT_CONFIDENCE_BONUS if track.consistent_reads >= 2 else 0.0
|
|
score = min(
|
|
read_confidence * 0.78 +
|
|
track.proposal.confidence * 0.12 +
|
|
regulatory_bonus +
|
|
repeat_bonus,
|
|
0.95,
|
|
)
|
|
return self._publishable_detection(Detection(speed_limit_mph, score))
|
|
|
|
def _detect_sign(self, frame_bgr):
|
|
self.latest_detector_proposal = None
|
|
if self.net is None:
|
|
if FULL_FRAME_OCR_FALLBACK_ENABLED:
|
|
return self._publishable_detection(self._detect_sign_from_ocr_candidates(frame_bgr))
|
|
return None
|
|
|
|
if self.model_mode == "detector_classifier" and self.classifier_net is not None:
|
|
detector_detection = self._detect_sign_from_detector_classifier(frame_bgr)
|
|
if detector_detection is not None:
|
|
return self._publishable_detection(detector_detection)
|
|
if FULL_FRAME_OCR_FALLBACK_ENABLED:
|
|
return self._publishable_detection(self._detect_sign_from_ocr_candidates(frame_bgr))
|
|
return None
|
|
|
|
model_detection = self._detect_sign_from_model_proposals(frame_bgr)
|
|
if model_detection is not None and model_detection.confidence >= MODEL_DETECTION_SHORT_CIRCUIT_CONFIDENCE:
|
|
return self._publishable_detection(model_detection)
|
|
|
|
if FULL_FRAME_OCR_FALLBACK_ENABLED:
|
|
ocr_detection = self._detect_sign_from_ocr_candidates(frame_bgr)
|
|
if ocr_detection is not None and (model_detection is None or ocr_detection.confidence > model_detection.confidence):
|
|
return self._publishable_detection(ocr_detection)
|
|
|
|
return self._publishable_detection(model_detection)
|
|
|
|
def _publishable_detection(self, detection):
|
|
if detection is None:
|
|
return None
|
|
if detection.speed_limit_mph < MIN_PUBLISHABLE_SPEED_LIMIT_MPH:
|
|
return None
|
|
return detection
|
|
|
|
def _is_regulatory_speed_sign(self, sign_crop):
|
|
if sign_crop.size == 0:
|
|
return False
|
|
|
|
crop_height, crop_width = sign_crop.shape[:2]
|
|
crop_area = crop_height * crop_width
|
|
if crop_area <= 0:
|
|
return False
|
|
|
|
hsv = cv2.cvtColor(sign_crop, cv2.COLOR_BGR2HSV)
|
|
hue = hsv[:, :, 0]
|
|
saturation = hsv[:, :, 1]
|
|
value = hsv[:, :, 2]
|
|
|
|
white_mask = ((value >= REGULATORY_WHITE_VALUE_MIN) & (saturation <= REGULATORY_WHITE_SAT_MAX)).astype(np.uint8)
|
|
dark_mask = ((value <= REGULATORY_DARK_VALUE_MAX) & (saturation <= REGULATORY_DARK_SAT_MAX)).astype(np.uint8)
|
|
yellow_mask = (
|
|
(hue >= REGULATORY_YELLOW_HUE_MIN) &
|
|
(hue <= REGULATORY_YELLOW_HUE_MAX) &
|
|
(saturation >= REGULATORY_YELLOW_SAT_MIN) &
|
|
(value >= REGULATORY_YELLOW_VALUE_MIN)
|
|
).astype(np.uint8)
|
|
red_mask = (
|
|
(((hue <= REGULATORY_RED_LOW_HUE_MAX) | (hue >= REGULATORY_RED_HIGH_HUE_MIN))) &
|
|
(saturation >= REGULATORY_RED_SAT_MIN) &
|
|
(value >= REGULATORY_RED_VALUE_MIN)
|
|
).astype(np.uint8)
|
|
green_mask = (
|
|
(hue >= REGULATORY_GREEN_HUE_MIN) &
|
|
(hue <= REGULATORY_GREEN_HUE_MAX) &
|
|
(saturation >= REGULATORY_COLORED_SAT_MIN) &
|
|
(value >= REGULATORY_COLORED_VALUE_MIN)
|
|
).astype(np.uint8)
|
|
blue_mask = (
|
|
(hue >= REGULATORY_BLUE_HUE_MIN) &
|
|
(hue <= REGULATORY_BLUE_HUE_MAX) &
|
|
(saturation >= REGULATORY_COLORED_SAT_MIN) &
|
|
(value >= REGULATORY_COLORED_VALUE_MIN)
|
|
).astype(np.uint8)
|
|
|
|
white_ratio = float(white_mask.mean())
|
|
dark_ratio = float(dark_mask.mean())
|
|
yellow_ratio = float(yellow_mask.mean())
|
|
red_ratio = float(red_mask.mean())
|
|
green_ratio = float(green_mask.mean())
|
|
blue_ratio = float(blue_mask.mean())
|
|
|
|
if white_ratio < REGULATORY_MIN_WHITE_RATIO or dark_ratio < REGULATORY_MIN_DARK_RATIO:
|
|
return False
|
|
if yellow_ratio > REGULATORY_MAX_YELLOW_RATIO and yellow_ratio > white_ratio * 0.45:
|
|
return False
|
|
if red_ratio > REGULATORY_MAX_RED_RATIO and red_ratio > white_ratio * 0.35:
|
|
return False
|
|
if green_ratio > REGULATORY_MAX_GREEN_RATIO and green_ratio > white_ratio * 0.60:
|
|
return False
|
|
if blue_ratio > REGULATORY_MAX_BLUE_RATIO and blue_ratio > white_ratio * 0.60:
|
|
return False
|
|
|
|
white_binary = (white_mask * 255).astype(np.uint8)
|
|
contours, _ = cv2.findContours(white_binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
|
min_component_area = crop_area * REGULATORY_MIN_WHITE_COMPONENT_RATIO
|
|
|
|
for contour in contours:
|
|
area = cv2.contourArea(contour)
|
|
if area < min_component_area:
|
|
continue
|
|
|
|
x, y, width, height = cv2.boundingRect(contour)
|
|
if height < crop_height * REGULATORY_MIN_COMPONENT_HEIGHT_RATIO:
|
|
continue
|
|
if width < crop_width * REGULATORY_MIN_COMPONENT_WIDTH_RATIO:
|
|
continue
|
|
|
|
aspect_ratio = width / max(height, 1)
|
|
if aspect_ratio < REGULATORY_MIN_ASPECT_RATIO or aspect_ratio > REGULATORY_MAX_ASPECT_RATIO:
|
|
continue
|
|
|
|
fill_ratio = area / max(width * height, 1)
|
|
if fill_ratio < REGULATORY_MIN_COMPONENT_FILL:
|
|
continue
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
@staticmethod
|
|
def _softmax(scores):
|
|
scores = scores.astype(np.float32)
|
|
scores = scores - np.max(scores)
|
|
exp_scores = np.exp(scores)
|
|
denominator = np.sum(exp_scores)
|
|
if denominator <= 0:
|
|
return exp_scores
|
|
return exp_scores / denominator
|
|
|
|
@staticmethod
|
|
def _normalize_classifier_output(scores):
|
|
scores = scores.astype(np.float32)
|
|
if scores.size == 0:
|
|
return scores
|
|
|
|
# Ultralytics classifier ONNX exports may already emit normalized probabilities.
|
|
if np.all(scores >= 0.0) and np.all(scores <= 1.0):
|
|
total = float(np.sum(scores))
|
|
if 0.99 <= total <= 1.01:
|
|
return scores
|
|
|
|
return SpeedLimitVisionDaemon._softmax(scores)
|
|
|
|
@staticmethod
|
|
def _square_resize(image, size=128, color=(114, 114, 114)):
|
|
image_height, image_width = image.shape[:2]
|
|
ratio = min(size / max(image_height, 1), size / max(image_width, 1))
|
|
resized_width = max(int(round(image_width * ratio)), 1)
|
|
resized_height = max(int(round(image_height * ratio)), 1)
|
|
image = cv2.resize(image, (resized_width, resized_height), interpolation=cv2.INTER_LINEAR)
|
|
|
|
canvas = np.full((size, size, image.shape[2]), color, dtype=image.dtype)
|
|
offset_x = (size - resized_width) // 2
|
|
offset_y = (size - resized_height) // 2
|
|
canvas[offset_y:offset_y + resized_height, offset_x:offset_x + resized_width] = image
|
|
return canvas
|
|
|
|
@staticmethod
|
|
def _crop_by_ratio(image, left_ratio, top_ratio, right_ratio, bottom_ratio):
|
|
image_height, image_width = image.shape[:2]
|
|
x1 = max(int(image_width * left_ratio), 0)
|
|
y1 = max(int(image_height * top_ratio), 0)
|
|
x2 = min(int(image_width * right_ratio), image_width)
|
|
y2 = min(int(image_height * bottom_ratio), image_height)
|
|
if x2 <= x1 or y2 <= y1:
|
|
return None
|
|
crop = image[y1:y2, x1:x2]
|
|
return crop if crop.size > 0 else None
|
|
|
|
def _iter_school_zone_read_crops(self, sign_crop):
|
|
yielded = set()
|
|
for left_ratio, top_ratio, right_ratio, bottom_ratio, weight in SCHOOL_ZONE_READ_VARIANTS:
|
|
crop = self._crop_by_ratio(sign_crop, left_ratio, top_ratio, right_ratio, bottom_ratio)
|
|
if crop is None:
|
|
continue
|
|
key = (crop.shape[1], crop.shape[0], left_ratio, top_ratio, right_ratio, bottom_ratio)
|
|
if key in yielded:
|
|
continue
|
|
yielded.add(key)
|
|
yield crop, weight
|
|
|
|
def _collect_model_proposals(self, frame_bgr):
|
|
if self.net is None:
|
|
return []
|
|
|
|
frame_height, frame_width = frame_bgr.shape[:2]
|
|
detector_shape = (self.detector_input_size, self.detector_input_size)
|
|
letterboxed, ratio, pad_width, pad_height = self._letterbox(frame_bgr, shape=detector_shape)
|
|
blob = cv2.dnn.blobFromImage(letterboxed, scalefactor=1 / 255.0, size=detector_shape, swapRB=True, crop=False)
|
|
self.net.setInput(blob)
|
|
|
|
predictions = np.squeeze(self.net.forward())
|
|
if predictions.ndim != 2:
|
|
return []
|
|
if predictions.shape[0] < predictions.shape[1]:
|
|
predictions = predictions.T
|
|
|
|
max_box_area = frame_width * frame_height * MODEL_PROPOSAL_MAX_AREA_RATIO
|
|
candidates = []
|
|
for prediction in predictions:
|
|
class_scores = prediction[4:]
|
|
class_id = int(np.argmax(class_scores))
|
|
confidence = float(class_scores[class_id])
|
|
if class_id not in SPEED_LIMIT_CLASSES:
|
|
continue
|
|
if confidence < MODEL_PROPOSAL_MIN_CONFIDENCE:
|
|
continue
|
|
|
|
center_x, center_y, width, height = prediction[:4]
|
|
x1 = max(int((center_x - width / 2 - pad_width) / ratio), 0)
|
|
y1 = max(int((center_y - height / 2 - pad_height) / ratio), 0)
|
|
x2 = min(int((center_x + width / 2 - pad_width) / ratio), frame_width)
|
|
y2 = min(int((center_y + height / 2 - pad_height) / ratio), frame_height)
|
|
if x2 <= x1 or y2 <= y1:
|
|
continue
|
|
|
|
box_width = x2 - x1
|
|
box_height = y2 - y1
|
|
if box_width < MODEL_PROPOSAL_MIN_WIDTH or box_height < MODEL_PROPOSAL_MIN_HEIGHT:
|
|
continue
|
|
if box_width * box_height > max_box_area:
|
|
continue
|
|
if (x1 + x2) / 2 < frame_width * MODEL_PROPOSAL_MIN_X_RATIO:
|
|
continue
|
|
if y1 > frame_height * MODEL_PROPOSAL_MAX_Y_RATIO:
|
|
continue
|
|
|
|
candidates.append((confidence, class_id, (x1, y1, x2, y2)))
|
|
|
|
return sorted(candidates, reverse=True)[:MODEL_PROPOSAL_MAX_COUNT]
|
|
|
|
def _collect_detector_classifier_proposals_from_region(self, frame_bgr, origin_x, origin_y, full_frame_width, full_frame_height, min_confidence):
|
|
if self.net is None:
|
|
return []
|
|
|
|
region_height, region_width = frame_bgr.shape[:2]
|
|
detector_shape = (self.detector_input_size, self.detector_input_size)
|
|
letterboxed, ratio, pad_width, pad_height = self._letterbox(frame_bgr, shape=detector_shape)
|
|
blob = cv2.dnn.blobFromImage(letterboxed, scalefactor=1 / 255.0, size=detector_shape, swapRB=True, crop=False)
|
|
self.net.setInput(blob)
|
|
|
|
forward_started_at = time.monotonic()
|
|
predictions = np.squeeze(self.net.forward())
|
|
self.last_detector_forward_count += 1
|
|
self.last_detector_forward_duration_s += time.monotonic() - forward_started_at
|
|
if predictions.ndim != 2:
|
|
return []
|
|
if predictions.shape[0] < predictions.shape[1]:
|
|
predictions = predictions.T
|
|
|
|
candidates = []
|
|
for prediction in predictions:
|
|
class_scores = prediction[4:]
|
|
class_id = int(np.argmax(class_scores))
|
|
confidence = float(class_scores[class_id])
|
|
if class_id not in US_DETECTOR_CLASSES:
|
|
continue
|
|
if confidence < min_confidence:
|
|
continue
|
|
|
|
center_x, center_y, width, height = prediction[:4]
|
|
x1 = max(int((center_x - width / 2 - pad_width) / ratio), 0)
|
|
y1 = max(int((center_y - height / 2 - pad_height) / ratio), 0)
|
|
x2 = min(int((center_x + width / 2 - pad_width) / ratio), region_width)
|
|
y2 = min(int((center_y + height / 2 - pad_height) / ratio), region_height)
|
|
if x2 <= x1 or y2 <= y1:
|
|
continue
|
|
|
|
x1 += origin_x
|
|
y1 += origin_y
|
|
x2 += origin_x
|
|
y2 += origin_y
|
|
|
|
box_width = x2 - x1
|
|
box_height = y2 - y1
|
|
if box_width < MODEL_PROPOSAL_MIN_WIDTH or box_height < MODEL_PROPOSAL_MIN_HEIGHT:
|
|
continue
|
|
if box_width * box_height > full_frame_width * full_frame_height * MODEL_PROPOSAL_MAX_AREA_RATIO:
|
|
continue
|
|
if (x1 + x2) / 2 < full_frame_width * MODEL_PROPOSAL_MIN_X_RATIO:
|
|
continue
|
|
if y1 > full_frame_height * MODEL_PROPOSAL_MAX_Y_RATIO:
|
|
continue
|
|
|
|
candidates.append((confidence, class_id, (x1, y1, x2, y2)))
|
|
|
|
return candidates
|
|
|
|
def _collect_detector_classifier_proposals(self, frame_bgr):
|
|
if self.net is None:
|
|
return []
|
|
|
|
frame_height, frame_width = frame_bgr.shape[:2]
|
|
candidates = []
|
|
if DETECTOR_CLASSIFIER_REGION_MODE in ("full", "full_and_right_roi"):
|
|
candidates.extend(self._collect_detector_classifier_proposals_from_region(
|
|
frame_bgr,
|
|
0,
|
|
0,
|
|
frame_width,
|
|
frame_height,
|
|
US_DETECTOR_MIN_CONFIDENCE,
|
|
))
|
|
|
|
# A second pass on a focused right-side ROI materially improves small U.S. sign reads.
|
|
if DETECTOR_CLASSIFIER_REGION_MODE in ("right_roi", "full_and_right_roi") and ROI_WINDOWS:
|
|
left_ratio, top_ratio, right_ratio, bottom_ratio = ROI_WINDOWS[-1]["bounds"]
|
|
left = int(frame_width * left_ratio)
|
|
top = int(frame_height * top_ratio)
|
|
right = int(frame_width * right_ratio)
|
|
bottom = int(frame_height * bottom_ratio)
|
|
roi = frame_bgr[top:bottom, left:right]
|
|
if roi.size > 0:
|
|
candidates.extend(self._collect_detector_classifier_proposals_from_region(
|
|
roi,
|
|
left,
|
|
top,
|
|
frame_width,
|
|
frame_height,
|
|
max(float(ROI_WINDOWS[-1]["min_confidence"]), US_DETECTOR_MIN_CONFIDENCE),
|
|
))
|
|
|
|
return sorted(candidates, reverse=True)[:MODEL_PROPOSAL_MAX_COUNT]
|
|
|
|
def _classify_speed_limit_from_model(self, sign_crop):
|
|
if self.classifier_net is None or sign_crop.size == 0:
|
|
return None
|
|
|
|
speed_class_count = len(US_CLASSIFIER_SPEED_VALUES)
|
|
|
|
if self.reject_classifier_net is not None:
|
|
normalized_mask = self._extract_value_template_mask(sign_crop)
|
|
if normalized_mask is not None:
|
|
reject_input = cv2.cvtColor(normalized_mask, cv2.COLOR_GRAY2BGR)
|
|
reject_crop = self._square_resize(reject_input, size=128)
|
|
reject_blob = cv2.dnn.blobFromImage(reject_crop, scalefactor=1 / 255.0, size=(128, 128), swapRB=True, crop=False)
|
|
self.reject_classifier_net.setInput(reject_blob)
|
|
reject_scores = np.array(self.reject_classifier_net.forward()).reshape(-1)
|
|
if reject_scores.size == speed_class_count + 1:
|
|
reject_probabilities = self._normalize_classifier_output(reject_scores)
|
|
if float(reject_probabilities[speed_class_count]) >= US_REJECT_CLASSIFIER_MIN_CONFIDENCE:
|
|
return None
|
|
|
|
input_size = self.classifier_input_size
|
|
padded_crop = self._square_resize(sign_crop, size=input_size)
|
|
blob = cv2.dnn.blobFromImage(padded_crop, scalefactor=1 / 255.0, size=(input_size, input_size), swapRB=True, crop=False)
|
|
self.classifier_net.setInput(blob)
|
|
|
|
forward_started_at = time.monotonic()
|
|
scores = np.array(self.classifier_net.forward()).reshape(-1)
|
|
self.last_classifier_forward_count += 1
|
|
self.last_classifier_forward_duration_s += time.monotonic() - forward_started_at
|
|
has_reject_class = scores.size == speed_class_count + 1
|
|
if scores.size != speed_class_count and not has_reject_class:
|
|
return None
|
|
|
|
probabilities = self._normalize_classifier_output(scores)
|
|
speed_probabilities = probabilities[:speed_class_count]
|
|
class_index = int(np.argmax(speed_probabilities))
|
|
confidence = float(speed_probabilities[class_index])
|
|
if has_reject_class and float(probabilities[speed_class_count]) >= max(confidence, US_CLASSIFIER_REJECT_MIN_CONFIDENCE):
|
|
return None
|
|
if confidence < US_CLASSIFIER_MIN_CONFIDENCE:
|
|
return None
|
|
|
|
return US_CLASSIFIER_SPEED_VALUES[class_index], confidence
|
|
|
|
def _detect_sign_from_detector_classifier(self, frame_bgr):
|
|
frame_height, frame_width = frame_bgr.shape[:2]
|
|
best_detection = None
|
|
best_score = 0.0
|
|
|
|
for proposal_confidence, class_id, (x1, y1, x2, y2) in self._collect_detector_classifier_proposals(frame_bgr):
|
|
if class_id == 1:
|
|
continue
|
|
|
|
box_width = x2 - x1
|
|
box_height = y2 - y1
|
|
if box_width <= 0 or box_height <= 0:
|
|
continue
|
|
is_small_box = box_width < DETECTOR_CLASSIFIER_MIN_ACCEPT_WIDTH or box_height < DETECTOR_CLASSIFIER_MIN_ACCEPT_HEIGHT
|
|
# Tiny far-away proposals are the main nighttime false-positive source.
|
|
# Keep a narrow rescue path for right-side high-confidence reads,
|
|
# then let temporal confirmation decide whether they are real.
|
|
if is_small_box and (
|
|
box_width < DETECTOR_CLASSIFIER_RESCUE_MIN_WIDTH or
|
|
box_height < DETECTOR_CLASSIFIER_RESCUE_MIN_HEIGHT or
|
|
x1 < frame_width * DETECTOR_CLASSIFIER_RESCUE_MIN_X_RATIO
|
|
):
|
|
continue
|
|
|
|
proposal_area_ratio = (box_width * box_height) / max(frame_width * frame_height, 1)
|
|
is_tiny_low_conf_box = (
|
|
class_id != 2 and
|
|
proposal_area_ratio < DETECTOR_CLASSIFIER_TINY_LOW_CONF_AREA_RATIO and
|
|
proposal_confidence < DETECTOR_CLASSIFIER_TINY_LOW_CONF_MIN_CONFIDENCE
|
|
)
|
|
self._remember_detector_proposal(proposal_confidence, class_id, (x1, y1, x2, y2))
|
|
|
|
if class_id == 2:
|
|
school_scores: dict[int, float] = {}
|
|
competing_scores: dict[int, float] = {}
|
|
school_best_confidences: dict[int, float] = {}
|
|
school_support_counts: dict[int, int] = {}
|
|
for expand_left, expand_top, expand_right, expand_bottom in SCHOOL_ZONE_DIRECT_EXPANSIONS:
|
|
expanded_x1 = max(int(x1 - box_width * expand_left), 0)
|
|
expanded_y1 = max(int(y1 - box_height * expand_top), 0)
|
|
expanded_x2 = min(int(x2 + box_width * expand_right), frame_width)
|
|
expanded_y2 = min(int(y2 + box_height * expand_bottom), frame_height)
|
|
sign_crop = frame_bgr[expanded_y1:expanded_y2, expanded_x1:expanded_x2]
|
|
if sign_crop.size == 0:
|
|
continue
|
|
|
|
for school_crop, crop_weight in self._iter_school_zone_read_crops(sign_crop):
|
|
read_result = self._classify_speed_limit_from_model(school_crop)
|
|
if read_result is None and DETECTOR_CLASSIFIER_CROP_OCR_ENABLED:
|
|
read_result = self._read_speed_limit_from_crop(school_crop)
|
|
if read_result is None:
|
|
continue
|
|
|
|
speed_limit_mph, read_confidence = read_result
|
|
if speed_limit_mph not in SCHOOL_ZONE_SPEED_VALUES:
|
|
competing_scores[speed_limit_mph] = competing_scores.get(speed_limit_mph, 0.0) + read_confidence * crop_weight
|
|
continue
|
|
|
|
school_scores[speed_limit_mph] = school_scores.get(speed_limit_mph, 0.0) + read_confidence * crop_weight
|
|
school_best_confidences[speed_limit_mph] = max(school_best_confidences.get(speed_limit_mph, 0.0), read_confidence)
|
|
school_support_counts[speed_limit_mph] = school_support_counts.get(speed_limit_mph, 0) + 1
|
|
|
|
if school_scores:
|
|
speed_limit_mph = max(
|
|
school_scores,
|
|
key=lambda speed: (
|
|
school_scores[speed] + max(school_support_counts[speed] - 1, 0) * SCHOOL_ZONE_SUPPORT_BONUS,
|
|
school_best_confidences[speed],
|
|
),
|
|
)
|
|
read_confidence = school_best_confidences[speed_limit_mph]
|
|
support_count = school_support_counts[speed_limit_mph]
|
|
if school_scores[speed_limit_mph] > max(competing_scores.values(), default=0.0):
|
|
if (
|
|
(support_count >= SCHOOL_ZONE_MIN_SUPPORT and read_confidence >= SCHOOL_ZONE_MIN_CONFIDENCE) or
|
|
read_confidence >= SCHOOL_ZONE_SINGLE_READ_CONFIDENCE
|
|
):
|
|
score = min(
|
|
read_confidence * 0.72 +
|
|
proposal_confidence * 0.22 +
|
|
max(support_count - 1, 0) * SCHOOL_ZONE_SUPPORT_BONUS +
|
|
0.04,
|
|
0.95,
|
|
)
|
|
if score >= SCHOOL_ZONE_SHORT_CIRCUIT_CONFIDENCE:
|
|
self._remember_detector_proposal(
|
|
proposal_confidence, class_id, (x1, y1, x2, y2), speed_limit_mph, preferred=True,
|
|
)
|
|
return Detection(speed_limit_mph, score)
|
|
|
|
speed_scores: dict[int, float] = {}
|
|
speed_best_confidences: dict[int, float] = {}
|
|
speed_support_counts: dict[int, int] = {}
|
|
speed_regulatory_support: dict[int, int] = {}
|
|
speed_trusted_model_support: dict[int, int] = {}
|
|
speed_model_only_rescue_support: dict[int, int] = {}
|
|
speed_direct_model_support: dict[int, int] = {}
|
|
speed_strong_model_support: dict[int, int] = {}
|
|
|
|
for expand_left, expand_top, expand_right, expand_bottom, expansion_weight in DETECTOR_CLASSIFIER_EXPANSIONS:
|
|
expanded_x1 = max(int(x1 - box_width * expand_left), 0)
|
|
expanded_y1 = max(int(y1 - box_height * expand_top), 0)
|
|
expanded_x2 = min(int(x2 + box_width * expand_right), frame_width)
|
|
expanded_y2 = min(int(y2 + box_height * expand_bottom), frame_height)
|
|
sign_crop = frame_bgr[expanded_y1:expanded_y2, expanded_x1:expanded_x2]
|
|
if sign_crop.size == 0:
|
|
continue
|
|
|
|
raw_is_regulatory = self._is_regulatory_speed_sign(sign_crop)
|
|
is_regulatory = raw_is_regulatory
|
|
if class_id == 2:
|
|
is_regulatory = True
|
|
|
|
model_read = self._classify_speed_limit_from_model(sign_crop)
|
|
ocr_read = None
|
|
trusted_model_read = (
|
|
class_id == 0 and
|
|
model_read is not None and
|
|
x1 >= frame_width * DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_X_RATIO and
|
|
box_height <= DETECTOR_CLASSIFIER_TRUSTED_MODEL_MAX_HEIGHT and
|
|
proposal_area_ratio <= DETECTOR_CLASSIFIER_TRUSTED_MODEL_MAX_AREA_RATIO and
|
|
proposal_confidence >= DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_PROPOSAL_CONFIDENCE and
|
|
model_read[1] >= DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_READ_CONFIDENCE
|
|
)
|
|
strong_model_read = (
|
|
class_id == 0 and
|
|
model_read is not None and
|
|
not is_small_box and
|
|
proposal_confidence >= DETECTOR_CLASSIFIER_STRONG_MODEL_MIN_PROPOSAL_CONFIDENCE and
|
|
model_read[1] >= DETECTOR_CLASSIFIER_STRONG_MODEL_MIN_READ_CONFIDENCE
|
|
)
|
|
strong_model_consensus_read = (
|
|
class_id == 0 and
|
|
model_read is not None and
|
|
not is_small_box and
|
|
proposal_confidence >= DETECTOR_CLASSIFIER_STRONG_MODEL_MIN_PROPOSAL_CONFIDENCE and
|
|
model_read[1] >= DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_MIN_READ_CONFIDENCE
|
|
)
|
|
model_only_consensus_read = (
|
|
not DETECTOR_CLASSIFIER_CROP_OCR_ENABLED and
|
|
class_id == 0 and
|
|
model_read is not None and
|
|
not is_small_box and
|
|
model_read[1] >= DETECTOR_CLASSIFIER_MODEL_ONLY_CONSENSUS_MIN_CONFIDENCE and
|
|
(is_regulatory or proposal_confidence >= DETECTOR_CLASSIFIER_STRONG_MODEL_MIN_PROPOSAL_CONFIDENCE)
|
|
)
|
|
needs_ocr_confirmation = (
|
|
class_id != 2 and
|
|
(not is_regulatory or is_tiny_low_conf_box) and
|
|
not trusted_model_read and
|
|
not strong_model_read
|
|
)
|
|
if DETECTOR_CLASSIFIER_CROP_OCR_ENABLED and (model_read is None or needs_ocr_confirmation):
|
|
ocr_read = self._read_speed_limit_from_crop(sign_crop)
|
|
read_result = model_read or ocr_read
|
|
if read_result is None:
|
|
continue
|
|
|
|
if needs_ocr_confirmation:
|
|
if DETECTOR_CLASSIFIER_CROP_OCR_ENABLED:
|
|
if model_read is None or ocr_read is None or model_read[0] != ocr_read[0]:
|
|
continue
|
|
read_result = (model_read[0], min(model_read[1], ocr_read[1]))
|
|
elif not trusted_model_read and not strong_model_read and not model_only_consensus_read:
|
|
continue
|
|
|
|
speed_limit_mph, read_confidence = read_result
|
|
score_is_regulatory = is_regulatory or trusted_model_read or strong_model_read
|
|
if (
|
|
class_id == 2 and
|
|
proposal_confidence < SCHOOL_ZONE_FALLBACK_MIN_CONFIDENCE and
|
|
not raw_is_regulatory
|
|
):
|
|
continue
|
|
|
|
score = read_confidence * expansion_weight
|
|
if score_is_regulatory:
|
|
score += DETECTOR_CLASSIFIER_REGULATORY_BONUS
|
|
elif proposal_area_ratio >= DETECTOR_CLASSIFIER_SMALL_BOX_AREA_RATIO:
|
|
score -= DETECTOR_CLASSIFIER_NON_REGULATORY_PENALTY
|
|
if class_id == 2:
|
|
score += SCHOOL_ZONE_SPEED_PRIOR if speed_limit_mph in SCHOOL_ZONE_SPEED_VALUES else -SCHOOL_ZONE_SPEED_PRIOR
|
|
|
|
speed_scores[speed_limit_mph] = speed_scores.get(speed_limit_mph, 0.0) + score
|
|
speed_best_confidences[speed_limit_mph] = max(speed_best_confidences.get(speed_limit_mph, 0.0), read_confidence)
|
|
speed_support_counts[speed_limit_mph] = speed_support_counts.get(speed_limit_mph, 0) + 1
|
|
if is_regulatory or class_id == 2 or strong_model_read:
|
|
speed_regulatory_support[speed_limit_mph] = speed_regulatory_support.get(speed_limit_mph, 0) + 1
|
|
if trusted_model_read:
|
|
speed_trusted_model_support[speed_limit_mph] = speed_trusted_model_support.get(speed_limit_mph, 0) + 1
|
|
if strong_model_consensus_read:
|
|
speed_strong_model_support[speed_limit_mph] = speed_strong_model_support.get(speed_limit_mph, 0) + 1
|
|
if needs_ocr_confirmation and model_only_consensus_read:
|
|
speed_model_only_rescue_support[speed_limit_mph] = speed_model_only_rescue_support.get(speed_limit_mph, 0) + 1
|
|
elif model_read is not None:
|
|
speed_direct_model_support[speed_limit_mph] = speed_direct_model_support.get(speed_limit_mph, 0) + 1
|
|
|
|
if not speed_scores:
|
|
continue
|
|
|
|
speed_limit_mph = max(
|
|
speed_scores,
|
|
key=lambda speed: (
|
|
speed_scores[speed] + max(speed_support_counts[speed] - 1, 0) * DETECTOR_CLASSIFIER_SUPPORT_BONUS,
|
|
speed_best_confidences[speed],
|
|
),
|
|
)
|
|
if class_id == 2 and speed_limit_mph not in SCHOOL_ZONE_SPEED_VALUES:
|
|
continue
|
|
model_only_rescue_support = speed_model_only_rescue_support.get(speed_limit_mph, 0)
|
|
if (
|
|
speed_direct_model_support.get(speed_limit_mph, 0) < 1 and
|
|
0 < model_only_rescue_support < DETECTOR_CLASSIFIER_MODEL_ONLY_CONSENSUS_MIN_SUPPORT
|
|
):
|
|
continue
|
|
if (
|
|
speed_regulatory_support.get(speed_limit_mph, 0) < 1 and
|
|
0 < speed_trusted_model_support.get(speed_limit_mph, 0) < DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_SUPPORT
|
|
):
|
|
continue
|
|
if class_id != 2 and speed_limit_mph in SCHOOL_ZONE_SPEED_VALUES:
|
|
competing_speed_limit_mph = max(
|
|
(speed for speed in speed_scores if speed not in SCHOOL_ZONE_SPEED_VALUES),
|
|
key=lambda speed: (speed_best_confidences[speed], speed_scores[speed]),
|
|
default=None,
|
|
)
|
|
if competing_speed_limit_mph is not None:
|
|
read_confidence = speed_best_confidences[speed_limit_mph]
|
|
competing_confidence = speed_best_confidences[competing_speed_limit_mph]
|
|
if (
|
|
competing_confidence >= NON_SCHOOL_LOW_SPEED_COMPETING_MIN_CONFIDENCE and
|
|
competing_confidence >= read_confidence
|
|
):
|
|
speed_limit_mph = competing_speed_limit_mph
|
|
read_confidence = speed_best_confidences[speed_limit_mph]
|
|
support_count = speed_support_counts[speed_limit_mph]
|
|
strong_rescue = (
|
|
DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_ENABLED and
|
|
speed_strong_model_support.get(speed_limit_mph, 0) >= DETECTOR_CLASSIFIER_STRONG_MODEL_CONSENSUS_MIN_SUPPORT
|
|
)
|
|
score = min(
|
|
read_confidence * 0.72 +
|
|
proposal_confidence * 0.24 +
|
|
max(support_count - 1, 0) * DETECTOR_CLASSIFIER_SUPPORT_BONUS,
|
|
0.95,
|
|
)
|
|
selection_score = score
|
|
published_score = score
|
|
if class_id == 2:
|
|
if speed_limit_mph in SCHOOL_ZONE_SPEED_VALUES:
|
|
selection_score = min(score + 0.06, 0.95)
|
|
published_score = selection_score
|
|
else:
|
|
selection_score = max(score - 0.06, 0.0)
|
|
published_score = selection_score
|
|
elif is_small_box:
|
|
if (
|
|
speed_regulatory_support.get(speed_limit_mph, 0) < 1 and
|
|
speed_trusted_model_support.get(speed_limit_mph, 0) < DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_SUPPORT
|
|
):
|
|
continue
|
|
if support_count < DETECTOR_CLASSIFIER_RESCUE_MIN_SUPPORT:
|
|
continue
|
|
if read_confidence < DETECTOR_CLASSIFIER_RESCUE_MIN_CONFIDENCE:
|
|
continue
|
|
strong_rescue = strong_rescue or (
|
|
speed_trusted_model_support.get(speed_limit_mph, 0) >= DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_SUPPORT and
|
|
proposal_confidence >= DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_PROPOSAL_CONFIDENCE and
|
|
read_confidence >= DETECTOR_CLASSIFIER_STRONG_RESCUE_MIN_READ_CONFIDENCE
|
|
)
|
|
rescue_max_score = (
|
|
DETECTOR_CLASSIFIER_STRONG_RESCUE_MAX_SCORE if strong_rescue else DETECTOR_CLASSIFIER_RESCUE_MAX_SCORE
|
|
)
|
|
published_score = min(score, rescue_max_score)
|
|
if speed_trusted_model_support.get(speed_limit_mph, 0) < DETECTOR_CLASSIFIER_TRUSTED_MODEL_MIN_SUPPORT:
|
|
selection_score = published_score
|
|
if selection_score > best_score:
|
|
best_score = selection_score
|
|
best_detection = Detection(speed_limit_mph, published_score, strong_rescue)
|
|
self._remember_detector_proposal(
|
|
proposal_confidence, class_id, (x1, y1, x2, y2), speed_limit_mph, preferred=True,
|
|
)
|
|
if best_detection is not None and best_detection.confidence >= MODEL_DETECTION_SHORT_CIRCUIT_CONFIDENCE:
|
|
return best_detection
|
|
|
|
return best_detection
|
|
|
|
def _detect_sign_from_model_proposals(self, frame_bgr):
|
|
frame_height, frame_width = frame_bgr.shape[:2]
|
|
best_detection = None
|
|
best_score = 0.0
|
|
|
|
for proposal_confidence, class_id, (x1, y1, x2, y2) in self._collect_model_proposals(frame_bgr):
|
|
proposal_crop = frame_bgr[y1:y2, x1:x2]
|
|
if proposal_crop.size == 0 or not self._is_regulatory_speed_sign(proposal_crop):
|
|
continue
|
|
|
|
box_width = x2 - x1
|
|
box_height = y2 - y1
|
|
|
|
for expand_left, expand_top, expand_right, expand_bottom in MODEL_PROPOSAL_EXPANSIONS:
|
|
expanded_x1 = max(int(x1 - box_width * expand_left), 0)
|
|
expanded_y1 = max(int(y1 - box_height * expand_top), 0)
|
|
expanded_x2 = min(int(x2 + box_width * expand_right), frame_width)
|
|
expanded_y2 = min(int(y2 + box_height * expand_bottom), frame_height)
|
|
if expanded_x2 <= expanded_x1 or expanded_y2 <= expanded_y1:
|
|
continue
|
|
|
|
for trim_bottom_ratio in MODEL_PROPOSAL_TRIM_BOTTOM_RATIOS:
|
|
trimmed_y2 = expanded_y1 + int((expanded_y2 - expanded_y1) * trim_bottom_ratio)
|
|
if trimmed_y2 <= expanded_y1:
|
|
continue
|
|
|
|
sign_crop = frame_bgr[expanded_y1:trimmed_y2, expanded_x1:expanded_x2]
|
|
if sign_crop.size == 0:
|
|
continue
|
|
|
|
read_result = self._read_speed_limit_from_crop(sign_crop)
|
|
if read_result is None:
|
|
continue
|
|
|
|
speed_limit_mph, read_confidence = read_result
|
|
score = read_confidence + min(proposal_confidence * 8.0, 0.08)
|
|
if class_id in SPEED_LIMIT_CLASSES and SPEED_LIMIT_CLASSES[class_id] == speed_limit_mph:
|
|
score += 0.03
|
|
|
|
if score > best_score:
|
|
best_score = score
|
|
best_detection = Detection(speed_limit_mph, min(score, 0.85))
|
|
if best_score >= MODEL_DETECTION_SHORT_CIRCUIT_CONFIDENCE:
|
|
return best_detection
|
|
|
|
return best_detection
|
|
|
|
def _detect_sign_from_ocr_candidates(self, frame_bgr):
|
|
frame_height, frame_width = frame_bgr.shape[:2]
|
|
best_detection = None
|
|
best_score = 0.0
|
|
|
|
for left_ratio, top_ratio, right_ratio, bottom_ratio in OCR_SEARCH_WINDOWS:
|
|
left = int(frame_width * left_ratio)
|
|
top = int(frame_height * top_ratio)
|
|
right = int(frame_width * right_ratio)
|
|
bottom = int(frame_height * bottom_ratio)
|
|
roi = frame_bgr[top:bottom, left:right]
|
|
if roi.size == 0:
|
|
continue
|
|
|
|
gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)
|
|
gray = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8)).apply(gray)
|
|
upscaled = cv2.resize(gray, None, fx=OCR_SEARCH_UPSCALE, fy=OCR_SEARCH_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
roi_area = upscaled.shape[0] * upscaled.shape[1]
|
|
if roi_area <= 0:
|
|
continue
|
|
|
|
for threshold in OCR_SEARCH_THRESHOLDS:
|
|
_, binary = cv2.threshold(upscaled, threshold, 255, cv2.THRESH_BINARY)
|
|
binary = cv2.morphologyEx(binary, cv2.MORPH_CLOSE, np.ones((5, 5), dtype=np.uint8))
|
|
contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
|
|
|
for contour in contours:
|
|
x, y, w, h = cv2.boundingRect(contour)
|
|
area = w * h
|
|
aspect = w / max(h, 1)
|
|
if area < roi_area * 0.002 or area > roi_area * 0.08:
|
|
continue
|
|
if aspect < 0.45 or aspect > 1.4:
|
|
continue
|
|
if y > upscaled.shape[0] * 0.82:
|
|
continue
|
|
|
|
for inset_left_ratio, inset_right_ratio, pad_ratio in OCR_SEARCH_CROP_VARIANTS:
|
|
inset_x = x + int(w * inset_left_ratio)
|
|
inset_width = max(int(w * (1 - inset_left_ratio - inset_right_ratio)), 20)
|
|
pad_x = int(inset_width * pad_ratio)
|
|
pad_y = int(h * pad_ratio)
|
|
|
|
x1 = max(inset_x - pad_x, 0)
|
|
y1 = max(y - pad_y, 0)
|
|
x2 = min(inset_x + inset_width + pad_x, upscaled.shape[1])
|
|
y2 = min(y + h + pad_y, upscaled.shape[0])
|
|
candidate_crop = upscaled[y1:y2, x1:x2]
|
|
if candidate_crop.size == 0:
|
|
continue
|
|
candidate_crop_bgr = cv2.cvtColor(candidate_crop, cv2.COLOR_GRAY2BGR)
|
|
if not self._is_regulatory_speed_sign(candidate_crop_bgr):
|
|
continue
|
|
|
|
ocr_result = self._read_speed_limit_from_crop(candidate_crop_bgr)
|
|
if ocr_result is None:
|
|
continue
|
|
|
|
speed_limit_mph, confidence = ocr_result
|
|
if confidence < OCR_FALLBACK_MIN_CONFIDENCE:
|
|
continue
|
|
|
|
score = confidence + min(area / roi_area, 0.03)
|
|
if score > best_score:
|
|
best_score = score
|
|
best_detection = Detection(speed_limit_mph, confidence)
|
|
|
|
return best_detection
|
|
|
|
def _read_speed_limit_from_crop(self, sign_crop):
|
|
gray = cv2.cvtColor(sign_crop, cv2.COLOR_BGR2GRAY)
|
|
height, width = gray.shape
|
|
roi = gray[int(height * 0.22):int(height * 0.92), int(width * 0.1):int(width * 0.9)]
|
|
if roi.size == 0:
|
|
return None
|
|
|
|
blurred = cv2.GaussianBlur(roi, (5, 5), 0)
|
|
_, binary = cv2.threshold(blurred, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
|
|
binary = cv2.morphologyEx(binary, cv2.MORPH_OPEN, np.ones((2, 2), dtype=np.uint8))
|
|
|
|
contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
|
digit_boxes = []
|
|
min_area = roi.shape[0] * roi.shape[1] * 0.01
|
|
min_height = roi.shape[0] * 0.35
|
|
min_width = roi.shape[1] * 0.05
|
|
|
|
for contour in contours:
|
|
x, y, w, h = cv2.boundingRect(contour)
|
|
area = cv2.contourArea(contour)
|
|
if area < min_area or h < min_height or w < min_width:
|
|
continue
|
|
if w / max(h, 1) > 1.1:
|
|
continue
|
|
if y + h < roi.shape[0] * 0.35:
|
|
continue
|
|
digit_boxes.append((x, y, w, h))
|
|
|
|
digit_boxes.sort(key=lambda box: box[0])
|
|
if len(digit_boxes) < 2:
|
|
return self._read_speed_limit_from_value_template(sign_crop)
|
|
if len(digit_boxes) > 3:
|
|
digit_boxes = sorted(digit_boxes, key=lambda box: box[3], reverse=True)[:3]
|
|
digit_boxes.sort(key=lambda box: box[0])
|
|
|
|
digits = []
|
|
confidences = []
|
|
for x, y, w, h in digit_boxes:
|
|
digit_mask = binary[y:y + h, x:x + w]
|
|
normalized = self._normalize_binary_digit(digit_mask)
|
|
if normalized is None:
|
|
return self._read_speed_limit_from_value_template(sign_crop)
|
|
digit, confidence = self._classify_digit(normalized)
|
|
if digit is None or confidence < OCR_MIN_CONFIDENCE:
|
|
return self._read_speed_limit_from_value_template(sign_crop)
|
|
digits.append(digit)
|
|
confidences.append(confidence)
|
|
|
|
if not 2 <= len(digits) <= 3:
|
|
return self._read_speed_limit_from_value_template(sign_crop)
|
|
|
|
speed_limit_mph = int("".join(digits))
|
|
if speed_limit_mph not in VALID_SPEED_LIMITS_MPH:
|
|
return self._read_speed_limit_from_value_template(sign_crop)
|
|
|
|
return speed_limit_mph, float(sum(confidences) / len(confidences))
|
|
|
|
def _read_speed_limit_from_value_template(self, sign_crop):
|
|
normalized = self._extract_value_template_mask(sign_crop)
|
|
if normalized is None:
|
|
return None
|
|
|
|
candidate = normalized > 0
|
|
best_match = None
|
|
for speed_limit, templates in self.speed_value_templates.items():
|
|
for template in templates:
|
|
template_mask = template > 0
|
|
union = np.logical_or(candidate, template_mask).sum()
|
|
if union == 0:
|
|
continue
|
|
intersection = np.logical_and(candidate, template_mask).sum()
|
|
iou = intersection / union
|
|
if best_match is None or iou > best_match[1]:
|
|
best_match = (int(speed_limit), iou)
|
|
|
|
if best_match is None or best_match[1] < VALUE_TEMPLATE_MIN_CONFIDENCE:
|
|
return None
|
|
return best_match
|
|
|
|
def _extract_value_template_mask(self, sign_crop):
|
|
gray = cv2.cvtColor(sign_crop, cv2.COLOR_BGR2GRAY)
|
|
height, width = gray.shape
|
|
best_mask = None
|
|
best_fill = 0.0
|
|
|
|
for top_ratio, bottom_ratio, left_ratio, right_ratio in VALUE_TEMPLATE_ROIS:
|
|
roi = gray[int(height * top_ratio):int(height * bottom_ratio), int(width * left_ratio):int(width * right_ratio)]
|
|
if roi.size == 0:
|
|
continue
|
|
|
|
clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8)).apply(roi)
|
|
_, binary = cv2.threshold(clahe, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
|
|
binary = cv2.morphologyEx(binary, cv2.MORPH_OPEN, np.ones((2, 2), dtype=np.uint8))
|
|
|
|
num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(binary, 8)
|
|
mask = np.zeros_like(binary)
|
|
for label_idx in range(1, num_labels):
|
|
x, y, w, h, area = stats[label_idx]
|
|
if area < roi.shape[0] * roi.shape[1] * 0.01:
|
|
continue
|
|
if y < binary.shape[0] * 0.08:
|
|
continue
|
|
if h < binary.shape[0] * 0.18:
|
|
continue
|
|
if w > binary.shape[1] * 0.75:
|
|
continue
|
|
mask[labels == label_idx] = 255
|
|
|
|
normalized = self._normalize_binary_digit(mask, size=(72, 96))
|
|
if normalized is None:
|
|
continue
|
|
|
|
fill_ratio = float(np.count_nonzero(normalized)) / normalized.size
|
|
if fill_ratio <= 0.02:
|
|
continue
|
|
if fill_ratio > best_fill:
|
|
best_fill = fill_ratio
|
|
best_mask = normalized
|
|
|
|
return best_mask
|
|
|
|
def _classify_digit(self, normalized_digit):
|
|
best_digit = None
|
|
best_iou = 0.0
|
|
|
|
candidate = normalized_digit > 0
|
|
for digit, templates in self.digit_templates.items():
|
|
for template in templates:
|
|
template_mask = template > 0
|
|
union = np.logical_or(candidate, template_mask).sum()
|
|
if union == 0:
|
|
continue
|
|
intersection = np.logical_and(candidate, template_mask).sum()
|
|
iou = intersection / union
|
|
if iou > best_iou:
|
|
best_iou = iou
|
|
best_digit = digit
|
|
|
|
return best_digit, best_iou
|
|
|
|
def _prune_history(self, now):
|
|
while self.history and now - self.history[0].created_at > HISTORY_SECONDS:
|
|
self.history.popleft()
|
|
|
|
def _confirm_detection(self):
|
|
if not self.history:
|
|
return None
|
|
|
|
counts = Counter(entry.speed_limit_mph for entry in self.history)
|
|
candidate_speed_limit, candidate_count = counts.most_common(1)[0]
|
|
matching_entries = [entry for entry in self.history if entry.speed_limit_mph == candidate_speed_limit]
|
|
matching_confidences = sorted((entry.confidence for entry in matching_entries), reverse=True)
|
|
best_confidence = max(entry.confidence for entry in matching_entries)
|
|
has_strong_consensus = any(entry.strong_consensus for entry in matching_entries)
|
|
current_speed_limit = self.published_speed_limit_mph
|
|
current_count = counts.get(current_speed_limit, 0) if current_speed_limit > 0 else 0
|
|
|
|
if current_speed_limit > 0 and candidate_speed_limit != current_speed_limit:
|
|
required_count = CHANGE_CONSISTENT_DETECTIONS
|
|
allow_single_frame_confirmation = (
|
|
has_strong_consensus or best_confidence >= CHANGE_SINGLE_READ_MIN_CONFIDENCE
|
|
)
|
|
if current_speed_limit >= 30 and candidate_speed_limit < 30:
|
|
required_count = LOW_SPEED_CHANGE_CONSISTENT_DETECTIONS
|
|
allow_single_frame_confirmation = (
|
|
best_confidence >= CHANGE_SINGLE_READ_MIN_CONFIDENCE or
|
|
(has_strong_consensus and LOW_SPEED_CHANGE_ALLOW_STRONG_CONSENSUS)
|
|
)
|
|
if best_confidence < LOW_SPEED_CHANGE_MIN_CONFIDENCE:
|
|
return None
|
|
if candidate_count < required_count and not allow_single_frame_confirmation:
|
|
return None
|
|
if (
|
|
not allow_single_frame_confirmation and
|
|
matching_confidences[required_count - 1] < CHANGE_REPEAT_MIN_CONFIDENCE
|
|
):
|
|
return None
|
|
if candidate_count <= current_count:
|
|
return None
|
|
return candidate_speed_limit, best_confidence
|
|
|
|
if has_strong_consensus or best_confidence >= STRONG_DETECTION_CONFIDENCE or candidate_count >= CONSISTENT_DETECTIONS:
|
|
return candidate_speed_limit, best_confidence
|
|
return None
|
|
|
|
def _clear_detection(self):
|
|
self.history.clear()
|
|
self.followup_until = 0.0
|
|
self._clear_proposal_track()
|
|
self.pending_auto_bookmark = None
|
|
self.pending_training_capture = None
|
|
self.previous_published_speed_limit_mph = self.published_speed_limit_mph
|
|
self.published_speed_limit_mph = 0
|
|
self.published_confidence = 0.0
|
|
self.last_publish_change_at = 0.0
|
|
self.last_published_support_at = 0.0
|
|
self.last_candidate_speed_limit_mph = 0
|
|
self.last_candidate_confidence = 0.0
|
|
self.last_candidate_at = 0.0
|
|
self.last_logged_candidate = None
|
|
if self.params_memory is not None:
|
|
self.params_memory.remove("VisionSpeedLimit")
|
|
self.params_memory.remove("VisionSpeedLimitConfidence")
|
|
|
|
def _publish_status(self, status, clear_speed=False):
|
|
if clear_speed:
|
|
self._clear_detection()
|
|
if self.params_memory is None:
|
|
return
|
|
self.params_memory.put("VisionSpeedLimitStatus", status)
|
|
if self.stream_name:
|
|
self.params_memory.put("VisionSpeedLimitStream", self.stream_name)
|
|
else:
|
|
self.params_memory.remove("VisionSpeedLimitStream")
|
|
if status != self.last_logged_status:
|
|
self.last_logged_status = status
|
|
self._write_debug_event("status", statusText=status)
|
|
|
|
def _publish_runtime_telemetry(self, now, phase, force=False, **fields):
|
|
if not self.use_runtime:
|
|
return
|
|
if not force and now - self.last_runtime_telemetry_at < RUNTIME_TELEMETRY_INTERVAL_SECONDS:
|
|
return
|
|
|
|
self.last_runtime_telemetry_at = now
|
|
try:
|
|
DEBUG_BASE_DIR.mkdir(parents=True, exist_ok=True)
|
|
except Exception as exc:
|
|
if self.params_memory is not None:
|
|
try:
|
|
self.params_memory.put("VisionSpeedLimitLastEvent", f"runtime telemetry mkdir error: {type(exc).__name__}"[:160])
|
|
except Exception:
|
|
pass
|
|
return
|
|
|
|
try:
|
|
live_pose_inputs_ok = True
|
|
started = False
|
|
cpu_usage = []
|
|
if self.sm is not None:
|
|
try:
|
|
started = bool(self.sm["deviceState"].started)
|
|
cpu_usage = [round(float(value), 1) for value in self.sm["deviceState"].cpuUsagePercent]
|
|
except Exception:
|
|
started = False
|
|
cpu_usage = []
|
|
try:
|
|
live_pose_inputs_ok = bool(self.sm["livePose"].inputsOK)
|
|
except Exception:
|
|
live_pose_inputs_ok = True
|
|
|
|
camera_connected = False
|
|
try:
|
|
camera_connected = bool(self.client is not None and self.client.is_connected())
|
|
except Exception:
|
|
camera_connected = False
|
|
|
|
telemetry = {
|
|
"phase": phase,
|
|
"wallTimeNs": time.time_ns(),
|
|
"monoTimeNs": time.monotonic_ns(),
|
|
"started": started,
|
|
"startedPrev": self.started_prev,
|
|
"modelMode": self.model_mode,
|
|
"detectorInputSize": self.detector_input_size,
|
|
"classifierInputSize": self.classifier_input_size,
|
|
"detectorRegionMode": DETECTOR_CLASSIFIER_REGION_MODE,
|
|
"separateRejectClassifierEnabled": SEPARATE_REJECT_CLASSIFIER_ENABLED,
|
|
"stream": self.stream_name,
|
|
"cameraConnected": camera_connected,
|
|
"debugSession": self.debug_session_id,
|
|
"loopCount": self.loop_count,
|
|
"inferenceCount": self.inference_count,
|
|
"detectorInferenceCount": self.detector_inference_count,
|
|
"intervalSkipCount": self.interval_skip_count,
|
|
"busySkipCount": self.busy_skip_count,
|
|
"cameraUnavailableCount": self.camera_unavailable_count,
|
|
"emptyFrameCount": self.empty_frame_count,
|
|
"detectionCount": self.detection_count,
|
|
"trackInferenceCount": self.track_inference_count,
|
|
"trackFailureCount": self.track_failure_count,
|
|
"trackStartCount": self.track_start_count,
|
|
"maxTrackProposalConfidence": round(self.max_track_proposal_confidence, 4),
|
|
"proposalTrackActive": self.proposal_track is not None,
|
|
"lastInferenceAgeS": round(max(now - self.last_inference_at, 0.0), 3),
|
|
"lastInferenceIntervalS": round(float(self.last_inference_interval), 3),
|
|
"lastInferenceIntervalReason": self.last_inference_interval_reason,
|
|
"cpuBusy": self.last_cpu_busy,
|
|
"lastFrameProcessDurationS": round(self.last_frame_process_duration_s, 3),
|
|
"lastDetectorForwardCount": self.last_detector_forward_count,
|
|
"lastDetectorForwardDurationS": round(self.last_detector_forward_duration_s, 3),
|
|
"lastClassifierForwardCount": self.last_classifier_forward_count,
|
|
"lastClassifierForwardDurationS": round(self.last_classifier_forward_duration_s, 3),
|
|
"cpuUsagePercent": cpu_usage,
|
|
"livePoseInputsOK": live_pose_inputs_ok,
|
|
"publishedSpeedLimitMph": self.published_speed_limit_mph,
|
|
"publishedConfidence": round(self.published_confidence, 4),
|
|
"lastCandidateSpeedLimitMph": self.last_candidate_speed_limit_mph,
|
|
"lastCandidateConfidence": round(self.last_candidate_confidence, 4),
|
|
"lastError": self.last_error,
|
|
}
|
|
telemetry.update(fields)
|
|
encoded = json.dumps(telemetry, separators=(",", ":")) + "\n"
|
|
except Exception as exc:
|
|
telemetry = {
|
|
"phase": phase,
|
|
"wallTimeNs": time.time_ns(),
|
|
"monoTimeNs": time.monotonic_ns(),
|
|
"loopCount": self.loop_count,
|
|
"telemetryError": f"{type(exc).__name__}: {exc}",
|
|
}
|
|
encoded = json.dumps(telemetry, separators=(",", ":")) + "\n"
|
|
if self.params_memory is not None:
|
|
try:
|
|
self.params_memory.put("VisionSpeedLimitLastEvent", f"runtime telemetry error: {type(exc).__name__}"[:160])
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
tmp_path = DEBUG_RUNTIME_STATUS_PATH.with_name(f"{DEBUG_RUNTIME_STATUS_PATH.name}.tmp")
|
|
tmp_path.write_text(encoded, encoding="utf-8")
|
|
tmp_path.replace(DEBUG_RUNTIME_STATUS_PATH)
|
|
except Exception as exc:
|
|
if self.params_memory is not None:
|
|
try:
|
|
self.params_memory.put("VisionSpeedLimitLastEvent", f"runtime telemetry write error: {type(exc).__name__}"[:160])
|
|
except Exception:
|
|
pass
|
|
return
|
|
|
|
if self.debug_log_path and now - self.last_debug_heartbeat_at >= DEBUG_HEARTBEAT_INTERVAL_SECONDS:
|
|
self.last_debug_heartbeat_at = now
|
|
self._write_debug_event("heartbeat", runtime=telemetry)
|
|
|
|
def _publish_detection(self, speed_limit_mph, confidence, status_prefix):
|
|
if speed_limit_mph != self.published_speed_limit_mph or abs(confidence - self.published_confidence) >= 0.05:
|
|
published_changed = speed_limit_mph != self.published_speed_limit_mph
|
|
if speed_limit_mph != self.published_speed_limit_mph:
|
|
self.previous_published_speed_limit_mph = self.published_speed_limit_mph
|
|
self.last_publish_change_at = time.monotonic()
|
|
self.last_published_support_at = self.last_publish_change_at
|
|
self.published_speed_limit_mph = speed_limit_mph
|
|
self.published_confidence = confidence
|
|
self._write_debug_event(
|
|
"publish",
|
|
frame_bgr=self.current_frame_bgr,
|
|
snapshot_prefix=f"publish_{speed_limit_mph:03d}",
|
|
speedLimitMph=speed_limit_mph,
|
|
confidence=round(confidence, 4),
|
|
)
|
|
if self.params_memory is not None:
|
|
self.params_memory.put_float("VisionSpeedLimit", speed_limit_mph * CV.MPH_TO_MS)
|
|
self.params_memory.put_float("VisionSpeedLimitConfidence", confidence)
|
|
if published_changed:
|
|
self.history.clear()
|
|
self.history.append(HistoryEntry(speed_limit_mph, confidence, time.monotonic()))
|
|
self._schedule_auto_bookmark(speed_limit_mph, confidence, self.last_publish_change_at)
|
|
|
|
status = f"{status_prefix} {speed_limit_mph} mph ({confidence * 100:.0f}%)"
|
|
self._publish_status(status, clear_speed=False)
|
|
|
|
def _should_hold_current_publish(self, speed_limit_mph, confidence, now):
|
|
if self.published_speed_limit_mph <= 0:
|
|
return False
|
|
if speed_limit_mph == self.published_speed_limit_mph:
|
|
return False
|
|
if speed_limit_mph != self.previous_published_speed_limit_mph:
|
|
return False
|
|
if now - self.last_publish_change_at >= PUBLISHED_CHANGE_COOLDOWN_SECONDS:
|
|
return False
|
|
return confidence < PUBLISHED_REVERT_CONFIDENCE
|
|
|
|
def _update_detection(self, detection):
|
|
now = time.monotonic()
|
|
self.last_detection_at = now
|
|
self.followup_until = max(self.followup_until, now + FOLLOWUP_WINDOW_SECONDS)
|
|
self.last_candidate_speed_limit_mph = detection.speed_limit_mph
|
|
self.last_candidate_confidence = detection.confidence
|
|
self.last_candidate_at = now
|
|
if detection.speed_limit_mph == self.published_speed_limit_mph:
|
|
self.last_published_support_at = now
|
|
self._schedule_training_capture(detection.speed_limit_mph, detection.confidence, now)
|
|
|
|
candidate_signature = (detection.speed_limit_mph, round(detection.confidence, 2))
|
|
if candidate_signature != self.last_logged_candidate:
|
|
self.last_logged_candidate = candidate_signature
|
|
self._write_debug_event(
|
|
"candidate",
|
|
candidateSpeedLimitMph=detection.speed_limit_mph,
|
|
candidateConfidence=round(detection.confidence, 4),
|
|
candidateStrongConsensus=detection.strong_consensus,
|
|
)
|
|
|
|
self.history.append(HistoryEntry(detection.speed_limit_mph, detection.confidence, now, detection.strong_consensus))
|
|
self._prune_history(now)
|
|
|
|
confirmed = self._confirm_detection()
|
|
if confirmed is not None:
|
|
speed_limit_mph, confidence = confirmed
|
|
if self._should_hold_current_publish(speed_limit_mph, confidence, now):
|
|
self._publish_status(
|
|
f"Candidate {speed_limit_mph} mph ({confidence * 100:.0f}%)",
|
|
clear_speed=False,
|
|
)
|
|
else:
|
|
self._publish_detection(speed_limit_mph, confidence, "Holding")
|
|
else:
|
|
self._publish_status(f"Candidate {detection.speed_limit_mph} mph ({detection.confidence * 100:.0f}%)", clear_speed=False)
|
|
|
|
def run(self):
|
|
if not self.use_runtime or self.sm is None:
|
|
raise RuntimeError("SpeedLimitVisionDaemon runtime loop requires use_runtime=True")
|
|
|
|
ratekeeper = self.Ratekeeper(RUNTIME_LOOP_HZ, None)
|
|
|
|
while True:
|
|
now = time.monotonic()
|
|
self.loop_count += 1
|
|
self.sm.update(0)
|
|
|
|
if self.sm.updated["userBookmark"]:
|
|
if self.ignore_next_user_bookmark:
|
|
self.ignore_next_user_bookmark = False
|
|
else:
|
|
self._record_bookmark()
|
|
|
|
if self.net is None:
|
|
self._publish_status(self.last_error or "Vision model unavailable", clear_speed=True)
|
|
self._publish_runtime_telemetry(now, "model_unavailable")
|
|
ratekeeper.keep_time()
|
|
continue
|
|
|
|
if not self.sm["deviceState"].started:
|
|
if self.started_prev:
|
|
self._write_debug_event("session_end", reason="offroad")
|
|
self._close_debug_session()
|
|
self.last_road_name = ""
|
|
self.started_prev = False
|
|
self.current_frame_bgr = None
|
|
self.pending_auto_bookmark = None
|
|
self._publish_status("Idle - offroad", clear_speed=True)
|
|
self._publish_runtime_telemetry(now, "offroad")
|
|
ratekeeper.keep_time()
|
|
continue
|
|
|
|
if self.sm.updated["livePose"] and not self.sm["livePose"].inputsOK:
|
|
self.last_live_pose_inputs_not_ok_at = now
|
|
|
|
if not self.started_prev:
|
|
self.started_prev = True
|
|
self._start_debug_session()
|
|
self._publish_runtime_telemetry(now, "onroad_start", force=True)
|
|
elif not self.debug_session_id:
|
|
self._start_debug_session()
|
|
self._write_debug_event("session_recovered", reason="missing_debug_session_while_onroad")
|
|
self._publish_runtime_telemetry(now, "session_recovered", force=True)
|
|
|
|
road_name = self.sm["mapdOut"].roadName
|
|
if self.last_road_name and road_name and road_name != self.last_road_name:
|
|
self._write_debug_event("road_change", previousRoadName=self.last_road_name, roadName=road_name)
|
|
self._clear_detection()
|
|
self.last_road_name = road_name or self.last_road_name
|
|
|
|
if not self._connect_camera():
|
|
self.camera_unavailable_count += 1
|
|
stale_cleared = self._clear_published_detection_if_stale(now, "camera_unavailable")
|
|
status = "Waiting for camera stream"
|
|
if self.published_speed_limit_mph > 0 and not stale_cleared:
|
|
status = f"{status}, holding {self.published_speed_limit_mph} mph"
|
|
self._publish_status(status, clear_speed=False)
|
|
self._publish_runtime_telemetry(now, "camera_unavailable")
|
|
ratekeeper.keep_time()
|
|
continue
|
|
|
|
inference_interval = self._inference_interval(now)
|
|
track_due = self._track_classification_due(now)
|
|
detector_interval = max(inference_interval, TRACK_DETECTOR_INTERVAL) if self.proposal_track is not None else inference_interval
|
|
detector_due = now - self.last_inference_at >= detector_interval
|
|
if not track_due and not detector_due:
|
|
self.interval_skip_count += 1
|
|
if self.last_inference_interval_reason == "cpu_busy":
|
|
self.busy_skip_count += 1
|
|
stale_cleared = self._clear_published_detection_if_stale(now, "inference_interval")
|
|
if self.published_speed_limit_mph > 0 and not stale_cleared:
|
|
self._publish_detection(self.published_speed_limit_mph, self.published_confidence, "Holding")
|
|
else:
|
|
self._publish_status(f"Scanning {self.stream_name}", clear_speed=False)
|
|
self._publish_runtime_telemetry(now, "interval_skip")
|
|
ratekeeper.keep_time()
|
|
continue
|
|
|
|
buffer = self.client.recv() if self.client is not None else None
|
|
self.inference_count += 1
|
|
inference_started_at = time.monotonic()
|
|
self.last_frame_process_duration_s = 0.0
|
|
self.last_detector_forward_count = 0
|
|
self.last_detector_forward_duration_s = 0.0
|
|
self.last_classifier_forward_count = 0
|
|
self.last_classifier_forward_duration_s = 0.0
|
|
if buffer is None or not buffer.data.any():
|
|
self.empty_frame_count += 1
|
|
stale_cleared = self._clear_published_detection_if_stale(now, "empty_frame")
|
|
if self.published_speed_limit_mph > 0 and not stale_cleared:
|
|
self._publish_status(f"Waiting for {self.stream_name}, holding {self.published_speed_limit_mph} mph", clear_speed=False)
|
|
else:
|
|
self._publish_status(f"Waiting for {self.stream_name}", clear_speed=False)
|
|
self._publish_runtime_telemetry(now, "empty_frame")
|
|
ratekeeper.keep_time()
|
|
continue
|
|
|
|
image = np.frombuffer(buffer.data, dtype=np.uint8).reshape((len(buffer.data) // self.client.stride, self.client.stride))
|
|
frame_bgr = cv2.cvtColor(image[:self.client.height * 3 // 2, :self.client.width], cv2.COLOR_YUV2BGR_NV12)
|
|
self.current_frame_bgr = frame_bgr
|
|
|
|
if detector_due:
|
|
self.detector_inference_count += 1
|
|
self.last_inference_at = now
|
|
detection = self._detect_sign(frame_bgr)
|
|
self._start_latest_detector_track(frame_bgr, now)
|
|
else:
|
|
detection = self._classify_proposal_track(frame_bgr, now)
|
|
self.last_frame_process_duration_s = time.monotonic() - inference_started_at
|
|
if detection is not None:
|
|
self.detection_count += 1
|
|
self._update_detection(detection)
|
|
self._publish_runtime_telemetry(now, "detection")
|
|
elif self._clear_published_detection_if_stale(now, "no_detection"):
|
|
self._publish_status(f"Scanning {self.stream_name}", clear_speed=False)
|
|
self._publish_runtime_telemetry(now, "stale_clear")
|
|
elif self.published_speed_limit_mph > 0:
|
|
self._publish_detection(self.published_speed_limit_mph, self.published_confidence, "Holding")
|
|
self._publish_runtime_telemetry(now, "holding")
|
|
else:
|
|
self._publish_status(f"Scanning {self.stream_name}", clear_speed=False)
|
|
self._publish_runtime_telemetry(now, "scanning")
|
|
|
|
self._maybe_commit_auto_bookmark(now)
|
|
self._maybe_commit_training_capture(now)
|
|
self._maybe_capture_map_transition_miss(now)
|
|
|
|
ratekeeper.keep_time()
|
|
|
|
|
|
def main():
|
|
# Keep this best-effort helper off the critical control/model/camera cores.
|
|
if not PC:
|
|
set_core_affinity(SPEED_LIMIT_VISION_AFFINITY_CORES)
|
|
|
|
# OpenCV may otherwise fan out across many worker threads and starve more
|
|
# important daemons during detection bursts.
|
|
cv2.setNumThreads(1)
|
|
|
|
daemon = SpeedLimitVisionDaemon()
|
|
daemon.run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|