mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-07-07 22:52:06 +08:00
Oh What A Night!
This commit is contained in:
+1
-1
@@ -99,7 +99,7 @@ Params::Params(const std::string &path, bool memory) {
|
||||
if (memory) {
|
||||
params_folder = Path::shm_path() + "/params";
|
||||
} else {
|
||||
cache_path = "/cache/params" + params_prefix + "/";
|
||||
cache_path = Path::params_cache() + params_prefix + "/";
|
||||
params_folder = path;
|
||||
}
|
||||
params_path = ensure_params_path(params_prefix, params_folder);
|
||||
|
||||
@@ -228,6 +228,7 @@ def supports_volt_auto_hold(CP, auto_hold_enabled: bool):
|
||||
stock_hold_safety_ready = bool(safety_param & GMSafetyFlags.FLAG_GM_PANDA_PADDLE_SCHED.value)
|
||||
return (
|
||||
auto_hold_enabled and
|
||||
getattr(CP, "openpilotLongitudinalControl", False) and
|
||||
stock_hold_safety_ready and
|
||||
CP.carFingerprint in AUTO_HOLD_VOLT_CARS
|
||||
)
|
||||
@@ -239,6 +240,7 @@ def supports_volt_one_pedal(CP, one_pedal_enabled: bool):
|
||||
stock_hold_safety_ready = bool(safety_param & GMSafetyFlags.FLAG_GM_PANDA_PADDLE_SCHED.value)
|
||||
return (
|
||||
one_pedal_enabled and
|
||||
getattr(CP, "openpilotLongitudinalControl", False) and
|
||||
stock_hold_safety_ready and
|
||||
getattr(CP, "transmissionType", None) == TransmissionType.direct and
|
||||
CP.carFingerprint in AUTO_HOLD_VOLT_CARS
|
||||
|
||||
@@ -691,6 +691,7 @@ class CarInterface(CarInterfaceBase):
|
||||
ret.safetyConfigs[0].safetyParam |= GMSafetyFlags.FLAG_GM_REMOTE_START_BOOTS_COMMA.value
|
||||
|
||||
volt_stock_friction_brake_safety = (
|
||||
ret.openpilotLongitudinalControl and
|
||||
(gm_auto_hold or volt_one_pedal_mode) and
|
||||
candidate in {
|
||||
CAR.CHEVROLET_VOLT,
|
||||
@@ -701,12 +702,14 @@ class CarInterface(CarInterfaceBase):
|
||||
)
|
||||
if volt_stock_friction_brake_safety:
|
||||
# Reuse the paddle-scheduler safety bit as a Volt stock friction-brake
|
||||
# marker on non-pedal paths. Both auto hold and one-pedal can run while
|
||||
# OP longitudinal is configured but not currently active, so the bit must
|
||||
# be present regardless of the current long-control mode.
|
||||
# marker on non-pedal paths. Auto hold and one-pedal can run while OP
|
||||
# longitudinal is configured but not currently active, so the bit must
|
||||
# be present regardless of the current long-control mode. Do not expose
|
||||
# the path at all when OP long is disabled in CarParams.
|
||||
ret.safetyConfigs[0].safetyParam |= GMSafetyFlags.FLAG_GM_PANDA_PADDLE_SCHED.value
|
||||
|
||||
volt_stock_one_pedal_safety = (
|
||||
ret.openpilotLongitudinalControl and
|
||||
volt_one_pedal_mode and
|
||||
candidate in {
|
||||
CAR.CHEVROLET_VOLT,
|
||||
@@ -719,6 +722,7 @@ class CarInterface(CarInterfaceBase):
|
||||
# Reuse the 3D1 scheduler bit as a Volt one-pedal marker on non-pedal
|
||||
# ACC paths. The bit is ignored by the actual 3D1 scheduler unless the
|
||||
# car is on a pedal-long CC-only path, so this stays isolated from Bolt.
|
||||
# Do not expose the path at all when OP long is disabled in CarParams.
|
||||
ret.safetyConfigs[0].safetyParam |= GMSafetyFlags.FLAG_GM_PANDA_3D1_SCHED.value
|
||||
|
||||
use_panda_3d1_sched = (
|
||||
|
||||
@@ -411,7 +411,7 @@ def test_volt_auto_hold_requires_toggle_supported_non_cc_only_volt_and_stock_saf
|
||||
),
|
||||
True,
|
||||
)
|
||||
assert supports_volt_auto_hold(
|
||||
assert not supports_volt_auto_hold(
|
||||
SimpleNamespace(
|
||||
carFingerprint=CAR.CHEVROLET_VOLT,
|
||||
openpilotLongitudinalControl=False,
|
||||
@@ -420,7 +420,7 @@ def test_volt_auto_hold_requires_toggle_supported_non_cc_only_volt_and_stock_saf
|
||||
),
|
||||
True,
|
||||
)
|
||||
assert supports_volt_auto_hold(
|
||||
assert not supports_volt_auto_hold(
|
||||
SimpleNamespace(
|
||||
carFingerprint=CAR.CHEVROLET_VOLT_2019,
|
||||
openpilotLongitudinalControl=False,
|
||||
@@ -464,6 +464,7 @@ def test_volt_one_pedal_requires_toggle_supported_volt_stock_safety_and_ev_trans
|
||||
assert supports_volt_one_pedal(
|
||||
SimpleNamespace(
|
||||
carFingerprint=CAR.CHEVROLET_VOLT_CAMERA,
|
||||
openpilotLongitudinalControl=True,
|
||||
safetyConfigs=stock_safety,
|
||||
transmissionType=structs.CarParams.TransmissionType.direct,
|
||||
),
|
||||
@@ -472,6 +473,7 @@ def test_volt_one_pedal_requires_toggle_supported_volt_stock_safety_and_ev_trans
|
||||
assert not supports_volt_one_pedal(
|
||||
SimpleNamespace(
|
||||
carFingerprint=CAR.CHEVROLET_VOLT_CAMERA,
|
||||
openpilotLongitudinalControl=True,
|
||||
safetyConfigs=no_safety,
|
||||
transmissionType=structs.CarParams.TransmissionType.direct,
|
||||
),
|
||||
@@ -480,6 +482,7 @@ def test_volt_one_pedal_requires_toggle_supported_volt_stock_safety_and_ev_trans
|
||||
assert not supports_volt_one_pedal(
|
||||
SimpleNamespace(
|
||||
carFingerprint=CAR.CHEVROLET_VOLT_CC,
|
||||
openpilotLongitudinalControl=True,
|
||||
safetyConfigs=stock_safety,
|
||||
transmissionType=structs.CarParams.TransmissionType.direct,
|
||||
),
|
||||
@@ -488,6 +491,7 @@ def test_volt_one_pedal_requires_toggle_supported_volt_stock_safety_and_ev_trans
|
||||
assert not supports_volt_one_pedal(
|
||||
SimpleNamespace(
|
||||
carFingerprint=CAR.CHEVROLET_VOLT_CAMERA,
|
||||
openpilotLongitudinalControl=True,
|
||||
safetyConfigs=stock_safety,
|
||||
transmissionType=structs.CarParams.TransmissionType.automatic,
|
||||
),
|
||||
@@ -496,6 +500,16 @@ def test_volt_one_pedal_requires_toggle_supported_volt_stock_safety_and_ev_trans
|
||||
assert not supports_volt_one_pedal(
|
||||
SimpleNamespace(
|
||||
carFingerprint=CAR.CHEVROLET_VOLT_CAMERA,
|
||||
openpilotLongitudinalControl=False,
|
||||
safetyConfigs=stock_safety,
|
||||
transmissionType=structs.CarParams.TransmissionType.direct,
|
||||
),
|
||||
True,
|
||||
)
|
||||
assert not supports_volt_one_pedal(
|
||||
SimpleNamespace(
|
||||
carFingerprint=CAR.CHEVROLET_VOLT_CAMERA,
|
||||
openpilotLongitudinalControl=True,
|
||||
safetyConfigs=stock_safety,
|
||||
transmissionType=structs.CarParams.TransmissionType.direct,
|
||||
),
|
||||
|
||||
@@ -168,6 +168,21 @@ class TestGMInterface:
|
||||
assert car_params.flags & GMFlags.NO_CAMERA.value
|
||||
assert car_params.safetyConfigs[0].safetyParam & GMSafetyFlags.FLAG_GM_NO_CAMERA.value
|
||||
|
||||
def test_volt_ascm_sparse_fingerprint_without_camera_does_not_set_no_camera(self):
|
||||
CarInterface = interfaces[CAR.CHEVROLET_VOLT_ASCM]
|
||||
fingerprint = {
|
||||
0: FINGERPRINTS[CAR.CHEVROLET_VOLT][0].copy(),
|
||||
1: {},
|
||||
}
|
||||
fingerprint[0][0x2FF] = 8 # SASCM detected
|
||||
|
||||
car_params = CarInterface.get_params(CAR.CHEVROLET_VOLT_ASCM, fingerprint, [], alpha_long=False, is_release=False,
|
||||
docs=False, starpilot_toggles=_test_starpilot_toggles())
|
||||
|
||||
assert not (car_params.flags & GMFlags.NO_CAMERA.value)
|
||||
assert not (car_params.safetyConfigs[0].safetyParam & GMSafetyFlags.FLAG_GM_NO_CAMERA.value)
|
||||
assert car_params.safetyConfigs[0].safetyParam & GMSafetyFlags.HW_ASCM_INT.value
|
||||
|
||||
def test_silverado_alpha_long_uses_trimmed_longitudinal_tune(self):
|
||||
CarInterface = interfaces[CAR.CHEVROLET_SILVERADO]
|
||||
fingerprint = _empty_fingerprint()
|
||||
@@ -235,6 +250,22 @@ class TestGMInterface:
|
||||
assert car_params.openpilotLongitudinalControl
|
||||
assert car_params.safetyConfigs[0].safetyParam & GMSafetyFlags.FLAG_GM_PANDA_PADDLE_SCHED.value
|
||||
|
||||
def test_volt_auto_hold_does_not_set_stock_hold_safety_bit_with_op_long_disabled(self):
|
||||
CarInterface = interfaces[CAR.CHEVROLET_VOLT_ASCM]
|
||||
fingerprint = _empty_fingerprint()
|
||||
fingerprint[0][0x2FF] = 8
|
||||
|
||||
params = Params()
|
||||
try:
|
||||
params.put_bool("GMAutoHold", True)
|
||||
car_params = CarInterface.get_params(CAR.CHEVROLET_VOLT_ASCM, fingerprint, [], alpha_long=False, is_release=False,
|
||||
docs=False, starpilot_toggles=_test_starpilot_toggles())
|
||||
finally:
|
||||
params.remove("GMAutoHold")
|
||||
|
||||
assert not car_params.openpilotLongitudinalControl
|
||||
assert not (car_params.safetyConfigs[0].safetyParam & GMSafetyFlags.FLAG_GM_PANDA_PADDLE_SCHED.value)
|
||||
|
||||
def test_volt_one_pedal_sets_stock_hold_safety_bit_without_auto_hold(self):
|
||||
CarInterface = interfaces[CAR.CHEVROLET_VOLT_ASCM]
|
||||
fingerprint = _empty_fingerprint()
|
||||
@@ -254,6 +285,25 @@ class TestGMInterface:
|
||||
assert car_params.safetyConfigs[0].safetyParam & GMSafetyFlags.FLAG_GM_PANDA_PADDLE_SCHED.value
|
||||
assert car_params.safetyConfigs[0].safetyParam & GMSafetyFlags.FLAG_GM_PANDA_3D1_SCHED.value
|
||||
|
||||
def test_volt_one_pedal_does_not_set_stock_hold_safety_bits_with_op_long_disabled(self):
|
||||
CarInterface = interfaces[CAR.CHEVROLET_VOLT_ASCM]
|
||||
fingerprint = _empty_fingerprint()
|
||||
fingerprint[0][0x2FF] = 8
|
||||
|
||||
params = Params()
|
||||
try:
|
||||
params.put_bool("GMAutoHold", False)
|
||||
params.put_bool("VoltOnePedalMode", True)
|
||||
car_params = CarInterface.get_params(CAR.CHEVROLET_VOLT_ASCM, fingerprint, [], alpha_long=False, is_release=False,
|
||||
docs=False, starpilot_toggles=_test_starpilot_toggles())
|
||||
finally:
|
||||
params.remove("GMAutoHold")
|
||||
params.remove("VoltOnePedalMode")
|
||||
|
||||
assert not car_params.openpilotLongitudinalControl
|
||||
assert not (car_params.safetyConfigs[0].safetyParam & GMSafetyFlags.FLAG_GM_PANDA_PADDLE_SCHED.value)
|
||||
assert not (car_params.safetyConfigs[0].safetyParam & GMSafetyFlags.FLAG_GM_PANDA_3D1_SCHED.value)
|
||||
|
||||
@parameterized.expand(VOLT_CARS)
|
||||
def test_volt_bsm_is_enabled_without_fingerprint_match(self, car_model):
|
||||
CarInterface = interfaces[car_model]
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import random
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
|
||||
import starpilot.system.speed_limit_vision as slv
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Evaluate runtime speed-limit ONNX models on a mined frame manifest.")
|
||||
parser.add_argument(
|
||||
"--models-dir",
|
||||
type=Path,
|
||||
default=Path("starpilot/assets/vision_models"),
|
||||
help="Directory containing speed_limit_us_detector.onnx and speed_limit_us_value_classifier.onnx.",
|
||||
)
|
||||
parser.add_argument("--manifest", type=Path, required=True, help="CSV manifest with dataset_image/frame_path and labels.")
|
||||
parser.add_argument("--split", action="append", help="Optional split filter. Repeat for multiple splits.")
|
||||
parser.add_argument("--max-rows", type=int, default=0, help="Optional cap after filtering.")
|
||||
parser.add_argument("--seed", type=int, default=0, help="Sampling seed used with --max-rows.")
|
||||
parser.add_argument("--output-csv", type=Path, help="Optional per-row prediction output.")
|
||||
parser.add_argument("--strict-positive-recall", type=float, help="Exit non-zero if positive exact recall is below this value.")
|
||||
parser.add_argument("--strict-negative-fpr", type=float, help="Exit non-zero if negative false-positive rate is above this value.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def first_present(row: dict[str, str], keys: tuple[str, ...]) -> str:
|
||||
for key in keys:
|
||||
value = row.get(key, "").strip()
|
||||
if value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def expected_value(row: dict[str, str]) -> int | None:
|
||||
value_text = first_present(row, ("speed_limit_mph", "dominant_value"))
|
||||
if value_text:
|
||||
try:
|
||||
return int(float(value_text))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
for key in ("full_detection", "model_read", "ocr_read"):
|
||||
read_text = row.get(key, "").strip()
|
||||
if "@" in read_text:
|
||||
try:
|
||||
return int(float(read_text.split("@", 1)[0]))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def is_negative(row: dict[str, str]) -> bool:
|
||||
sample_type = row.get("sample_type", "").lower()
|
||||
if "negative" in sample_type:
|
||||
return True
|
||||
return expected_value(row) is None
|
||||
|
||||
|
||||
def load_rows(manifest_path: Path, splits: set[str] | None) -> list[dict[str, str]]:
|
||||
with manifest_path.open("r", encoding="utf-8", newline="") as manifest_file:
|
||||
reader = csv.DictReader(manifest_file)
|
||||
rows = []
|
||||
for row in reader:
|
||||
if splits is not None and row.get("split", "") not in splits:
|
||||
continue
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
models_dir = args.models_dir.expanduser().resolve()
|
||||
detector_path = models_dir / "speed_limit_us_detector.onnx"
|
||||
classifier_path = models_dir / "speed_limit_us_value_classifier.onnx"
|
||||
if not detector_path.is_file():
|
||||
raise FileNotFoundError(detector_path)
|
||||
if not classifier_path.is_file():
|
||||
raise FileNotFoundError(classifier_path)
|
||||
|
||||
rows = load_rows(args.manifest.expanduser().resolve(), set(args.split) if args.split else None)
|
||||
if args.max_rows > 0 and len(rows) > args.max_rows:
|
||||
rng = random.Random(args.seed)
|
||||
rows = rng.sample(rows, args.max_rows)
|
||||
|
||||
slv.US_DETECTOR_MODEL_PATH = detector_path
|
||||
slv.US_CLASSIFIER_MODEL_PATH = classifier_path
|
||||
daemon = slv.SpeedLimitVisionDaemon(use_runtime=False)
|
||||
|
||||
output_rows: list[dict[str, str]] = []
|
||||
positive_count = 0
|
||||
positive_exact = 0
|
||||
positive_detected = 0
|
||||
negative_count = 0
|
||||
negative_false_positive = 0
|
||||
unreadable_count = 0
|
||||
|
||||
for row in rows:
|
||||
image_text = first_present(row, ("dataset_image", "frame_path", "source_frame"))
|
||||
if not image_text:
|
||||
unreadable_count += 1
|
||||
continue
|
||||
|
||||
image_path = Path(image_text).expanduser().resolve()
|
||||
frame_bgr = cv2.imread(str(image_path))
|
||||
if frame_bgr is None:
|
||||
unreadable_count += 1
|
||||
continue
|
||||
|
||||
detection = daemon._detect_sign(frame_bgr)
|
||||
predicted_value = detection.speed_limit_mph if detection is not None else None
|
||||
confidence = detection.confidence if detection is not None else None
|
||||
expected = expected_value(row)
|
||||
negative = is_negative(row)
|
||||
|
||||
if negative:
|
||||
negative_count += 1
|
||||
if predicted_value is not None:
|
||||
negative_false_positive += 1
|
||||
else:
|
||||
positive_count += 1
|
||||
if predicted_value is not None:
|
||||
positive_detected += 1
|
||||
if predicted_value == expected:
|
||||
positive_exact += 1
|
||||
|
||||
if args.output_csv:
|
||||
output_rows.append({
|
||||
"record_key": row.get("record_key", ""),
|
||||
"split": row.get("split", ""),
|
||||
"sample_type": row.get("sample_type", ""),
|
||||
"image_path": str(image_path),
|
||||
"expected_speed_limit_mph": "" if expected is None else str(expected),
|
||||
"predicted_speed_limit_mph": "" if predicted_value is None else str(predicted_value),
|
||||
"confidence": "" if confidence is None else f"{confidence:.6f}",
|
||||
"negative": str(negative),
|
||||
})
|
||||
|
||||
positive_exact_recall = positive_exact / positive_count if positive_count else 0.0
|
||||
positive_any_recall = positive_detected / positive_count if positive_count else 0.0
|
||||
negative_fpr = negative_false_positive / negative_count if negative_count else 0.0
|
||||
|
||||
print(f"Rows evaluated: {positive_count + negative_count}")
|
||||
print(f"Unreadable rows: {unreadable_count}")
|
||||
print(
|
||||
f"Positive exact: {positive_exact}/{positive_count} "
|
||||
f"({positive_exact_recall:.3f}); any detection: {positive_detected}/{positive_count} ({positive_any_recall:.3f})"
|
||||
)
|
||||
print(f"Negative false positives: {negative_false_positive}/{negative_count} ({negative_fpr:.3f})")
|
||||
|
||||
if args.output_csv:
|
||||
args.output_csv.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output_csv.open("w", encoding="utf-8", newline="") as output_file:
|
||||
fieldnames = (
|
||||
"record_key",
|
||||
"split",
|
||||
"sample_type",
|
||||
"image_path",
|
||||
"expected_speed_limit_mph",
|
||||
"predicted_speed_limit_mph",
|
||||
"confidence",
|
||||
"negative",
|
||||
)
|
||||
writer = csv.DictWriter(output_file, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(output_rows)
|
||||
print(f"Wrote {args.output_csv}")
|
||||
|
||||
failed = False
|
||||
if args.strict_positive_recall is not None and positive_exact_recall < args.strict_positive_recall:
|
||||
failed = True
|
||||
if args.strict_negative_fpr is not None and negative_fpr > args.strict_negative_fpr:
|
||||
failed = True
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import random
|
||||
import re
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
|
||||
if __package__ in (None, ""):
|
||||
import sys
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from build_value_dataset import crop_box, parse_yolo_labels # type: ignore
|
||||
from common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore
|
||||
from generate_value_roi_classifier_dataset import augment_mask, extract_value_mask # type: ignore
|
||||
else:
|
||||
from .build_value_dataset import crop_box, parse_yolo_labels
|
||||
from .common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace
|
||||
from .generate_value_roi_classifier_dataset import augment_mask, extract_value_mask
|
||||
|
||||
|
||||
READ_RE = re.compile(r"^\s*(\d+)(?:@|$)")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Import mined comma speed-limit crops as ROI classifier digit masks.")
|
||||
parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Target training workspace root.")
|
||||
parser.add_argument("--manifest", type=Path, action="append", required=True, help="CSV manifest to import. May be passed more than once.")
|
||||
parser.add_argument("--variants-per-example", type=int, default=3, help="Augmented mask variants to generate per imported crop.")
|
||||
parser.add_argument("--default-padding", type=float, default=0.12, help="Crop padding when a manifest row does not specify one.")
|
||||
parser.add_argument("--val-modulo", type=int, default=5, help="Hash modulo for rows that do not specify train/val. 0 sends them to train.")
|
||||
parser.add_argument("--val-remainder", type=int, default=0, help="Hash remainder for rows that do not specify train/val.")
|
||||
parser.add_argument("--max-rows", type=int, default=0, help="Optional maximum rows to attempt across all manifests.")
|
||||
parser.add_argument("--seed", type=int, default=20260630, help="Random seed.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def safe_stem(text: str) -> str:
|
||||
cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "_", text.strip())
|
||||
return cleaned.strip("._")[:180] or "sample"
|
||||
|
||||
|
||||
def read_rows(path: Path) -> list[dict[str, str]]:
|
||||
with path.open("r", encoding="utf-8", newline="") as csv_file:
|
||||
return list(csv.DictReader(csv_file))
|
||||
|
||||
|
||||
def parse_speed_from_read(text: str) -> int:
|
||||
match = READ_RE.match(text or "")
|
||||
if not match:
|
||||
return 0
|
||||
value = int(match.group(1))
|
||||
return value if value in DEFAULT_SPEED_VALUES else 0
|
||||
|
||||
|
||||
def row_speed(row: dict[str, str]) -> int:
|
||||
for field in ("speed_limit_mph", "speed_limit", "posted_speed"):
|
||||
text = (row.get(field) or "").strip()
|
||||
if text.isdigit():
|
||||
value = int(text)
|
||||
if value in DEFAULT_SPEED_VALUES:
|
||||
return value
|
||||
for field in ("full_detection", "model_read", "ocr_read"):
|
||||
value = parse_speed_from_read(row.get(field, ""))
|
||||
if value:
|
||||
return value
|
||||
return 0
|
||||
|
||||
|
||||
def row_split(row: dict[str, str], key_text: str, val_modulo: int, val_remainder: int) -> str:
|
||||
split = (row.get("split") or "").strip().lower()
|
||||
if split in ("train", "val"):
|
||||
return split
|
||||
if val_modulo <= 0:
|
||||
return "train"
|
||||
digest = hashlib.sha1(key_text.encode("utf-8")).hexdigest()
|
||||
return "val" if int(digest[:8], 16) % val_modulo == val_remainder else "train"
|
||||
|
||||
|
||||
def resolve_existing_path(path_text: str, manifest_path: Path) -> Path | None:
|
||||
text = (path_text or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
path = Path(text).expanduser()
|
||||
candidates = [path]
|
||||
if not path.is_absolute():
|
||||
candidates.append((manifest_path.parent / path).resolve())
|
||||
for candidate in candidates:
|
||||
if candidate.is_file():
|
||||
return candidate.resolve()
|
||||
return None
|
||||
|
||||
|
||||
def parse_xyxy(text: str) -> tuple[int, int, int, int] | None:
|
||||
if not text:
|
||||
return None
|
||||
parts = [part.strip() for part in text.replace(";", ",").split(",") if part.strip()]
|
||||
if len(parts) != 4:
|
||||
return None
|
||||
try:
|
||||
x1, y1, x2, y2 = (int(round(float(part))) for part in parts)
|
||||
except ValueError:
|
||||
return None
|
||||
if x2 <= x1 or y2 <= y1:
|
||||
return None
|
||||
return x1, y1, x2, y2
|
||||
|
||||
|
||||
def crop_from_xyxy(image, box: tuple[int, int, int, int], padding: float):
|
||||
height, width = image.shape[:2]
|
||||
x1, y1, x2, y2 = box
|
||||
pad_x = int(round((x2 - x1) * padding))
|
||||
pad_y = int(round((y2 - y1) * padding))
|
||||
x1 = max(x1 - pad_x, 0)
|
||||
y1 = max(y1 - pad_y, 0)
|
||||
x2 = min(x2 + pad_x, width)
|
||||
y2 = min(y2 + pad_y, height)
|
||||
if x2 <= x1 or y2 <= y1:
|
||||
return None
|
||||
return image[y1:y2, x1:x2]
|
||||
|
||||
|
||||
def load_crop(row: dict[str, str], manifest_path: Path, default_padding: float):
|
||||
crop_path = resolve_existing_path(row.get("crop_path", ""), manifest_path)
|
||||
if crop_path is not None:
|
||||
crop = cv2.imread(str(crop_path))
|
||||
if crop is not None and crop.size:
|
||||
return crop
|
||||
|
||||
image_path = (
|
||||
resolve_existing_path(row.get("image_path", ""), manifest_path) or
|
||||
resolve_existing_path(row.get("dataset_image", ""), manifest_path) or
|
||||
resolve_existing_path(row.get("frame_path", ""), manifest_path)
|
||||
)
|
||||
if image_path is None:
|
||||
return None
|
||||
|
||||
image = cv2.imread(str(image_path))
|
||||
if image is None or not image.size:
|
||||
return None
|
||||
|
||||
padding_text = (row.get("padding") or "").strip()
|
||||
padding = float(padding_text) if padding_text else default_padding
|
||||
box = parse_xyxy(row.get("bbox", "") or row.get("box", ""))
|
||||
if box is not None:
|
||||
return crop_from_xyxy(image, box, padding)
|
||||
|
||||
label_path = (
|
||||
resolve_existing_path(row.get("label_path", ""), manifest_path) or
|
||||
resolve_existing_path(row.get("dataset_label", ""), manifest_path)
|
||||
)
|
||||
if label_path is None:
|
||||
return image
|
||||
|
||||
bbox_index = int((row.get("bbox_index") or "0").strip() or "0")
|
||||
boxes = parse_yolo_labels(label_path)
|
||||
if bbox_index >= len(boxes):
|
||||
return None
|
||||
return crop_box(image, boxes[bbox_index], padding)
|
||||
|
||||
|
||||
def write_mask(workspace: Path, split: str, speed_value: int, stem: str, image_bgr) -> None:
|
||||
output_dir = ensure_dir(workspace / "classifier" / split / str(speed_value))
|
||||
cv2.imwrite(str(output_dir / f"{stem}.png"), image_bgr)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
workspace = resolve_workspace(args.workspace)
|
||||
rng = random.Random(args.seed)
|
||||
|
||||
attempted = 0
|
||||
imported = 0
|
||||
skipped_no_speed = 0
|
||||
skipped_no_crop = 0
|
||||
skipped_no_mask = 0
|
||||
written = 0
|
||||
|
||||
for manifest_path in [path.expanduser().resolve() for path in args.manifest]:
|
||||
rows = read_rows(manifest_path)
|
||||
for row_index, row in enumerate(rows):
|
||||
if args.max_rows > 0 and attempted >= args.max_rows:
|
||||
break
|
||||
attempted += 1
|
||||
|
||||
speed_value = row_speed(row)
|
||||
if not speed_value:
|
||||
skipped_no_speed += 1
|
||||
continue
|
||||
|
||||
key_text = "|".join(
|
||||
row.get(field, "")
|
||||
for field in ("record_key", "image_path", "dataset_image", "crop_path", "frame_path", "session_id", "bookmark_number")
|
||||
) or f"{manifest_path}:{row_index}"
|
||||
split = row_split(row, key_text, args.val_modulo, args.val_remainder)
|
||||
crop = load_crop(row, manifest_path, args.default_padding)
|
||||
if crop is None:
|
||||
skipped_no_crop += 1
|
||||
continue
|
||||
|
||||
mask = extract_value_mask(crop)
|
||||
if mask is None:
|
||||
skipped_no_mask += 1
|
||||
continue
|
||||
|
||||
manifest_stem = safe_stem(manifest_path.stem)
|
||||
source_stem = safe_stem(key_text)
|
||||
base_stem = f"manifest_{manifest_stem}_{row_index:06d}_{source_stem}"
|
||||
base_mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)
|
||||
write_mask(workspace, split, speed_value, f"{base_stem}_base", base_mask)
|
||||
written += 1
|
||||
|
||||
for variant_index in range(max(args.variants_per_example, 0)):
|
||||
augmented = augment_mask(mask, rng)
|
||||
write_mask(workspace, split, speed_value, f"{base_stem}_var{variant_index:02d}", augmented)
|
||||
written += 1
|
||||
imported += 1
|
||||
if args.max_rows > 0 and attempted >= args.max_rows:
|
||||
break
|
||||
|
||||
print(
|
||||
"Imported manifest classifier masks: "
|
||||
f"attempted={attempted} imported={imported} written={written} "
|
||||
f"skipped_no_speed={skipped_no_speed} skipped_no_crop={skipped_no_crop} skipped_no_mask={skipped_no_mask}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -28,6 +28,23 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--patience", type=int, default=20, help="Early stopping patience.")
|
||||
parser.add_argument("--cache", action="store_true", help="Cache images in RAM if supported.")
|
||||
parser.add_argument("--exist-ok", action="store_true", help="Allow overwriting an existing run directory.")
|
||||
parser.add_argument("--optimizer", help="Ultralytics optimizer name, for example SGD, Adam, or AdamW.")
|
||||
parser.add_argument("--lr0", type=float, help="Initial learning rate passed to Ultralytics.")
|
||||
parser.add_argument("--lrf", type=float, help="Final LR fraction passed to Ultralytics.")
|
||||
parser.add_argument("--warmup-epochs", type=float, help="Warmup epochs passed to Ultralytics.")
|
||||
parser.add_argument("--weight-decay", type=float, help="Weight decay passed to Ultralytics.")
|
||||
parser.add_argument("--cos-lr", action="store_true", help="Use cosine LR scheduling.")
|
||||
parser.add_argument("--close-mosaic", type=int, help="Disable mosaic augmentation for the final N epochs.")
|
||||
parser.add_argument("--mosaic", type=float, help="Mosaic augmentation probability.")
|
||||
parser.add_argument("--mixup", type=float, help="MixUp augmentation probability.")
|
||||
parser.add_argument("--copy-paste", type=float, help="Copy-paste augmentation probability.")
|
||||
parser.add_argument("--degrees", type=float, help="Rotation augmentation degrees.")
|
||||
parser.add_argument("--translate", type=float, help="Translation augmentation fraction.")
|
||||
parser.add_argument("--scale", type=float, help="Scale augmentation gain.")
|
||||
parser.add_argument("--shear", type=float, help="Shear augmentation degrees.")
|
||||
parser.add_argument("--perspective", type=float, help="Perspective augmentation fraction.")
|
||||
parser.add_argument("--fliplr", type=float, help="Horizontal flip augmentation probability.")
|
||||
parser.add_argument("--freeze", type=int, help="Freeze the first N model layers.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -44,19 +61,44 @@ def main() -> int:
|
||||
"Ultralytics is not installed. Run `uv sync --extra speedvision` in the repo root before training."
|
||||
) from exc
|
||||
|
||||
train_kwargs = {
|
||||
"data": str(data_path),
|
||||
"epochs": args.epochs,
|
||||
"imgsz": args.imgsz,
|
||||
"batch": args.batch,
|
||||
"workers": args.workers,
|
||||
"device": args.device,
|
||||
"project": str(project_path),
|
||||
"name": args.name,
|
||||
"patience": args.patience,
|
||||
"cache": args.cache,
|
||||
"exist_ok": args.exist_ok,
|
||||
}
|
||||
optional_kwargs = {
|
||||
"optimizer": args.optimizer,
|
||||
"lr0": args.lr0,
|
||||
"lrf": args.lrf,
|
||||
"warmup_epochs": args.warmup_epochs,
|
||||
"weight_decay": args.weight_decay,
|
||||
"close_mosaic": args.close_mosaic,
|
||||
"mosaic": args.mosaic,
|
||||
"mixup": args.mixup,
|
||||
"copy_paste": args.copy_paste,
|
||||
"degrees": args.degrees,
|
||||
"translate": args.translate,
|
||||
"scale": args.scale,
|
||||
"shear": args.shear,
|
||||
"perspective": args.perspective,
|
||||
"fliplr": args.fliplr,
|
||||
"freeze": args.freeze,
|
||||
}
|
||||
train_kwargs.update({key: value for key, value in optional_kwargs.items() if value is not None})
|
||||
if args.cos_lr:
|
||||
train_kwargs["cos_lr"] = True
|
||||
|
||||
model = YOLO(args.model)
|
||||
model.train(
|
||||
data=str(data_path),
|
||||
epochs=args.epochs,
|
||||
imgsz=args.imgsz,
|
||||
batch=args.batch,
|
||||
workers=args.workers,
|
||||
device=args.device,
|
||||
project=str(project_path),
|
||||
name=args.name,
|
||||
patience=args.patience,
|
||||
cache=args.cache,
|
||||
exist_ok=args.exist_ok,
|
||||
**train_kwargs,
|
||||
)
|
||||
print(f"Detector training complete under {project_path / args.name}")
|
||||
return 0
|
||||
|
||||
@@ -28,6 +28,22 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--patience", type=int, default=15, help="Early stopping patience.")
|
||||
parser.add_argument("--cache", action="store_true", help="Cache images in RAM if supported.")
|
||||
parser.add_argument("--exist-ok", action="store_true", help="Allow overwriting an existing run directory.")
|
||||
parser.add_argument("--optimizer", help="Ultralytics optimizer name, for example SGD, Adam, or AdamW.")
|
||||
parser.add_argument("--lr0", type=float, help="Initial learning rate passed to Ultralytics.")
|
||||
parser.add_argument("--lrf", type=float, help="Final LR fraction passed to Ultralytics.")
|
||||
parser.add_argument("--warmup-epochs", type=float, help="Warmup epochs passed to Ultralytics.")
|
||||
parser.add_argument("--weight-decay", type=float, help="Weight decay passed to Ultralytics.")
|
||||
parser.add_argument("--cos-lr", action="store_true", help="Use cosine LR scheduling.")
|
||||
parser.add_argument("--degrees", type=float, help="Rotation augmentation degrees.")
|
||||
parser.add_argument("--translate", type=float, help="Translation augmentation fraction.")
|
||||
parser.add_argument("--scale", type=float, help="Scale augmentation gain.")
|
||||
parser.add_argument("--shear", type=float, help="Shear augmentation degrees.")
|
||||
parser.add_argument("--perspective", type=float, help="Perspective augmentation fraction.")
|
||||
parser.add_argument("--fliplr", type=float, help="Horizontal flip augmentation probability.")
|
||||
parser.add_argument("--erasing", type=float, help="Random erasing augmentation probability.")
|
||||
parser.add_argument("--auto-augment", help="Ultralytics classification auto-augment policy.")
|
||||
parser.add_argument("--dropout", type=float, help="Classification head dropout.")
|
||||
parser.add_argument("--freeze", type=int, help="Freeze the first N model layers.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -44,19 +60,45 @@ def main() -> int:
|
||||
"Ultralytics is not installed. Run `uv sync --extra speedvision` in the repo root before training."
|
||||
) from exc
|
||||
|
||||
train_kwargs = {
|
||||
"data": str(data_path),
|
||||
"epochs": args.epochs,
|
||||
"imgsz": args.imgsz,
|
||||
"batch": args.batch,
|
||||
"workers": args.workers,
|
||||
"device": args.device,
|
||||
"project": str(project_path),
|
||||
"name": args.name,
|
||||
"patience": args.patience,
|
||||
"cache": args.cache,
|
||||
"exist_ok": args.exist_ok,
|
||||
}
|
||||
optional_kwargs = {
|
||||
"optimizer": args.optimizer,
|
||||
"lr0": args.lr0,
|
||||
"lrf": args.lrf,
|
||||
"warmup_epochs": args.warmup_epochs,
|
||||
"weight_decay": args.weight_decay,
|
||||
"degrees": args.degrees,
|
||||
"translate": args.translate,
|
||||
"scale": args.scale,
|
||||
"shear": args.shear,
|
||||
"perspective": args.perspective,
|
||||
"fliplr": args.fliplr,
|
||||
"erasing": args.erasing,
|
||||
"auto_augment": args.auto_augment,
|
||||
"dropout": args.dropout,
|
||||
"freeze": args.freeze,
|
||||
}
|
||||
train_kwargs.update({key: value for key, value in optional_kwargs.items() if value is not None})
|
||||
if args.auto_augment is not None and args.auto_augment.lower() in ("none", "off", "false", "0"):
|
||||
train_kwargs["auto_augment"] = None
|
||||
if args.cos_lr:
|
||||
train_kwargs["cos_lr"] = True
|
||||
|
||||
model = YOLO(args.model)
|
||||
model.train(
|
||||
data=str(data_path),
|
||||
epochs=args.epochs,
|
||||
imgsz=args.imgsz,
|
||||
batch=args.batch,
|
||||
workers=args.workers,
|
||||
device=args.device,
|
||||
project=str(project_path),
|
||||
name=args.name,
|
||||
patience=args.patience,
|
||||
cache=args.cache,
|
||||
exist_ok=args.exist_ok,
|
||||
**train_kwargs,
|
||||
)
|
||||
print(f"Classifier training complete under {project_path / args.name}")
|
||||
return 0
|
||||
|
||||
@@ -11,6 +11,7 @@ from cereal import car, custom, log
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.realtime import config_realtime_process, Priority, Ratekeeper
|
||||
from openpilot.common.swaglog import cloudlog, ForwardingHandler
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
|
||||
from opendbc.car import DT_CTRL, ButtonType, structs
|
||||
from opendbc.car.can_definitions import CanData, CanRecvCallable, CanSendCallable
|
||||
@@ -136,8 +137,9 @@ class Car:
|
||||
if self.CP.secOcRequired and not is_release:
|
||||
# Copy user key if available
|
||||
try:
|
||||
with open("/cache/params/SecOCKey") as f:
|
||||
user_key = f.readline().strip()
|
||||
user_key = Params(Paths.params_cache_root()).get("SecOCKey")
|
||||
if user_key is not None:
|
||||
user_key = user_key.strip()
|
||||
if len(user_key) == 32:
|
||||
self.params.put("SecOCKey", user_key)
|
||||
except Exception:
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -2,16 +2,13 @@ from pathlib import Path
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.athena.registration import register
|
||||
from openpilot.system.hardware import PC
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
|
||||
from openpilot.starpilot.common.starpilot_utilities import use_konik_server
|
||||
|
||||
|
||||
def _cache_params_path() -> str:
|
||||
if PC:
|
||||
return str(Path(Paths.comma_home()) / "cache" / "params")
|
||||
return "/cache/params"
|
||||
return Paths.params_cache_root()
|
||||
|
||||
|
||||
def _normalize_dongle_id(value):
|
||||
|
||||
@@ -160,6 +160,7 @@ SCHOOL_ZONE_MIN_SUPPORT = 2
|
||||
SCHOOL_ZONE_MIN_CONFIDENCE = 0.70
|
||||
SCHOOL_ZONE_SINGLE_READ_CONFIDENCE = 0.975
|
||||
SCHOOL_ZONE_SHORT_CIRCUIT_CONFIDENCE = 0.78
|
||||
SCHOOL_ZONE_FALLBACK_MIN_CONFIDENCE = 0.35
|
||||
DEBUG_BASE_DIR = Path("/data/media/0/vision_speed_limit_debug")
|
||||
DEBUG_CAPTURE_DIRNAME = "captures"
|
||||
SNAPSHOT_JPEG_QUALITY = 85
|
||||
@@ -1162,6 +1163,7 @@ class SpeedLimitVisionDaemon:
|
||||
|
||||
if class_id == 2:
|
||||
school_scores: dict[int, float] = {}
|
||||
competing_scores: dict[int, float] = {}
|
||||
school_best_confidences: dict[int, float] = {}
|
||||
school_support_counts: dict[int, int] = {}
|
||||
for expand_left, expand_top, expand_right, expand_bottom in SCHOOL_ZONE_DIRECT_EXPANSIONS:
|
||||
@@ -1182,6 +1184,7 @@ class SpeedLimitVisionDaemon:
|
||||
|
||||
speed_limit_mph, read_confidence = read_result
|
||||
if speed_limit_mph not in SCHOOL_ZONE_SPEED_VALUES:
|
||||
competing_scores[speed_limit_mph] = competing_scores.get(speed_limit_mph, 0.0) + read_confidence * crop_weight
|
||||
continue
|
||||
|
||||
school_scores[speed_limit_mph] = school_scores.get(speed_limit_mph, 0.0) + read_confidence * crop_weight
|
||||
@@ -1198,19 +1201,20 @@ class SpeedLimitVisionDaemon:
|
||||
)
|
||||
read_confidence = school_best_confidences[speed_limit_mph]
|
||||
support_count = school_support_counts[speed_limit_mph]
|
||||
if (
|
||||
(support_count >= SCHOOL_ZONE_MIN_SUPPORT and read_confidence >= SCHOOL_ZONE_MIN_CONFIDENCE) or
|
||||
read_confidence >= SCHOOL_ZONE_SINGLE_READ_CONFIDENCE
|
||||
):
|
||||
score = min(
|
||||
read_confidence * 0.72 +
|
||||
proposal_confidence * 0.22 +
|
||||
max(support_count - 1, 0) * SCHOOL_ZONE_SUPPORT_BONUS +
|
||||
0.04,
|
||||
0.95,
|
||||
)
|
||||
if score >= SCHOOL_ZONE_SHORT_CIRCUIT_CONFIDENCE:
|
||||
return Detection(speed_limit_mph, score)
|
||||
if school_scores[speed_limit_mph] > max(competing_scores.values(), default=0.0):
|
||||
if (
|
||||
(support_count >= SCHOOL_ZONE_MIN_SUPPORT and read_confidence >= SCHOOL_ZONE_MIN_CONFIDENCE) or
|
||||
read_confidence >= SCHOOL_ZONE_SINGLE_READ_CONFIDENCE
|
||||
):
|
||||
score = min(
|
||||
read_confidence * 0.72 +
|
||||
proposal_confidence * 0.22 +
|
||||
max(support_count - 1, 0) * SCHOOL_ZONE_SUPPORT_BONUS +
|
||||
0.04,
|
||||
0.95,
|
||||
)
|
||||
if score >= SCHOOL_ZONE_SHORT_CIRCUIT_CONFIDENCE:
|
||||
return Detection(speed_limit_mph, score)
|
||||
|
||||
proposal_area_ratio = (box_width * box_height) / max(frame_width * frame_height, 1)
|
||||
speed_scores: dict[int, float] = {}
|
||||
@@ -1227,7 +1231,8 @@ class SpeedLimitVisionDaemon:
|
||||
if sign_crop.size == 0:
|
||||
continue
|
||||
|
||||
is_regulatory = self._is_regulatory_speed_sign(sign_crop)
|
||||
raw_is_regulatory = self._is_regulatory_speed_sign(sign_crop)
|
||||
is_regulatory = raw_is_regulatory
|
||||
if class_id == 2:
|
||||
is_regulatory = True
|
||||
|
||||
@@ -1243,6 +1248,13 @@ class SpeedLimitVisionDaemon:
|
||||
read_result = (model_read[0], min(model_read[1], ocr_read[1]))
|
||||
|
||||
speed_limit_mph, read_confidence = read_result
|
||||
if (
|
||||
class_id == 2 and
|
||||
proposal_confidence < SCHOOL_ZONE_FALLBACK_MIN_CONFIDENCE and
|
||||
not raw_is_regulatory
|
||||
):
|
||||
continue
|
||||
|
||||
score = read_confidence * expansion_weight
|
||||
if is_regulatory:
|
||||
score += DETECTOR_CLASSIFIER_REGULATORY_BONUS
|
||||
|
||||
@@ -715,6 +715,7 @@ _fast_update_state = {
|
||||
|
||||
_FACTORY_RESET_WIPE_PATHS = [
|
||||
"/data/params",
|
||||
"/cache/starpilot/params",
|
||||
"/cache/params",
|
||||
"/data/media/0/realdata",
|
||||
"/data/media/0/realdata_HD",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
namespace {
|
||||
|
||||
std::string cacheParamsPath() {
|
||||
return Hardware::PC() ? Path::comma_home() + "/cache/params" : "/cache/params";
|
||||
return Path::params_cache();
|
||||
}
|
||||
|
||||
void prepareKonikServerSwitch(bool use_konik) {
|
||||
|
||||
@@ -48,6 +48,14 @@ namespace Path {
|
||||
return util::getenv("PARAMS_ROOT", Hardware::PC() ? (Path::comma_home() + "/params") : "/data/params");
|
||||
}
|
||||
|
||||
inline std::string params_cache() {
|
||||
return Hardware::PC() ? Path::comma_home() + "/cache/starpilot/params" : "/cache/starpilot/params";
|
||||
}
|
||||
|
||||
inline std::string legacy_params_cache() {
|
||||
return Hardware::PC() ? Path::comma_home() + "/cache/params" : "/cache/params";
|
||||
}
|
||||
|
||||
inline std::string rsa_file() {
|
||||
return Hardware::PC() ? Path::comma_home() + "/persist/comma/id_rsa" : "/persist/comma/id_rsa";
|
||||
}
|
||||
|
||||
@@ -62,6 +62,18 @@ class Paths:
|
||||
else:
|
||||
return "/tmp/.comma"
|
||||
|
||||
@staticmethod
|
||||
def params_cache_root() -> str:
|
||||
if PC:
|
||||
return str(Path(Paths.comma_home()) / "cache" / "starpilot" / "params")
|
||||
return "/cache/starpilot/params"
|
||||
|
||||
@staticmethod
|
||||
def legacy_params_cache_root() -> str:
|
||||
if PC:
|
||||
return str(Path(Paths.comma_home()) / "cache" / "params")
|
||||
return "/cache/params"
|
||||
|
||||
@staticmethod
|
||||
def shm_path() -> str:
|
||||
if PC and platform.system() == "Darwin":
|
||||
|
||||
@@ -40,6 +40,8 @@ STARPILOT_PRIORITIZE_SMOOTH_FOLLOWING_MIGRATION_FLAG = Path("/data") / "starpilo
|
||||
STARPILOT_PARAM_RENAME_MIGRATION_FLAG = Path("/data") / "starpilot_param_rename_v1"
|
||||
STARPILOT_PARAM_CANONICALIZATION_MIGRATION_FLAG = Path("/data") / "starpilot_param_canonicalization_v1"
|
||||
STARPILOT_PC_ROOT_MIGRATION_FLAG = Path("/data") / "starpilot_pc_root_v1"
|
||||
STARPILOT_PARAMS_CACHE_MIGRATION_FLAG = Path("/data") / "starpilot_params_cache_v1"
|
||||
STARPILOT_LEGACY_CACHE_MARKER_KEYS = ("RemapCancelToDistance",)
|
||||
STARPILOT_REMOVED_PARAM_KEYS = ("HumanFollowing",)
|
||||
LEGACY_CARMODEL_MIGRATIONS = {
|
||||
"CHEVROLET_BOLT_CC_2019_2021": "CHEVROLET_BOLT_CC_2018_2021",
|
||||
@@ -206,6 +208,68 @@ def _remove_persisted_param_file(params: Params, key: str | bytes) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _params_store_path(root: str | Path) -> Path:
|
||||
return Path(root) / os.environ.get("OPENPILOT_PREFIX", "d")
|
||||
|
||||
|
||||
def _cache_store_has_starpilot_marker(cache_root: str | Path) -> bool:
|
||||
store_path = _params_store_path(cache_root)
|
||||
return any((store_path / key).is_file() for key in STARPILOT_LEGACY_CACHE_MARKER_KEYS)
|
||||
|
||||
|
||||
def _copy_param_store_without_overwrite(source: Path, destination: Path) -> int:
|
||||
if not source.is_dir():
|
||||
return 0
|
||||
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
copied_entries = 0
|
||||
for path in source.iterdir():
|
||||
if not path.is_file() or path.name == ".lock" or path.name.startswith(".tmp_"):
|
||||
continue
|
||||
|
||||
target = destination / path.name
|
||||
if target.exists():
|
||||
continue
|
||||
|
||||
shutil.copy2(path, target)
|
||||
copied_entries += 1
|
||||
|
||||
return copied_entries
|
||||
|
||||
|
||||
def migrate_legacy_starpilot_params_cache(params: Params, legacy_cache_root: str | Path, cache_root: str | Path) -> None:
|
||||
if STARPILOT_PARAMS_CACHE_MIGRATION_FLAG.exists():
|
||||
return
|
||||
|
||||
legacy_store = _params_store_path(legacy_cache_root)
|
||||
cache_store = _params_store_path(cache_root)
|
||||
active_marker = any(_has_persisted_param_file(params, key) for key in STARPILOT_LEGACY_CACHE_MARKER_KEYS)
|
||||
cache_marker = _cache_store_has_starpilot_marker(legacy_cache_root)
|
||||
|
||||
migration_succeeded = True
|
||||
copied_entries = 0
|
||||
if active_marker or cache_marker:
|
||||
try:
|
||||
copied_entries = _copy_param_store_without_overwrite(legacy_store, cache_store)
|
||||
except Exception:
|
||||
migration_succeeded = False
|
||||
cloudlog.exception(f"Failed to migrate legacy StarPilot params cache from {legacy_store} to {cache_store}")
|
||||
elif legacy_store.exists():
|
||||
cloudlog.warning(f"Skipped legacy params cache import without StarPilot marker: {legacy_store}")
|
||||
|
||||
if not migration_succeeded:
|
||||
return
|
||||
|
||||
if copied_entries:
|
||||
cloudlog.warning(f"Migrated {copied_entries} legacy StarPilot params cache entries from {legacy_store} to {cache_store}")
|
||||
|
||||
try:
|
||||
STARPILOT_PARAMS_CACHE_MIGRATION_FLAG.parent.mkdir(parents=True, exist_ok=True)
|
||||
STARPILOT_PARAMS_CACHE_MIGRATION_FLAG.write_text(f"{datetime.datetime.now(datetime.UTC).isoformat()}\n")
|
||||
except Exception:
|
||||
cloudlog.exception(f"Failed to write migration flag: {STARPILOT_PARAMS_CACHE_MIGRATION_FLAG}")
|
||||
|
||||
|
||||
def cleanup_removed_starpilot_params(params: Params, params_cache: Params) -> None:
|
||||
removed_keys = []
|
||||
for key in STARPILOT_REMOVED_PARAM_KEYS:
|
||||
@@ -668,9 +732,8 @@ def manager_init() -> None:
|
||||
build_metadata = get_build_metadata()
|
||||
|
||||
params = Params()
|
||||
cache_params_path = "/cache/params"
|
||||
if HARDWARE.get_device_type() == "pc":
|
||||
cache_params_path = os.path.join(Paths.comma_home(), "cache", "params")
|
||||
cache_params_path = Paths.params_cache_root()
|
||||
migrate_legacy_starpilot_params_cache(params, Paths.legacy_params_cache_root(), cache_params_path)
|
||||
params_cache = Params(cache_params_path, return_defaults=True)
|
||||
|
||||
# Legacy FrogPilot params are unknown to the renamed schema and would be
|
||||
|
||||
@@ -198,6 +198,58 @@ class TestManager:
|
||||
assert not Path(params.get_param_path("HumanFollowing")).exists()
|
||||
assert not Path(params_cache.get_param_path("HumanFollowing")).exists()
|
||||
|
||||
def test_migrate_legacy_starpilot_params_cache_copies_marker_sources(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(manager, "STARPILOT_PARAMS_CACHE_MIGRATION_FLAG", tmp_path / "starpilot_params_cache_v1")
|
||||
|
||||
params = FileBackedFakeParams(tmp_path / "params")
|
||||
legacy_cache = tmp_path / "legacy_cache"
|
||||
new_cache = tmp_path / "new_cache"
|
||||
legacy_store = manager._params_store_path(legacy_cache)
|
||||
legacy_store.mkdir(parents=True)
|
||||
(legacy_store / "RemapCancelToDistance").write_text("0")
|
||||
(legacy_store / "ClusterOffset").write_text("1.02")
|
||||
|
||||
manager.migrate_legacy_starpilot_params_cache(params, legacy_cache, new_cache)
|
||||
|
||||
new_store = manager._params_store_path(new_cache)
|
||||
assert (new_store / "RemapCancelToDistance").read_text() == "0"
|
||||
assert (new_store / "ClusterOffset").read_text() == "1.02"
|
||||
assert manager.STARPILOT_PARAMS_CACHE_MIGRATION_FLAG.exists()
|
||||
|
||||
def test_migrate_legacy_starpilot_params_cache_skips_without_marker(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(manager, "STARPILOT_PARAMS_CACHE_MIGRATION_FLAG", tmp_path / "starpilot_params_cache_v1")
|
||||
|
||||
params = FileBackedFakeParams(tmp_path / "params")
|
||||
legacy_cache = tmp_path / "legacy_cache"
|
||||
new_cache = tmp_path / "new_cache"
|
||||
legacy_store = manager._params_store_path(legacy_cache)
|
||||
legacy_store.mkdir(parents=True)
|
||||
(legacy_store / "ClusterOffset").write_text("1.02")
|
||||
|
||||
manager.migrate_legacy_starpilot_params_cache(params, legacy_cache, new_cache)
|
||||
|
||||
assert not (manager._params_store_path(new_cache) / "ClusterOffset").exists()
|
||||
assert manager.STARPILOT_PARAMS_CACHE_MIGRATION_FLAG.exists()
|
||||
|
||||
def test_migrate_legacy_starpilot_params_cache_does_not_overwrite_new_cache(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(manager, "STARPILOT_PARAMS_CACHE_MIGRATION_FLAG", tmp_path / "starpilot_params_cache_v1")
|
||||
|
||||
params = FileBackedFakeParams(tmp_path / "params")
|
||||
legacy_cache = tmp_path / "legacy_cache"
|
||||
new_cache = tmp_path / "new_cache"
|
||||
legacy_store = manager._params_store_path(legacy_cache)
|
||||
new_store = manager._params_store_path(new_cache)
|
||||
legacy_store.mkdir(parents=True)
|
||||
new_store.mkdir(parents=True)
|
||||
(legacy_store / "RemapCancelToDistance").write_text("0")
|
||||
(legacy_store / "ClusterOffset").write_text("1.02")
|
||||
(new_store / "ClusterOffset").write_text("1.0")
|
||||
|
||||
manager.migrate_legacy_starpilot_params_cache(params, legacy_cache, new_cache)
|
||||
|
||||
assert (new_store / "ClusterOffset").read_text() == "1.0"
|
||||
assert (new_store / "RemapCancelToDistance").read_text() == "0"
|
||||
|
||||
def test_migrate_cluster_offset_default_resets_legacy_default_only(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(manager, "STARPILOT_CLUSTER_OFFSET_MIGRATION_FLAG", tmp_path / "starpilot_cluster_offset_v1")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user