diff --git a/common/api.py b/common/api.py index f2e5e61b8..703176562 100644 --- a/common/api.py +++ b/common/api.py @@ -2,10 +2,15 @@ import jwt import os import requests from datetime import datetime, timedelta, UTC +from functools import cache +from openpilot.common.params import Params from openpilot.system.hardware.hw import Paths from openpilot.system.version import get_version -from openpilot.starpilot.common.starpilot_utilities import use_konik_server + +@cache +def use_konik_server() -> bool: + return Params().get_bool("UseKonikServer") API_HOST = os.getenv('API_HOST', f"https://api.{'konik.ai' if use_konik_server() else 'commadotai.com'}") diff --git a/opendbc_repo/opendbc/car/toyota/carcontroller.py b/opendbc_repo/opendbc/car/toyota/carcontroller.py index 918672906..5cc8485a7 100644 --- a/opendbc_repo/opendbc/car/toyota/carcontroller.py +++ b/opendbc_repo/opendbc/car/toyota/carcontroller.py @@ -34,6 +34,7 @@ TOYOTA_COAST_BRAKE_ENABLE_ACCEL = -0.10 # m/s^2 TOYOTA_COAST_BRAKE_DISABLE_ACCEL = -0.06 # m/s^2 TOYOTA_NO_LEAD_COAST_BRAKE_ACCEL = -0.30 # m/s^2 TOYOTA_INTERCEPTOR_COMFORT_TARGET_ACCEL = 2.0 # m/s^2 +TOYOTA_NO_LEAD_CRUISE_SIGN_FLIP_MIN_SET_SPEED_ERROR = 0.35 # m/s # LKA limits # EPS faults if you apply torque while the steering rate is above 100 deg/s for too long @@ -168,6 +169,19 @@ def limit_prius_stopping_accel(pcm_accel_cmd: float, target_accel: float, stoppi return max(pcm_accel_cmd, max(stop_floor, planner_floor)) +def limit_no_lead_cruise_sign_flip(pcm_accel_cmd: float, target_accel: float, stopping: bool, v_ego: float, + set_speed: float, lead_visible: bool) -> float: + if stopping or lead_visible or pcm_accel_cmd >= 0.0 or v_ego < TOYOTA_COAST_BRAKE_MIN_SPEED: + return pcm_accel_cmd + if target_accel < -0.02 or set_speed <= 0.0: + return pcm_accel_cmd + + if float(set_speed) - float(v_ego) >= TOYOTA_NO_LEAD_CRUISE_SIGN_FLIP_MIN_SET_SPEED_ERROR: + return max(pcm_accel_cmd, 0.0) + + return pcm_accel_cmd + + class CarController(CarControllerBase): def __init__(self, dbc_names, CP): super().__init__(dbc_names, CP) @@ -464,8 +478,11 @@ class CarController(CarControllerBase): if self.CP.enableGasInterceptorDEPRECATED: pcm_accel_cmd = limit_interceptor_pcm_accel(pcm_accel_cmd, actuators.accel, stopping, CS.out.vEgo) pcm_accel_cmd = limit_interceptor_stopping_accel(pcm_accel_cmd, actuators.accel, stopping, CS.out.vEgo, bool(hud_control.leadVisible)) - elif self.CP.carFingerprint == CAR.TOYOTA_PRIUS: - pcm_accel_cmd = limit_prius_stopping_accel(pcm_accel_cmd, actuators.accel, stopping, CS.out.vEgo, lead) + else: + pcm_accel_cmd = limit_no_lead_cruise_sign_flip(pcm_accel_cmd, actuators.accel, stopping, CS.out.vEgo, + CS.out.cruiseState.speed, bool(hud_control.leadVisible)) + if self.CP.carFingerprint == CAR.TOYOTA_PRIUS: + pcm_accel_cmd = limit_prius_stopping_accel(pcm_accel_cmd, actuators.accel, stopping, CS.out.vEgo, lead) pcm_accel_cmd = float(np.clip(pcm_accel_cmd, self.params.ACCEL_MIN, self.params.ACCEL_MAX)) diff --git a/opendbc_repo/opendbc/car/toyota/tests/test_toyota.py b/opendbc_repo/opendbc/car/toyota/tests/test_toyota.py index be3071ef8..bfb9be070 100644 --- a/opendbc_repo/opendbc/car/toyota/tests/test_toyota.py +++ b/opendbc_repo/opendbc/car/toyota/tests/test_toyota.py @@ -8,7 +8,8 @@ from opendbc.car.structs import CarParams from opendbc.car.fw_versions import build_fw_dict from opendbc.car.toyota import toyotacan from opendbc.car.toyota.carcontroller import CarController, get_prius_positive_feedforward_scale, limit_interceptor_pcm_accel, \ - limit_interceptor_stopping_accel, limit_prius_stopping_accel, update_permit_braking + limit_interceptor_stopping_accel, limit_no_lead_cruise_sign_flip, \ + limit_prius_stopping_accel, update_permit_braking from opendbc.car.toyota.carstate import calculate_interceptor_gas_pressed from opendbc.car.toyota.fingerprints import FW_VERSIONS from opendbc.car.toyota.interface import CarInterface @@ -292,6 +293,18 @@ class TestToyotaCarController: assert update_permit_braking(False, 0.10, True, True, 25.0, False) is True assert update_permit_braking(False, 0.10, False, False, 25.0, False) is True + def test_no_lead_cruise_sign_flip_clamps_negative_pulse_when_set_speed_is_ahead(self): + limited = limit_no_lead_cruise_sign_flip(-0.44, 0.0, False, 23.3, 25.0, False) + assert limited == 0.0 + + def test_no_lead_cruise_sign_flip_keeps_real_decel_requests(self): + limited = limit_no_lead_cruise_sign_flip(-0.44, -0.15, False, 23.3, 25.0, False) + assert limited == -0.44 + + def test_no_lead_cruise_sign_flip_keeps_lead_follow_brake(self): + limited = limit_no_lead_cruise_sign_flip(-0.44, 0.0, False, 23.3, 25.0, True) + assert limited == -0.44 + def test_prius_stopping_accel_unwinds_stale_stop_hold(self): limited = limit_prius_stopping_accel(-3.28, -0.05, True, 0.0, True) assert -1.5 < limited < 0.0 diff --git a/scripts/speed_limit_vision/import_manual_review_queue.py b/scripts/speed_limit_vision/import_manual_review_queue.py new file mode 100644 index 000000000..1781ac6f1 --- /dev/null +++ b/scripts/speed_limit_vision/import_manual_review_queue.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import shutil + +from pathlib import Path + +import cv2 + +if __package__ in (None, ""): + import sys + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace # type: ignore +else: + from .common import DEFAULT_SPEED_VALUES, DEFAULT_WORKSPACE, ensure_dir, resolve_workspace + + +CLASSIFIER_FIELDNAMES = [ + "record_key", + "split", + "speed_limit_mph", + "review_sign_type", + "crop_path", + "frame_path", + "bbox", + "crop_bbox", + "review_status", + "candidate_speed_limit_mph", + "candidate_confidence", + "detector_class", +] + +RUNTIME_FIELDNAMES = [ + "record_key", + "split", + "sample_type", + "dataset_image", + "speed_limit_mph", + "review_status", + "review_sign_type", + "detector_class", + "candidate_speed_limit_mph", + "candidate_confidence", +] + +DETECTOR_MANIFEST_FIELDNAMES = [ + "record_key", + "split", + "sample_type", + "speed_limit_mph", + "review_sign_type", + "source_frame", + "dataset_image", + "dataset_label", + "bbox", + "class_id", + "review_status", + "detector_class", +] + +POSITIVE_STATUSES = {"accepted", "corrected"} +NEGATIVE_STATUS = "ignore" +SIGN_TYPE_CLASS_IDS = { + "regulatory": 0, + "advisory": 1, + "school_zone": 2, + "construction": 0, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import manually reviewed speed-limit rows into training/eval manifests.") + parser.add_argument("--workspace", type=Path, default=DEFAULT_WORKSPACE, help="Training workspace root.") + parser.add_argument("--queue", type=Path, required=True, help="manual_review_queue.csv from build_manual_review_queue.py.") + parser.add_argument("--labels", type=Path, help="manual_review_labels.csv. Defaults to /manual_review_labels.csv.") + parser.add_argument("--classifier-manifest-out", type=Path, help="Positive crop manifest for import_manifest_classifier_masks.py.") + parser.add_argument("--runtime-manifest-out", type=Path, help="Full-frame eval manifest including positives and true negatives.") + parser.add_argument("--detector-manifest-out", type=Path, help="Manifest of imported detector examples.") + parser.add_argument("--source-name", default="manual_review", help="Filename prefix for detector dataset imports.") + parser.add_argument("--mode", choices=("symlink", "copy"), default="symlink", help="How to place detector images.") + parser.add_argument("--val-modulo", type=int, default=5, help="Hash modulo for validation split. 0 sends everything to train.") + parser.add_argument("--val-remainder", type=int, default=0, help="Hash remainder used as validation split.") + parser.add_argument("--max-detector-negatives", type=int, default=0, help="Cap true negative detector imports. 0 keeps all.") + parser.add_argument("--overwrite", action="store_true", help="Replace existing detector image links/files and labels.") + return parser.parse_args() + + +def read_csv(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, object]]) -> None: + ensure_dir(path.parent) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + +def split_for_key(key: str, val_modulo: int, val_remainder: int) -> str: + if val_modulo <= 0: + return "train" + digest = hashlib.sha1(key.encode("utf-8")).hexdigest() + return "val" if int(digest[:8], 16) % val_modulo == val_remainder else "train" + + +def parse_speed(text: str) -> int: + text = (text or "").strip() + if not text: + return 0 + try: + value = int(float(text)) + except ValueError: + return 0 + return value if value in DEFAULT_SPEED_VALUES else 0 + + +def parse_bbox(text: str) -> tuple[int, int, int, int] | None: + parts = [part.strip() for part in (text or "").replace(";", ",").split(",") if part.strip()] + if len(parts) != 4: + return None + try: + x1, y1, x2, y2 = (int(round(float(part))) for part in parts) + except ValueError: + return None + if x2 <= x1 or y2 <= y1: + return None + return x1, y1, x2, y2 + + +def detector_class_id(row: dict[str, str]) -> int: + sign_type = effective_sign_type(row) + if sign_type in SIGN_TYPE_CLASS_IDS: + return SIGN_TYPE_CLASS_IDS[sign_type] + + class_text = (row.get("class_id") or "").strip() + if class_text.isdigit(): + class_id = int(class_text) + if class_id in (0, 1, 2): + return class_id + + detector_class = row.get("detector_class", "") + if detector_class == "school_zone_speed_limit": + return 2 + if detector_class == "advisory_speed_limit": + return 1 + return 0 + + +def effective_sign_type(row: dict[str, str]) -> str: + sign_type = (row.get("review_sign_type") or "").strip() + if sign_type and sign_type != "not_speed_limit": + return sign_type + + detector_class = row.get("detector_class", "") + if detector_class == "school_zone_speed_limit": + return "school_zone" + if detector_class == "advisory_speed_limit": + return "advisory" + if detector_class == "negative_empty": + return "not_speed_limit" + return "regulatory" + + +def detector_label_line(class_id: int, bbox: tuple[int, int, int, int], image_shape: tuple[int, int, int]) -> str: + image_h, image_w = image_shape[:2] + x1, y1, x2, y2 = bbox + x_center = ((x1 + x2) / 2) / image_w + y_center = ((y1 + y2) / 2) / image_h + width = (x2 - x1) / image_w + height = (y2 - y1) / image_h + return f"{class_id} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}\n" + + +def stage_image(source_path: Path, dest_path: Path, mode: str, overwrite: bool) -> None: + ensure_dir(dest_path.parent) + if dest_path.exists() or dest_path.is_symlink(): + if not overwrite: + return + dest_path.unlink() + if mode == "copy": + shutil.copy2(source_path, dest_path) + else: + dest_path.symlink_to(source_path.resolve()) + + +def safe_stem(text: str) -> str: + keep = [] + for char in text: + keep.append(char if char.isalnum() or char in "._-" else "_") + cleaned = "".join(keep).strip("._") + return cleaned[:180] or "sample" + + +def merged_review_rows(queue_path: Path, labels_path: Path) -> list[dict[str, str]]: + queue_rows = read_csv(queue_path) + labels_by_key = {row["record_key"]: row for row in read_csv(labels_path) if row.get("record_key")} + merged = [] + for row in queue_rows: + label = labels_by_key.get(row.get("record_key", "")) + if not label: + continue + item = dict(row) + item.update({ + "review_status": label.get("review_status", ""), + "review_speed_limit_mph": label.get("review_speed_limit_mph", ""), + "review_sign_type": label.get("review_sign_type", ""), + "review_bbox": label.get("review_bbox", ""), + "review_ignore_reason": label.get("review_ignore_reason", ""), + "review_notes": label.get("review_notes", ""), + }) + merged.append(item) + return merged + + +def is_positive(row: dict[str, str]) -> bool: + if row.get("review_status") not in POSITIVE_STATUSES: + return False + if not parse_speed(row.get("review_speed_limit_mph", "")): + return False + return Path(row.get("crop_path", "")).is_file() and Path(row.get("frame_path", "")).is_file() + + +def is_true_negative(row: dict[str, str]) -> bool: + if row.get("review_status") != NEGATIVE_STATUS: + return False + if row.get("detector_class") != "negative_empty": + return False + return Path(row.get("frame_path", "")).is_file() + + +def positive_classifier_row(row: dict[str, str], split: str) -> dict[str, object]: + speed = parse_speed(row.get("review_speed_limit_mph", "")) + return { + "record_key": row["record_key"], + "split": split, + "speed_limit_mph": speed, + "review_sign_type": effective_sign_type(row), + "crop_path": row.get("crop_path", ""), + "frame_path": row.get("frame_path", ""), + "bbox": row.get("bbox", ""), + "crop_bbox": row.get("crop_bbox", ""), + "review_status": row.get("review_status", ""), + "candidate_speed_limit_mph": row.get("candidate_speed_limit_mph", ""), + "candidate_confidence": row.get("candidate_confidence", ""), + "detector_class": row.get("detector_class", ""), + } + + +def runtime_row(row: dict[str, str], split: str, sample_type: str) -> dict[str, object]: + speed = parse_speed(row.get("review_speed_limit_mph", "")) if sample_type == "positive" else 0 + return { + "record_key": row["record_key"], + "split": split, + "sample_type": sample_type, + "dataset_image": row.get("frame_path", ""), + "speed_limit_mph": "" if speed == 0 else speed, + "review_status": row.get("review_status", ""), + "review_sign_type": effective_sign_type(row), + "detector_class": row.get("detector_class", ""), + "candidate_speed_limit_mph": row.get("candidate_speed_limit_mph", ""), + "candidate_confidence": row.get("candidate_confidence", ""), + } + + +def import_detector_example( + workspace: Path, + row: dict[str, str], + split: str, + source_name: str, + sample_type: str, + mode: str, + overwrite: bool, +) -> dict[str, object] | None: + source_frame = Path(row.get("frame_path", "")).expanduser() + if not source_frame.is_file(): + return None + + stem = f"{safe_stem(source_name)}_{safe_stem(row['record_key'])}" + image_path = workspace / "detector" / "images" / split / f"{stem}{source_frame.suffix.lower() or '.jpg'}" + label_path = workspace / "detector" / "labels" / split / f"{stem}.txt" + stage_image(source_frame, image_path, mode, overwrite) + ensure_dir(label_path.parent) + + class_id = "" + bbox_text = "" + if sample_type == "positive": + bbox = parse_bbox(row.get("review_bbox") or row.get("bbox", "")) + if bbox is None: + return None + image = cv2.imread(str(source_frame)) + if image is None: + return None + class_id_int = detector_class_id(row) + label_path.write_text(detector_label_line(class_id_int, bbox, image.shape), encoding="utf-8") + class_id = str(class_id_int) + bbox_text = ",".join(str(value) for value in bbox) + else: + label_path.write_text("", encoding="utf-8") + + return { + "record_key": row["record_key"], + "split": split, + "sample_type": sample_type, + "speed_limit_mph": parse_speed(row.get("review_speed_limit_mph", "")) if sample_type == "positive" else "", + "review_sign_type": effective_sign_type(row), + "source_frame": str(source_frame), + "dataset_image": str(image_path), + "dataset_label": str(label_path), + "bbox": bbox_text, + "class_id": class_id, + "review_status": row.get("review_status", ""), + "detector_class": row.get("detector_class", ""), + } + + +def main() -> int: + args = parse_args() + workspace = resolve_workspace(args.workspace) + queue_path = args.queue.expanduser().resolve() + labels_path = args.labels.expanduser().resolve() if args.labels else queue_path.with_name("manual_review_labels.csv") + output_dir = queue_path.parent + classifier_manifest = args.classifier_manifest_out.expanduser().resolve() if args.classifier_manifest_out else output_dir / "manual_review_classifier_manifest.csv" + runtime_manifest = args.runtime_manifest_out.expanduser().resolve() if args.runtime_manifest_out else output_dir / "manual_review_runtime_eval_manifest.csv" + detector_manifest = args.detector_manifest_out.expanduser().resolve() if args.detector_manifest_out else output_dir / "manual_review_detector_import_manifest.csv" + + rows = merged_review_rows(queue_path, labels_path) + positive_rows = [row for row in rows if is_positive(row)] + true_negative_rows = [row for row in rows if is_true_negative(row)] + if args.max_detector_negatives > 0: + true_negative_rows = true_negative_rows[:args.max_detector_negatives] + + classifier_rows: list[dict[str, object]] = [] + runtime_rows: list[dict[str, object]] = [] + detector_rows: list[dict[str, object]] = [] + + for row in positive_rows: + split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder) + classifier_rows.append(positive_classifier_row(row, split)) + runtime_rows.append(runtime_row(row, split, "positive")) + detector_row = import_detector_example(workspace, row, split, args.source_name, "positive", args.mode, args.overwrite) + if detector_row is not None: + detector_rows.append(detector_row) + + for row in true_negative_rows: + split = split_for_key(row["record_key"], args.val_modulo, args.val_remainder) + runtime_rows.append(runtime_row(row, split, "negative_empty")) + detector_row = import_detector_example(workspace, row, split, args.source_name, "negative_empty", args.mode, args.overwrite) + if detector_row is not None: + detector_rows.append(detector_row) + + write_csv(classifier_manifest, CLASSIFIER_FIELDNAMES, classifier_rows) + write_csv(runtime_manifest, RUNTIME_FIELDNAMES, runtime_rows) + write_csv(detector_manifest, DETECTOR_MANIFEST_FIELDNAMES, detector_rows) + + summary = { + "queue": str(queue_path), + "labels": str(labels_path), + "reviewed_rows": len(rows), + "positive_rows": len(positive_rows), + "true_negative_rows": len(true_negative_rows), + "classifier_manifest": str(classifier_manifest), + "runtime_manifest": str(runtime_manifest), + "detector_manifest": str(detector_manifest), + "detector_imported": len(detector_rows), + } + summary_path = output_dir / "manual_review_import_summary.json" + summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + print( + "Imported manual review queue: " + f"reviewed={len(rows)} positives={len(positive_rows)} true_negatives={len(true_negative_rows)} " + f"detector_imported={len(detector_rows)}" + ) + print(f"Classifier manifest: {classifier_manifest}") + print(f"Runtime eval manifest: {runtime_manifest}") + print(f"Detector import manifest: {detector_manifest}") + print(f"Summary: {summary_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/speed_limit_vision/serve_manual_review_queue.py b/scripts/speed_limit_vision/serve_manual_review_queue.py index 7f040bdbc..0eb41c442 100644 --- a/scripts/speed_limit_vision/serve_manual_review_queue.py +++ b/scripts/speed_limit_vision/serve_manual_review_queue.py @@ -76,7 +76,7 @@ HTML = r""" - Keys: j/k next/prev, 0 ignore, s school, r regulatory, a advisory + Keys: Space/p accept model, type speed to correct, i/x ignore, Enter save correction, j/k next/prev, s school, r regulatory, a advisory
@@ -101,11 +101,9 @@ HTML = r""" -

Status

+

Action

- - - +
@@ -113,8 +111,8 @@ HTML = r"""
- - + +
@@ -124,6 +122,8 @@ let rows = []; let index = 0; let current = null; let draft = {}; +let speedBuffer = ""; +let speedBufferTimer = null; function qs(sel) { return document.querySelector(sel); } function qsa(sel) { return Array.from(document.querySelectorAll(sel)); } @@ -141,6 +141,27 @@ function setActive(selector, attr, value) { qsa(selector).forEach(btn => btn.classList.toggle("active", btn.dataset[attr] === String(value))); } +function clearSpeedBuffer() { + speedBuffer = ""; + if (speedBufferTimer) clearTimeout(speedBufferTimer); + speedBufferTimer = null; +} + +function inferredType(row) { + if (!row) return ""; + if (row.detector_class === "school_zone_speed_limit") return "school_zone"; + if (row.detector_class === "advisory_speed_limit") return "advisory"; + if (row.detector_class === "negative_empty") return "not_speed_limit"; + return "regulatory"; +} + +function ensureSpeedSignType() { + if (draft.review_sign_type === "not_speed_limit") { + draft.review_sign_type = inferredType(current); + setActive("#typeButtons button", "type", draft.review_sign_type); + } +} + function renderSpeedButtons() { const root = qs("#speedButtons"); root.innerHTML = ""; @@ -149,13 +170,38 @@ function renderSpeedButtons() { btn.textContent = speed; btn.dataset.speed = speed; btn.onclick = () => { - draft.review_speed_limit_mph = String(speed); - setActive("#speedButtons button", "speed", speed); + setSpeed(speed, false); }; root.appendChild(btn); } } +function setSpeed(speed, shouldSave) { + draft.review_speed_limit_mph = String(speed); + ensureSpeedSignType(); + setActive("#speedButtons button", "speed", speed); + if (shouldSave) save(true, "corrected"); +} + +function handleDigitShortcut(digit) { + speedBuffer += digit; + if (speedBuffer.length > 2) speedBuffer = speedBuffer.slice(-2); + if (speedBufferTimer) clearTimeout(speedBufferTimer); + speedBufferTimer = setTimeout(clearSpeedBuffer, 1000); + + if (speedBuffer.length < 2) return; + + const speed = Number(speedBuffer); + clearSpeedBuffer(); + if (speeds.includes(speed)) { + setSpeed(speed, true); + return; + } + + speedBuffer = digit; + speedBufferTimer = setTimeout(clearSpeedBuffer, 1000); +} + function render() { current = rows[index] || null; if (!current) { @@ -167,8 +213,8 @@ function render() { } draft = { review_status: current.review_status || "", - review_speed_limit_mph: current.review_speed_limit_mph || "", - review_sign_type: current.review_sign_type || "", + review_speed_limit_mph: current.review_speed_limit_mph || current.candidate_speed_limit_mph || "", + review_sign_type: current.review_sign_type || inferredType(current), review_bbox: current.review_bbox || current.bbox || "", review_ignore_reason: current.review_ignore_reason || "", review_notes: current.review_notes || "", @@ -193,8 +239,14 @@ function render() { ].map(([k,v]) => `
${k}: ${String(v || "")}
`).join(""); } -async function save(moveNext = true) { +function manualReviewStatus() { + if (draft.review_status === "ignore" || draft.review_status === "needs_later") return draft.review_status; + return "corrected"; +} + +async function save(moveNext = true, forcedStatus = null) { if (!current) return; + draft.review_status = forcedStatus || manualReviewStatus(); draft.review_ignore_reason = qs("#ignoreReason").value; draft.review_notes = qs("#notes").value; const payload = {record_key: current.record_key, ...draft}; @@ -218,12 +270,9 @@ qs("#prevBtn").onclick = prev; qs("#saveBtn").onclick = () => save(true); qs("#acceptPredBtn").onclick = () => { if (!current) return; - draft.review_status = "accepted"; draft.review_speed_limit_mph = current.candidate_speed_limit_mph || ""; - draft.review_sign_type = current.detector_class === "school_zone_speed_limit" ? "school_zone" : - current.detector_class === "advisory_speed_limit" ? "advisory" : - current.detector_class === "negative_empty" ? "not_speed_limit" : "regulatory"; - save(true); + draft.review_sign_type = inferredType(current); + save(true, "accepted"); }; qsa("#typeButtons button").forEach(btn => btn.onclick = () => { draft.review_sign_type = btn.dataset.type; @@ -235,13 +284,33 @@ qsa("#statusButtons button").forEach(btn => btn.onclick = () => { }); document.addEventListener("keydown", ev => { if (ev.target.tagName === "TEXTAREA" || ev.target.tagName === "INPUT") return; - if (ev.key === "j") next(); - if (ev.key === "k") prev(); - if (ev.key === "s") { draft.review_sign_type = "school_zone"; setActive("#typeButtons button", "type", "school_zone"); } - if (ev.key === "r") { draft.review_sign_type = "regulatory"; setActive("#typeButtons button", "type", "regulatory"); } - if (ev.key === "a") { draft.review_sign_type = "advisory"; setActive("#typeButtons button", "type", "advisory"); } - if (ev.key === "0") { draft.review_status = "ignore"; draft.review_sign_type = "not_speed_limit"; setActive("#statusButtons button", "status", "ignore"); setActive("#typeButtons button", "type", "not_speed_limit"); } - if (ev.key === "Enter") save(true); + const key = ev.key.toLowerCase(); + if (/^[0-9]$/.test(ev.key)) { + ev.preventDefault(); + handleDigitShortcut(ev.key); + return; + } + if (key === "j") { clearSpeedBuffer(); next(); } + if (key === "k") { clearSpeedBuffer(); prev(); } + if (ev.key === " " || key === "p") { + ev.preventDefault(); + clearSpeedBuffer(); + qs("#acceptPredBtn").click(); + return; + } + if (key === "s") { clearSpeedBuffer(); draft.review_sign_type = "school_zone"; setActive("#typeButtons button", "type", "school_zone"); } + if (key === "r") { clearSpeedBuffer(); draft.review_sign_type = "regulatory"; setActive("#typeButtons button", "type", "regulatory"); } + if (key === "a") { clearSpeedBuffer(); draft.review_sign_type = "advisory"; setActive("#typeButtons button", "type", "advisory"); } + if (key === "i" || key === "x") { + clearSpeedBuffer(); + draft.review_status = "ignore"; + draft.review_sign_type = "not_speed_limit"; + setActive("#statusButtons button", "status", "ignore"); + setActive("#typeButtons button", "type", "not_speed_limit"); + save(true, "ignore"); + return; + } + if (key === "enter") { clearSpeedBuffer(); save(true); } }); loadQueue(); diff --git a/selfdrive/controls/lib/latcontrol_vehicle_tunes.py b/selfdrive/controls/lib/latcontrol_vehicle_tunes.py index 02df13bad..b51ab0450 100644 --- a/selfdrive/controls/lib/latcontrol_vehicle_tunes.py +++ b/selfdrive/controls/lib/latcontrol_vehicle_tunes.py @@ -514,8 +514,8 @@ IONIQ_6_DIRECTIONAL_TAPER_UNWIND_LEFT = 2.15 IONIQ_6_DIRECTIONAL_TAPER_UNWIND_RIGHT = 4.25 IONIQ_6_DIRECTIONAL_TAPER_FLOOR_LEFT = 0.48 IONIQ_6_DIRECTIONAL_TAPER_FLOOR_RIGHT = 0.52 -IONIQ_6_DIRECTIONAL_TAPER_UNWIND_FLOOR_LEFT = 0.16 -IONIQ_6_DIRECTIONAL_TAPER_UNWIND_FLOOR_RIGHT = 0.04 +IONIQ_6_DIRECTIONAL_TAPER_UNWIND_FLOOR_LEFT = 0.20 +IONIQ_6_DIRECTIONAL_TAPER_UNWIND_FLOOR_RIGHT = 0.10 IONIQ_6_DIRECTIONAL_TAPER_JERK_ONSET = 0.60 IONIQ_6_DIRECTIONAL_TAPER_JERK_WIDTH = 0.14 IONIQ_6_DIRECTIONAL_TAPER_LOW_SPEED_RELIEF = 0.98 @@ -523,6 +523,8 @@ IONIQ_6_DIRECTIONAL_TAPER_LOW_SPEED_RELIEF_SPEED = 11.2 IONIQ_6_DIRECTIONAL_TAPER_LOW_SPEED_RELIEF_SPEED_WIDTH = 1.5 IONIQ_6_DIRECTIONAL_TAPER_LOW_SPEED_RELIEF_LAT = 0.10 IONIQ_6_DIRECTIONAL_TAPER_LOW_SPEED_RELIEF_LAT_WIDTH = 0.06 +IONIQ_6_UNWIND_HIGH_SPEED_SPEED = 23.2 +IONIQ_6_UNWIND_HIGH_SPEED_SPEED_WIDTH = 1.7 IONIQ_6_CRAWL_TURN_IN_FF_BOOST_LEFT = 0.18 IONIQ_6_CRAWL_TURN_IN_FF_BOOST_RIGHT = 0.24 IONIQ_6_CRAWL_TURN_IN_FF_SPEED = 5.3 @@ -1786,10 +1788,11 @@ def get_ioniq_6_friction_threshold(v_ego: float, desired_lateral_accel: float = phase = _ioniq_6_transition_phase(desired_lateral_accel, desired_lateral_jerk) turn_in_weight = max(phase, 0.0) unwind_weight = max(-phase, 0.0) + unwind_speed_weight = _ioniq_6_sigmoid((v_ego - IONIQ_6_UNWIND_HIGH_SPEED_SPEED) / IONIQ_6_UNWIND_HIGH_SPEED_SPEED_WIDTH) threshold_scale = 1.0 - (_ioniq_6_side_value(desired_lateral_accel, IONIQ_6_TURN_IN_THRESHOLD_REDUCTION_LEFT, IONIQ_6_TURN_IN_THRESHOLD_REDUCTION_RIGHT) * transition_envelope * turn_in_weight) threshold_scale += (_ioniq_6_side_value(desired_lateral_accel, IONIQ_6_UNWIND_THRESHOLD_INCREASE_LEFT, IONIQ_6_UNWIND_THRESHOLD_INCREASE_RIGHT) * - transition_envelope * unwind_weight) + transition_envelope * unwind_weight * unwind_speed_weight) return base_threshold * min(max(threshold_scale, 0.82), 1.18) @@ -1798,11 +1801,12 @@ def get_ioniq_6_friction_scale(v_ego: float, desired_lateral_accel: float, desir phase = _ioniq_6_transition_phase(desired_lateral_accel, desired_lateral_jerk) turn_in_weight = max(phase, 0.0) unwind_weight = max(-phase, 0.0) + unwind_speed_weight = _ioniq_6_sigmoid((v_ego - IONIQ_6_UNWIND_HIGH_SPEED_SPEED) / IONIQ_6_UNWIND_HIGH_SPEED_SPEED_WIDTH) friction_scale = IONIQ_6_FRICTION_MULT friction_scale += (_ioniq_6_side_value(desired_lateral_accel, IONIQ_6_TURN_IN_FRICTION_BOOST_LEFT, IONIQ_6_TURN_IN_FRICTION_BOOST_RIGHT) * transition_envelope * turn_in_weight) friction_scale -= (_ioniq_6_side_value(desired_lateral_accel, IONIQ_6_UNWIND_FRICTION_REDUCTION_LEFT, IONIQ_6_UNWIND_FRICTION_REDUCTION_RIGHT) * - transition_envelope * unwind_weight) + transition_envelope * unwind_weight * unwind_speed_weight) return min(max(friction_scale, 0.82), 1.08) diff --git a/selfdrive/controls/lib/longcontrol.py b/selfdrive/controls/lib/longcontrol.py index 45a5928b6..5a5917ef6 100644 --- a/selfdrive/controls/lib/longcontrol.py +++ b/selfdrive/controls/lib/longcontrol.py @@ -45,6 +45,21 @@ def get_bolt_acc_pedal_friction_bias(output_accel, a_target, v_ego): return float(min(authority_gap * 0.30, max_bias) * speed_factor) +def get_bolt_acc_pedal_friction_floor(a_target, v_ego, pedal_regen_limit): + if v_ego <= 5.0 or a_target >= (pedal_regen_limit - 0.05): + return None + + friction_request = max(0.0, pedal_regen_limit - a_target) + if friction_request <= 0.10: + return None + + speed_factor = interp(v_ego, [5.0, 8.0, 12.0, 18.0, 25.0], [0.0, 0.45, 0.75, 0.90, 1.0]) + demand_factor = interp(friction_request, [0.10, 0.25, 0.50, 0.90, 1.30], [0.0, 0.22, 0.50, 0.78, 1.0]) + floor_fraction = float(clip(speed_factor * demand_factor, 0.0, 1.0)) + + return float(pedal_regen_limit - (friction_request * floor_fraction)) + + def get_bolt_acc_pedal_feedforward_gain(feedforward_gain, a_target, v_ego, pedal_regen_limit, last_output_accel): effective_gain = feedforward_gain if a_target >= 0.0: @@ -336,8 +351,12 @@ class LongControl: authority_gap = max(0.0, abs(a_target) - abs(output_accel)) if self.is_bolt_acc_pedal_friction_car: + pedal_regen_limit = float(interp(CS.vEgo, BOLT_ACC_PEDAL_REGEN_LIMIT_BP, BOLT_ACC_PEDAL_REGEN_LIMIT_V)) bias = get_bolt_acc_pedal_friction_bias(output_accel, a_target, CS.vEgo) - return output_accel - float(bias) + floor = get_bolt_acc_pedal_friction_floor(a_target, CS.vEgo, pedal_regen_limit) + if floor is not None: + bias = max(bias, output_accel - floor) + return output_accel - float(max(bias, 0.0)) if authority_gap <= 0.40: return output_accel diff --git a/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py b/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py index 051953633..4f9defce0 100755 --- a/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py +++ b/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py @@ -85,6 +85,10 @@ STABLE_FOLLOW_CRUISE_MIN_HEADWAY = 0.95 STABLE_FOLLOW_CRUISE_HEADWAY_BELOW_TARGET = 0.35 STABLE_FOLLOW_CRUISE_HEADWAY_ABOVE_TARGET = 0.90 STABLE_FOLLOW_CRUISE_MAX_LEAD_BRAKE = 0.35 +STABLE_FOLLOW_CRUISE_PULLAWAY_MAX_REL_SPEED = 2.0 +STABLE_FOLLOW_CRUISE_PULLAWAY_MAX_HEADWAY_MARGIN = 0.35 +STABLE_FOLLOW_CRUISE_PULLAWAY_MIN_HEADWAY_MARGIN = -0.10 +STABLE_FOLLOW_CRUISE_PULLAWAY_HYSTERESIS_MAX = 1.75 NEAR_DUPLICATE_LEAD_SOURCE_MIN_SPEED = 20.0 NEAR_DUPLICATE_IDENTICAL_RADAR_SOURCE_MIN_SPEED = 10.0 NEAR_DUPLICATE_LEAD_SOURCE_MIN_MODEL_PROB = 0.9 @@ -596,6 +600,8 @@ class LongitudinalMpc: return 0.0 lead_radar = bool(getattr(lead, "radar", False)) + relative_speed = float(v_ego) - float(lead.vLead) + actual_headway = None lead_brake = max(0.0, -float(getattr(lead, "aLeadK", 0.0))) if lead_brake > STABLE_FOLLOW_CRUISE_MAX_LEAD_BRAKE: return 0.0 @@ -603,7 +609,6 @@ class LongitudinalMpc: if lead_radar: if float(t_follow) <= 0.0 or float(v_ego) < STABLE_FOLLOW_CRUISE_MIN_SPEED: return 0.0 - relative_speed = float(v_ego) - float(lead.vLead) if abs(relative_speed) > STABLE_FOLLOW_CRUISE_MAX_REL_SPEED: return 0.0 @@ -624,9 +629,23 @@ class LongitudinalMpc: min_speed=STABLE_FOLLOW_CRUISE_MIN_SPEED, ): return 0.0 + actual_headway = float(lead.dRel) / max(float(v_ego), 1e-3) - return max(STABLE_FOLLOW_CRUISE_HYSTERESIS_MIN, - STABLE_FOLLOW_CRUISE_HYSTERESIS_GAIN * float(v_ego)) + hysteresis = max(STABLE_FOLLOW_CRUISE_HYSTERESIS_MIN, + STABLE_FOLLOW_CRUISE_HYSTERESIS_GAIN * float(v_ego)) + + if relative_speed < 0.0: + headway_margin = actual_headway - float(t_follow) + if headway_margin <= STABLE_FOLLOW_CRUISE_PULLAWAY_MAX_HEADWAY_MARGIN: + rel_speed_factor = float(np.clip((-relative_speed) / STABLE_FOLLOW_CRUISE_PULLAWAY_MAX_REL_SPEED, 0.0, 1.0)) + headway_factor = float(np.clip( + (STABLE_FOLLOW_CRUISE_PULLAWAY_MAX_HEADWAY_MARGIN - headway_margin) / + max(STABLE_FOLLOW_CRUISE_PULLAWAY_MAX_HEADWAY_MARGIN - STABLE_FOLLOW_CRUISE_PULLAWAY_MIN_HEADWAY_MARGIN, 1e-3), + 0.0, 1.0, + )) + hysteresis += STABLE_FOLLOW_CRUISE_PULLAWAY_HYSTERESIS_MAX * rel_speed_factor * headway_factor + + return hysteresis @staticmethod def leads_share_identical_radar_track(lead_one, lead_two): diff --git a/selfdrive/controls/tests/test_longcontrol.py b/selfdrive/controls/tests/test_longcontrol.py index cd2dcccd1..71ba1748e 100644 --- a/selfdrive/controls/tests/test_longcontrol.py +++ b/selfdrive/controls/tests/test_longcontrol.py @@ -520,6 +520,46 @@ def test_bolt_acc_pedal_friction_feedforward_blends_back_in_for_small_friction_r assert lc._get_longitudinal_feedforward(a_target, 20.0) == pytest.approx(expected) +def test_bolt_acc_pedal_friction_floor_holds_friction_only_authority(): + pedal_regen_limit = float(longcontrol.interp(9.85, longcontrol.BOLT_ACC_PEDAL_REGEN_LIMIT_BP, + longcontrol.BOLT_ACC_PEDAL_REGEN_LIMIT_V)) + floor = longcontrol.get_bolt_acc_pedal_friction_floor(-3.47, 9.85, pedal_regen_limit) + + assert floor is not None + assert floor < pedal_regen_limit + assert floor > -3.47 + + +def test_bolt_acc_pedal_friction_bias_applies_floor_only_on_experimental_fingerprint(): + pedal_cp = make_longcontrol_cp( + brand="gm", + enableGasInterceptorDEPRECATED=True, + flags=GMFlags.PEDAL_LONG.value, + carFingerprint=CAR.CHEVROLET_BOLT_ACC_2022_2023_PEDAL, + ) + bolt_cc_cp = make_longcontrol_cp( + brand="gm", + enableGasInterceptorDEPRECATED=True, + flags=GMFlags.PEDAL_LONG.value, + carFingerprint=CAR.CHEVROLET_BOLT_CC_2022_2023, + ) + + pedal_lc = LongControl(pedal_cp) + bolt_cc_lc = LongControl(bolt_cc_cp) + CS = car.CarState.new_message(vEgo=9.85, aEgo=-2.0, brakePressed=False) + + pedal_regen_limit = float(longcontrol.interp(CS.vEgo, longcontrol.BOLT_ACC_PEDAL_REGEN_LIMIT_BP, + longcontrol.BOLT_ACC_PEDAL_REGEN_LIMIT_V)) + floor = longcontrol.get_bolt_acc_pedal_friction_floor(-3.47, CS.vEgo, pedal_regen_limit) + assert floor is not None + + pedal_biased = pedal_lc._apply_pedal_long_brake_bias(-1.85, -3.47, CS) + bolt_cc_biased = bolt_cc_lc._apply_pedal_long_brake_bias(-1.85, -3.47, CS) + + assert pedal_biased == pytest.approx(floor) + assert bolt_cc_biased > pedal_biased + 0.5 + + def test_bolt_acc_pedal_feedforward_gain_stays_base_for_mild_regen(): gain = longcontrol.get_bolt_acc_pedal_feedforward_gain(0.2, -1.0, 10.0, -2.75, -0.4) diff --git a/selfdrive/controls/tests/test_longitudinal_planner.py b/selfdrive/controls/tests/test_longitudinal_planner.py index 2042d8500..2f13032e0 100644 --- a/selfdrive/controls/tests/test_longitudinal_planner.py +++ b/selfdrive/controls/tests/test_longitudinal_planner.py @@ -2988,6 +2988,20 @@ def test_stable_follow_cruise_hysteresis_applies_for_radar_lead(): assert hysteresis > 0.0 +def test_stable_follow_cruise_hysteresis_holds_pullaway_lead_longer_near_target_gap(): + v_ego = 15.0 + t_follow = 1.45 + CP = CarInterface.get_non_essential_params(CAR.HONDA_CIVIC) + planner = LongitudinalPlanner(CP, init_v=v_ego) + lead_matched = make_lead(status=True, d_rel=22.0, v_lead=15.0, a_lead=0.02, radar=True, model_prob=1.0) + lead_pullaway = make_lead(status=True, d_rel=22.0, v_lead=16.4, a_lead=0.02, radar=True, model_prob=1.0) + + matched_hysteresis = planner.mpc.get_stable_follow_cruise_hysteresis(lead_matched, v_ego, t_follow) + pullaway_hysteresis = planner.mpc.get_stable_follow_cruise_hysteresis(lead_pullaway, v_ego, t_follow) + + assert pullaway_hysteresis > matched_hysteresis + + def test_stable_follow_cruise_hysteresis_skips_fast_closing_radar_lead(): v_ego = 27.0 CP = CarInterface.get_non_essential_params(CAR.HONDA_CIVIC) diff --git a/selfdrive/ui/mici/layouts/main.py b/selfdrive/ui/mici/layouts/main.py index 95258e279..d32513668 100644 --- a/selfdrive/ui/mici/layouts/main.py +++ b/selfdrive/ui/mici/layouts/main.py @@ -1,7 +1,6 @@ import pyray as rl import cereal.messaging as messaging from openpilot.selfdrive.ui.mici.layouts.home import MiciHomeLayout -from openpilot.selfdrive.ui.mici.layouts.settings.settings import SettingsLayout from openpilot.selfdrive.ui.mici.layouts.offroad_alerts import MiciOffroadAlerts from openpilot.selfdrive.ui.mici.onroad.augmented_road_view import AugmentedRoadView from openpilot.selfdrive.ui.ui_state import device, ui_state @@ -28,11 +27,11 @@ class MiciMainLayout(Scroller): # Initialize widgets self._home_layout = MiciHomeLayout() self._alerts_layout = MiciOffroadAlerts() - self._settings_layout = SettingsLayout() + self._settings_layout = None self._onroad_layout = AugmentedRoadView(bookmark_callback=self._on_bookmark_clicked) # Initialize widget rects - for widget in (self._home_layout, self._settings_layout, self._alerts_layout, self._onroad_layout): + for widget in (self._home_layout, self._alerts_layout, self._onroad_layout): # TODO: set parent rect and use it if never passed rect from render (like in Scroller) widget.set_rect(rl.Rectangle(0, 0, gui_app.width, gui_app.height)) @@ -58,10 +57,18 @@ class MiciMainLayout(Scroller): gui_app.push_widget(self._onboarding_window) def _setup_callbacks(self): - self._home_layout.set_callbacks(on_settings=lambda: gui_app.push_widget(self._settings_layout)) + self._home_layout.set_callbacks(on_settings=self._open_settings) self._onroad_layout.set_click_callback(lambda: self._scroll_to(self._home_layout)) device.add_interactive_timeout_callback(self._on_interactive_timeout) + def _open_settings(self): + if self._settings_layout is None: + from openpilot.selfdrive.ui.mici.layouts.settings.settings import SettingsLayout + + self._settings_layout = SettingsLayout() + self._settings_layout.set_rect(rl.Rectangle(0, 0, gui_app.width, gui_app.height)) + gui_app.push_widget(self._settings_layout) + def _scroll_to(self, layout: Widget): layout_x = int(layout.rect.x) self._scroller.scroll_to(layout_x, smooth=True) diff --git a/selfdrive/ui/mici/layouts/settings/driving_model.py b/selfdrive/ui/mici/layouts/settings/driving_model.py index 677701efa..e2bc259ae 100644 --- a/selfdrive/ui/mici/layouts/settings/driving_model.py +++ b/selfdrive/ui/mici/layouts/settings/driving_model.py @@ -5,22 +5,22 @@ import threading import time from dataclasses import dataclass from collections.abc import Callable +from pathlib import Path from openpilot.common.params import Params -from openpilot.starpilot.assets.model_manager import ( - CANCEL_DOWNLOAD_PARAM, - DOWNLOAD_PROGRESS_PARAM, - ModelManager, -) -from openpilot.starpilot.common.starpilot_variables import MODELS_PATH from openpilot.selfdrive.ui.mici.widgets.button import BigButton from openpilot.selfdrive.ui.mici.widgets.dialog import BigDialog, BigDialogBase, BigMultiOptionDialog from openpilot.selfdrive.ui.ui_state import ui_state +from openpilot.system.hardware import PC +from openpilot.system.hardware.hw import Paths from openpilot.system.ui.lib.application import gui_app, FontWeight from openpilot.system.ui.widgets import Widget from openpilot.system.ui.widgets.label import gui_label import pyray as rl +CANCEL_DOWNLOAD_PARAM = "CancelModelDownload" +DOWNLOAD_PROGRESS_PARAM = "ModelDownloadProgress" +MODELS_PATH = Path(Paths.comma_home()) / "starpilot" / "data" / "models" if PC else Path("/data/models") MANIFEST_STALE_SECONDS = 60 * 60 _PROGRESS_HOLD_SECONDS = 2.5 _DOWNLOAD_DIALOG_CLOSE_SECONDS = 1.0 @@ -280,7 +280,7 @@ class DrivingModelBigButton(BigButton): super().__init__("driving model", "", gui_app.texture("icons_mici/settings/device/lkas.png", 72, 56)) self._params = Params() self._params_memory = Params(memory=True) - self._model_manager = ModelManager(self._params, self._params_memory) + self._model_manager = None self._worker_thread: threading.Thread | None = None self._active_job = "" @@ -294,6 +294,13 @@ class DrivingModelBigButton(BigButton): self.set_click_callback(self._open_manager_menu) self.refresh() + def _get_model_manager(self): + if self._model_manager is None: + from openpilot.starpilot.assets.model_manager import ModelManager + + self._model_manager = ModelManager(self._params, self._params_memory) + return self._model_manager + def show_event(self): super().show_event() self._sub_label.reset_scroll() @@ -515,7 +522,7 @@ class DrivingModelBigButton(BigButton): def _run_download_one(self, model_key: str): self._params_memory.put_bool(CANCEL_DOWNLOAD_PARAM, False) - self._model_manager.download_model(model_key) + self._get_model_manager().download_model(model_key) entries = {entry.key: entry for entry in self._load_model_entries()} entry = entries.get(model_key) @@ -525,10 +532,10 @@ class DrivingModelBigButton(BigButton): def _run_download_all(self): self._params_memory.put_bool(CANCEL_DOWNLOAD_PARAM, False) - self._model_manager.download_all_models() + self._get_model_manager().download_all_models() def _run_manifest_refresh(self): - self._model_manager.update_models() + self._get_model_manager().update_models() def _switch_model(self, model_key: str): entries = {entry.key: entry for entry in self._load_model_entries()} diff --git a/selfdrive/ui/mici/layouts/settings/software.py b/selfdrive/ui/mici/layouts/settings/software.py index caec6bdca..909623609 100644 --- a/selfdrive/ui/mici/layouts/settings/software.py +++ b/selfdrive/ui/mici/layouts/settings/software.py @@ -2,6 +2,8 @@ import os import threading from collections.abc import Callable from enum import IntEnum +from functools import cache +from pathlib import Path import pyray as rl @@ -11,7 +13,7 @@ from openpilot.selfdrive.ui.mici.layouts.settings.device import EngagedConfirmat from openpilot.selfdrive.ui.mici.widgets.button import BigButton, BigParamControl from openpilot.selfdrive.ui.mici.widgets.dialog import BigConfirmationDialog, BigDialog from openpilot.selfdrive.ui.ui_state import ui_state -from openpilot.starpilot.common.starpilot_utilities import is_FrogsGoMoo +from openpilot.system.hardware.hw import Paths from openpilot.system.ui.lib.application import FontWeight, MousePos, gui_app from openpilot.system.ui.lib.multilang import tr from openpilot.system.ui.widgets import Widget @@ -21,6 +23,11 @@ from openpilot.system.ui.widgets.scroller import NavScroller UPDATER_TIMEOUT = 10.0 +@cache +def _is_frogs_go_moo() -> bool: + return (Path(Paths.persist_root()) / "frogsgomoo.py").is_file() + + def _split_description(desc: str) -> tuple[str, str, str, str] | None: parts = [p.strip() for p in desc.split(" / ")] if len(parts) != 4: @@ -226,7 +233,7 @@ class BranchSelectPage(NavScroller): branches_str = params.get("UpdaterAvailableBranches") or "" branches = [b for b in branches_str.split(",") if b] - if not is_FrogsGoMoo(): + if not _is_frogs_go_moo(): for hidden_branch in ("StarPilot-Vetting", "MAKE-PRS-HERE"): if hidden_branch in branches: branches.remove(hidden_branch) diff --git a/starpilot/common/connect_server.py b/starpilot/common/connect_server.py index 7941d6a40..19aa0ba2e 100644 --- a/starpilot/common/connect_server.py +++ b/starpilot/common/connect_server.py @@ -1,11 +1,10 @@ from pathlib import Path from openpilot.common.params import Params +from openpilot.common.api import use_konik_server from openpilot.system.athena.registration import register from openpilot.system.hardware.hw import Paths -from openpilot.starpilot.common.starpilot_utilities import use_konik_server - def _cache_params_path() -> str: return Paths.params_cache_root() diff --git a/system/athena/athenad.py b/system/athena/athenad.py index ed167c417..d77c8da7e 100755 --- a/system/athena/athenad.py +++ b/system/athena/athenad.py @@ -30,7 +30,7 @@ from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutExce import cereal.messaging as messaging from cereal import log from cereal.services import SERVICE_LIST -from openpilot.common.api import Api, get_key_pair +from openpilot.common.api import Api, get_key_pair, use_konik_server from openpilot.common.utils import CallbackReader, get_upload_stream from openpilot.common.params import Params from openpilot.common.realtime import set_core_affinity @@ -41,9 +41,6 @@ from openpilot.system.athena.registration import UNREGISTERED_DONGLE_ID from openpilot.system.version import get_build_metadata from openpilot.system.hardware.hw import Paths -from openpilot.starpilot.common.starpilot_utilities import use_konik_server - - ATHENA_HOST = os.getenv('ATHENA_HOST', f"wss://athena.{'konik.ai' if use_konik_server() else 'comma.ai'}") HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4")) LOCAL_PORT_WHITELIST = {22, } # SSH diff --git a/system/manager/manager.py b/system/manager/manager.py index 55cae6cb2..34dbcda6c 100755 --- a/system/manager/manager.py +++ b/system/manager/manager.py @@ -8,6 +8,7 @@ import sys import threading import time import traceback +from types import SimpleNamespace _MANAGER_IMPORT_START = time.monotonic() _BOOT_TIMING_LOG_PATH = os.environ.get("SP_BOOT_TIMING_LOG", "/tmp/starpilot_boot_timing.log") @@ -37,14 +38,15 @@ _MANAGER_CORE_IMPORT_DONE = time.monotonic() from openpilot.starpilot.common.starpilot_functions import starpilot_boot_functions, install_starpilot, uninstall_starpilot _MANAGER_IMPORT_DONE = time.monotonic() -_manager_import_timing_line = ( - "SP_BOOT_TIMING manager_import " - f"core={_MANAGER_CORE_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s " - f"starpilot={_MANAGER_IMPORT_DONE - _MANAGER_CORE_IMPORT_DONE:.3f}s " - f"total={_MANAGER_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s" -) -print(_manager_import_timing_line, flush=True) -_append_boot_timing_line(_manager_import_timing_line) +if __name__ == "__main__": + _manager_import_timing_line = ( + "SP_BOOT_TIMING manager_import " + f"core={_MANAGER_CORE_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s " + f"starpilot={_MANAGER_IMPORT_DONE - _MANAGER_CORE_IMPORT_DONE:.3f}s " + f"total={_MANAGER_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s" + ) + print(_manager_import_timing_line, flush=True) + _append_boot_timing_line(_manager_import_timing_line) LEGACY_BOLT_FP_MIGRATION_FLAG = Path("/data") / "legacy_bolt_fp_migration_v1" @@ -58,8 +60,15 @@ STARPILOT_PC_ROOT_MIGRATION_FLAG = Path("/data") / "starpilot_pc_root_v1" STARPILOT_PARAMS_CACHE_MIGRATION_FLAG = Path("/data") / "starpilot_params_cache_v1" STARPILOT_LEGACY_CACHE_MARKER_KEYS = ("RemapCancelToDistance",) STARPILOT_REMOVED_PARAM_KEYS = ("HumanFollowing",) +PARAMS_CACHE_RESTORE_SKIP_FLAGS = ( + ParamKeyFlag.CLEAR_ON_MANAGER_START + | ParamKeyFlag.CLEAR_ON_ONROAD_TRANSITION + | ParamKeyFlag.CLEAR_ON_OFFROAD_TRANSITION + | ParamKeyFlag.CLEAR_ON_IGNITION_ON +) UNREGISTERED_DONGLE_ID = "UnregisteredDevice" POWER_WATCHDOG_PATH = "/var/tmp/power_watchdog" +_PENDING_PARAMS_CACHE_SYNC: tuple[str, list[str]] | None = None LEGACY_CARMODEL_MIGRATIONS = { "CHEVROLET_BOLT_CC_2019_2021": "CHEVROLET_BOLT_CC_2018_2021", } @@ -88,16 +97,86 @@ def _log_boot_timing(scope: str, label: str, start: float, previous: float | Non return now -def _sync_params_cache_async(cache_params_path: str, values: list[tuple[bytes | str, object]]) -> None: +def _sync_params_cache_keys_async(cache_params_path: str, keys: list[str]) -> None: try: + params = Params() params_cache = Params(cache_params_path, return_defaults=True) - for key, value in values: - if params_cache.get(key) != value: - params_cache.put(key, value) + for key in keys: + current_value = params.get(key) + if current_value is not None and params_cache.get(key) != current_value: + params_cache.put(key, current_value) except Exception: cloudlog.exception("failed to sync params cache") +def _schedule_params_cache_sync(cache_params_path: str, keys: list[str]) -> None: + timer = threading.Timer(5.0, _sync_params_cache_keys_async, args=(cache_params_path, keys)) + timer.daemon = True + timer.start() + + +def _schedule_pending_params_cache_sync() -> None: + global _PENDING_PARAMS_CACHE_SYNC + + pending_sync = _PENDING_PARAMS_CACHE_SYNC + _PENDING_PARAMS_CACHE_SYNC = None + if pending_sync is not None and pending_sync[1]: + _schedule_params_cache_sync(*pending_sync) + + +def _iter_param_store_keys(store_path: str | Path) -> set[str]: + try: + path = Path(store_path) + if not path.is_dir(): + return set() + + with os.scandir(path) as entries: + return { + entry.name + for entry in entries + if entry.is_file(follow_symlinks=False) and entry.name != ".lock" and not entry.name.startswith(".tmp_") + } + except Exception: + cloudlog.exception(f"failed to list params store: {store_path}") + return set() + + +def _param_key_to_text(key: bytes | str) -> str: + return key.decode("utf-8", errors="ignore") if isinstance(key, bytes) else str(key) + + +def _should_restore_param_from_cache(params: Params, key: bytes | str) -> bool: + try: + return not (params.get_key_flag(key) & PARAMS_CACHE_RESTORE_SKIP_FLAGS) + except Exception: + return False + + +def _restore_missing_params_from_cache(params: Params, params_cache: Params, active_keys: set[str] | None = None) -> list[str]: + active_keys = active_keys if active_keys is not None else _iter_param_store_keys(params.get_param_path()) + restored_keys: list[str] = [] + + for raw_key in params.all_keys(): + key = _param_key_to_text(raw_key) + if key in active_keys: + continue + + if not _should_restore_param_from_cache(params, raw_key): + continue + + cached_value = params_cache.get(raw_key) + if cached_value is None: + continue + + try: + params.put(raw_key, cached_value) + restored_keys.append(key) + except Exception: + cloudlog.exception(f"failed to restore param from cache: {key}") + + return restored_keys + + def _init_sentry_async() -> None: def fn() -> None: try: @@ -143,6 +222,29 @@ def _get_starpilot_toggles(sm=None): return get_starpilot_toggles(sm) +def _get_manager_startup_toggles(params: Params | None = None) -> SimpleNamespace: + params = params or Params() + force_onroad = params.get_bool("ForceOnroad") + device_management = params.get_bool("DeviceManagement") + vetting_branch = os.environ.get("GIT_BRANCH") == "StarPilot-Vetting" + + no_logging = False + no_uploads = False + if device_management and not vetting_branch: + no_logging = params.get_bool("NoLogging") + no_uploads = params.get_bool("NoUploads") + + return SimpleNamespace( + force_offroad=params.get_bool("ForceOffroad"), + force_onroad=force_onroad, + no_logging=no_logging or (force_onroad and HARDWARE.get_device_type() == "pc"), + no_uploads=no_uploads, + no_onroad_uploads=params.get_bool("DisableOnroadUploads") if no_uploads else False, + speed_limit_filler=params.get_bool("SpeedLimitFiller"), + vision_speed_limit_detection=params.get_bool("VisionSpeedLimitDetection"), + ) + + def _to_text(value): if value is None: return None @@ -824,11 +926,12 @@ def migrate_legacy_experimental_longitudinal(params: Params, params_cache: Param def manager_init() -> None: + global _PENDING_PARAMS_CACHE_SYNC + manager_init_start = time.monotonic() last_timing = _log_boot_timing("manager_init", "start", manager_init_start, manager_init_start) - save_bootlog() - last_timing = _log_boot_timing("manager_init", "save_bootlog", manager_init_start, last_timing) + last_timing = _log_boot_timing("manager_init", "save_bootlog_deferred", manager_init_start, last_timing) build_metadata = get_build_metadata() last_timing = _log_boot_timing("manager_init", "build_metadata", manager_init_start, last_timing) @@ -874,17 +977,11 @@ def manager_init() -> None: last_timing = _log_boot_timing("manager_init", "starpilot_migrations", manager_init_start, last_timing) # set unset params to their default value - params_cache_updates = [] - for k in params.all_keys(): - current_value = params.get(k) - if current_value is None: - cached_value = params_cache.get(k) - if cached_value is not None: - params.put(k, cached_value) - else: - params_cache_updates.append((k, current_value)) - if params_cache_updates: - threading.Thread(target=_sync_params_cache_async, args=(cache_params_path, params_cache_updates), daemon=True).start() + active_param_keys = _iter_param_store_keys(params.get_param_path()) + last_timing = _log_boot_timing("manager_init", f"params_store_snapshot_{len(active_param_keys)}", manager_init_start, last_timing) + restored_param_keys = _restore_missing_params_from_cache(params, params_cache, active_param_keys) + last_timing = _log_boot_timing("manager_init", f"params_restore_{len(restored_param_keys)}", manager_init_start, last_timing) + params_cache_update_keys = sorted(active_param_keys) last_timing = _log_boot_timing("manager_init", "params_defaults_cache_sync", manager_init_start, last_timing) # Create folders needed for msgq @@ -938,7 +1035,6 @@ def manager_init() -> None: commit=build_metadata.openpilot.git_commit, dirty=build_metadata.openpilot.is_dirty, device=HARDWARE.get_device_type()) - _init_sentry_async() last_timing = _log_boot_timing("manager_init", "logging_ready", manager_init_start, last_timing) # Preimporting every process serializes a lot of import work before manager can @@ -956,6 +1052,7 @@ def manager_init() -> None: last_timing = _log_boot_timing("manager_init", "install_starpilot", manager_init_start, last_timing) starpilot_boot_functions(build_metadata, params) _log_boot_timing("manager_init", "starpilot_boot_functions", manager_init_start, last_timing) + _PENDING_PARAMS_CACHE_SYNC = (cache_params_path, params_cache_update_keys) def manager_cleanup() -> None: @@ -993,7 +1090,7 @@ def manager_thread() -> None: last_timing = _log_boot_timing("manager_thread", "messaging", manager_thread_start, last_timing) write_onroad_params(False, params) - initial_toggles = _get_starpilot_toggles() + initial_toggles = _get_manager_startup_toggles(params) last_timing = _log_boot_timing("manager_thread", "initial_toggles", manager_thread_start, last_timing) ensure_running(managed_processes.values(), False, params=params, CP=sm['carParams'], not_run=ignore, starpilot_toggles=initial_toggles) last_timing = _log_boot_timing("manager_thread", "initial_ensure_running", manager_thread_start, last_timing) @@ -1009,9 +1106,12 @@ def manager_thread() -> None: params_memory = Params(memory=True) - starpilot_toggles = _get_starpilot_toggles() + starpilot_toggles = _get_manager_startup_toggles(params) last_timing = _log_boot_timing("manager_thread", "loop_toggles", manager_thread_start, last_timing) _log_boot_timing("manager_thread", "loop_ready", manager_thread_start, last_timing) + save_bootlog() + _init_sentry_async() + _schedule_pending_params_cache_sync() while True: sm.update(1000) diff --git a/system/manager/process.py b/system/manager/process.py index 11e1cbdfc..aadf38f82 100644 --- a/system/manager/process.py +++ b/system/manager/process.py @@ -6,6 +6,8 @@ import struct import threading import time import subprocess +import multiprocessing +import sys from pathlib import Path from collections.abc import Callable, ValuesView from abc import ABC, abstractmethod @@ -22,6 +24,7 @@ from openpilot.common.swaglog import cloudlog from openpilot.common.watchdog import WATCHDOG_FN ENABLE_WATCHDOG = os.getenv("NO_WATCHDOG") is None +PYTHON_PROCESS_START_METHOD = os.getenv("PYTHON_PROCESS_START_METHOD", "subprocess") DEBUG_ENV_KEYS = ( "XDG_RUNTIME_DIR", @@ -465,11 +468,33 @@ def join_process(process: Process, timeout: float) -> None: time.sleep(0.001) +class SubprocessProcess: + def __init__(self, proc: subprocess.Popen): + self._proc = proc + + @property + def pid(self) -> int | None: + return self._proc.pid + + @property + def exitcode(self) -> int | None: + return self._proc.poll() + + def is_alive(self) -> bool: + return self._proc.poll() is None + + def join(self, timeout: float | None = None) -> None: + try: + self._proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + pass + + class ManagerProcess(ABC): daemon = False sigkill = False should_run: Callable[[bool, Params, car.CarParams, SimpleNamespace], bool] - proc: Process | None = None + proc: Process | SubprocessProcess | None = None enabled = True name = "" @@ -657,8 +682,19 @@ class PythonProcess(ManagerProcess): name = self.name if "modeld" not in self.name else "MainProcess" cloudlog.info(f"starting python {self.module}") - self.proc = Process(name=name, target=self.launcher, args=(self.module, self.name, self.nice)) - self.proc.start() + if PYTHON_PROCESS_START_METHOD == "subprocess": + launcher_code = ( + "from openpilot.system.manager.process import launcher; " + f"launcher({self.module!r}, {self.name!r}, {self.nice!r})" + ) + self.proc = SubprocessProcess(subprocess.Popen([sys.executable, "-c", launcher_code])) + else: + self.proc = multiprocessing.get_context(PYTHON_PROCESS_START_METHOD).Process( + name=name, + target=self.launcher, + args=(self.module, self.name, self.nice), + ) + self.proc.start() self.last_watchdog_time = 0 self.watchdog_seen = False self.shutting_down = False diff --git a/system/manager/test/test_manager.py b/system/manager/test/test_manager.py index fb6447de2..7b908807b 100644 --- a/system/manager/test/test_manager.py +++ b/system/manager/test/test_manager.py @@ -6,7 +6,7 @@ from pathlib import Path import json from cereal import car -from openpilot.common.params import Params +from openpilot.common.params import Params, ParamKeyFlag import openpilot.system.manager.manager as manager from openpilot.system.manager.process import ensure_running from openpilot.system.manager.process_config import managed_processes, procs @@ -19,15 +19,29 @@ BLACKLIST_PROCS = ['manage_athenad', 'pandad', 'pigeond'] class FileBackedFakeParams: - def __init__(self, root: Path, values: dict[str, object] | None = None): + def __init__( + self, + root: Path, + values: dict[str, object] | None = None, + keys: set[str] | None = None, + flags: dict[str, ParamKeyFlag] | None = None, + ): self.root = root + self.keys = set(keys or []) + self.flags = dict(flags or {}) self.root.mkdir(parents=True, exist_ok=True) for key, value in (values or {}).items(): self.put(key, value) - def get_param_path(self, key): + def get_param_path(self, key=""): return str(self.root / (key.decode() if isinstance(key, bytes) else str(key))) + def all_keys(self): + return sorted(self.keys) + + def get_key_flag(self, key): + return self.flags.get(key.decode() if isinstance(key, bytes) else str(key), ParamKeyFlag.PERSISTENT) + def get(self, key): path = Path(self.get_param_path(key)) if not path.is_file(): @@ -48,6 +62,7 @@ class FileBackedFakeParams: return str(value).strip().lower() in ("1", "true", "yes", "on") def put(self, key, value): + self.keys.add(key.decode() if isinstance(key, bytes) else str(key)) path = Path(self.get_param_path(key)) path.parent.mkdir(parents=True, exist_ok=True) @@ -96,6 +111,59 @@ class TestManager: assert names.index("the_galaxy") < ui_idx assert names.index("galaxy") < ui_idx + def test_manager_startup_toggles_use_params_only(self, tmp_path, monkeypatch): + monkeypatch.setenv("GIT_BRANCH", "StarPilot") + params = FileBackedFakeParams(tmp_path / "params", { + "DeviceManagement": True, + "NoLogging": True, + "NoUploads": True, + "DisableOnroadUploads": True, + "SpeedLimitFiller": True, + "VisionSpeedLimitDetection": True, + "ForceOffroad": True, + "ForceOnroad": False, + }) + + toggles = manager._get_manager_startup_toggles(params) + + assert toggles.no_logging is True + assert toggles.no_uploads is True + assert toggles.no_onroad_uploads is True + assert toggles.speed_limit_filler is True + assert toggles.vision_speed_limit_detection is True + assert toggles.force_offroad is True + assert toggles.force_onroad is False + + def test_restore_missing_params_from_cache_preserves_live_values(self, tmp_path): + params = FileBackedFakeParams( + tmp_path / "params", + {"ExistingParam": "live"}, + keys={"ExistingParam", "RestoredParam", "MissingParam", "TransientParam"}, + flags={"TransientParam": ParamKeyFlag.CLEAR_ON_MANAGER_START}, + ) + params_cache = FileBackedFakeParams( + tmp_path / "cache", + {"ExistingParam": "cached", "RestoredParam": "restored", "TransientParam": "stale"}, + ) + + restored_keys = manager._restore_missing_params_from_cache(params, params_cache) + + assert restored_keys == ["RestoredParam"] + assert params.get("ExistingParam") == "live" + assert params.get("RestoredParam") == "restored" + assert params.get("MissingParam") is None + assert params.get("TransientParam") is None + + def test_iter_param_store_keys_skips_lock_and_temp_files(self, tmp_path): + store_path = tmp_path / "params" + store_path.mkdir() + (store_path / "GoodParam").write_text("1") + (store_path / ".lock").write_text("") + (store_path / ".tmp_value_abc").write_text("stale") + (store_path / "nested").mkdir() + + assert manager._iter_param_store_keys(store_path) == {"GoodParam"} + def test_blacklisted_procs(self): # TODO: ensure there are blacklisted procs until we have a dedicated test assert len(BLACKLIST_PROCS), "No blacklisted procs to test not_run" diff --git a/system/tests/test_version.py b/system/tests/test_version.py new file mode 100644 index 000000000..47f259837 --- /dev/null +++ b/system/tests/test_version.py @@ -0,0 +1,67 @@ +import json + +from openpilot.system import version + + +def _write_minimal_git_checkout(path, branch="StarPilot", commit="1" * 40, origin="https://example.com/openpilot"): + git_folder = path / ".git" + git_folder.mkdir() + (git_folder / "refs" / "heads").mkdir(parents=True) + (git_folder / "HEAD").write_text(f"ref: refs/heads/{branch}\n", encoding="utf-8") + (git_folder / "refs" / "heads" / branch).write_text(f"{commit}\n", encoding="utf-8") + (git_folder / "config").write_text( + f"""[remote "origin"] + url = {origin} +[branch "{branch}"] + remote = origin + merge = refs/heads/{branch} +""", + encoding="utf-8", + ) + return { + "branch": branch, + "commit": commit, + "origin": origin, + } + + +def test_read_git_metadata_reads_head_ref_and_origin(tmp_path): + git_metadata = _write_minimal_git_checkout(tmp_path, branch="Dom", commit="a" * 40) + + assert version._read_git_metadata(str(tmp_path)) == git_metadata + + +def test_get_build_metadata_uses_prebuilt_git_cache(tmp_path, monkeypatch): + git_metadata = _write_minimal_git_checkout(tmp_path, branch="Dom", commit="b" * 40) + cache_root = tmp_path / "cache" + monkeypatch.setenv("OPENPILOT_BUILD_METADATA_CACHE_ROOT", str(cache_root)) + (tmp_path / "prebuilt").write_text("", encoding="utf-8") + (tmp_path / "common").mkdir() + (tmp_path / "common" / "version.h").write_text('#define COMMA_VERSION "1.2.3"\n', encoding="utf-8") + (tmp_path / "RELEASES.md").write_text("Release notes\n\nOlder notes\n", encoding="utf-8") + + cached_payload = { + "path": str(tmp_path.resolve()), + "git_metadata": git_metadata, + "build_metadata": { + "channel": "Dom", + "openpilot": { + "version": "1.2.3", + "release_notes": "Release notes", + "git_commit": git_metadata["commit"], + "git_origin": git_metadata["origin"], + "git_commit_date": "123 1970-01-01 00:02:03 +0000", + "build_style": "unknown", + "is_dirty": False, + }, + }, + } + cache_root.mkdir() + (cache_root / version.BUILD_METADATA_CACHE_FILENAME).write_text(json.dumps(cached_payload), encoding="utf-8") + monkeypatch.setattr(version, "get_commit_date", lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("cache miss"))) + + build_metadata = version.get_build_metadata(str(tmp_path)) + + assert build_metadata.channel == "Dom" + assert build_metadata.openpilot.git_commit == git_metadata["commit"] + assert build_metadata.openpilot.git_commit_date == "123 1970-01-01 00:02:03 +0000" diff --git a/system/version.py b/system/version.py index ead7aa1e4..2c49d1cbe 100644 --- a/system/version.py +++ b/system/version.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from dataclasses import dataclass +from dataclasses import asdict, dataclass from functools import cache import json import os @@ -14,6 +14,7 @@ RELEASE_BRANCHES = ['StarPilot', 'StarPilot-Vetting'] TESTED_BRANCHES = RELEASE_BRANCHES + ['StarPilot-Staging', 'StarPilot-Testing'] BUILD_METADATA_FILENAME = "build.json" +BUILD_METADATA_CACHE_FILENAME = "starpilot_build_metadata_cache.json" training_version: str = "0.2.0" terms_version: str = "2" @@ -132,6 +133,133 @@ def build_metadata_from_dict(build_metadata: dict) -> BuildMetadata: is_dirty=False)) +def _read_text_file(path: pathlib.Path) -> str | None: + try: + return path.read_text(encoding="utf-8").strip() + except OSError: + return None + + +def _read_git_ref(git_folder: pathlib.Path, ref: str) -> str | None: + ref_value = _read_text_file(git_folder / ref) + if ref_value: + return ref_value.splitlines()[0].strip() + + packed_refs = _read_text_file(git_folder / "packed-refs") + if packed_refs is None: + return None + + for line in packed_refs.splitlines(): + if not line or line.startswith(("#", "^")): + continue + + parts = line.split() + if len(parts) == 2 and parts[1] == ref: + return parts[0] + + return None + + +def _read_git_config_value(git_folder: pathlib.Path, section: str, key: str) -> str | None: + config = _read_text_file(git_folder / "config") + if config is None: + return None + + current_section = None + for raw_line in config.splitlines(): + line = raw_line.strip() + if not line or line.startswith(("#", ";")): + continue + + if line.startswith("[") and line.endswith("]"): + current_section = line[1:-1] + continue + + if current_section != section or "=" not in line: + continue + + config_key, value = line.split("=", 1) + if config_key.strip() == key: + return value.strip() + + return None + + +def _read_git_metadata(path: str) -> dict[str, str] | None: + git_folder = pathlib.Path(path) / ".git" + if not git_folder.is_dir(): + return None + + head = _read_text_file(git_folder / "HEAD") + if not head: + return None + + branch = "HEAD" + commit = head.splitlines()[0].strip() + if head.startswith("ref:"): + ref = head.split(":", 1)[1].strip() + branch = ref.rsplit("/", 1)[-1] + commit = _read_git_ref(git_folder, ref) + + if not commit: + return None + + remote = _read_git_config_value(git_folder, f'branch "{branch}"', "remote") or "origin" + origin = _read_git_config_value(git_folder, f'remote "{remote}"', "url") + if origin is None and remote != "origin": + origin = _read_git_config_value(git_folder, 'remote "origin"', "url") + + return { + "branch": branch, + "commit": commit, + "origin": origin or "", + } + + +def _build_metadata_cache_path(path: str) -> pathlib.Path | None: + if not (pathlib.Path(path) / ".git").is_dir(): + return None + + cache_root = pathlib.Path(os.environ.get("OPENPILOT_BUILD_METADATA_CACHE_ROOT", "/cache/starpilot")) + return cache_root / BUILD_METADATA_CACHE_FILENAME + + +def _read_cached_build_metadata(path: str, git_metadata: dict[str, str]) -> BuildMetadata | None: + cache_path = _build_metadata_cache_path(path) + if cache_path is None: + return None + + try: + cache_payload = json.loads(cache_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + if cache_payload.get("path") != str(pathlib.Path(path).resolve()) or cache_payload.get("git_metadata") != git_metadata: + return None + + build_metadata = cache_payload.get("build_metadata") + if not isinstance(build_metadata, dict): + return None + + return build_metadata_from_dict(build_metadata) + + +def _write_cached_build_metadata(path: str, git_metadata: dict[str, str], build_metadata: BuildMetadata) -> None: + cache_path = _build_metadata_cache_path(path) + if cache_path is None: + return + + try: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(json.dumps({ + "path": str(pathlib.Path(path).resolve()), + "git_metadata": git_metadata, + "build_metadata": asdict(build_metadata), + }, separators=(",", ":")), encoding="utf-8") + except OSError: + pass + + def get_build_metadata(path: str = BASEDIR) -> BuildMetadata: build_metadata_path = pathlib.Path(path) / BUILD_METADATA_FILENAME @@ -142,15 +270,34 @@ def get_build_metadata(path: str = BASEDIR) -> BuildMetadata: git_folder = pathlib.Path(path) / ".git" if git_folder.exists(): + prebuilt = is_prebuilt(path) + git_metadata = _read_git_metadata(path) + if prebuilt and git_metadata is not None: + cached_metadata = _read_cached_build_metadata(path, git_metadata) + if cached_metadata is not None: + return cached_metadata + + build_metadata = BuildMetadata(git_metadata["branch"], + OpenpilotMetadata( + version=get_version(path), + release_notes=get_release_notes(path), + git_commit=git_metadata["commit"], + git_origin=git_metadata["origin"] or get_origin(path), + git_commit_date=get_commit_date(path, git_metadata["commit"]), + build_style="unknown", + is_dirty=False)) + _write_cached_build_metadata(path, git_metadata, build_metadata) + return build_metadata + return BuildMetadata(get_short_branch(path), - OpenpilotMetadata( - version=get_version(path), - release_notes=get_release_notes(path), - git_commit=get_commit(path), - git_origin=get_origin(path), - git_commit_date=get_commit_date(path), - build_style="unknown", - is_dirty=is_dirty(path))) + OpenpilotMetadata( + version=get_version(path), + release_notes=get_release_notes(path), + git_commit=get_commit(path), + git_origin=get_origin(path), + git_commit_date=get_commit_date(path), + build_style="unknown", + is_dirty=is_dirty(path))) cloudlog.exception("unable to get build metadata") raise Exception("invalid build metadata")