From ea2aa532af564fa78f52820523f1cb46cbf1daa7 Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:38:13 -0500 Subject: [PATCH] I AM SCREAMING --- common/api.py | 7 +- launch_chffrplus.sh | 79 +---- starpilot/common/connect_server.py | 3 +- starpilot/common/starpilot_functions.py | 306 +++--------------- starpilot/common/starpilot_variables.py | 9 +- system/athena/athenad.py | 5 +- system/athena/registration.py | 65 +--- system/manager/helpers.py | 48 ++- system/manager/manager.py | 256 +++------------ system/manager/process.py | 55 +--- system/manager/process_config.py | 64 +--- .../test/test_launch_param_migrations.py | 4 +- system/manager/test/test_manager.py | 106 +----- system/tests/test_version.py | 67 ---- system/ui/reset.py | 67 +--- system/ui/tests/test_reset.py | 42 --- system/updated/updated.py | 12 +- system/version.py | 165 +--------- 18 files changed, 173 insertions(+), 1187 deletions(-) delete mode 100644 system/tests/test_version.py delete mode 100644 system/ui/tests/test_reset.py diff --git a/common/api.py b/common/api.py index 703176562..f2e5e61b8 100644 --- a/common/api.py +++ b/common/api.py @@ -2,15 +2,10 @@ import jwt import os import requests from datetime import datetime, timedelta, UTC -from functools import cache -from openpilot.common.params import Params from openpilot.system.hardware.hw import Paths from openpilot.system.version import get_version - -@cache -def use_konik_server() -> bool: - return Params().get_bool("UseKonikServer") +from openpilot.starpilot.common.starpilot_utilities import use_konik_server API_HOST = os.getenv('API_HOST', f"https://api.{'konik.ai' if use_konik_server() else 'commadotai.com'}") diff --git a/launch_chffrplus.sh b/launch_chffrplus.sh index 39ec1c291..1fa0fcb0c 100755 --- a/launch_chffrplus.sh +++ b/launch_chffrplus.sh @@ -6,7 +6,6 @@ source "$DIR/launch_env.sh" export SP_BOOT_TIMING_LOG="${SP_BOOT_TIMING_LOG:-/tmp/starpilot_boot_timing.log}" : > "$SP_BOOT_TIMING_LOG" 2>/dev/null || true -PREBUILT_COMPAT_CACHE="${SP_PREBUILT_COMPAT_CACHE:-/cache/starpilot_prebuilt_runtime_compatible_v1}" SP_LAUNCH_LAST_SECONDS=$SECONDS function sp_boot_timing_line { @@ -21,54 +20,6 @@ function sp_launch_timing { SP_LAUNCH_LAST_SECONDS=$now } -function launch_param_migrations_needed { - local marker_dir="/data/params/.starpilot_param_migrations/${OPENPILOT_PREFIX:-d}" - [ ! -f "$marker_dir/.starpilot_launch_param_migrations_v2" ] && return 0 - [ ! -f "$marker_dir/.starpilot_branch_defaults_migrations_v1" ] && return 0 - [ ! -f "$marker_dir/.starpilot_acceleration_profile_default_v1" ] && return 0 - return 1 -} - -function prebuilt_runtime_artifacts { - printf '%s\n' \ - "$DIR/common/params_pyx.so" \ - "$DIR/msgq/ipc_pyx.so" \ - "$DIR/msgq/visionipc/visionipc_pyx.so" \ - "$DIR/common/transformations/transformations.so" \ - "$DIR/selfdrive/modeld/models/driving_tinygrad.pkl" \ - "$DIR/selfdrive/modeld/models/dmonitoring_model_metadata.pkl" \ - "$DIR/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl" \ - "$DIR/selfdrive/modeld/models/dm_warp_1928x1208_tinygrad.pkl" \ - "$DIR/selfdrive/modeld/models/dm_warp_1344x760_tinygrad.pkl" \ - "$DIR/selfdrive/pandad/pandad_api_impl.so" \ - "$DIR/selfdrive/controls/lib/lateral_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so" \ - "$DIR/selfdrive/controls/lib/lateral_mpc_lib/c_generated_code/libacados_ocp_solver_lat.so" \ - "$DIR/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so" \ - "$DIR/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/libacados_ocp_solver_long.so" \ - "$DIR/opendbc_repo/opendbc/dbc/gm_global_a_powertrain_generated.dbc" -} - -function prebuilt_compat_cache_valid { - [ -f "$PREBUILT_COMPAT_CACHE" ] || return 1 - [ -f "$DIR/prebuilt" ] || return 1 - [ "$DIR/prebuilt" -nt "$PREBUILT_COMPAT_CACHE" ] && return 1 - - while IFS= read -r artifact; do - [ -e "$artifact" ] || return 1 - [ "$artifact" -nt "$PREBUILT_COMPAT_CACHE" ] && return 1 - done < <(prebuilt_runtime_artifacts) - - return 0 -} - -function prebuilt_artifacts_present { - while IFS= read -r artifact; do - [ -e "$artifact" ] || return 1 - done < <(prebuilt_runtime_artifacts) - - return 0 -} - function agnos_init { sp_launch_timing "agnos_init_start" @@ -175,15 +126,11 @@ function launch { # start manager cd system/manager - if launch_param_migrations_needed; then - sp_launch_timing "launch_param_migrations_start" - if ! python3 ./launch_param_migrations.py; then - echo "Launch param migrations failed; continuing boot." - fi - sp_launch_timing "launch_param_migrations_done" - else - sp_launch_timing "launch_param_migrations_skipped" + sp_launch_timing "launch_param_migrations_start" + if ! python3 ./launch_param_migrations.py; then + echo "Launch param migrations failed; continuing boot." fi + sp_launch_timing "launch_param_migrations_done" # Bootstrap runtime (e.g. /usr/comma after reset/uninstall) must go straight # to manager/setup flow. Do not run StarPilot prebuilt checks/builds here. @@ -266,21 +213,9 @@ PY fi sp_launch_timing "prebuilt_decision_done" - if [ "$USE_PREBUILT" = "1" ] && [ -f $DIR/prebuilt ]; then - if prebuilt_compat_cache_valid; then - sp_launch_timing "prebuilt_compat_cache_hit" - elif [ "${SP_STRICT_PREBUILT_CHECK:-0}" != "1" ] && prebuilt_artifacts_present; then - sp_launch_timing "prebuilt_compat_fast_path" - mkdir -p "$(dirname "$PREBUILT_COMPAT_CACHE")" 2>/dev/null || true - touch "$PREBUILT_COMPAT_CACHE" 2>/dev/null || true - elif prebuilt_runtime_compatible; then - mkdir -p "$(dirname "$PREBUILT_COMPAT_CACHE")" 2>/dev/null || true - touch "$PREBUILT_COMPAT_CACHE" 2>/dev/null || true - else - echo "Prebuilt runtime artifacts are incompatible on this device; rebuilding locally." - rm -f "$PREBUILT_COMPAT_CACHE" 2>/dev/null || true - USE_PREBUILT=0 - fi + if [ "$USE_PREBUILT" = "1" ] && [ -f $DIR/prebuilt ] && ! prebuilt_runtime_compatible; then + echo "Prebuilt runtime artifacts are incompatible on this device; rebuilding locally." + USE_PREBUILT=0 fi sp_launch_timing "prebuilt_compat_done" diff --git a/starpilot/common/connect_server.py b/starpilot/common/connect_server.py index 19aa0ba2e..7941d6a40 100644 --- a/starpilot/common/connect_server.py +++ b/starpilot/common/connect_server.py @@ -1,10 +1,11 @@ from pathlib import Path from openpilot.common.params import Params -from openpilot.common.api import use_konik_server from openpilot.system.athena.registration import register from openpilot.system.hardware.hw import Paths +from openpilot.starpilot.common.starpilot_utilities import use_konik_server + def _cache_params_path() -> str: return Paths.params_cache_root() diff --git a/starpilot/common/starpilot_functions.py b/starpilot/common/starpilot_functions.py index 80195e263..ff94af244 100644 --- a/starpilot/common/starpilot_functions.py +++ b/starpilot/common/starpilot_functions.py @@ -1,205 +1,33 @@ #!/usr/bin/env python3 import dataclasses import json -import os +import requests import threading import time from pathlib import Path from types import SimpleNamespace +from cereal import messaging from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params from openpilot.common.time_helpers import system_time_valid from openpilot.system.hardware import HARDWARE +from openpilot.system.version import get_build_metadata -STARPILOT_API = os.getenv("STARPILOT_API", "https://frogpilot.com/api") -BOOT_LOGO_STATE_PATH = Path("/cache/starpilot_boot_logo_state_v1") -ACTIVE_THEME_PATH = Path(BASEDIR) / "starpilot/assets/active_theme" -STOCK_THEME_PATH = Path(BASEDIR) / "starpilot/assets/stock_theme" -ACTIVE_THEME_REQUIRED_PATHS = ( - ACTIVE_THEME_PATH / "colors/colors.json", - ACTIVE_THEME_PATH / "icons", - ACTIVE_THEME_PATH / "distance_icons", - ACTIVE_THEME_PATH / "sounds", - ACTIVE_THEME_PATH / "steering_wheel", +from openpilot.starpilot.assets.theme_manager import ThemeManager +from openpilot.starpilot.common.starpilot_backups import backup_starpilot +from openpilot.starpilot.common.connect_server import sync_konik_dongle_id +from openpilot.starpilot.common.maps_catalog import normalize_schedule_value, sanitize_selected_locations_csv +from openpilot.starpilot.common.theme_asset_names import find_matching_theme_asset_file +from openpilot.starpilot.common.starpilot_utilities import get_starpilot_api_info, is_FrogsGoMoo, is_url_pingable, run_cmd +from openpilot.starpilot.common.starpilot_variables import ( + ERROR_LOGS_PATH, STARPILOT_API, FROGS_GO_MOO_PATH, HD_LOGS_PATH, KONIK_LOGS_PATH, MAPS_PATH, THEME_SAVE_PATH, + StarPilotVariables, get_starpilot_toggles ) -ACTIVE_THEME_BOOTSTRAP_LINKS = ( - ("colors", STOCK_THEME_PATH / "colors"), - ("icons", STOCK_THEME_PATH / "icons"), - ("distance_icons", STOCK_THEME_PATH / "distance_icons"), - ("sounds", STOCK_THEME_PATH / "sounds"), - ("steering_wheel", STOCK_THEME_PATH / "steering_wheel"), -) - - -def _starpilot_data_root(): - if HARDWARE.get_device_type() == "pc": - from openpilot.system.hardware.hw import Paths - - return Path(Paths.comma_home()) / "starpilot" / "data" - return Path("/data") - - -def _starpilot_persist_root(): - if HARDWARE.get_device_type() == "pc": - from openpilot.system.hardware.hw import Paths - - return Path(Paths.persist_root()) - return Path("/persist") - - -def _theme_save_path(): - return _starpilot_data_root() / "themes" - - -def _maps_path(): - return _starpilot_data_root() / "media/0/osm/offline" - - -def _run_cmd(*args, **kwargs): - from openpilot.starpilot.common.starpilot_utilities import run_cmd - - return run_cmd(*args, **kwargs) - - -def _path_has_content(path): - if path.is_symlink() and not path.exists(): - return False - if path.is_file(): - return True - if not path.is_dir(): - return False - try: - next(path.iterdir()) - return True - except (OSError, StopIteration): - return False - - -def _active_theme_ready(): - return all(_path_has_content(path) for path in ACTIVE_THEME_REQUIRED_PATHS) - - -def _replace_with_symlink(path, target): - if path.is_symlink() or path.is_file(): - try: - path.unlink() - except OSError: - return - elif path.exists(): - import shutil - - try: - shutil.rmtree(path) - except OSError: - return - - path.parent.mkdir(parents=True, exist_ok=True) - try: - path.symlink_to(target, target_is_directory=target.is_dir()) - except OSError: - pass - - -def _ensure_minimal_active_theme(): - for name, target in ACTIVE_THEME_BOOTSTRAP_LINKS: - if target.exists() and not _path_has_content(ACTIVE_THEME_PATH / name): - _replace_with_symlink(ACTIVE_THEME_PATH / name, target) - - signals_path = ACTIVE_THEME_PATH / "signals" - if signals_path.is_symlink() and not signals_path.exists(): - try: - signals_path.unlink() - except OSError: - pass - - -def _normalize_dongle_id(value): - if isinstance(value, bytes): - value = value.decode("utf-8", errors="ignore") - if value is None: - return None - value = str(value).strip() - return value or None - - -def _read_persisted_stock_dongle_id(): - persisted_dongle_id_path = _starpilot_persist_root() / "comma" / "dongle_id" - if not persisted_dongle_id_path.is_file(): - return None - return _normalize_dongle_id(persisted_dongle_id_path.read_text()) - - -def _ensure_stock_dongle_id_fast(params): - current_dongle_id = _normalize_dongle_id(params.get("DongleId")) - konik_dongle_id = _normalize_dongle_id(params.get("KonikDongleId")) - stock_dongle_id = _normalize_dongle_id(params.get("StockDongleId")) - - if stock_dongle_id not in (None, konik_dongle_id): - return stock_dongle_id - - candidate = _read_persisted_stock_dongle_id() - if candidate in (None, konik_dongle_id): - candidate = current_dongle_id if current_dongle_id != konik_dongle_id else None - - if candidate is not None and candidate != stock_dongle_id: - params.put("StockDongleId", candidate) - - return candidate - - -def _sync_cached_konik_dongle_id(params): - current_dongle_id = _normalize_dongle_id(params.get("DongleId")) - konik_dongle_id = _normalize_dongle_id(params.get("KonikDongleId")) - stock_dongle_id = _ensure_stock_dongle_id_fast(params) - - if params.get_bool("UseKonikServer"): - if konik_dongle_id is not None and current_dongle_id != konik_dongle_id: - params.put("DongleId", konik_dongle_id) - elif current_dongle_id == konik_dongle_id and stock_dongle_id is not None: - params.put("DongleId", stock_dongle_id) - - -def _boot_logo_cache_key(selected_logo, source_logo): - try: - source_stat = source_logo.stat() - source_path = source_logo.resolve() - except OSError: - return None - - selected = selected_logo.decode("utf-8", "ignore") if isinstance(selected_logo, (bytes, bytearray)) else str(selected_logo or "") - return "\n".join([ - selected.strip().lower(), - str(source_path), - str(source_stat.st_mtime_ns), - str(source_stat.st_size), - "", - ]) - - -def _boot_logo_cache_matches(cache_key): - if cache_key is None: - return False - try: - return BOOT_LOGO_STATE_PATH.read_text() == cache_key - except OSError: - return False - - -def _write_boot_logo_cache(cache_key): - if cache_key is None: - return - try: - BOOT_LOGO_STATE_PATH.parent.mkdir(parents=True, exist_ok=True) - BOOT_LOGO_STATE_PATH.write_text(cache_key) - except OSError: - pass def seed_desktop_theme_assets(): - from openpilot.starpilot.assets.theme_manager import ThemeManager - params = Params() params_memory = Params(memory=True) params_defaults = Params(return_defaults=True) @@ -231,41 +59,7 @@ def seed_desktop_theme_assets(): def starpilot_boot_functions(build_metadata, params): - params.put("BuildMetadata", json.dumps(dataclasses.asdict(build_metadata))) - _sync_cached_konik_dongle_id(params) - - def boot_thread(): - try: - _finish_starpilot_boot(build_metadata) - except Exception as error: - print(f"StarPilot boot functions failed: {error}") - - if _active_theme_ready(): - threading.Thread(target=boot_thread, daemon=True).start() - return - - # Brand-new or partially migrated installs need basic active theme links - # before UI starts. Use stock links here and let the full theme refresh run - # off the manager hot path. - _ensure_minimal_active_theme() - threading.Thread(target=boot_thread, daemon=True).start() - - -def _refresh_active_theme(params): - from openpilot.starpilot.assets.theme_manager import ThemeManager - from openpilot.starpilot.common.starpilot_variables import StarPilotVariables - params_memory = Params(memory=True) - starpilot_toggles = StarPilotVariables().starpilot_toggles - ThemeManager(params, params_memory, boot_run=True).update_active_theme(time_validated=system_time_valid(), starpilot_toggles=starpilot_toggles, boot_run=True) - - -def _finish_starpilot_boot(build_metadata): - from openpilot.starpilot.common.connect_server import sync_konik_dongle_id - from openpilot.starpilot.common.maps_catalog import sanitize_selected_locations_csv - from openpilot.starpilot.common.starpilot_backups import backup_starpilot - - params = Params() maps_selected_raw = params.get("MapsSelected") maps_selected = sanitize_selected_locations_csv(maps_selected_raw) @@ -276,24 +70,28 @@ def _finish_starpilot_boot(build_metadata): params.put("BuildMetadata", json.dumps(dataclasses.asdict(build_metadata))) - _refresh_active_theme(params) + StarPilotVariables() + ThemeManager(params, params_memory, boot_run=True).update_active_theme(time_validated=system_time_valid(), starpilot_toggles=get_starpilot_toggles(), boot_run=True) + sync_konik_dongle_id(params) - while not system_time_valid(): - print("Waiting for system time to become valid...") - time.sleep(1) + def boot_thread(): + while not system_time_valid(): + print("Waiting for system time to become valid...") + time.sleep(1) - backup_starpilot(build_metadata, params) + backup_starpilot(build_metadata, params) + + threading.Thread(target=boot_thread, daemon=True).start() def install_starpilot(build_metadata, params): - data_root = _starpilot_data_root() paths = [ - data_root / "error_logs", - data_root / "media/0/realdata_HD", - data_root / "media/0/realdata_konik", - data_root / "media/0/osm/offline", - _theme_save_path() + ERROR_LOGS_PATH, + HD_LOGS_PATH, + KONIK_LOGS_PATH, + MAPS_PATH, + THEME_SAVE_PATH ] for path in paths: path.mkdir(parents=True, exist_ok=True) @@ -302,27 +100,15 @@ def install_starpilot(build_metadata, params): update_boot_logo(starpilot=True, selected_logo=params.get("BootLogo")) - frogs_go_moo_path = _starpilot_persist_root() / "frogsgomoo.py" - if frogs_go_moo_path.is_file(): - mount_options = _run_cmd(["findmnt", "-n", "-o", "OPTIONS", "/persist"], "Successfully retrieved mount options", "Failed to retrieve mount options") - _run_cmd(["sudo", "mount", "-o", "remount,rw", "/persist"], "Successfully remounted /persist as read-write", "Failed to remount /persist") - _run_cmd(["sudo", "python3", frogs_go_moo_path], "Successfully ran frogsgomoo.py", "Failed to run frogsgomoo.py") - _run_cmd(["sudo", "mount", "-o", f"remount,{mount_options}", "/persist"], "Successfully restored /persist mount options", "Failed to restore /persist mount options") + if is_FrogsGoMoo(): + mount_options = run_cmd(["findmnt", "-n", "-o", "OPTIONS", "/persist"], "Successfully retrieved mount options", "Failed to retrieve mount options") + run_cmd(["sudo", "mount", "-o", "remount,rw", "/persist"], "Successfully remounted /persist as read-write", "Failed to remount /persist") + run_cmd(["sudo", "python3", FROGS_GO_MOO_PATH], "Successfully ran frogsgomoo.py", "Failed to run frogsgomoo.py") + run_cmd(["sudo", "mount", "-o", f"remount,{mount_options}", "/persist"], "Successfully restored /persist mount options", "Failed to restore /persist mount options") def register_device(build_metadata, params): def register_thread(): - import requests - - def is_url_pingable(url): - if not url: - return False - try: - response = requests.head(url, timeout=10, allow_redirects=True, headers={"User-Agent": "starpilot-ping-test/1.0 (https://github.com/FrogAi/StarPilot)"}) - return response.ok - except Exception: - return False - dongle_id = params.get("DongleId") if isinstance(dongle_id, bytes): dongle_id = dongle_id.decode("utf-8", errors="ignore") @@ -377,9 +163,7 @@ def update_boot_logo(starpilot=False, stock=False, selected_logo=None): selected = selected_logo.decode("utf-8", "ignore") if isinstance(selected_logo, (bytes, bytearray)) else str(selected_logo) selected = selected.strip() if selected.lower() not in {"", "stock", "default"}: - from openpilot.starpilot.common.theme_asset_names import find_matching_theme_asset_file - - matched_logo = find_matching_theme_asset_file(_theme_save_path() / "bootlogos", selected) + matched_logo = find_matching_theme_asset_file(THEME_SAVE_PATH / "bootlogos", selected) if matched_logo is not None: target_logo = matched_logo elif stock: @@ -392,10 +176,6 @@ def update_boot_logo(starpilot=False, stock=False, selected_logo=None): print(f"Error: Target logo file not found at {target_logo}") return - cache_key = _boot_logo_cache_key(selected_logo, target_logo) - if boot_logo_location.is_file() and _boot_logo_cache_matches(cache_key): - return - source_logo = target_logo staged_logo = Path("/tmp/starpilot_boot_logo.jpg") try: @@ -416,18 +196,13 @@ def update_boot_logo(starpilot=False, stock=False, selected_logo=None): current_logo = boot_logo_location.read_bytes() if boot_logo_location.is_file() else b"" desired_logo = source_logo.read_bytes() if current_logo != desired_logo: - mount_options = _run_cmd(["findmnt", "-n", "-o", "OPTIONS", "/"], "Successfully retrieved mount options", "Failed to retrieve mount options") - _run_cmd(["sudo", "mount", "-o", "remount,rw", "/"], "Successfully remounted / as read-write", "Failed to remount /") - if _run_cmd(["sudo", "cp", source_logo, boot_logo_location], "Successfully replaced boot logo", "Failed to replace boot logo") is None: - return - _run_cmd(["sudo", "mount", "-o", f"remount,{mount_options}", "/"], "Successfully restored / mount options", "Failed to restore / mount options") - _write_boot_logo_cache(cache_key) + mount_options = run_cmd(["findmnt", "-n", "-o", "OPTIONS", "/"], "Successfully retrieved mount options", "Failed to retrieve mount options") + run_cmd(["sudo", "mount", "-o", "remount,rw", "/"], "Successfully remounted / as read-write", "Failed to remount /") + run_cmd(["sudo", "cp", source_logo, boot_logo_location], "Successfully replaced boot logo", "Failed to replace boot logo") + run_cmd(["sudo", "mount", "-o", f"remount,{mount_options}", "/"], "Successfully restored / mount options", "Failed to restore / mount options") def update_maps(now, params, params_memory, manual_update=False): - from cereal import messaging - from openpilot.starpilot.common.maps_catalog import normalize_schedule_value, sanitize_selected_locations_csv - maps_selected_raw = params.get("MapsSelected") maps_selected = sanitize_selected_locations_csv(maps_selected_raw) if not maps_selected: @@ -442,8 +217,7 @@ def update_maps(now, params, params_memory, manual_update=False): is_sunday = now.weekday() == 6 schedule = normalize_schedule_value(params.get("PreferredSchedule")) - maps_path = _maps_path() - maps_downloaded = maps_path.exists() and any(path.is_file() for path in maps_path.rglob("*")) + maps_downloaded = MAPS_PATH.exists() and any(path.is_file() for path in MAPS_PATH.rglob("*")) if maps_downloaded and (schedule == 0 or (schedule == 1 and not is_sunday) or (schedule == 2 and not is_first)) and not manual_update: return @@ -491,7 +265,7 @@ def update_maps(now, params, params_memory, manual_update=False): def update_openpilot(thread_manager, params): def update_available(): - _run_cmd(["pkill", "-SIGUSR1", "-f", "system.updated.updated"], "Checking for updates...", "Failed to check for update...", report=False) + run_cmd(["pkill", "-SIGUSR1", "-f", "system.updated.updated"], "Checking for updates...", "Failed to check for update...", report=False) while params.get("UpdaterState") != "checking...": time.sleep(1) @@ -505,7 +279,7 @@ def update_openpilot(thread_manager, params): while params.get_bool("IsOnroad") or thread_manager.is_thread_alive("lock_doors"): time.sleep(60) - _run_cmd(["pkill", "-SIGHUP", "-f", "system.updated.updated"], "Update available, downloading...", "Failed to download update...", report=False) + run_cmd(["pkill", "-SIGHUP", "-f", "system.updated.updated"], "Update available, downloading...", "Failed to download update...", report=False) while not params.get_bool("UpdateAvailable"): time.sleep(60) diff --git a/starpilot/common/starpilot_variables.py b/starpilot/common/starpilot_variables.py index 530c3225d..fb91f55dc 100644 --- a/starpilot/common/starpilot_variables.py +++ b/starpilot/common/starpilot_variables.py @@ -15,7 +15,6 @@ import numpy as np from cereal import car, custom, log from opendbc.car import gen_empty_fingerprint from opendbc.car.car_helpers import interfaces -from opendbc.car.chrysler.values import JEEPS as CHRYSLER_JEEPS from opendbc.car.gm.values import CAR as GM_CAR, EV_CAR as GM_EV_CAR, GMFlags from opendbc.car.hyundai.values import CAR as HYUNDAI_CAR, EV_CAR as HYUNDAI_EV_CAR, HyundaiFlags, HyundaiStarPilotSafetyFlags from opendbc.car.interfaces import TORQUE_SUBSTITUTE_PATH, CarInterfaceBase, GearShifter @@ -317,12 +316,7 @@ def default_ev_tuning_enabled(CP): ev_vehicle |= getattr(CP, "transmissionType", None) == car.CarParams.TransmissionType.direct return bool(ev_vehicle) -def get_starpilot_toggles(sm=None): - if sm is None: - if not hasattr(get_starpilot_toggles, "_sm"): - get_starpilot_toggles._sm = messaging.SubMaster(["starpilotPlan"]) - sm = get_starpilot_toggles._sm - +def get_starpilot_toggles(sm=messaging.SubMaster(["starpilotPlan"])): toggles_text = sm["starpilotPlan"].starpilotToggles if toggles_text: get_starpilot_toggles._last_toggles_text = toggles_text @@ -1413,7 +1407,6 @@ class StarPilotVariables: toggle.volt_one_pedal_mode = self.get_value("VoltOnePedalMode", condition=gm_auto_hold_supported) toggle.volt_sng = self.get_value("VoltSNG", condition=toggle.car_model in LEGACY_VOLT_STOCK_ACC_CARS) - toggle.jeep_brake_hold = self.get_value("JeepBrakeHold", condition=toggle.car_model in CHRYSLER_JEEPS) process_starpilot_toggles.cache_clear() self.params_memory.remove("StarPilotTogglesUpdated") diff --git a/system/athena/athenad.py b/system/athena/athenad.py index d77c8da7e..ed167c417 100755 --- a/system/athena/athenad.py +++ b/system/athena/athenad.py @@ -30,7 +30,7 @@ from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutExce import cereal.messaging as messaging from cereal import log from cereal.services import SERVICE_LIST -from openpilot.common.api import Api, get_key_pair, use_konik_server +from openpilot.common.api import Api, get_key_pair from openpilot.common.utils import CallbackReader, get_upload_stream from openpilot.common.params import Params from openpilot.common.realtime import set_core_affinity @@ -41,6 +41,9 @@ from openpilot.system.athena.registration import UNREGISTERED_DONGLE_ID from openpilot.system.version import get_build_metadata from openpilot.system.hardware.hw import Paths +from openpilot.starpilot.common.starpilot_utilities import use_konik_server + + ATHENA_HOST = os.getenv('ATHENA_HOST', f"wss://athena.{'konik.ai' if use_konik_server() else 'comma.ai'}") HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4")) LOCAL_PORT_WHITELIST = {22, } # SSH diff --git a/system/athena/registration.py b/system/athena/registration.py index 759bd40f0..d20de70b9 100644 --- a/system/athena/registration.py +++ b/system/athena/registration.py @@ -1,57 +1,29 @@ #!/usr/bin/env python3 import time +import json +import jwt +import random +import string +from typing import cast from pathlib import Path +from datetime import datetime, timedelta, UTC +from openpilot.common.api import api_get, get_key_pair from openpilot.common.params import Params +from openpilot.common.spinner import Spinner +from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert from openpilot.system.hardware import HARDWARE, PC from openpilot.system.hardware.hw import Paths from openpilot.common.swaglog import cloudlog UNREGISTERED_DONGLE_ID = "UnregisteredDevice" -OFFROAD_UNREGISTERED_ALERT = "Offroad_UnregisteredHardware" -KEYS = {"id_rsa": "RS256", - "id_ecdsa": "ES256"} - - -def api_get(*args, **kwargs): - from openpilot.common.api import api_get as api_get_impl - - return api_get_impl(*args, **kwargs) - - -def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]: - for key, algorithm in KEYS.items(): - private_path = Path(Paths.persist_root()) / "comma" / key - public_path = Path(Paths.persist_root()) / "comma" / f"{key}.pub" - if private_path.is_file() and public_path.is_file(): - return algorithm, private_path.read_text(), public_path.read_text() - return None, None, None - - -def _has_key_pair() -> bool: - for key in KEYS: - private_path = Path(Paths.persist_root()) / "comma" / key - public_path = Path(Paths.persist_root()) / "comma" / f"{key}.pub" - if private_path.is_file() and public_path.is_file(): - return True - return False def is_registered_device() -> bool: dongle = Params().get("DongleId") return dongle not in (None, UNREGISTERED_DONGLE_ID) -def _set_registration_alert(params: Params, dongle_id: str | None) -> None: - show_alert = (dongle_id == UNREGISTERED_DONGLE_ID) and not PC - if show_alert: - from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert - - set_offroad_alert(OFFROAD_UNREGISTERED_ALERT, True) - else: - params.remove(OFFROAD_UNREGISTERED_ALERT) - - def register(show_spinner=False, register_konik=False) -> str | None: """ All devices built since March 2024 come with all @@ -70,18 +42,13 @@ def register(show_spinner=False, register_konik=False) -> str | None: with open(Paths.persist_root()+"/comma/dongle_id") as f: dongle_id = f.read().strip() - if not _has_key_pair() and not register_konik: + # Create registration token, in the future, this key will make JWTs directly + jwt_algo, private_key, public_key = get_key_pair() + + if not public_key and not register_konik: dongle_id = UNREGISTERED_DONGLE_ID cloudlog.warning("missing public key") elif dongle_id is None or register_konik: - import json - import jwt - from datetime import datetime, timedelta, UTC - from openpilot.common.spinner import Spinner - - # Create registration token, in the future, this key will make JWTs directly - jwt_algo, private_key, public_key = get_key_pair() - if show_spinner: spinner = Spinner() spinner.update("registering device") @@ -106,14 +73,14 @@ def register(show_spinner=False, register_konik=False) -> str | None: while True: try: register_token = jwt.encode({'register': True, 'exp': datetime.now(UTC).replace(tzinfo=None) + timedelta(hours=1)}, - private_key, algorithm=jwt_algo) + cast(str, private_key), algorithm=jwt_algo) cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token) if resp.status_code in (402, 403): cloudlog.info(f"Unable to register device, got {resp.status_code}") - dongle_id = UNREGISTERED_DONGLE_ID + dongle_id = ''.join(random.choices(string.ascii_lowercase + string.digits, k=16)) else: dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] @@ -132,7 +99,7 @@ def register(show_spinner=False, register_konik=False) -> str | None: if not register_konik and dongle_id != params.get("KonikDongleId"): params.put("DongleId", dongle_id) params.put("StockDongleId", dongle_id) - _set_registration_alert(params, dongle_id) + set_offroad_alert("Offroad_UnregisteredHardware", (dongle_id == UNREGISTERED_DONGLE_ID) and not PC) return dongle_id diff --git a/system/manager/helpers.py b/system/manager/helpers.py index a096f0687..047d0ac2d 100644 --- a/system/manager/helpers.py +++ b/system/manager/helpers.py @@ -1,11 +1,18 @@ +import errno +import fcntl import os +import sys +import pathlib +import shutil +import signal +import subprocess +import tempfile +import threading + +from openpilot.common.basedir import BASEDIR +from openpilot.common.params import Params def unblock_stdout() -> None: - import errno - import fcntl - import signal - import sys - # get a non-blocking stdout child_pid, child_pty = os.forkpty() if child_pid != 0: # parent @@ -44,30 +51,17 @@ def write_onroad_params(started, params): def save_bootlog(): - import threading + # copy current params + tmp = tempfile.mkdtemp() + params_dirname = pathlib.Path(Params().get_param_path()).name + params_dir = os.path.join(tmp, params_dirname) + shutil.copytree(Params().get_param_path(), params_dir, dirs_exist_ok=True) - def fn(): - import pathlib - import shutil - import subprocess - import tempfile - - from openpilot.common.basedir import BASEDIR - from openpilot.common.params import Params - - tmpdir = tempfile.mkdtemp() + def fn(tmpdir): env = os.environ.copy() env['PARAMS_COPY_PATH'] = tmpdir - - try: - params = Params() - params_dirname = pathlib.Path(params.get_param_path()).name - params_dir = os.path.join(tmpdir, params_dirname) - shutil.copytree(params.get_param_path(), params_dir, dirs_exist_ok=True) - subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "system/loggerd"), env=env) - finally: - shutil.rmtree(tmpdir, ignore_errors=True) - - t = threading.Thread(target=fn) + subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "system/loggerd"), env=env) + shutil.rmtree(tmpdir) + t = threading.Thread(target=fn, args=(tmp, )) t.daemon = True t.start() diff --git a/system/manager/manager.py b/system/manager/manager.py index 34dbcda6c..d3875f21b 100755 --- a/system/manager/manager.py +++ b/system/manager/manager.py @@ -2,13 +2,12 @@ import datetime import json import os +import shutil from pathlib import Path import signal import sys -import threading import time import traceback -from types import SimpleNamespace _MANAGER_IMPORT_START = time.monotonic() _BOOT_TIMING_LOG_PATH = os.environ.get("SP_BOOT_TIMING_LOG", "/tmp/starpilot_boot_timing.log") @@ -24,29 +23,37 @@ def _append_boot_timing_line(line: str) -> None: from cereal import car, log import cereal.messaging as messaging +import openpilot.system.sentry as sentry +from openpilot.common.utils import atomic_write from openpilot.common.params import Params, ParamKeyFlag, ParamKeyType +from openpilot.common.text_window import TextWindow from openpilot.system.hardware import HARDWARE from openpilot.system.manager.helpers import unblock_stdout, write_onroad_params, save_bootlog from openpilot.system.manager.process import ensure_running from openpilot.system.manager.process_config import managed_processes -from openpilot.common.swaglog import cloudlog +from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID +from openpilot.common.swaglog import cloudlog, add_file_handler from openpilot.system.version import get_build_metadata, terms_version, training_version from openpilot.system.hardware.hw import Paths _MANAGER_CORE_IMPORT_DONE = time.monotonic() from openpilot.starpilot.common.starpilot_functions import starpilot_boot_functions, install_starpilot, uninstall_starpilot +from openpilot.starpilot.common.starpilot_variables import ( + LEGACY_STARPILOT_PARAM_RENAMES, + LEGACY_STARPILOT_STATS_KEY_RENAMES, + get_starpilot_toggles, +) _MANAGER_IMPORT_DONE = time.monotonic() -if __name__ == "__main__": - _manager_import_timing_line = ( - "SP_BOOT_TIMING manager_import " - f"core={_MANAGER_CORE_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s " - f"starpilot={_MANAGER_IMPORT_DONE - _MANAGER_CORE_IMPORT_DONE:.3f}s " - f"total={_MANAGER_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s" - ) - print(_manager_import_timing_line, flush=True) - _append_boot_timing_line(_manager_import_timing_line) +_manager_import_timing_line = ( + "SP_BOOT_TIMING manager_import " + f"core={_MANAGER_CORE_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s " + f"starpilot={_MANAGER_IMPORT_DONE - _MANAGER_CORE_IMPORT_DONE:.3f}s " + f"total={_MANAGER_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s" +) +print(_manager_import_timing_line, flush=True) +_append_boot_timing_line(_manager_import_timing_line) LEGACY_BOLT_FP_MIGRATION_FLAG = Path("/data") / "legacy_bolt_fp_migration_v1" @@ -60,30 +67,9 @@ STARPILOT_PC_ROOT_MIGRATION_FLAG = Path("/data") / "starpilot_pc_root_v1" STARPILOT_PARAMS_CACHE_MIGRATION_FLAG = Path("/data") / "starpilot_params_cache_v1" STARPILOT_LEGACY_CACHE_MARKER_KEYS = ("RemapCancelToDistance",) STARPILOT_REMOVED_PARAM_KEYS = ("HumanFollowing",) -PARAMS_CACHE_RESTORE_SKIP_FLAGS = ( - ParamKeyFlag.CLEAR_ON_MANAGER_START - | ParamKeyFlag.CLEAR_ON_ONROAD_TRANSITION - | ParamKeyFlag.CLEAR_ON_OFFROAD_TRANSITION - | ParamKeyFlag.CLEAR_ON_IGNITION_ON -) -UNREGISTERED_DONGLE_ID = "UnregisteredDevice" -POWER_WATCHDOG_PATH = "/var/tmp/power_watchdog" -_PENDING_PARAMS_CACHE_SYNC: tuple[str, list[str]] | None = None LEGACY_CARMODEL_MIGRATIONS = { "CHEVROLET_BOLT_CC_2019_2021": "CHEVROLET_BOLT_CC_2018_2021", } -LEGACY_STARPILOT_PARAM_RENAMES = { - "FrogPilotApiToken": "StarPilotApiToken", - "FrogPilotCarParams": "StarPilotCarParams", - "FrogPilotCarParamsPersistent": "StarPilotCarParamsPersistent", - "FrogPilotDongleId": "StarPilotDongleId", - "FrogPilotStats": "StarPilotStats", -} -LEGACY_STARPILOT_STATS_KEY_RENAMES = { - "FrogPilotDrives": "StarPilotDrives", - "FrogPilotMeters": "StarPilotMeters", - "FrogPilotSeconds": "StarPilotSeconds", -} STARPILOT_STATS_DROP_KEYS = {"CurrentMonthsKilometers", "ResetStats"} STARPILOT_STATS_MAX_KEYS = {"LongestDistanceWithoutOverride", "MaxAcceleration"} @@ -97,154 +83,6 @@ def _log_boot_timing(scope: str, label: str, start: float, previous: float | Non return now -def _sync_params_cache_keys_async(cache_params_path: str, keys: list[str]) -> None: - try: - params = Params() - params_cache = Params(cache_params_path, return_defaults=True) - for key in keys: - current_value = params.get(key) - if current_value is not None and params_cache.get(key) != current_value: - params_cache.put(key, current_value) - except Exception: - cloudlog.exception("failed to sync params cache") - - -def _schedule_params_cache_sync(cache_params_path: str, keys: list[str]) -> None: - timer = threading.Timer(5.0, _sync_params_cache_keys_async, args=(cache_params_path, keys)) - timer.daemon = True - timer.start() - - -def _schedule_pending_params_cache_sync() -> None: - global _PENDING_PARAMS_CACHE_SYNC - - pending_sync = _PENDING_PARAMS_CACHE_SYNC - _PENDING_PARAMS_CACHE_SYNC = None - if pending_sync is not None and pending_sync[1]: - _schedule_params_cache_sync(*pending_sync) - - -def _iter_param_store_keys(store_path: str | Path) -> set[str]: - try: - path = Path(store_path) - if not path.is_dir(): - return set() - - with os.scandir(path) as entries: - return { - entry.name - for entry in entries - if entry.is_file(follow_symlinks=False) and entry.name != ".lock" and not entry.name.startswith(".tmp_") - } - except Exception: - cloudlog.exception(f"failed to list params store: {store_path}") - return set() - - -def _param_key_to_text(key: bytes | str) -> str: - return key.decode("utf-8", errors="ignore") if isinstance(key, bytes) else str(key) - - -def _should_restore_param_from_cache(params: Params, key: bytes | str) -> bool: - try: - return not (params.get_key_flag(key) & PARAMS_CACHE_RESTORE_SKIP_FLAGS) - except Exception: - return False - - -def _restore_missing_params_from_cache(params: Params, params_cache: Params, active_keys: set[str] | None = None) -> list[str]: - active_keys = active_keys if active_keys is not None else _iter_param_store_keys(params.get_param_path()) - restored_keys: list[str] = [] - - for raw_key in params.all_keys(): - key = _param_key_to_text(raw_key) - if key in active_keys: - continue - - if not _should_restore_param_from_cache(params, raw_key): - continue - - cached_value = params_cache.get(raw_key) - if cached_value is None: - continue - - try: - params.put(raw_key, cached_value) - restored_keys.append(key) - except Exception: - cloudlog.exception(f"failed to restore param from cache: {key}") - - return restored_keys - - -def _init_sentry_async() -> None: - def fn() -> None: - try: - import openpilot.system.sentry as sentry - - sentry.init(sentry.SentryProject.SELFDRIVE) - except Exception: - cloudlog.exception("failed to initialize sentry") - - threading.Thread(target=fn, daemon=True).start() - - -def _capture_manager_exception() -> None: - try: - import openpilot.system.sentry as sentry - - sentry.capture_exception() - except Exception: - cloudlog.exception("failed to capture manager exception") - - -def _write_power_watchdog(timestamp: float) -> None: - import tempfile - - tmp_file_name = None - try: - with tempfile.NamedTemporaryFile("w", dir=os.path.dirname(POWER_WATCHDOG_PATH), delete=False) as f: - tmp_file_name = f.name - f.write(str(timestamp)) - os.replace(tmp_file_name, POWER_WATCHDOG_PATH) - tmp_file_name = None - finally: - if tmp_file_name is not None: - try: - os.unlink(tmp_file_name) - except FileNotFoundError: - pass - - -def _get_starpilot_toggles(sm=None): - from openpilot.starpilot.common.starpilot_variables import get_starpilot_toggles - - return get_starpilot_toggles(sm) - - -def _get_manager_startup_toggles(params: Params | None = None) -> SimpleNamespace: - params = params or Params() - force_onroad = params.get_bool("ForceOnroad") - device_management = params.get_bool("DeviceManagement") - vetting_branch = os.environ.get("GIT_BRANCH") == "StarPilot-Vetting" - - no_logging = False - no_uploads = False - if device_management and not vetting_branch: - no_logging = params.get_bool("NoLogging") - no_uploads = params.get_bool("NoUploads") - - return SimpleNamespace( - force_offroad=params.get_bool("ForceOffroad"), - force_onroad=force_onroad, - no_logging=no_logging or (force_onroad and HARDWARE.get_device_type() == "pc"), - no_uploads=no_uploads, - no_onroad_uploads=params.get_bool("DisableOnroadUploads") if no_uploads else False, - speed_limit_filler=params.get_bool("SpeedLimitFiller"), - vision_speed_limit_detection=params.get_bool("VisionSpeedLimitDetection"), - ) - - def _to_text(value): if value is None: return None @@ -416,8 +254,6 @@ def _copy_param_store_without_overwrite(source: Path, destination: Path) -> int: if not source.is_dir(): return 0 - import shutil - destination.mkdir(parents=True, exist_ok=True) copied_entries = 0 for path in source.iterdir(): @@ -535,8 +371,6 @@ def _merge_tree_without_overwrite(source: Path, destination: Path) -> int: if not source.exists(): return moved_entries - import shutil - if not destination.exists(): destination.parent.mkdir(parents=True, exist_ok=True) shutil.move(str(source), str(destination)) @@ -926,12 +760,11 @@ def migrate_legacy_experimental_longitudinal(params: Params, params_cache: Param def manager_init() -> None: - global _PENDING_PARAMS_CACHE_SYNC - manager_init_start = time.monotonic() last_timing = _log_boot_timing("manager_init", "start", manager_init_start, manager_init_start) - last_timing = _log_boot_timing("manager_init", "save_bootlog_deferred", manager_init_start, last_timing) + save_bootlog() + last_timing = _log_boot_timing("manager_init", "save_bootlog", manager_init_start, last_timing) build_metadata = get_build_metadata() last_timing = _log_boot_timing("manager_init", "build_metadata", manager_init_start, last_timing) @@ -977,11 +810,14 @@ def manager_init() -> None: last_timing = _log_boot_timing("manager_init", "starpilot_migrations", manager_init_start, last_timing) # set unset params to their default value - active_param_keys = _iter_param_store_keys(params.get_param_path()) - last_timing = _log_boot_timing("manager_init", f"params_store_snapshot_{len(active_param_keys)}", manager_init_start, last_timing) - restored_param_keys = _restore_missing_params_from_cache(params, params_cache, active_param_keys) - last_timing = _log_boot_timing("manager_init", f"params_restore_{len(restored_param_keys)}", manager_init_start, last_timing) - params_cache_update_keys = sorted(active_param_keys) + for k in params.all_keys(): + current_value = params.get(k) + if current_value is None: + cached_value = params_cache.get(k) + if cached_value is not None: + params.put(k, cached_value) + else: + params_cache.put(k, current_value) last_timing = _log_boot_timing("manager_init", "params_defaults_cache_sync", manager_init_start, last_timing) # Create folders needed for msgq @@ -1011,8 +847,6 @@ def manager_init() -> None: last_timing = _log_boot_timing("manager_init", "version_params", manager_init_start, last_timing) # set dongle id - from openpilot.system.athena.registration import register - reg_res = register(show_spinner=True) if reg_res: dongle_id = reg_res @@ -1028,6 +862,7 @@ def manager_init() -> None: os.environ['CLEAN'] = '1' # init logging + sentry.init(sentry.SentryProject.SELFDRIVE) cloudlog.bind_global(dongle_id=dongle_id, version=build_metadata.openpilot.version, origin=build_metadata.openpilot.git_normalized_origin, @@ -1037,22 +872,16 @@ def manager_init() -> None: device=HARDWARE.get_device_type()) last_timing = _log_boot_timing("manager_init", "logging_ready", manager_init_start, last_timing) - # Preimporting every process serializes a lot of import work before manager can - # start the always-on processes. Keep the old behavior available for debugging, - # but optimize normal boot by letting children import their modules in parallel. - if os.getenv("SP_MANAGER_PREIMPORT", "0").lower() in ("1", "true", "yes", "on"): - for p in managed_processes.values(): - p.prepare() - last_timing = _log_boot_timing("manager_init", "preimport_processes", manager_init_start, last_timing) - else: - last_timing = _log_boot_timing("manager_init", "preimport_processes_skipped", manager_init_start, last_timing) + # preimport all processes + for p in managed_processes.values(): + p.prepare() + last_timing = _log_boot_timing("manager_init", "preimport_processes", manager_init_start, last_timing) # StarPilot variables install_starpilot(build_metadata, params) last_timing = _log_boot_timing("manager_init", "install_starpilot", manager_init_start, last_timing) starpilot_boot_functions(build_metadata, params) _log_boot_timing("manager_init", "starpilot_boot_functions", manager_init_start, last_timing) - _PENDING_PARAMS_CACHE_SYNC = (cache_params_path, params_cache_update_keys) def manager_cleanup() -> None: @@ -1090,7 +919,7 @@ def manager_thread() -> None: last_timing = _log_boot_timing("manager_thread", "messaging", manager_thread_start, last_timing) write_onroad_params(False, params) - initial_toggles = _get_manager_startup_toggles(params) + initial_toggles = get_starpilot_toggles() last_timing = _log_boot_timing("manager_thread", "initial_toggles", manager_thread_start, last_timing) ensure_running(managed_processes.values(), False, params=params, CP=sm['carParams'], not_run=ignore, starpilot_toggles=initial_toggles) last_timing = _log_boot_timing("manager_thread", "initial_ensure_running", manager_thread_start, last_timing) @@ -1106,12 +935,9 @@ def manager_thread() -> None: params_memory = Params(memory=True) - starpilot_toggles = _get_manager_startup_toggles(params) + starpilot_toggles = get_starpilot_toggles() last_timing = _log_boot_timing("manager_thread", "loop_toggles", manager_thread_start, last_timing) _log_boot_timing("manager_thread", "loop_ready", manager_thread_start, last_timing) - save_bootlog() - _init_sentry_async() - _schedule_pending_params_cache_sync() while True: sm.update(1000) @@ -1159,7 +985,8 @@ def manager_thread() -> None: # kick AGNOS power monitoring watchdog try: if sm.all_checks(['deviceState']): - _write_power_watchdog(time.monotonic()) + with atomic_write("/var/tmp/power_watchdog", "w", overwrite=True) as f: + f.write(str(time.monotonic())) except Exception: pass @@ -1182,7 +1009,7 @@ def manager_thread() -> None: break # StarPilot variables - starpilot_toggles = _get_starpilot_toggles(sm) + starpilot_toggles = get_starpilot_toggles(sm) def main() -> None: @@ -1197,7 +1024,7 @@ def main() -> None: manager_thread() except Exception: traceback.print_exc() - _capture_manager_exception() + sentry.capture_exception() finally: manager_cleanup() @@ -1221,9 +1048,6 @@ if __name__ == "__main__": except KeyboardInterrupt: print("got CTRL-C, exiting") except Exception: - from openpilot.common.swaglog import add_file_handler - from openpilot.common.text_window import TextWindow - add_file_handler(cloudlog) cloudlog.exception("Manager failed to start") diff --git a/system/manager/process.py b/system/manager/process.py index a85355fb5..516d34159 100644 --- a/system/manager/process.py +++ b/system/manager/process.py @@ -6,8 +6,6 @@ import struct import threading import time import subprocess -import multiprocessing -import sys from pathlib import Path from collections.abc import Callable, ValuesView from abc import ABC, abstractmethod @@ -18,6 +16,7 @@ from setproctitle import setproctitle from cereal import car, log import cereal.messaging as messaging +import openpilot.system.sentry as sentry from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog @@ -430,6 +429,7 @@ def launcher(proc: str, name: str, nice: int | None = None) -> None: # add daemon name tag to logs cloudlog.bind(daemon=name) + sentry.set_tag("daemon", name) # exec the process mod.main() @@ -438,13 +438,7 @@ def launcher(proc: str, name: str, nice: int | None = None) -> None: except Exception: # can't install the crash handler because sys.excepthook doesn't play nice # with threads, so catch it here. - try: - import openpilot.system.sentry as sentry - - sentry.set_tag("daemon", name) - sentry.capture_exception() - except Exception: - cloudlog.exception(f"failed to capture exception for child {proc}") + sentry.capture_exception() raise @@ -467,33 +461,11 @@ def join_process(process: Process, timeout: float) -> None: time.sleep(0.001) -class SubprocessProcess: - def __init__(self, proc: subprocess.Popen): - self._proc = proc - - @property - def pid(self) -> int | None: - return self._proc.pid - - @property - def exitcode(self) -> int | None: - return self._proc.poll() - - def is_alive(self) -> bool: - return self._proc.poll() is None - - def join(self, timeout: float | None = None) -> None: - try: - self._proc.wait(timeout=timeout) - except subprocess.TimeoutExpired: - pass - - class ManagerProcess(ABC): daemon = False sigkill = False should_run: Callable[[bool, Params, car.CarParams, SimpleNamespace], bool] - proc: Process | SubprocessProcess | None = None + proc: Process | None = None enabled = True name = "" @@ -643,7 +615,7 @@ class NativeProcess(ManagerProcess): class PythonProcess(ManagerProcess): - def __init__(self, name, module, should_run, enabled=True, sigkill=False, watchdog_max_dt=None, nice=None, start_method=None): + def __init__(self, name, module, should_run, enabled=True, sigkill=False, watchdog_max_dt=None, nice=None): self.name = name self.module = module self.should_run = should_run @@ -651,7 +623,6 @@ class PythonProcess(ManagerProcess): self.sigkill = sigkill self.watchdog_max_dt = watchdog_max_dt self.nice = nice - self.start_method = start_method self.launcher = launcher def prepare(self) -> None: @@ -682,20 +653,8 @@ class PythonProcess(ManagerProcess): name = self.name if "modeld" not in self.name else "MainProcess" cloudlog.info(f"starting python {self.module}") - start_method = self.start_method or os.getenv("PYTHON_PROCESS_START_METHOD", "fork") - if start_method == "subprocess": - launcher_code = ( - "from openpilot.system.manager.process import launcher; " - f"launcher({self.module!r}, {self.name!r}, {self.nice!r})" - ) - self.proc = SubprocessProcess(subprocess.Popen([sys.executable, "-c", launcher_code])) - else: - self.proc = multiprocessing.get_context(start_method).Process( - name=name, - target=self.launcher, - args=(self.module, self.name, self.nice), - ) - self.proc.start() + self.proc = Process(name=name, target=self.launcher, args=(self.module, self.name, self.nice)) + self.proc.start() self.last_watchdog_time = 0 self.watchdog_seen = False self.shutting_down = False diff --git a/system/manager/process_config.py b/system/manager/process_config.py index 22f5cd755..6f6bee01b 100644 --- a/system/manager/process_config.py +++ b/system/manager/process_config.py @@ -7,42 +7,10 @@ from types import SimpleNamespace from cereal import car from openpilot.common.params import Params from openpilot.system.hardware import HARDWARE, PC, TICI +from openpilot.system.manager.process import PythonProcess, NativeProcess, DaemonProcess WEBCAM = os.getenv("USE_WEBCAM") is not None UI_WATCHDOG_MAX_DT = int(os.getenv("UI_WATCHDOG_MAX_DT", "10")) -device_type = HARDWARE.get_device_type() - - -def _env_bool(name: str) -> bool | None: - value = os.getenv(name) - if value is None: - return None - return value.strip().lower() in ("1", "true", "yes", "on") - - -def python_ui_enabled(device_type: str) -> bool: - for env_name in ("USE_PYTHON_UI", "USE_RAYLIB_UI"): - enabled = _env_bool(env_name) - if enabled is not None: - return enabled - - native_ui = _env_bool("USE_NATIVE_UI") - if native_ui is not None: - return not native_ui - - return device_type not in ("tici", "tizi") - - -def python_ui_process_start_method(uses_python_ui: bool, is_pc: bool = PC) -> str: - return "fork" if is_pc or not uses_python_ui else "subprocess" - - -PYTHON_UI = python_ui_enabled(device_type) -PYTHON_UI_PROCESS_START_METHOD = python_ui_process_start_method(PYTHON_UI) -THE_GALAXY_PROCESS_START_METHOD = "fork" if PC else "subprocess" -UPDATED_PROCESS_START_METHOD = "fork" if PC else "subprocess" - -from openpilot.system.manager.process import PythonProcess, NativeProcess, DaemonProcess def driverview(started: bool, params: Params, CP: car.CarParams, starpilot_toggles: SimpleNamespace) -> bool: return started or params.get_bool("IsDriverViewEnabled") @@ -157,13 +125,7 @@ procs = [ PythonProcess("radard", "selfdrive.controls.radard", only_onroad), PythonProcess("hardwared", "system.hardware.hardwared", always_run), PythonProcess("tombstoned", "system.tombstoned", always_run, enabled=not PC), - PythonProcess( - "updated", - "system.updated.updated", - always_run, - enabled=not PC, - start_method=UPDATED_PROCESS_START_METHOD, - ), + PythonProcess("updated", "system.updated.updated", always_run, enabled=not PC), PythonProcess("uploader", "system.loggerd.uploader", allow_uploads), PythonProcess("statsd", "system.statsd", always_run), PythonProcess("feedbackd", "selfdrive.ui.feedback.feedbackd", only_onroad), @@ -177,26 +139,16 @@ procs = [ # StarPilot variables procs += [ - PythonProcess( - "the_galaxy", - "starpilot.system.the_galaxy.the_galaxy", - always_run, - nice=19, - start_method=THE_GALAXY_PROCESS_START_METHOD, - ), + PythonProcess("the_galaxy", "starpilot.system.the_galaxy.the_galaxy", always_run, nice=19), PythonProcess("galaxy", "starpilot.system.galaxy.galaxy", always_run, nice=19), ] -if PYTHON_UI: - procs.append(PythonProcess( - "ui", - "selfdrive.ui.ui", - always_run, - watchdog_max_dt=UI_WATCHDOG_MAX_DT, - start_method=PYTHON_UI_PROCESS_START_METHOD, - )) -else: +device_type = HARDWARE.get_device_type() +if device_type in ("tici", "tizi"): procs.append(NativeProcess("ui", "selfdrive/ui", ["./ui"], always_run, watchdog_max_dt=UI_WATCHDOG_MAX_DT)) +else: + # C4 (mici) runs the Python raylib UI path. + procs.append(PythonProcess("ui", "selfdrive.ui.ui", always_run, watchdog_max_dt=UI_WATCHDOG_MAX_DT)) procs += [ PythonProcess("device_syncd", "starpilot.system.device_syncd", always_run), diff --git a/system/manager/test/test_launch_param_migrations.py b/system/manager/test/test_launch_param_migrations.py index 67d3fa356..5f1701948 100644 --- a/system/manager/test/test_launch_param_migrations.py +++ b/system/manager/test/test_launch_param_migrations.py @@ -50,9 +50,7 @@ class FileBackedFakeParams: def marker_path(tmp_path: Path, marker_name: str) -> Path: - path = tmp_path / MARKER_DIRNAME / "params" / marker_name - path.parent.mkdir(parents=True, exist_ok=True) - return path + return tmp_path / MARKER_DIRNAME / "params" / marker_name def test_apply_launch_param_migrations_sets_branch_defaults_once(tmp_path): diff --git a/system/manager/test/test_manager.py b/system/manager/test/test_manager.py index b23547ca4..fb6447de2 100644 --- a/system/manager/test/test_manager.py +++ b/system/manager/test/test_manager.py @@ -6,16 +6,10 @@ from pathlib import Path import json from cereal import car -from openpilot.common.params import Params, ParamKeyFlag +from openpilot.common.params import Params import openpilot.system.manager.manager as manager from openpilot.system.manager.process import ensure_running -from openpilot.system.manager.process import PythonProcess -from openpilot.system.manager.process_config import ( - managed_processes, - procs, - python_ui_enabled, - python_ui_process_start_method, -) +from openpilot.system.manager.process_config import managed_processes, procs from openpilot.system.hardware import HARDWARE os.environ['FAKEUPLOAD'] = "1" @@ -25,29 +19,15 @@ BLACKLIST_PROCS = ['manage_athenad', 'pandad', 'pigeond'] class FileBackedFakeParams: - def __init__( - self, - root: Path, - values: dict[str, object] | None = None, - keys: set[str] | None = None, - flags: dict[str, ParamKeyFlag] | None = None, - ): + def __init__(self, root: Path, values: dict[str, object] | None = None): self.root = root - self.keys = set(keys or []) - self.flags = dict(flags or {}) self.root.mkdir(parents=True, exist_ok=True) for key, value in (values or {}).items(): self.put(key, value) - def get_param_path(self, key=""): + def get_param_path(self, key): return str(self.root / (key.decode() if isinstance(key, bytes) else str(key))) - def all_keys(self): - return sorted(self.keys) - - def get_key_flag(self, key): - return self.flags.get(key.decode() if isinstance(key, bytes) else str(key), ParamKeyFlag.PERSISTENT) - def get(self, key): path = Path(self.get_param_path(key)) if not path.is_file(): @@ -68,7 +48,6 @@ class FileBackedFakeParams: return str(value).strip().lower() in ("1", "true", "yes", "on") def put(self, key, value): - self.keys.add(key.decode() if isinstance(key, bytes) else str(key)) path = Path(self.get_param_path(key)) path.parent.mkdir(parents=True, exist_ok=True) @@ -117,83 +96,6 @@ class TestManager: assert names.index("the_galaxy") < ui_idx assert names.index("galaxy") < ui_idx - def test_python_ui_process_start_method_follows_ui_implementation(self): - assert python_ui_process_start_method(False, False) == "fork" - assert python_ui_process_start_method(True, False) == "subprocess" - assert python_ui_process_start_method(True, True) == "fork" - - def test_python_ui_subprocess_is_scoped_to_ui(self): - ui_proc = managed_processes["ui"] - uses_python_ui = python_ui_enabled(HARDWARE.get_device_type()) - subprocess_scoped_procs = {"the_galaxy", "updated"} - - assert isinstance(ui_proc, PythonProcess) == uses_python_ui - if uses_python_ui: - assert ui_proc.start_method == python_ui_process_start_method(uses_python_ui) - for proc in procs: - if isinstance(proc, PythonProcess) and proc.name not in subprocess_scoped_procs | {"ui"}: - assert proc.start_method is None - - def test_python_ui_env_override(self, monkeypatch): - monkeypatch.setenv("USE_RAYLIB_UI", "1") - assert python_ui_enabled("tici") is True - - monkeypatch.setenv("USE_RAYLIB_UI", "0") - assert python_ui_enabled("mici") is False - - def test_manager_startup_toggles_use_params_only(self, tmp_path, monkeypatch): - monkeypatch.setenv("GIT_BRANCH", "StarPilot") - params = FileBackedFakeParams(tmp_path / "params", { - "DeviceManagement": True, - "NoLogging": True, - "NoUploads": True, - "DisableOnroadUploads": True, - "SpeedLimitFiller": True, - "VisionSpeedLimitDetection": True, - "ForceOffroad": True, - "ForceOnroad": False, - }) - - toggles = manager._get_manager_startup_toggles(params) - - assert toggles.no_logging is True - assert toggles.no_uploads is True - assert toggles.no_onroad_uploads is True - assert toggles.speed_limit_filler is True - assert toggles.vision_speed_limit_detection is True - assert toggles.force_offroad is True - assert toggles.force_onroad is False - - def test_restore_missing_params_from_cache_preserves_live_values(self, tmp_path): - params = FileBackedFakeParams( - tmp_path / "params", - {"ExistingParam": "live"}, - keys={"ExistingParam", "RestoredParam", "MissingParam", "TransientParam"}, - flags={"TransientParam": ParamKeyFlag.CLEAR_ON_MANAGER_START}, - ) - params_cache = FileBackedFakeParams( - tmp_path / "cache", - {"ExistingParam": "cached", "RestoredParam": "restored", "TransientParam": "stale"}, - ) - - restored_keys = manager._restore_missing_params_from_cache(params, params_cache) - - assert restored_keys == ["RestoredParam"] - assert params.get("ExistingParam") == "live" - assert params.get("RestoredParam") == "restored" - assert params.get("MissingParam") is None - assert params.get("TransientParam") is None - - def test_iter_param_store_keys_skips_lock_and_temp_files(self, tmp_path): - store_path = tmp_path / "params" - store_path.mkdir() - (store_path / "GoodParam").write_text("1") - (store_path / ".lock").write_text("") - (store_path / ".tmp_value_abc").write_text("stale") - (store_path / "nested").mkdir() - - assert manager._iter_param_store_keys(store_path) == {"GoodParam"} - def test_blacklisted_procs(self): # TODO: ensure there are blacklisted procs until we have a dedicated test assert len(BLACKLIST_PROCS), "No blacklisted procs to test not_run" diff --git a/system/tests/test_version.py b/system/tests/test_version.py deleted file mode 100644 index 47f259837..000000000 --- a/system/tests/test_version.py +++ /dev/null @@ -1,67 +0,0 @@ -import json - -from openpilot.system import version - - -def _write_minimal_git_checkout(path, branch="StarPilot", commit="1" * 40, origin="https://example.com/openpilot"): - git_folder = path / ".git" - git_folder.mkdir() - (git_folder / "refs" / "heads").mkdir(parents=True) - (git_folder / "HEAD").write_text(f"ref: refs/heads/{branch}\n", encoding="utf-8") - (git_folder / "refs" / "heads" / branch).write_text(f"{commit}\n", encoding="utf-8") - (git_folder / "config").write_text( - f"""[remote "origin"] - url = {origin} -[branch "{branch}"] - remote = origin - merge = refs/heads/{branch} -""", - encoding="utf-8", - ) - return { - "branch": branch, - "commit": commit, - "origin": origin, - } - - -def test_read_git_metadata_reads_head_ref_and_origin(tmp_path): - git_metadata = _write_minimal_git_checkout(tmp_path, branch="Dom", commit="a" * 40) - - assert version._read_git_metadata(str(tmp_path)) == git_metadata - - -def test_get_build_metadata_uses_prebuilt_git_cache(tmp_path, monkeypatch): - git_metadata = _write_minimal_git_checkout(tmp_path, branch="Dom", commit="b" * 40) - cache_root = tmp_path / "cache" - monkeypatch.setenv("OPENPILOT_BUILD_METADATA_CACHE_ROOT", str(cache_root)) - (tmp_path / "prebuilt").write_text("", encoding="utf-8") - (tmp_path / "common").mkdir() - (tmp_path / "common" / "version.h").write_text('#define COMMA_VERSION "1.2.3"\n', encoding="utf-8") - (tmp_path / "RELEASES.md").write_text("Release notes\n\nOlder notes\n", encoding="utf-8") - - cached_payload = { - "path": str(tmp_path.resolve()), - "git_metadata": git_metadata, - "build_metadata": { - "channel": "Dom", - "openpilot": { - "version": "1.2.3", - "release_notes": "Release notes", - "git_commit": git_metadata["commit"], - "git_origin": git_metadata["origin"], - "git_commit_date": "123 1970-01-01 00:02:03 +0000", - "build_style": "unknown", - "is_dirty": False, - }, - }, - } - cache_root.mkdir() - (cache_root / version.BUILD_METADATA_CACHE_FILENAME).write_text(json.dumps(cached_payload), encoding="utf-8") - monkeypatch.setattr(version, "get_commit_date", lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("cache miss"))) - - build_metadata = version.get_build_metadata(str(tmp_path)) - - assert build_metadata.channel == "Dom" - assert build_metadata.openpilot.git_commit == git_metadata["commit"] - assert build_metadata.openpilot.git_commit_date == "123 1970-01-01 00:02:03 +0000" diff --git a/system/ui/reset.py b/system/ui/reset.py index af47693c0..c32504a5b 100755 --- a/system/ui/reset.py +++ b/system/ui/reset.py @@ -1,69 +1,14 @@ #!/usr/bin/env python3 -from pathlib import Path - -from openpilot.system.hardware import HARDWARE - -BIG_UI_DEVICE_TYPES = ("tici", "tizi") -SMALL_UI_DEVICE_TYPE = "mici" - - -def _normalize_device_type(raw: str) -> str: - device_type = raw.replace("\x00", "").strip().lower() - if "comma " in device_type: - device_type = device_type.rsplit("comma ", 1)[-1].strip() - return device_type - - -def _device_tree_device_type() -> str | None: - model_path = Path("/sys/firmware/devicetree/base/model") - if not model_path.is_file(): - return None - - try: - return _normalize_device_type(model_path.read_text(encoding="utf-8", errors="ignore")) - except Exception: - return None - - -def _reported_device_type() -> str | None: - try: - return _normalize_device_type(HARDWARE.get_device_type()) - except Exception: - return None - - -def _ui_device_type() -> str: - device_tree_type = _device_tree_device_type() - if device_tree_type in BIG_UI_DEVICE_TYPES: - return device_tree_type - if device_tree_type: - return SMALL_UI_DEVICE_TYPE - - reported_type = _reported_device_type() - if reported_type in BIG_UI_DEVICE_TYPES: - return reported_type - return SMALL_UI_DEVICE_TYPE - - -def _patch_hardware_device_type(device_type: str) -> None: - HARDWARE.get_device_type = lambda: device_type - try: - import openpilot.system.hardware.tici.hardware as tici_hardware - tici_hardware.get_device_type = lambda: device_type - except Exception: - pass +from openpilot.system.ui.lib.application import gui_app +import openpilot.system.ui.tici_reset as tici_reset +import openpilot.system.ui.mici_reset as mici_reset def main(): - device_type = _ui_device_type() - _patch_hardware_device_type(device_type) - - if device_type in BIG_UI_DEVICE_TYPES: - import openpilot.system.ui.tici_reset as reset_impl + if gui_app.big_ui(): + tici_reset.main() else: - import openpilot.system.ui.mici_reset as reset_impl - - reset_impl.main() + mici_reset.main() if __name__ == "__main__": diff --git a/system/ui/tests/test_reset.py b/system/ui/tests/test_reset.py deleted file mode 100644 index 70d423c50..000000000 --- a/system/ui/tests/test_reset.py +++ /dev/null @@ -1,42 +0,0 @@ -import importlib -import sys -import types -from unittest.mock import patch - -import pytest - - -@pytest.fixture -def reset_module(monkeypatch): - hardware = types.ModuleType("openpilot.system.hardware") - hardware.HARDWARE = types.SimpleNamespace(get_device_type=lambda: "tici") - monkeypatch.setitem(sys.modules, "openpilot.system.hardware", hardware) - sys.modules.pop("openpilot.system.ui.reset", None) - module = importlib.import_module("openpilot.system.ui.reset") - yield module - sys.modules.pop("openpilot.system.ui.reset", None) - - -def test_device_tree_tici_uses_big_ui(reset_module): - with patch.object(reset_module, "_device_tree_device_type", return_value="tici"), \ - patch.object(reset_module, "_reported_device_type", return_value="mici"): - assert reset_module._ui_device_type() == "tici" - - -def test_device_tree_tizi_uses_big_ui(reset_module): - with patch.object(reset_module, "_device_tree_device_type", return_value="tizi"), \ - patch.object(reset_module, "_reported_device_type", return_value="mici"): - assert reset_module._ui_device_type() == "tizi" - - -@pytest.mark.parametrize("device_tree_type", ["mici", "comma 4", "comma four"]) -def test_non_tici_device_tree_uses_small_ui(reset_module, device_tree_type): - with patch.object(reset_module, "_device_tree_device_type", return_value=device_tree_type), \ - patch.object(reset_module, "_reported_device_type", return_value="tici"): - assert reset_module._ui_device_type() == "mici" - - -def test_reported_tici_used_when_no_device_tree(reset_module): - with patch.object(reset_module, "_device_tree_device_type", return_value=None), \ - patch.object(reset_module, "_reported_device_type", return_value="tici"): - assert reset_module._ui_device_type() == "tici" diff --git a/system/updated/updated.py b/system/updated/updated.py index b7c4c45fe..8e7c8890c 100644 --- a/system/updated/updated.py +++ b/system/updated/updated.py @@ -26,7 +26,6 @@ from openpilot.starpilot.common.starpilot_variables import BACKUP_PATH, get_star LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock") STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging") -GIT_CLEANUP_TIMEOUT = int(os.getenv("UPDATER_GIT_CLEANUP_TIMEOUT", "30")) OVERLAY_UPPER = os.path.join(STAGING_ROOT, "upper") OVERLAY_METADATA = os.path.join(STAGING_ROOT, "metadata") @@ -234,12 +233,10 @@ def finalize_update() -> None: cloudlog.info("Starting git cleanup in finalized update") t = time.monotonic() try: - subprocess.run(["git", "gc", "--auto"], cwd=FINALIZED, check=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, encoding="utf8", timeout=GIT_CLEANUP_TIMEOUT) - subprocess.run(["git", "lfs", "prune"], cwd=FINALIZED, check=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, encoding="utf8", timeout=GIT_CLEANUP_TIMEOUT) + run(["git", "gc"], FINALIZED) + run(["git", "lfs", "prune"], FINALIZED) cloudlog.event("Done git cleanup", duration=time.monotonic() - t) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + except subprocess.CalledProcessError: cloudlog.exception(f"Failed git cleanup, took {time.monotonic() - t:.3f} s") set_consistent_flag(True) @@ -495,6 +492,9 @@ def main() -> None: update_failed_count = 0 # TODO: Load from param? wait_helper = WaitTimeHelper() + # invalidate old finalized update + set_consistent_flag(False) + # set initial state params.put("UpdaterState", "idle") diff --git a/system/version.py b/system/version.py index 2c49d1cbe..ead7aa1e4 100644 --- a/system/version.py +++ b/system/version.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from dataclasses import asdict, dataclass +from dataclasses import dataclass from functools import cache import json import os @@ -14,7 +14,6 @@ RELEASE_BRANCHES = ['StarPilot', 'StarPilot-Vetting'] TESTED_BRANCHES = RELEASE_BRANCHES + ['StarPilot-Staging', 'StarPilot-Testing'] BUILD_METADATA_FILENAME = "build.json" -BUILD_METADATA_CACHE_FILENAME = "starpilot_build_metadata_cache.json" training_version: str = "0.2.0" terms_version: str = "2" @@ -133,133 +132,6 @@ def build_metadata_from_dict(build_metadata: dict) -> BuildMetadata: is_dirty=False)) -def _read_text_file(path: pathlib.Path) -> str | None: - try: - return path.read_text(encoding="utf-8").strip() - except OSError: - return None - - -def _read_git_ref(git_folder: pathlib.Path, ref: str) -> str | None: - ref_value = _read_text_file(git_folder / ref) - if ref_value: - return ref_value.splitlines()[0].strip() - - packed_refs = _read_text_file(git_folder / "packed-refs") - if packed_refs is None: - return None - - for line in packed_refs.splitlines(): - if not line or line.startswith(("#", "^")): - continue - - parts = line.split() - if len(parts) == 2 and parts[1] == ref: - return parts[0] - - return None - - -def _read_git_config_value(git_folder: pathlib.Path, section: str, key: str) -> str | None: - config = _read_text_file(git_folder / "config") - if config is None: - return None - - current_section = None - for raw_line in config.splitlines(): - line = raw_line.strip() - if not line or line.startswith(("#", ";")): - continue - - if line.startswith("[") and line.endswith("]"): - current_section = line[1:-1] - continue - - if current_section != section or "=" not in line: - continue - - config_key, value = line.split("=", 1) - if config_key.strip() == key: - return value.strip() - - return None - - -def _read_git_metadata(path: str) -> dict[str, str] | None: - git_folder = pathlib.Path(path) / ".git" - if not git_folder.is_dir(): - return None - - head = _read_text_file(git_folder / "HEAD") - if not head: - return None - - branch = "HEAD" - commit = head.splitlines()[0].strip() - if head.startswith("ref:"): - ref = head.split(":", 1)[1].strip() - branch = ref.rsplit("/", 1)[-1] - commit = _read_git_ref(git_folder, ref) - - if not commit: - return None - - remote = _read_git_config_value(git_folder, f'branch "{branch}"', "remote") or "origin" - origin = _read_git_config_value(git_folder, f'remote "{remote}"', "url") - if origin is None and remote != "origin": - origin = _read_git_config_value(git_folder, 'remote "origin"', "url") - - return { - "branch": branch, - "commit": commit, - "origin": origin or "", - } - - -def _build_metadata_cache_path(path: str) -> pathlib.Path | None: - if not (pathlib.Path(path) / ".git").is_dir(): - return None - - cache_root = pathlib.Path(os.environ.get("OPENPILOT_BUILD_METADATA_CACHE_ROOT", "/cache/starpilot")) - return cache_root / BUILD_METADATA_CACHE_FILENAME - - -def _read_cached_build_metadata(path: str, git_metadata: dict[str, str]) -> BuildMetadata | None: - cache_path = _build_metadata_cache_path(path) - if cache_path is None: - return None - - try: - cache_payload = json.loads(cache_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return None - - if cache_payload.get("path") != str(pathlib.Path(path).resolve()) or cache_payload.get("git_metadata") != git_metadata: - return None - - build_metadata = cache_payload.get("build_metadata") - if not isinstance(build_metadata, dict): - return None - - return build_metadata_from_dict(build_metadata) - - -def _write_cached_build_metadata(path: str, git_metadata: dict[str, str], build_metadata: BuildMetadata) -> None: - cache_path = _build_metadata_cache_path(path) - if cache_path is None: - return - - try: - cache_path.parent.mkdir(parents=True, exist_ok=True) - cache_path.write_text(json.dumps({ - "path": str(pathlib.Path(path).resolve()), - "git_metadata": git_metadata, - "build_metadata": asdict(build_metadata), - }, separators=(",", ":")), encoding="utf-8") - except OSError: - pass - - def get_build_metadata(path: str = BASEDIR) -> BuildMetadata: build_metadata_path = pathlib.Path(path) / BUILD_METADATA_FILENAME @@ -270,34 +142,15 @@ def get_build_metadata(path: str = BASEDIR) -> BuildMetadata: git_folder = pathlib.Path(path) / ".git" if git_folder.exists(): - prebuilt = is_prebuilt(path) - git_metadata = _read_git_metadata(path) - if prebuilt and git_metadata is not None: - cached_metadata = _read_cached_build_metadata(path, git_metadata) - if cached_metadata is not None: - return cached_metadata - - build_metadata = BuildMetadata(git_metadata["branch"], - OpenpilotMetadata( - version=get_version(path), - release_notes=get_release_notes(path), - git_commit=git_metadata["commit"], - git_origin=git_metadata["origin"] or get_origin(path), - git_commit_date=get_commit_date(path, git_metadata["commit"]), - build_style="unknown", - is_dirty=False)) - _write_cached_build_metadata(path, git_metadata, build_metadata) - return build_metadata - return BuildMetadata(get_short_branch(path), - OpenpilotMetadata( - version=get_version(path), - release_notes=get_release_notes(path), - git_commit=get_commit(path), - git_origin=get_origin(path), - git_commit_date=get_commit_date(path), - build_style="unknown", - is_dirty=is_dirty(path))) + OpenpilotMetadata( + version=get_version(path), + release_notes=get_release_notes(path), + git_commit=get_commit(path), + git_origin=get_origin(path), + git_commit_date=get_commit_date(path), + build_style="unknown", + is_dirty=is_dirty(path))) cloudlog.exception("unable to get build metadata") raise Exception("invalid build metadata")