I AM SCREAMING

system/manager/manager.py

@@ -2,13 +2,12 @@
 import datetime
 import json
 import os
+import shutil
 from pathlib import Path
 import signal
 import sys
-import threading
 import time
 import traceback
-from types import SimpleNamespace
 
 _MANAGER_IMPORT_START = time.monotonic()
 _BOOT_TIMING_LOG_PATH = os.environ.get("SP_BOOT_TIMING_LOG", "/tmp/starpilot_boot_timing.log")
@@ -24,29 +23,37 @@ def _append_boot_timing_line(line: str) -> None:
 
 from cereal import car, log
 import cereal.messaging as messaging
+import openpilot.system.sentry as sentry
+from openpilot.common.utils import atomic_write
 from openpilot.common.params import Params, ParamKeyFlag, ParamKeyType
+from openpilot.common.text_window import TextWindow
 from openpilot.system.hardware import HARDWARE
 from openpilot.system.manager.helpers import unblock_stdout, write_onroad_params, save_bootlog
 from openpilot.system.manager.process import ensure_running
 from openpilot.system.manager.process_config import managed_processes
-from openpilot.common.swaglog import cloudlog
+from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID
+from openpilot.common.swaglog import cloudlog, add_file_handler
 from openpilot.system.version import get_build_metadata, terms_version, training_version
 from openpilot.system.hardware.hw import Paths
 
 _MANAGER_CORE_IMPORT_DONE = time.monotonic()
 
 from openpilot.starpilot.common.starpilot_functions import starpilot_boot_functions, install_starpilot, uninstall_starpilot
+from openpilot.starpilot.common.starpilot_variables import (
+  LEGACY_STARPILOT_PARAM_RENAMES,
+  LEGACY_STARPILOT_STATS_KEY_RENAMES,
+  get_starpilot_toggles,
+)
 
 _MANAGER_IMPORT_DONE = time.monotonic()
-if __name__ == "__main__":
-  _manager_import_timing_line = (
-    "SP_BOOT_TIMING manager_import "
-    f"core={_MANAGER_CORE_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s "
-    f"starpilot={_MANAGER_IMPORT_DONE - _MANAGER_CORE_IMPORT_DONE:.3f}s "
-    f"total={_MANAGER_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s"
-  )
-  print(_manager_import_timing_line, flush=True)
-  _append_boot_timing_line(_manager_import_timing_line)
+_manager_import_timing_line = (
+  "SP_BOOT_TIMING manager_import "
+  f"core={_MANAGER_CORE_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s "
+  f"starpilot={_MANAGER_IMPORT_DONE - _MANAGER_CORE_IMPORT_DONE:.3f}s "
+  f"total={_MANAGER_IMPORT_DONE - _MANAGER_IMPORT_START:.3f}s"
+)
+print(_manager_import_timing_line, flush=True)
+_append_boot_timing_line(_manager_import_timing_line)
 
 
 LEGACY_BOLT_FP_MIGRATION_FLAG = Path("/data") / "legacy_bolt_fp_migration_v1"
@@ -60,30 +67,9 @@ STARPILOT_PC_ROOT_MIGRATION_FLAG = Path("/data") / "starpilot_pc_root_v1"
 STARPILOT_PARAMS_CACHE_MIGRATION_FLAG = Path("/data") / "starpilot_params_cache_v1"
 STARPILOT_LEGACY_CACHE_MARKER_KEYS = ("RemapCancelToDistance",)
 STARPILOT_REMOVED_PARAM_KEYS = ("HumanFollowing",)
-PARAMS_CACHE_RESTORE_SKIP_FLAGS = (
-  ParamKeyFlag.CLEAR_ON_MANAGER_START
-  | ParamKeyFlag.CLEAR_ON_ONROAD_TRANSITION
-  | ParamKeyFlag.CLEAR_ON_OFFROAD_TRANSITION
-  | ParamKeyFlag.CLEAR_ON_IGNITION_ON
-)
-UNREGISTERED_DONGLE_ID = "UnregisteredDevice"
-POWER_WATCHDOG_PATH = "/var/tmp/power_watchdog"
-_PENDING_PARAMS_CACHE_SYNC: tuple[str, list[str]] | None = None
 LEGACY_CARMODEL_MIGRATIONS = {
   "CHEVROLET_BOLT_CC_2019_2021": "CHEVROLET_BOLT_CC_2018_2021",
 }
-LEGACY_STARPILOT_PARAM_RENAMES = {
-  "FrogPilotApiToken": "StarPilotApiToken",
-  "FrogPilotCarParams": "StarPilotCarParams",
-  "FrogPilotCarParamsPersistent": "StarPilotCarParamsPersistent",
-  "FrogPilotDongleId": "StarPilotDongleId",
-  "FrogPilotStats": "StarPilotStats",
-}
-LEGACY_STARPILOT_STATS_KEY_RENAMES = {
-  "FrogPilotDrives": "StarPilotDrives",
-  "FrogPilotMeters": "StarPilotMeters",
-  "FrogPilotSeconds": "StarPilotSeconds",
-}
 STARPILOT_STATS_DROP_KEYS = {"CurrentMonthsKilometers", "ResetStats"}
 STARPILOT_STATS_MAX_KEYS = {"LongestDistanceWithoutOverride", "MaxAcceleration"}
 
@@ -97,154 +83,6 @@ def _log_boot_timing(scope: str, label: str, start: float, previous: float | Non
   return now
 
 
-def _sync_params_cache_keys_async(cache_params_path: str, keys: list[str]) -> None:
-  try:
-    params = Params()
-    params_cache = Params(cache_params_path, return_defaults=True)
-    for key in keys:
-      current_value = params.get(key)
-      if current_value is not None and params_cache.get(key) != current_value:
-        params_cache.put(key, current_value)
-  except Exception:
-    cloudlog.exception("failed to sync params cache")
-
-
-def _schedule_params_cache_sync(cache_params_path: str, keys: list[str]) -> None:
-  timer = threading.Timer(5.0, _sync_params_cache_keys_async, args=(cache_params_path, keys))
-  timer.daemon = True
-  timer.start()
-
-
-def _schedule_pending_params_cache_sync() -> None:
-  global _PENDING_PARAMS_CACHE_SYNC
-
-  pending_sync = _PENDING_PARAMS_CACHE_SYNC
-  _PENDING_PARAMS_CACHE_SYNC = None
-  if pending_sync is not None and pending_sync[1]:
-    _schedule_params_cache_sync(*pending_sync)
-
-
-def _iter_param_store_keys(store_path: str | Path) -> set[str]:
-  try:
-    path = Path(store_path)
-    if not path.is_dir():
-      return set()
-
-    with os.scandir(path) as entries:
-      return {
-        entry.name
-        for entry in entries
-        if entry.is_file(follow_symlinks=False) and entry.name != ".lock" and not entry.name.startswith(".tmp_")
-      }
-  except Exception:
-    cloudlog.exception(f"failed to list params store: {store_path}")
-    return set()
-
-
-def _param_key_to_text(key: bytes | str) -> str:
-  return key.decode("utf-8", errors="ignore") if isinstance(key, bytes) else str(key)
-
-
-def _should_restore_param_from_cache(params: Params, key: bytes | str) -> bool:
-  try:
-    return not (params.get_key_flag(key) & PARAMS_CACHE_RESTORE_SKIP_FLAGS)
-  except Exception:
-    return False
-
-
-def _restore_missing_params_from_cache(params: Params, params_cache: Params, active_keys: set[str] | None = None) -> list[str]:
-  active_keys = active_keys if active_keys is not None else _iter_param_store_keys(params.get_param_path())
-  restored_keys: list[str] = []
-
-  for raw_key in params.all_keys():
-    key = _param_key_to_text(raw_key)
-    if key in active_keys:
-      continue
-
-    if not _should_restore_param_from_cache(params, raw_key):
-      continue
-
-    cached_value = params_cache.get(raw_key)
-    if cached_value is None:
-      continue
-
-    try:
-      params.put(raw_key, cached_value)
-      restored_keys.append(key)
-    except Exception:
-      cloudlog.exception(f"failed to restore param from cache: {key}")
-
-  return restored_keys
-
-
-def _init_sentry_async() -> None:
-  def fn() -> None:
-    try:
-      import openpilot.system.sentry as sentry
-
-      sentry.init(sentry.SentryProject.SELFDRIVE)
-    except Exception:
-      cloudlog.exception("failed to initialize sentry")
-
-  threading.Thread(target=fn, daemon=True).start()
-
-
-def _capture_manager_exception() -> None:
-  try:
-    import openpilot.system.sentry as sentry
-
-    sentry.capture_exception()
-  except Exception:
-    cloudlog.exception("failed to capture manager exception")
-
-
-def _write_power_watchdog(timestamp: float) -> None:
-  import tempfile
-
-  tmp_file_name = None
-  try:
-    with tempfile.NamedTemporaryFile("w", dir=os.path.dirname(POWER_WATCHDOG_PATH), delete=False) as f:
-      tmp_file_name = f.name
-      f.write(str(timestamp))
-    os.replace(tmp_file_name, POWER_WATCHDOG_PATH)
-    tmp_file_name = None
-  finally:
-    if tmp_file_name is not None:
-      try:
-        os.unlink(tmp_file_name)
-      except FileNotFoundError:
-        pass
-
-
-def _get_starpilot_toggles(sm=None):
-  from openpilot.starpilot.common.starpilot_variables import get_starpilot_toggles
-
-  return get_starpilot_toggles(sm)
-
-
-def _get_manager_startup_toggles(params: Params | None = None) -> SimpleNamespace:
-  params = params or Params()
-  force_onroad = params.get_bool("ForceOnroad")
-  device_management = params.get_bool("DeviceManagement")
-  vetting_branch = os.environ.get("GIT_BRANCH") == "StarPilot-Vetting"
-
-  no_logging = False
-  no_uploads = False
-  if device_management and not vetting_branch:
-    no_logging = params.get_bool("NoLogging")
-    no_uploads = params.get_bool("NoUploads")
-
-  return SimpleNamespace(
-    force_offroad=params.get_bool("ForceOffroad"),
-    force_onroad=force_onroad,
-    no_logging=no_logging or (force_onroad and HARDWARE.get_device_type() == "pc"),
-    no_uploads=no_uploads,
-    no_onroad_uploads=params.get_bool("DisableOnroadUploads") if no_uploads else False,
-    speed_limit_filler=params.get_bool("SpeedLimitFiller"),
-    vision_speed_limit_detection=params.get_bool("VisionSpeedLimitDetection"),
-  )
-
-
 def _to_text(value):
   if value is None:
     return None
@@ -416,8 +254,6 @@ def _copy_param_store_without_overwrite(source: Path, destination: Path) -> int:
   if not source.is_dir():
     return 0
 
-  import shutil
-
   destination.mkdir(parents=True, exist_ok=True)
   copied_entries = 0
   for path in source.iterdir():
@@ -535,8 +371,6 @@ def _merge_tree_without_overwrite(source: Path, destination: Path) -> int:
   if not source.exists():
     return moved_entries
 
-  import shutil
-
   if not destination.exists():
     destination.parent.mkdir(parents=True, exist_ok=True)
     shutil.move(str(source), str(destination))
@@ -926,12 +760,11 @@ def migrate_legacy_experimental_longitudinal(params: Params, params_cache: Param
 
 
 def manager_init() -> None:
-  global _PENDING_PARAMS_CACHE_SYNC
-
   manager_init_start = time.monotonic()
   last_timing = _log_boot_timing("manager_init", "start", manager_init_start, manager_init_start)
 
-  last_timing = _log_boot_timing("manager_init", "save_bootlog_deferred", manager_init_start, last_timing)
+  save_bootlog()
+  last_timing = _log_boot_timing("manager_init", "save_bootlog", manager_init_start, last_timing)
 
   build_metadata = get_build_metadata()
   last_timing = _log_boot_timing("manager_init", "build_metadata", manager_init_start, last_timing)
@@ -977,11 +810,14 @@ def manager_init() -> None:
   last_timing = _log_boot_timing("manager_init", "starpilot_migrations", manager_init_start, last_timing)
 
   # set unset params to their default value
-  active_param_keys = _iter_param_store_keys(params.get_param_path())
-  last_timing = _log_boot_timing("manager_init", f"params_store_snapshot_{len(active_param_keys)}", manager_init_start, last_timing)
-  restored_param_keys = _restore_missing_params_from_cache(params, params_cache, active_param_keys)
-  last_timing = _log_boot_timing("manager_init", f"params_restore_{len(restored_param_keys)}", manager_init_start, last_timing)
-  params_cache_update_keys = sorted(active_param_keys)
+  for k in params.all_keys():
+    current_value = params.get(k)
+    if current_value is None:
+      cached_value = params_cache.get(k)
+      if cached_value is not None:
+        params.put(k, cached_value)
+    else:
+      params_cache.put(k, current_value)
   last_timing = _log_boot_timing("manager_init", "params_defaults_cache_sync", manager_init_start, last_timing)
 
   # Create folders needed for msgq
@@ -1011,8 +847,6 @@ def manager_init() -> None:
   last_timing = _log_boot_timing("manager_init", "version_params", manager_init_start, last_timing)
 
   # set dongle id
-  from openpilot.system.athena.registration import register
-
   reg_res = register(show_spinner=True)
   if reg_res:
     dongle_id = reg_res
@@ -1028,6 +862,7 @@ def manager_init() -> None:
     os.environ['CLEAN'] = '1'
 
   # init logging
+  sentry.init(sentry.SentryProject.SELFDRIVE)
   cloudlog.bind_global(dongle_id=dongle_id,
                        version=build_metadata.openpilot.version,
                        origin=build_metadata.openpilot.git_normalized_origin,
@@ -1037,22 +872,16 @@ def manager_init() -> None:
                        device=HARDWARE.get_device_type())
   last_timing = _log_boot_timing("manager_init", "logging_ready", manager_init_start, last_timing)
 
-  # Preimporting every process serializes a lot of import work before manager can
-  # start the always-on processes. Keep the old behavior available for debugging,
-  # but optimize normal boot by letting children import their modules in parallel.
-  if os.getenv("SP_MANAGER_PREIMPORT", "0").lower() in ("1", "true", "yes", "on"):
-    for p in managed_processes.values():
-      p.prepare()
-    last_timing = _log_boot_timing("manager_init", "preimport_processes", manager_init_start, last_timing)
-  else:
-    last_timing = _log_boot_timing("manager_init", "preimport_processes_skipped", manager_init_start, last_timing)
+  # preimport all processes
+  for p in managed_processes.values():
+    p.prepare()
+  last_timing = _log_boot_timing("manager_init", "preimport_processes", manager_init_start, last_timing)
 
   # StarPilot variables
   install_starpilot(build_metadata, params)
   last_timing = _log_boot_timing("manager_init", "install_starpilot", manager_init_start, last_timing)
   starpilot_boot_functions(build_metadata, params)
   _log_boot_timing("manager_init", "starpilot_boot_functions", manager_init_start, last_timing)
-  _PENDING_PARAMS_CACHE_SYNC = (cache_params_path, params_cache_update_keys)
 
 
 def manager_cleanup() -> None:
@@ -1090,7 +919,7 @@ def manager_thread() -> None:
   last_timing = _log_boot_timing("manager_thread", "messaging", manager_thread_start, last_timing)
 
   write_onroad_params(False, params)
-  initial_toggles = _get_manager_startup_toggles(params)
+  initial_toggles = get_starpilot_toggles()
   last_timing = _log_boot_timing("manager_thread", "initial_toggles", manager_thread_start, last_timing)
   ensure_running(managed_processes.values(), False, params=params, CP=sm['carParams'], not_run=ignore, starpilot_toggles=initial_toggles)
   last_timing = _log_boot_timing("manager_thread", "initial_ensure_running", manager_thread_start, last_timing)
@@ -1106,12 +935,9 @@ def manager_thread() -> None:
 
   params_memory = Params(memory=True)
 
-  starpilot_toggles = _get_manager_startup_toggles(params)
+  starpilot_toggles = get_starpilot_toggles()
   last_timing = _log_boot_timing("manager_thread", "loop_toggles", manager_thread_start, last_timing)
   _log_boot_timing("manager_thread", "loop_ready", manager_thread_start, last_timing)
-  save_bootlog()
-  _init_sentry_async()
-  _schedule_pending_params_cache_sync()
 
   while True:
     sm.update(1000)
@@ -1159,7 +985,8 @@ def manager_thread() -> None:
     # kick AGNOS power monitoring watchdog
     try:
       if sm.all_checks(['deviceState']):
-        _write_power_watchdog(time.monotonic())
+        with atomic_write("/var/tmp/power_watchdog", "w", overwrite=True) as f:
+          f.write(str(time.monotonic()))
     except Exception:
       pass
 
@@ -1182,7 +1009,7 @@ def manager_thread() -> None:
       break
 
     # StarPilot variables
-    starpilot_toggles = _get_starpilot_toggles(sm)
+    starpilot_toggles = get_starpilot_toggles(sm)
 
 
 def main() -> None:
@@ -1197,7 +1024,7 @@ def main() -> None:
     manager_thread()
   except Exception:
     traceback.print_exc()
-    _capture_manager_exception()
+    sentry.capture_exception()
   finally:
     manager_cleanup()
 
@@ -1221,9 +1048,6 @@ if __name__ == "__main__":
   except KeyboardInterrupt:
     print("got CTRL-C, exiting")
   except Exception:
-    from openpilot.common.swaglog import add_file_handler
-    from openpilot.common.text_window import TextWindow
-
     add_file_handler(cloudlog)
     cloudlog.exception("Manager failed to start")