StarPilot/openpilot/system/manager/manager.py

#!/usr/bin/env python3
import datetime
import os
import signal
import sys
import time
import traceback

from openpilot.starpilot.drive_state.owner import DriveStateOwner
from openpilot.cereal import log
import openpilot.cereal.messaging as messaging
from openpilot.common.utils import atomic_write
from openpilot.common.params import Params, ParamKeyFlag, UnknownKeyName
from openpilot.common.text_window import TextWindow
from openpilot.common.hardware import HARDWARE, PC
from openpilot.system.manager.helpers import unblock_stdout, save_bootlog
from openpilot.system.manager.process import ensure_running
from openpilot.system.manager.process_config import managed_processes
from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID
from openpilot.common.swaglog import cloudlog, add_file_handler
from openpilot.common.version import get_build_metadata
from openpilot.common.hardware.hw import Paths
from openpilot.system import sentry
from openpilot.starpilot.state_migration import prepare_manager_start
from openpilot.starpilot.storage import starpilot_storage_root


def manager_init() -> None:
  params = Params()
  prepare_manager_start(params, starpilot_storage_root(), auto_migrate=True)
  if not PC:
    from openpilot.starpilot.external_preferences_reset import reset_external_preferences
    from openpilot.starpilot.models.manager import ROOT as model_preferences_root
    for issue in reset_external_preferences(starpilot_storage_root(), model_preferences_root):
      cloudlog.warning('Fresh software preferences: %s', issue)
  from openpilot.starpilot.settings_retirement import retire_settings
  try:
    for issue in retire_settings(params):
      cloudlog.warning("Settings migration: %s", issue)
  except (OSError, ValueError, KeyError, UnknownKeyName):
    cloudlog.exception("Settings migration interrupted; retrying at next startup")
  from openpilot.starpilot.audio.default_enrollment import enroll_default_sounds
  try:
    enroll_default_sounds(params, starpilot_storage_root())
  except (OSError, ValueError):
    cloudlog.warning('Sound default enrollment interrupted; retrying at next startup')
  from openpilot.starpilot.connect.provider import activate_at_boot
  try:
    activate_at_boot(params)
  except (OSError, KeyError, TypeError, ValueError):
    os.environ['STARPILOT_CLOUD_DISABLED'] = '1'
    cloudlog.warning('Cloud provider state needs repair; cloud services disabled for this session')
  save_bootlog()

  build_metadata = get_build_metadata()

  params.clear_all(ParamKeyFlag.CLEAR_ON_MANAGER_START)
  params.clear_all(ParamKeyFlag.CLEAR_ON_ONROAD_TRANSITION)
  params.clear_all(ParamKeyFlag.CLEAR_ON_OFFROAD_TRANSITION)
  params.clear_all(ParamKeyFlag.CLEAR_ON_IGNITION_ON)
  if build_metadata.release_channel:
    params.clear_all(ParamKeyFlag.DEVELOPMENT_ONLY)

  if params.get_bool("RecordFrontLock"):
    params.put_bool("RecordFront", True, block=True)

  drive_state = DriveStateOwner(params, starpilot_storage_root() / "drive-state")
  try:
    drive_state.initialize_manager()
  except (OSError, KeyError, TypeError, ValueError, UnknownKeyName):
    cloudlog.warning("Force drive state is unavailable")

  # Initialize absent settings only. Typed reads can return None for malformed
  # saved data; replacing it here would bypass feature validation and recovery.
  for k in params.all_keys():
    if k == b"LKASButtonControl":
      continue  # Card enrolls the vehicle-specific default after identification.
    default_value = params.get_default_value(k)
    if default_value is not None and not os.path.lexists(params.get_param_path(k)):
      params.put(k, default_value, block=True)

  # Create folders needed for msgq
  try:
    os.mkdir(Paths.shm_path())
  except FileExistsError:
    pass
  except PermissionError:
    print(f"WARNING: failed to make {Paths.shm_path()}")

  # set params
  serial = HARDWARE.get_serial()
  params.put("Version", build_metadata.openpilot.version, block=True)
  params.put("GitCommit", build_metadata.openpilot.git_commit, block=True)
  params.put("GitCommitDate", build_metadata.openpilot.git_commit_date, block=True)
  params.put("GitBranch", build_metadata.channel, block=True)
  params.put("GitRemote", build_metadata.openpilot.git_origin, block=True)
  params.put_bool("IsTestedBranch", build_metadata.tested_channel, block=True)
  params.put_bool("IsReleaseBranch", build_metadata.release_channel, block=True)
  params.put("HardwareSerial", serial, block=True)

  # set dongle id
  reg_res = register(show_spinner=True)
  if reg_res:
    dongle_id = reg_res
  else:
    raise Exception(f"Registration failed for device {serial}")
  os.environ['DONGLE_ID'] = dongle_id  # Needed for swaglog
  os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_normalized_origin # Needed for swaglog
  os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog
  os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog

  if not build_metadata.openpilot.is_dirty:
    os.environ['CLEAN'] = '1'

  # init logging
  cloudlog.bind_global(dongle_id=dongle_id,
                       version=build_metadata.openpilot.version,
                       origin=build_metadata.openpilot.git_normalized_origin,
                       branch=build_metadata.channel,
                       commit=build_metadata.openpilot.git_commit,
                       dirty=build_metadata.openpilot.is_dirty,
                       device=HARDWARE.get_device_type())

def manager_cleanup() -> None:
  # send signals to kill all procs
  for p in managed_processes.values():
    p.stop(block=False)

  # ensure all are killed
  for p in managed_processes.values():
    p.stop(block=True)

  cloudlog.info("everything is dead")


def manager_thread() -> None:
  cloudlog.bind(daemon="manager")
  cloudlog.info("manager start")
  cloudlog.info({"environ": os.environ})

  params = Params()

  ignore: list[str] = []
  from openpilot.starpilot.connect.provider import active_provider
  if os.getenv('STARPILOT_CLOUD_DISABLED') == '1' or (params.get("DongleId") in (None, UNREGISTERED_DONGLE_ID) and
      active_provider().name != 'konik'):
    ignore += ["manage_athenad", "uploader"]
  if os.getenv("NOBOARD") is not None:
    ignore.append("pandad")
  ignore += [x for x in os.getenv("BLOCK", "").split(",") if len(x) > 0]

  sm = messaging.SubMaster(['deviceState', 'carParams', 'pandaStates', 'modelV2', 'drivingModelData',
                               'narrowRoadCameraState', 'wideRoadCameraState'], poll='deviceState')
  pm = messaging.PubMaster(['managerState'])

  params.put_bool("IsOffroad", True, block=True)
  ensure_running(managed_processes.values(), False, params=params, CP=sm['carParams'], not_run=ignore)

  started_prev = False
  ignition_prev = False

  while True:
    sm.update(1000)

    started = sm['deviceState'].started

    if started and not started_prev:
      params.clear_all(ParamKeyFlag.CLEAR_ON_ONROAD_TRANSITION)
    elif not started and started_prev:
      params.clear_all(ParamKeyFlag.CLEAR_ON_OFFROAD_TRANSITION)

    ignition = any(ps.ignitionLine or ps.ignitionCan for ps in sm['pandaStates'] if ps.pandaType != log.PandaState.PandaType.unknown)
    if ignition and not ignition_prev:
      params.clear_all(ParamKeyFlag.CLEAR_ON_IGNITION_ON)

    # update offroad state for services that don't subscribe to deviceState
    if started != started_prev:
      params.put_bool("IsOffroad", not started, block=True)

    started_prev = started
    ignition_prev = ignition

    managed_processes['modeld'].observe(sm, started)
    ensure_running(managed_processes.values(), started, params=params, CP=sm['carParams'], not_run=ignore)

    running = ' '.join("{}{}\u001b[0m".format("\u001b[32m" if p.proc.is_alive() else "\u001b[31m", p.name)
                       for p in managed_processes.values() if p.proc)
    print(running)
    cloudlog.debug(running)

    # send managerState
    msg = messaging.new_message('managerState', valid=True)
    msg.managerState.processes = [p.get_process_state_msg() for p in managed_processes.values()]
    pm.send('managerState', msg)

    # kick AGNOS power monitoring watchdog
    try:
      if sm.all_checks(['deviceState']):
        with atomic_write("/var/tmp/power_watchdog", "w", overwrite=True) as f:
          f.write(str(time.monotonic()))
    except Exception:
      pass

    # Exit main loop when uninstall/shutdown/reboot is needed
    shutdown = False
    for param in ("DoUninstall", "DoShutdown", "DoReboot"):
      if params.get_bool(param):
        shutdown = True
        params.put("LastManagerExitReason", f"{param} {datetime.datetime.now()}", block=True)
        cloudlog.warning(f"Shutting down manager - {param} set")

    if shutdown:
      break


def main() -> None:
  sentry.init()
  sentry.set_tag("daemon", "manager")
  try:
    manager_init()
  except Exception:
    sentry.capture_exception()
    sentry.flush()
    raise
  sentry.init()
  if os.getenv("PREPAREONLY") is not None:
    return

  # SystemExit on sigterm
  signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(1))

  try:
    manager_thread()
  except Exception:
    traceback.print_exc()
    cloudlog.exception("crash")
    sentry.capture_exception()
    sentry.flush()
  finally:
    try:
      from openpilot.starpilot.system.jetlink.lifecycle import shutdown as shutdown_jetlink
      shutdown_jetlink(Params())
    except Exception:
      cloudlog.exception("Jetlink shutdown handoff failed")
    manager_cleanup()

  params = Params()
  if params.get_bool("DoUninstall"):
    cloudlog.warning("uninstalling")
    HARDWARE.uninstall()
  elif params.get_bool("DoReboot"):
    cloudlog.warning("reboot")
    HARDWARE.reboot()
  elif params.get_bool("DoShutdown"):
    cloudlog.warning("shutdown")
    HARDWARE.shutdown()


if __name__ == "__main__":
  unblock_stdout()

  try:
    main()
  except KeyboardInterrupt:
    print("got CTRL-C, exiting")
  except Exception:
    add_file_handler(cloudlog)
    cloudlog.exception("Manager failed to start")

    try:
      managed_processes['ui'].stop()
    except Exception:
      pass

    # Show last 3 lines of traceback
    error = traceback.format_exc(-3)
    error = "Manager failed to start\n\n" + error
    with TextWindow(error) as t:
      t.wait_for_exit()

    raise

  # manual exit because we are forked
  sys.exit(0)