StarPilot/openpilot/starpilot/models/recovery_process.py

"""Manager adapter for process-isolated modeld recovery; no GPU context here."""
import os
import signal
import time
from multiprocessing import Process

from openpilot.common.swaglog import cloudlog
from openpilot.starpilot.models.receipt import process_start_ticks, read_receipt
from openpilot.starpilot.models.recovery import KILL_GRACE_NS, STARTUP_FALLBACK_EXIT, ModelRecoveryOwner, RecoveryEvidence
from openpilot.starpilot.models.status import ModelVariant
from openpilot.system.manager.process import PythonProcess, join_process, launcher

SMALL_ONLY_ENV = 'STARPILOT_MODELD_RECOVERY_SMALL_ONLY'


def modeld_launcher(module, name, small_only):
  # Set in this fresh child before importing modeld; never alter manager's env.
  if small_only:
    os.environ[SMALL_ONLY_ENV] = '1'
  else:
    os.environ.pop(SMALL_ONLY_ENV, None)
  launcher(module, name)


class ModeldProcess(PythonProcess):
  def __init__(self, name, module, should_run, enabled=True):
    super().__init__(name, module, should_run, enabled=enabled)
    self.recovery = ModelRecoveryOwner()
    self.context_ready = False
    self.launch_drive_id = 0

  def observe(self, sm, started):
    now = time.monotonic_ns()
    alive = self.proc is not None and self.proc.is_alive()
    pid = self.proc.pid if self.proc is not None else 0
    ticks = process_start_ticks(pid or 0) if alive else 0
    identity = (pid, ticks) if pid and ticks else None
    load = read_receipt() if alive else None
    matches = load is not None and identity == (load.pid, load.process_start_ticks)
    cameras = []
    for service in ('narrowRoadCameraState', 'wideRoadCameraState'):
      if sm.seen[service] and sm.valid[service] and sm.alive[service]:
        cameras.append((int(sm.logMonoTime[service]), int(sm[service].frameId)))
    evidence = RecoveryEvidence(
      now, int(sm['deviceState'].startedMonoTime) if started else 0, identity, alive,
      bool(matches and load.variant == ModelVariant.CHESTNUT), load.loaded_mono_ns if matches else 0,
      int(sm.logMonoTime['modelV2']), int(sm.logMonoTime['drivingModelData']),
      bool(sm.seen['modelV2'] and sm.seen['drivingModelData'] and sm.valid['modelV2'] and sm.valid['drivingModelData']),
      bool(sm['modelV2'].big), tuple(cameras),
    )
    device_fresh = (sm.seen['deviceState'] and sm.valid['deviceState'] and sm.alive['deviceState'] and
                    0 < int(sm.logMonoTime['deviceState']) <= now <= int(sm.logMonoTime['deviceState']) + 1_000_000_000)
    self.context_ready = bool(device_fresh and started)
    if (self.context_ready and self.proc is not None and not alive and not self.shutting_down and
        self.proc.exitcode == STARTUP_FALLBACK_EXIT and self.recovery.phase == 'monitor' and
        not self.recovery.small_only and 0 < self.launch_drive_id == evidence.drive_id):
      # Only our explicitly failed startup may request this one fresh process.
      # Never reuse the old GPU context or retry an arbitrary Small crash.
      self.proc.join(timeout=0)
      self.proc = None
      self.recovery.drive_id = evidence.drive_id
      self.recovery.small_only = True
      self.recovery.phase = 'small'
      cloudlog.event('modeld.recovery', action='small-after-startup-exit', pid=pid, drive=evidence.drive_id)
      return
    if not device_fresh and self.recovery.phase == 'monitor':
      self.recovery._withdraw()
      return
    action = self.recovery.step(evidence)
    if action in ('interrupt', 'kill'):
      # Owner identity is checked above; SIGKILL gets a bounded exit observation,
      # never an unbounded join and never a concurrent Small publisher.
      try:
        self.signal(signal.SIGINT if action == 'interrupt' else signal.SIGKILL)
      except ProcessLookupError:
        pass  # Exit is confirmed on the next owner observation.
      cloudlog.event('modeld.recovery', action=action, pid=pid, drive=evidence.drive_id)
    elif action == 'failed':
      cloudlog.error('modeld recovery could not confirm process exit; Small restart blocked')
    elif action in ('small', 'exit'):
      assert self.proc is not None and not self.proc.is_alive()
      self.proc.join(timeout=0)
      self.proc = None
      self.shutting_down = False
      cloudlog.event('modeld.recovery', action='small-after-exit', pid=pid, drive=evidence.drive_id)

  def stop(self, retry=True, block=True, sig=None):
    if self.recovery.phase in ('sigint', 'sigkill', 'failed'):
      # Recovery owns teardown even when manager transitions offroad. Never
      # enter the generic five-second/unbounded join for this known stalled PID.
      if self.proc is None:
        return None
      if self.proc.is_alive() and block:
        # manager_cleanup has no future observation tick. Kill this owned child
        # now and bound the exit wait instead of abandoning a stalled process.
        try:
          self.signal(signal.SIGKILL)
        except ProcessLookupError:
          pass
        join_process(self.proc, KILL_GRACE_NS / 1e9)
      if not self.proc.is_alive():
        self.proc.join(timeout=0)
        ret = self.proc.exitcode
        self.proc = None
        self.shutting_down = False
        return ret
      if block:
        self.recovery.phase = 'failed'
        cloudlog.error('modeld shutdown could not confirm stalled process exit')
      return None
    return super().stop(retry=retry, block=block, sig=sig)

  def start(self):
    if not self.context_ready or self.recovery.phase in ('sigint', 'sigkill', 'failed'):
      return
    if self.shutting_down:
      self.stop()
    if self.proc is not None:
      return
    self.proc = Process(name=self.name, target=modeld_launcher,
                        args=(self.module, self.name, self.recovery.small_only))
    self.launch_drive_id = self.recovery.drive_id
    self.proc.start()
    self.shutting_down = False