StarPilot/openpilot/starpilot/models/recovery.py

"""Drive-scoped process recovery after an established Big publisher stops.

This owner never touches GPU buffers or retained model selection. Small may start
only after the exact failed modeld process has exited.
"""
from dataclasses import dataclass

ARM_SPAN_NS = 1_000_000_000
ARM_MAX_GAP_NS = 1_000_000_000
STALL_NS = 2_000_000_000
CAMERA_MAX_AGE_NS = 200_000_000
INT_GRACE_NS = 500_000_000
KILL_GRACE_NS = 1_000_000_000
STARTUP_FALLBACK_EXIT = 75


@dataclass(frozen=True)
class RecoveryEvidence:
  now_ns: int
  drive_id: int
  identity: tuple[int, int] | None
  alive: bool
  big_receipt: bool
  loaded_ns: int
  model_ns: int
  driving_ns: int
  output_valid: bool
  output_big: bool
  cameras: tuple[tuple[int, int], ...]  # producer timestamp, frameId


class ModelRecoveryOwner:
  def __init__(self):
    self.drive_id = 0
    self.small_only = False
    self.phase = 'monitor'
    self.identity = None
    self.first_output_ns = 0
    self.last_output = (0, 0)
    self.last_progress_ns = 0
    self.last_cameras = ()
    self.camera_progress_since = 0
    self.deadline_ns = 0
    self.armed = False

  def _withdraw(self):
    self.identity = None
    self.first_output_ns = 0
    self.last_output = (0, 0)
    self.last_progress_ns = 0
    self.last_cameras = ()
    self.camera_progress_since = 0
    self.armed = False

  def step(self, e: RecoveryEvidence) -> str | None:
    if self.phase in ('sigint', 'sigkill', 'failed'):
      if not e.alive:
        if self.phase == 'failed' and e.drive_id == self.drive_id:
          return None
        if e.drive_id == self.drive_id and e.drive_id > 0:
          self.phase = 'small'
          return 'small'
        self.__init__()
        self.drive_id = e.drive_id
        return 'exit'
      if e.identity != self.identity:
        self.phase = 'failed'
        return None
      if self.phase != 'failed' and e.now_ns >= self.deadline_ns:
        if self.phase == 'sigint':
          self.phase = 'sigkill'
          self.deadline_ns = e.now_ns + KILL_GRACE_NS
          return 'kill'
        self.phase = 'failed'
        return 'failed'
      return None
    if e.drive_id <= 0:
      self.__init__()
      return None
    if e.drive_id != self.drive_id:
      self.__init__()
      self.drive_id = e.drive_id
    if self.phase != 'monitor' or self.small_only:
      return None
    if (not e.alive or e.identity is None or min(e.identity) <= 0 or not e.big_receipt or
        not self.drive_id <= e.loaded_ns <= e.now_ns):
      self._withdraw()
      return None
    if e.identity != self.identity:
      self._withdraw()
      self.identity = e.identity
    cameras_fresh = (len(e.cameras) == 2 and all(self.drive_id <= t <= e.now_ns and
                     e.now_ns - t <= CAMERA_MAX_AGE_NS and f >= 0 for t, f in e.cameras))
    if not cameras_fresh:
      self.camera_progress_since = 0
      self.last_cameras = ()
      return None
    cameras_advance = bool(self.last_cameras and all(t > pt and f > pf for (t, f), (pt, pf) in zip(e.cameras, self.last_cameras, strict=True)))
    if cameras_advance:
      if not self.camera_progress_since:
        self.camera_progress_since = e.now_ns
    else:
      self.camera_progress_since = 0
    self.last_cameras = e.cameras
    output = (e.model_ns, e.driving_ns)
    valid_output = (e.output_valid and e.output_big and all(e.loaded_ns <= t <= e.now_ns for t in output) and
                    e.now_ns - min(output) <= CAMERA_MAX_AGE_NS)
    if valid_output and all(t > old for t, old in zip(output, self.last_output, strict=True)):
      if not self.armed and self.last_progress_ns and e.now_ns - self.last_progress_ns > ARM_MAX_GAP_NS:
        self.first_output_ns = 0
      if not self.first_output_ns:
        self.first_output_ns = min(output)
      self.last_output = output
      self.last_progress_ns = e.now_ns
      self.armed = min(output) - self.first_output_ns >= ARM_SPAN_NS
      return None
    if (self.armed and self.last_progress_ns and e.now_ns - self.last_progress_ns >= STALL_NS and
        self.camera_progress_since and e.now_ns - self.camera_progress_since >= STALL_NS):
      self.small_only = True
      self.phase = 'sigint'
      self.deadline_ns = e.now_ns + INT_GRACE_NS
      return 'interrupt'
    return None