Add watchdog check to manager (#20277)

* this should work but doesnt

* Only offroad

* works

* make it work offorad

* reduce diff

* cleanup

* need util

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: a94ba4fb8b704a5462352af4501290f63c987754

selfdrive/manager/process.py

@@ -12,10 +12,14 @@ import cereal.messaging as messaging
 import selfdrive.crash as crash
 from common.basedir import BASEDIR
 from common.params import Params
+from common.realtime import sec_since_boot
 from selfdrive.swaglog import cloudlog
 from selfdrive.hardware import HARDWARE
 from cereal import log
 
+WATCHDOG_FN = "/dev/shm/wd_"
+ENABLE_WATCHDOG = os.getenv("NO_WATCHDOG") is None
+
 
 def launcher(proc):
   try:
@@ -61,6 +65,10 @@ class ManagerProcess(ABC):
   enabled = True
   name = ""
 
+  last_watchdog_time = 0
+  watchdog_max_dt = None
+  watchdog_seen = False
+
   @abstractmethod
   def prepare(self):
     pass
@@ -69,6 +77,30 @@ class ManagerProcess(ABC):
   def start(self):
     pass
 
+  def restart(self):
+    self.stop()
+    self.start()
+
+  def check_watchdog(self, started):
+    if self.watchdog_max_dt is None or self.proc is None:
+      return
+
+    try:
+      fn = WATCHDOG_FN + str(self.proc.pid)
+      self.last_watchdog_time = int(open(fn).read())
+    except Exception:
+      pass
+
+    dt = sec_since_boot() - self.last_watchdog_time / 1e9
+
+    if dt > self.watchdog_max_dt:
+      # Only restart while offroad for now
+      if self.watchdog_seen and ENABLE_WATCHDOG and (not started):
+        cloudlog.error(f"Watchdog timeout for {self.name}, restarting")
+        self.restart()
+    else:
+      self.watchdog_seen = True
+
   def stop(self, retry=True):
     if self.proc is None:
       return
@@ -128,7 +160,7 @@ class ManagerProcess(ABC):
 
 
 class NativeProcess(ManagerProcess):
-  def __init__(self, name, cwd, cmdline, enabled=True, persistent=False, driverview=False, unkillable=False, sigkill=False):
+  def __init__(self, name, cwd, cmdline, enabled=True, persistent=False, driverview=False, unkillable=False, sigkill=False, watchdog_max_dt=None):
     self.name = name
     self.cwd = cwd
     self.cmdline = cmdline
@@ -137,6 +169,7 @@ class NativeProcess(ManagerProcess):
     self.driverview = driverview
     self.unkillable = unkillable
     self.sigkill = sigkill
+    self.watchdog_max_dt = watchdog_max_dt
 
   def prepare(self):
     pass
@@ -149,10 +182,11 @@ class NativeProcess(ManagerProcess):
     cloudlog.info("starting process %s" % self.name)
     self.proc = Process(name=self.name, target=nativelauncher, args=(self.cmdline, cwd))
     self.proc.start()
+    self.watchdog_seen = False
 
 
 class PythonProcess(ManagerProcess):
-  def __init__(self, name, module, enabled=True, persistent=False, driverview=False, unkillable=False, sigkill=False):
+  def __init__(self, name, module, enabled=True, persistent=False, driverview=False, unkillable=False, sigkill=False, watchdog_max_dt=None):
     self.name = name
     self.module = module
     self.enabled = enabled
@@ -160,6 +194,7 @@ class PythonProcess(ManagerProcess):
     self.driverview = driverview
     self.unkillable = unkillable
     self.sigkill = sigkill
+    self.watchdog_max_dt = watchdog_max_dt
 
   def prepare(self):
     if self.enabled:
@@ -173,6 +208,7 @@ class PythonProcess(ManagerProcess):
     cloudlog.info("starting python %s" % self.module)
     self.proc = Process(name=self.name, target=launcher, args=(self.module,))
     self.proc.start()
+    self.watchdog_seen = False
 
 
 class DaemonProcess(ManagerProcess):
@@ -234,3 +270,6 @@ def ensure_running(procs, started, driverview=False, not_run=None):
       p.start()
     else:
       p.stop()
+
+    p.check_watchdog(started)
+