Update process.py

openpilot/sunnypilot/modeld_v2/modeld.py

@@ -319,6 +319,9 @@ class ModelState(ModelStateBase):
     return log.ModelDataV2.Action(desiredCurvature=float(desired_curvature), desiredAcceleration=float(desired_accel), shouldStop=bool(stop))
 
 
+LINK_UP_TIMEOUT = 300  # seconds to wait for the 12V socket, driver may be walking to a remote-started car
+
+
 def main(demo=False):
   cloudlog.warning("modeld init")
 
@@ -364,9 +367,24 @@ def main(demo=False):
   model = None
   if USBGPU:
     import threading
+    from openpilot.system.hardware.chestnut.flash import link_up
+    # the asm enumerates off the comma's USB alone, but the link is only usable once the 12V socket is
+    # live and the pcie link is trained. a modeld SIGKILLed mid device_fini (tinygrad finalizes over USB
+    # from an atexit hook, which outlasts manager's 5s SIGINT grace) leaves it wedged, so retrain on
+    # retry. loading with the link down leaks tinygrad's am_usb lock fd onto the System singleton, which
+    # then masks every later error with "failed to acquire lock file".
+    for i in range(LINK_UP_TIMEOUT):
+      if link_up(reset=i > 0):
+        break
+      time.sleep(1)
+    else:
+      cloudlog.error("chestnut pcie link never came up")
     def load():
       nonlocal model
-      model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, usbgpu=True)
+      try:
+        model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, usbgpu=True)
+      except Exception:
+        cloudlog.exception("eGPU model load failed")  # the raise below only says "timed out", log the real cause
     t = threading.Thread(target=load, daemon=True)
     t.start()
     t.join(60)

openpilot/system/hardware/chestnut/flash.py

@@ -105,7 +105,7 @@ def open_device(path):
   return os.open(f"/dev/bus/usb/{bus:03d}/{dev:03d}", os.O_RDWR)
 
 
-def link_up() -> bool:
+def link_up(reset: bool = False) -> bool:
   # asm enumerates on USB-C alone, gpu is only usable once pcie link is up
   try:
     path, _, _ = find_chestnut()
@@ -115,6 +115,10 @@ def link_up() -> bool:
   except (OSError, RuntimeError):
     return False
   try:
+    if reset:
+      # a consumer killed mid device_fini leaves the link wedged, drop power to force a retrain
+      fcntl.ioctl(fd, USBDEVFS_CONTROL, Ctrl(0x40, 0xF3, 0, 0, 0, 2000, None))
+      time.sleep(0.5)
     fcntl.ioctl(fd, USBDEVFS_CONTROL, Ctrl(0x40, 0xF3, 1, 0, 0, 2000, None))
     buf = (ctypes.c_ubyte * 1)()
     fcntl.ioctl(fd, USBDEVFS_CONTROL, Ctrl(0xC0, 0xE4, 0xB450, 0, 1, 1000, ctypes.cast(buf, ctypes.c_void_p)))

openpilot/system/manager/process.py

@@ -105,6 +105,12 @@ class ManagerProcess(ABC):
 
     return ret
 
+  def reap(self) -> None:
+    # a process that exited on its own is otherwise never restarted: start() early-returns while proc is
+    # set, and only stop() clears it. modeld dying on a bad eGPU load left it dead for the whole drive.
+    if self.proc is not None and self.proc.exitcode is not None:
+      self.stop()
+
   def signal(self, sig: int) -> None:
     if self.proc is None:
       return
@@ -145,6 +151,7 @@ class NativeProcess(ManagerProcess):
     # In case we only tried a non blocking stop we need to stop it before restarting
     if self.shutting_down:
       self.stop()
+    self.reap()
 
     if self.proc is not None:
       return
@@ -169,6 +176,7 @@ class PythonProcess(ManagerProcess):
     # In case we only tried a non blocking stop we need to stop it before restarting
     if self.shutting_down:
       self.stop()
+    self.reap()
 
     if self.proc is not None:
       return