interesting

openpilot/sunnypilot/modeld_v2/modeld.py

@@ -319,6 +319,10 @@ class ModelState(ModelStateBase):
     return log.ModelDataV2.Action(desiredCurvature=float(desired_curvature), desiredAcceleration=float(desired_accel), shouldStop=bool(stop))
 
 
+BIG_MODEL_TIMEOUT = 60.
+BIG_MODEL_RETRY_INTERVAL = 30.  # ponytail: blind retry, no cheap way to poll eGPU power from outside modeld
+
+
 def main(demo=False):
   cloudlog.warning("modeld init")
 
@@ -362,23 +366,32 @@ def main(demo=False):
   st = time.monotonic()
 
   model = None
+  big_pending = False
   if USBGPU:
     import threading
+    big_model = None
     def load():
-      nonlocal model
-      model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, usbgpu=True)
+      nonlocal big_model
+      try:
+        big_model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, usbgpu=True)
+      except Exception:
+        cloudlog.exception("eGPU model load failed")
     t = threading.Thread(target=load, daemon=True)
     t.start()
-    t.join(60)
-    if model is None:
-      params.put_bool("UsbGpuActive", False)
-      raise RuntimeError("eGPU model load failed or timed out (60s)")
-    params.put_bool("UsbGpuActive", True)
-  else:
+    t.join(BIG_MODEL_TIMEOUT)
+    model = big_model
+    if model is not None:
+      params.put_bool("UsbGpuActive", True)
+    else:
+      # the eGPU enumerates over USB before the 12V socket is live (e.g. remote start), so the GPU
+      # can be dead on the first try. run the small model and retry while the car is still parked.
+      big_pending = True
+  if model is None:
     model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, usbgpu=False)
 
-  params.put_bool("UsbGpuLoading", False)
+  params.put_bool("UsbGpuLoading", big_pending)  # blocks engagement (NO_ENTRY) while we retry
   cloudlog.warning(f"models loaded in {time.monotonic() - st:.1f}s, modeld starting")
+  big_retry_t = time.monotonic() + BIG_MODEL_RETRY_INTERVAL
 
   # messaging
   pub_socks = ["modelV2", "drivingModelData", "cameraOdometry", "modelDataV2SP"] + (["chestnutState"] if USBGPU else [])
@@ -386,7 +399,7 @@ def main(demo=False):
   sm = SubMaster(["deviceState", "carState", "narrowRoadCameraState", "extrinsicsCalibration", "driverMonitoringState", "carControl", "lateralDelay"])
 
   publish_state = PublishState()
-  chestnut_state = ChestnutState(pm, USBGPU) if USBGPU else None
+  chestnut_state = ChestnutState(pm, model.usbgpu) if USBGPU else None
 
   # setup filter to track dropped frames
   frame_dropped_filter = FirstOrderFilter(0., 10., 1. / model.constants.MODEL_FREQ)
@@ -455,6 +468,15 @@ def main(demo=False):
     is_rhd = sm["driverMonitoringState"].isRHD
     frame_id = sm["narrowRoadCameraState"].frameId
     v_ego = max(sm["carState"].vEgo, 0.)
+
+    if big_pending:
+      if v_ego > 0.1 or sm["carState"].brakePressed or sm["carState"].gasPressed:
+        big_pending = False  # driver is here, stop retrying and commit to the small model
+        params.put_bool("UsbGpuActive", False)
+        params.put_bool("UsbGpuLoading", False)
+      elif time.monotonic() > big_retry_t:
+        cloudlog.warning("still parked, exiting to retry the eGPU model load")
+        return
     if sm.frame % 60 == 0:
       model.lat_delay = get_lat_delay(params, sm["lateralDelay"].lateralDelay)
       model.PLANPLUS_CONTROL = params.get("PlanplusControl", return_default=True)