controlsd: handle radar missing + cleanup system error handling (#25179)

* controlsd: handle radar missing + cleanup system erorr handling

* fix carla

* update refs
old-commit-hash: 505bbce29d45c246be177d81e618f9289e78aa16

selfdrive/controls/controlsd.py

@@ -96,7 +96,11 @@ class Controls:
 
     self.sm = sm
     if self.sm is None:
-      ignore = ['driverCameraState', 'managerState'] if SIMULATION else None
+      ignore = []
+      if SIMULATION:
+        ignore += ['driverCameraState', 'managerState']
+      if params.get_bool('WideCameraOnly'):
+        ignore += ['roadCameraState']
       self.sm = messaging.SubMaster(['deviceState', 'pandaStates', 'peripheralState', 'modelV2', 'liveCalibration',
                                      'driverMonitoringState', 'longitudinalPlan', 'lateralPlan', 'liveLocationKalman',
                                      'managerState', 'liveParameters', 'radarState'] + self.camera_packets + joystick_packet,
@@ -224,12 +228,8 @@ class Controls:
     if not self.CP.notCar:
       self.events.add_from_msg(self.sm['driverMonitoringState'].events)
 
-    # Handle car events. Ignore when CAN is invalid
-    if CS.canTimeout:
-      self.events.add(EventName.canBusMissing)
-    elif not CS.canValid:
-      self.events.add(EventName.canError)
-    else:
+    # Add car events, ignore if CAN isn't valid
+    if CS.canValid:
       self.events.add_from_msg(CS.events)
 
     # Create events for temperature, disk space, and memory
@@ -309,14 +309,19 @@ class Controls:
           self.events.add(EventName.cameraFrameRate)
     if self.rk.lagging:
       self.events.add(EventName.controlsdLagging)
-    if len(self.sm['radarState'].radarErrors):
+    if len(self.sm['radarState'].radarErrors) or not self.sm.all_checks(['radarState']):
       self.events.add(EventName.radarFault)
     if not self.sm.valid['pandaStates']:
       self.events.add(EventName.usbError)
+    if CS.canTimeout:
+      self.events.add(EventName.canBusMissing)
+    elif not CS.canValid:
+      self.events.add(EventName.canError)
 
     # generic catch-all. ideally, a more specific event should be added above instead
-    no_system_errors = len(self.events) != num_events
-    if (not self.sm.all_checks() or self.can_rcv_error) and no_system_errors and CS.canValid and not CS.canTimeout:
+    has_disable_events = self.events.any(ET.NO_ENTRY) and (self.events.any(ET.SOFT_DISABLE) or self.events.any(ET.IMMEDIATE_DISABLE))
+    no_system_errors = (not has_disable_events) or (len(self.events) == num_events)
+    if (not self.sm.all_checks() or self.can_rcv_error) and no_system_errors:
       if not self.sm.all_alive():
         self.events.add(EventName.commIssue)
       elif not self.sm.all_freq_ok():

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-fa52fa6c6703269e23610b1c6aba8a56b911fbbb
+7c1168af0311d2fef67b82812cd863a0e97c030e

selfdrive/test/process_replay/regen.py

@@ -263,7 +263,7 @@ def regen_segment(lr, frs=None, outdir=FAKEDATA, disable_tqdm=False):
   seg_path = os.path.join(outdir, segment)
   # check to make sure openpilot is engaged in the route
   if not check_enabled(LogReader(os.path.join(seg_path, "rlog"))):
-    raise Exception(f"Route never enabled: {segment}")
+    raise Exception(f"Route did not engage for long enough: {segment}")
 
   return seg_path
 

selfdrive/test/process_replay/test_processes.py

@@ -38,18 +38,18 @@ original_segments = [
 
 segments = [
   ("BODY", "regen660D86654BA|2022-07-06--14-27-15--0"),
-  ("HYUNDAI", "regen657E25856BB|2022-07-06--14-26-51--0"),
+  ("HYUNDAI", "regen114E5FF24D8|2022-07-14--17-08-47--0"),
   ("HYUNDAI", "d824e27e8c60172c|2022-07-08--21-21-15--0"),
   ("TOYOTA", "regenBA97410FBEC|2022-07-06--14-26-49--0"),
   ("TOYOTA2", "regenDEDB1D9C991|2022-07-06--14-54-08--0"),
   ("TOYOTA3", "regenDDC1FE60734|2022-07-06--14-32-06--0"),
-  ("HONDA", "regen17B09D158B8|2022-07-06--14-31-46--0"),
-  ("HONDA2", "regen041739C3E9A|2022-07-06--15-08-02--0"),
-  ("CHRYSLER", "regenBB2F9C1425C|2022-07-06--14-31-41--0"),
+  ("HONDA", "regenE62960EEC38|2022-07-14--19-33-24--0"),
+  ("HONDA2", "regenC3EBD92F029|2022-07-14--19-29-47--0"),
+  ("CHRYSLER", "regen38346FB33D0|2022-07-14--18-05-26--0"),
   ("RAM", "2f4452b03ccb98f0|2022-07-07--08-01-56--3"),
-  ("SUBARU", "regen732B69F33B1|2022-07-06--14-36-18--0"),
+  ("SUBARU", "regen54A1E2BE5AA|2022-07-14--18-07-50--0"),
   ("GM", "regen01D09D915B5|2022-07-06--14-36-20--0"),
-  ("NISSAN", "regenEA6FB2773F5|2022-07-06--14-58-23--0"),
+  ("NISSAN", "regenCA0B0DC946E|2022-07-14--18-10-17--0"),
   ("VOLKSWAGEN", "regen007098CA0EF|2022-07-06--15-01-26--0"),
   ("MAZDA", "regen61BA413D53B|2022-07-06--14-39-42--0"),
 ]

tools/sim/start_carla.sh

@@ -22,6 +22,7 @@ if [[ "$DETACH" ]]; then
   EXTRA_ARGS="-d"
 fi
 
+docker kill carla_sim || true
 docker run \
   --name carla_sim \
   --rm \