alert for errors in camera stack (#20932)

* camera error alert

* check crcs

* cleanup

* bump cereal

Co-authored-by: Comma Device <device@comma.ai>
old-commit-hash: 2defc730db18f7f9c0b08b0c8b0f6806cc513ac4

selfdrive/controls/controlsd.py

@@ -53,12 +53,16 @@ class Controls:
       self.pm = messaging.PubMaster(['sendcan', 'controlsState', 'carState',
                                      'carControl', 'carEvents', 'carParams'])
 
+    self.camera_packets = ["roadCameraState", "driverCameraState"]
+    if TICI:
+      self.camera_packets.append("wideRoadCameraState")
+
     self.sm = sm
     if self.sm is None:
       ignore = ['driverCameraState', 'managerState'] if SIMULATION else None
       self.sm = messaging.SubMaster(['deviceState', 'pandaState', 'modelV2', 'liveCalibration',
                                      'driverMonitoringState', 'longitudinalPlan', 'lateralPlan', 'liveLocationKalman',
-                                     'roadCameraState', 'driverCameraState', 'managerState', 'liveParameters', 'radarState'],
+                                     'managerState', 'liveParameters', 'radarState'] + self.camera_packets,
                                      ignore_alive=ignore, ignore_avg_freq=['radarState', 'longitudinalPlan'])
 
     self.can_sock = can_sock
@@ -66,6 +70,9 @@ class Controls:
       can_timeout = None if os.environ.get('NO_CAN_TIMEOUT', False) else 100
       self.can_sock = messaging.sub_sock('can', timeout=can_timeout)
 
+    if TICI:
+      self.log_sock = messaging.sub_sock('androidLog')
+
     # wait for one pandaState and one CAN packet
     print("Waiting for CAN messages...")
     get_one_can(self.can_sock)
@@ -249,6 +256,21 @@ class Controls:
     if self.sm['longitudinalPlan'].fcw:
       self.events.add(EventName.fcw)
 
+    if TICI and self.enable_lte_onroad:
+      logs = messaging.drain_sock(self.log_sock, wait_for_one=False)
+      messages = []
+      for m in logs:
+        try:
+          messages.append(m.androidLog.message)
+        except UnicodeDecodeError:
+          pass
+
+      for err in ["ERROR_CRC", "ERROR_ECC", "ERROR_STREAM_UNDERFLOW", "APPLY FAILED"]:
+        err_cnt = sum(err in m for m in messages)
+        if err_cnt:
+          self.events.add(EventName.cameraError)
+          break
+
     # TODO: fix simulator
     if not SIMULATION:
       if not NOSENSOR:
@@ -256,7 +278,7 @@ class Controls:
           (not TICI or self.enable_lte_onroad):
           # Not show in first 1 km to allow for driving out of garage. This event shows after 5 minutes
           self.events.add(EventName.noGps)
-      if not self.sm.all_alive(['roadCameraState', 'driverCameraState']):
+      if not self.sm.all_alive(self.camera_packets):
         self.events.add(EventName.cameraMalfunction)
       if self.sm['modelV2'].frameDropPerc > 20:
         self.events.add(EventName.modeldLagging)

selfdrive/controls/lib/events.py

@@ -672,6 +672,10 @@ EVENTS: Dict[int, Dict[str, Union[Alert, Callable[[Any, messaging.SubMaster, boo
     ET.IMMEDIATE_DISABLE: ImmediateDisableAlert("Controls Mismatch"),
   },
 
+  EventName.cameraError: {
+    ET.PERMANENT: NormalPermanentAlert("CRC Error", ""),
+  },
+
   EventName.usbError: {
     ET.SOFT_DISABLE: SoftDisableAlert("USB Error: Reboot Your Device"),
     ET.PERMANENT: NormalPermanentAlert("USB Error: Reboot Your Device", ""),