FrogPilot features - openpilot crashed alert

selfdrive/controls/controlsd.py

@@ -6,6 +6,7 @@ import threading
 from typing import SupportsFloat
 
 import cereal.messaging as messaging
+import openpilot.system.sentry as sentry
 
 from cereal import car, custom, log
 from msgq.visionipc import VisionIpcClient, VisionStreamType
@@ -180,6 +181,7 @@ class Controls:
     # FrogPilot variables
     self.frogpilot_toggles = FrogPilotVariables.toggles
 
+    self.openpilot_crashed_triggered = False
     self.update_toggles = False
 
     self.display_timer = 0
@@ -398,6 +400,9 @@ class Controls:
       if self.sm['modelV2'].frameDropPerc > 20:
         self.events.add(EventName.modeldLagging)
 
+    # Update FrogPilot events
+    self.update_frogpilot_events(CS)
+
   def data_sample(self):
     """Receive data from sockets"""
 
@@ -871,6 +876,11 @@ class Controls:
       e.set()
       t.join()
 
+  def update_frogpilot_events(self, CS):
+    if not self.openpilot_crashed_triggered and os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')):
+      self.events.add(EventName.openpilotCrashed)
+      self.openpilot_crashed_triggered = True
+
   def update_frogpilot_variables(self, CS):
     FPCC = custom.FrogPilotCarControl.new_message()
 

selfdrive/controls/lib/events.py

@@ -961,6 +961,14 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
     ET.NO_ENTRY: NoEntryAlert("Please don't use the 'Development' branch!"),
   },
 
+  EventName.openpilotCrashed: {
+    ET.PERMANENT: Alert(
+      "openpilot crashed",
+      "Please post the error log in the FrogPilot Discord!",
+      AlertStatus.normal, AlertSize.mid,
+      Priority.HIGH, VisualAlert.none, AudibleAlert.none, 10.),
+  },
+
   EventName.pedalInterceptorNoBrake: {
     ET.WARNING: Alert(
       "Braking Unavailable",

system/manager/manager.py

@@ -414,6 +414,11 @@ def manager_thread() -> None:
 
     if started and not started_prev:
       params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
+
+      error_log = os.path.join(sentry.CRASHES_DIR, 'error.txt')
+      if os.path.isfile(error_log):
+        os.remove(error_log)
+
     elif not started and started_prev:
       params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
       params_memory.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)

system/sentry.py

@@ -1,11 +1,14 @@
 """Install exception handler for process crash."""
 import http.client
+import os
 import sentry_sdk
 import socket
 import time
+import traceback
 import urllib.request
 import urllib.error
 
+from datetime import datetime
 from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
@@ -15,6 +18,7 @@ from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.version import get_build_metadata, get_version
 
+CRASHES_DIR = "/data/community/crashes/"
 
 class SentryProject(Enum):
   # python project
@@ -118,6 +122,16 @@ def capture_fingerprint(candidate, params, blocked=False):
 
 
 def capture_exception(*args, **kwargs) -> None:
+  exc_text = traceback.format_exc()
+
+  phrases_to_check = [
+    "To overwrite it, set 'overwrite' to True.",
+  ]
+
+  if any(phrase in exc_text for phrase in phrases_to_check):
+    return
+
+  save_exception(exc_text)
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
   FrogPilot = "frogai" in get_build_metadata().openpilot.git_origin.lower()
@@ -132,6 +146,26 @@ def capture_exception(*args, **kwargs) -> None:
     cloudlog.exception("sentry exception")
 
 
+def save_exception(exc_text: str) -> None:
+  if not os.path.exists(CRASHES_DIR):
+    os.makedirs(CRASHES_DIR)
+
+  files = [
+    os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')),
+    os.path.join(CRASHES_DIR, 'error.txt')
+  ]
+
+  for file in files:
+    with open(file, 'w') as f:
+      if file.endswith("error.txt"):
+        lines = exc_text.splitlines()[-10:]
+        f.write("\n".join(lines))
+      else:
+        f.write(exc_text)
+
+  print('Logged current crash to {}'.format(files))
+
+
 def set_tag(key: str, value: str) -> None:
   sentry_sdk.set_tag(key, value)