openpilot crashed alert

cereal/car.capnp

@@ -125,6 +125,7 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     laneChangeBlockedLoud @128;
     leadDeparting @129;
     noLaneAvailable @130;
+    openpilotCrashed @131;
     torqueNNLoad @135;
 
     radarCanErrorDEPRECATED @15;

selfdrive/controls/controlsd.py

@@ -7,6 +7,7 @@ from types import SimpleNamespace
 from typing import SupportsFloat
 
 import cereal.messaging as messaging
+import openpilot.selfdrive.sentry as sentry
 
 from cereal import car, custom, log
 from cereal.visionipc import VisionIpcClient, VisionStreamType
@@ -947,6 +948,11 @@ class Controls:
       if lead_departing:
         self.events.add(EventName.leadDeparting)
 
+    if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')) and not self.openpilot_crashed_triggered:
+      self.events.add(EventName.openpilotCrashed)
+
+      self.openpilot_crashed_triggered = True
+
     if self.sm.frame == 550 and self.CP.lateralTuning.which() == 'torque' and self.CI.use_nnff:
       self.events.add(EventName.torqueNNLoad)
 

selfdrive/controls/lib/events.py

@@ -1056,6 +1056,14 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
     ET.PERMANENT: no_lane_available_alert,
   },
 
+  EventName.openpilotCrashed: {
+    ET.PERMANENT: Alert(
+      "openpilot crashed",
+      "Please post the error log in the FrogPilot Discord!",
+      AlertStatus.normal, AlertSize.mid,
+      Priority.HIGHEST, VisualAlert.none, AudibleAlert.none, 10.),
+  },
+
   EventName.torqueNNLoad: {
     ET.PERMANENT: torque_nn_load_alert,
   },

selfdrive/frogpilot/controls/lib/frogpilot_functions.py

@@ -5,6 +5,7 @@ import numpy as np
 import os
 import shutil
 import subprocess
+import time
 
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.numpy_fast import interp
@@ -116,6 +117,22 @@ class FrogPilotFunctions:
       cmd = ['sudo', 'cp', '-a', '/data/params/.', f"{backup_path}/"]
       cls.run_cmd(cmd, f"Successfully backed up toggles to {backup_folder_name}.", f"Failed to backup toggles to {backup_folder_name}.")
 
+  @classmethod
+  def delete_logs(cls):
+    directories_to_check = []
+    for root, dirs, files in os.walk('/data/media/0/realdata/', topdown=False):
+      for name in files:
+        filepath = os.path.join(root, name)
+        if time.time() - max(os.path.getctime(filepath), os.path.getmtime(filepath)) < 3600:
+          os.remove(filepath)
+          if root not in directories_to_check:
+            directories_to_check.append(root)
+    for directory in directories_to_check:
+      try:
+        os.rmdir(directory)
+      except OSError:
+        pass
+
   @classmethod
   def setup_frogpilot(cls):
     remount_cmd = ['sudo', 'mount', '-o', 'remount,rw', '/persist']

selfdrive/manager/manager.py

@@ -360,7 +360,7 @@ def manager_cleanup() -> None:
   cloudlog.info("everything is dead")
 
 
-def manager_thread() -> None:
+def manager_thread(frogpilot_functions) -> None:
   cloudlog.bind(daemon="manager")
   cloudlog.info("manager start")
   cloudlog.info({"environ": os.environ})
@@ -386,10 +386,18 @@ def manager_thread() -> None:
   while True:
     sm.update(1000)
 
+    openpilot_crashed = os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt'))
+    if openpilot_crashed:
+      frogpilot_functions.delete_logs()
+
     started = sm['deviceState'].started
 
     if started and not started_prev:
       params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
+
+      if openpilot_crashed:
+        os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt'))
+
     elif not started and started_prev:
       params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
       params_memory.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
@@ -441,7 +449,7 @@ def main() -> None:
   signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(1))
 
   try:
-    manager_thread()
+    manager_thread(frogpilot_functions)
   except Exception:
     traceback.print_exc()
     sentry.capture_exception()

selfdrive/sentry.py

@@ -1,5 +1,9 @@
 """Install exception handler for process crash."""
+import os
 import sentry_sdk
+import traceback
+
+from datetime import datetime
 from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
@@ -10,6 +14,7 @@ from openpilot.common.swaglog import cloudlog
 from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \
                               is_comma_remote, is_dirty, is_tested_branch
 
+CRASHES_DIR = "/data/community/crashes/"
 
 class SentryProject(Enum):
   # python project
@@ -29,6 +34,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None:
 
 
 def capture_exception(*args, **kwargs) -> None:
+  save_exception(traceback.format_exc())
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
   try:
@@ -38,6 +44,26 @@ def capture_exception(*args, **kwargs) -> None:
     cloudlog.exception("sentry exception")
 
 
+def save_exception(exc_text: str) -> None:
+  if not os.path.exists(CRASHES_DIR):
+    os.makedirs(CRASHES_DIR)
+
+  files = [
+    os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')),
+    os.path.join(CRASHES_DIR, 'error.txt')
+  ]
+
+  for file in files:
+    with open(file, 'w') as f:
+      if file.endswith("error.txt"):
+        lines = exc_text.splitlines()[-10:]
+        f.write("\n".join(lines))
+      else:
+        f.write(exc_text)
+
+  print('Logged current crash to {}'.format(files))
+
+
 def set_tag(key: str, value: str) -> None:
   sentry_sdk.set_tag(key, value)
 

selfdrive/ui/ui.h

@@ -76,7 +76,12 @@ struct Alert {
       const int controls_missing = (nanos_since_boot() - sm.rcv_time("controlsState")) / 1e9;
 
       // Handle controls timeout
-      if (controls_frame < started_frame) {
+      if (std::ifstream("/data/community/crashes/error.txt")) {
+        alert = {"openpilot crashed", "Please post the error log in the FrogPilot Discord!",
+                 "controlsWaiting", cereal::ControlsState::AlertSize::MID,
+                 cereal::ControlsState::AlertStatus::NORMAL,
+                 AudibleAlert::NONE};
+      } else if (controls_frame < started_frame) {
         // car is started, but controlsState hasn't been seen at all
         alert = {"openpilot Unavailable", "Waiting for controls to start",
                  "controlsWaiting", cereal::ControlsState::AlertSize::MID,

system/loggerd/uploader.py

@@ -14,6 +14,7 @@ from collections.abc import Iterator
 
 from cereal import log
 import cereal.messaging as messaging
+import openpilot.selfdrive.sentry as sentry
 from openpilot.common.api import Api
 from openpilot.common.params import Params
 from openpilot.common.realtime import set_core_affinity
@@ -251,7 +252,8 @@ def main(exit_event: threading.Event = None) -> None:
     offroad = params.get_bool("IsOffroad")
     network_type = sm['deviceState'].networkType if not force_wifi else NetworkType.wifi
     at_home = not params.get_bool("DisableOnroadUploads") or offroad and network_type in (NetworkType.ethernet, NetworkType.wifi)
-    if network_type == NetworkType.none or not at_home:
+    openpilot_crashed = os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt'))
+    if network_type == NetworkType.none or not at_home or openpilot_crashed:
       if allow_sleep:
         time.sleep(60 if offroad else 5)
       continue