Error logs

frogpilot/common/frogpilot_functions.py

@@ -15,7 +15,7 @@ from openpilot.frogpilot.assets.theme_manager import ThemeManager
 from openpilot.frogpilot.common.frogpilot_backups import backup_frogpilot
 from openpilot.frogpilot.common.frogpilot_utilities import is_FrogsGoMoo, run_cmd
 from openpilot.frogpilot.common.frogpilot_variables import (
-  FROGS_GO_MOO_PATH, THEME_SAVE_PATH,
+  ERROR_LOGS_PATH, FROGS_GO_MOO_PATH, THEME_SAVE_PATH,
   FrogPilotVariables, get_frogpilot_toggles
 )
 
@@ -38,6 +38,7 @@ def frogpilot_boot_functions(build_metadata, params):
 
 def install_frogpilot(build_metadata, params):
   paths = [
+    ERROR_LOGS_PATH,
     THEME_SAVE_PATH
   ]
   for path in paths:

frogpilot/controls/frogpilot_card.py

@@ -5,7 +5,7 @@ from openpilot.selfdrive.car.cruise import CRUISE_LONG_PRESS, ButtonType
 from openpilot.selfdrive.selfdrived.events import ET
 
 from openpilot.frogpilot.common.frogpilot_utilities import is_FrogsGoMoo
-from openpilot.frogpilot.common.frogpilot_variables import NON_DRIVING_GEARS
+from openpilot.frogpilot.common.frogpilot_variables import ERROR_LOGS_PATH, NON_DRIVING_GEARS
 from openpilot.frogpilot.controls.lib.conditional_experimental_mode import CEStatus
 
 class FrogPilotCard:
@@ -28,6 +28,8 @@ class FrogPilotCard:
     self.long_press_threshold = CRUISE_LONG_PRESS * (1.5 if self.CP.brand == "gm" else 1)
     self.very_long_press_threshold = CRUISE_LONG_PRESS * 5
 
+    self.error_log = ERROR_LOGS_PATH / "error.txt"
+
   def handle_button_event(self, key, sm, frogpilot_toggles):
     if sm["carControl"].longActive and getattr(frogpilot_toggles, f"experimental_mode_via_{key}"):
       self.handle_experimental_mode(sm, frogpilot_toggles)
@@ -61,6 +63,7 @@ class FrogPilotCard:
     self.always_on_lateral_enabled &= sm["liveCalibration"].calPerc >= 1
     self.always_on_lateral_enabled &= (ET.IMMEDIATE_DISABLE not in sm["selfdriveState"].alertType + sm["frogpilotSelfdriveState"].alertType) or self.frogs_go_moo
     self.always_on_lateral_enabled &= not (carState.brakePressed and carState.vEgo < frogpilot_toggles.always_on_lateral_pause_speed) or carState.standstill
+    self.always_on_lateral_enabled &= not self.error_log.is_file() or self.frogs_go_moo
 
     if sm.updated["frogpilotPlan"] or any(be.type in (ButtonType.accelCruise, ButtonType.resumeCruise) for be in carState.buttonEvents):
       self.accel_pressed = any(be.type in (ButtonType.accelCruise, ButtonType.resumeCruise) for be in carState.buttonEvents)

frogpilot/controls/frogpilot_planner.py

@@ -20,13 +20,13 @@ from openpilot.frogpilot.controls.lib.frogpilot_following import FrogPilotFollow
 from openpilot.frogpilot.controls.lib.frogpilot_vcruise import FrogPilotVCruise
 
 class FrogPilotPlanner:
-  def __init__(self):
+  def __init__(self, error_log):
     self.params = Params(return_defaults=True)
     self.params_memory = Params(memory=True)
 
     self.frogpilot_acceleration = FrogPilotAcceleration(self)
     self.frogpilot_cem = ConditionalExperimentalMode(self)
-    self.frogpilot_events = FrogPilotEvents(self)
+    self.frogpilot_events = FrogPilotEvents(self, error_log)
     self.frogpilot_following = FrogPilotFollowing(self)
     self.frogpilot_vcruise = FrogPilotVCruise(self)
 

frogpilot/controls/lib/frogpilot_events.py

@@ -2,7 +2,7 @@
 from openpilot.selfdrive.selfdrived.events import ET, EVENT_NAME, FROGPILOT_EVENT_NAME, EventName, FrogPilotEventName, Events
 
 class FrogPilotEvents:
-  def __init__(self, FrogPilotPlanner):
+  def __init__(self, FrogPilotPlanner, error_log):
     self.frogpilot_planner = FrogPilotPlanner
 
     self.events = Events(frogpilot=True)
@@ -13,6 +13,8 @@ class FrogPilotEvents:
 
     self.played_events = set()
 
+    self.error_log = error_log
+
   def update(self, v_cruise, sm, frogpilot_toggles):
     current_alert = sm["selfdriveState"].alertType
     current_frogpilot_alert = sm["selfdriveState"].alertType
@@ -28,6 +30,9 @@ class FrogPilotEvents:
     else:
       self.max_acceleration = 0
 
+    if self.error_log.is_file():
+      self.events.add(FrogPilotEventName.openpilotCrashed)
+
     self.startup_seen |= sm["frogpilotSelfdriveState"].alertText1 == frogpilot_toggles.startup_alert_top and sm["frogpilotSelfdriveState"].alertText2 == frogpilot_toggles.startup_alert_bottom
 
     self.played_events.update(FROGPILOT_EVENT_NAME[event] for event in self.events.names)

frogpilot/frogpilot_process.py

@@ -12,7 +12,7 @@ from openpilot.frogpilot.assets.theme_manager import THEME_COMPONENT_PARAMS, The
 from openpilot.frogpilot.common.frogpilot_backups import backup_toggles
 from openpilot.frogpilot.common.frogpilot_functions import update_openpilot
 from openpilot.frogpilot.common.frogpilot_utilities import ThreadManager, is_url_pingable
-from openpilot.frogpilot.common.frogpilot_variables import FrogPilotVariables
+from openpilot.frogpilot.common.frogpilot_variables import ERROR_LOGS_PATH, FrogPilotVariables
 from openpilot.frogpilot.controls.frogpilot_planner import FrogPilotPlanner
 from openpilot.frogpilot.system.frogpilot_stats import send_stats
 from openpilot.frogpilot.system.frogpilot_tracking import FrogPilotTracking
@@ -31,7 +31,9 @@ def transition_offroad(frogpilot_planner, thread_manager, time_validated, sm, pa
   if time_validated:
     thread_manager.run_with_lock(send_stats, (params, frogpilot_toggles))
 
-def transition_onroad():
+def transition_onroad(error_log):
+  if error_log.is_file():
+    error_log.unlink()
 
 def update_checks(now, theme_manager, thread_manager, params, params_memory, frogpilot_toggles, boot_run=False):
   while not (is_url_pingable("https://github.com") or is_url_pingable("https://gitlab.com")):
@@ -80,6 +82,10 @@ def frogpilot_thread():
   started_previously = False
   time_validated = False
 
+  error_log = ERROR_LOGS_PATH / "error.txt"
+  if error_log.is_file():
+    error_log.unlink()
+
   while True:
     sm.update()
 
@@ -93,10 +99,10 @@ def frogpilot_thread():
 
       run_update_checks = True
     elif started and not started_previously:
-      frogpilot_planner = FrogPilotPlanner()
+      frogpilot_planner = FrogPilotPlanner(error_log)
       frogpilot_tracking = FrogPilotTracking(frogpilot_planner, frogpilot_toggles)
 
-      transition_onroad()
+      transition_onroad(error_log)
 
     if started and sm.updated["modelV2"]:
       frogpilot_planner.update(now, time_validated, sm, frogpilot_toggles)

frogpilot/ui/qt/offroad/data_settings.cc

@@ -57,6 +57,36 @@ FrogPilotDataPanel::FrogPilotDataPanel(FrogPilotSettingsWindow *parent, bool for
   }
   dataMainList->addItem(deleteDrivingDataButton);
 
+  ButtonControl *deleteErrorLogsButton = new ButtonControl(tr("Delete Error Logs"), tr("DELETE"), tr("<b>Delete collected error logs</b> to free up space and clear old crash records."));
+  QObject::connect(deleteErrorLogsButton, &ButtonControl::clicked, [=]() {
+    QDir errorLogsDir("/data/error_logs");
+
+    if (ConfirmationDialog::confirm(tr("Delete all error logs?"), tr("Delete"), this)) {
+      std::thread([=]() mutable {
+        parent->keepScreenOn = true;
+
+        deleteErrorLogsButton->setEnabled(false);
+        deleteErrorLogsButton->setValue(tr("Deleting..."));
+
+        errorLogsDir.removeRecursively();
+        errorLogsDir.mkpath(".");
+
+        deleteErrorLogsButton->setValue(tr("Deleted!"));
+
+        util::sleep_for(2500);
+
+        deleteErrorLogsButton->setEnabled(true);
+        deleteErrorLogsButton->setValue("");
+
+        parent->keepScreenOn = false;
+      }).detach();
+    }
+  });
+  if (forceOpenDescriptions) {
+    deleteErrorLogsButton->showDescription();
+  }
+  dataMainList->addItem(deleteErrorLogsButton);
+
   FrogPilotButtonsControl *frogpilotBackupButton = new FrogPilotButtonsControl(tr("FrogPilot Backups"), tr("<b>Create, delete, or restore FrogPilot backups.</b>"), "", {tr("BACKUP"), tr("DELETE"), tr("DELETE ALL"), tr("RESTORE")});
   QObject::connect(frogpilotBackupButton, &FrogPilotButtonsControl::buttonClicked, [=](int id) {
     QDir backupDir("/data/backups");

selfdrive/selfdrived/events.py

@@ -1054,6 +1054,20 @@ FROGPILOT_EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
   FrogPilotEventName.customStartupAlert: {
     ET.PERMANENT: custom_startup_alert,
   },
+
+  FrogPilotEventName.openpilotCrashed: {
+    ET.IMMEDIATE_DISABLE: Alert(
+      "openpilot crashed",
+      "Please post the 'Error Log' in the FrogPilot Discord!",
+      AlertStatus.critical, AlertSize.mid,
+      Priority.HIGHEST, VisualAlert.none, AudibleAlert.prompt, .1),
+
+    ET.NO_ENTRY: Alert(
+      "openpilot crashed",
+      "Please post the 'Error Log' in the FrogPilot Discord!",
+      AlertStatus.critical, AlertSize.mid,
+      Priority.HIGHEST, VisualAlert.none, AudibleAlert.prompt, .1),
+  },
 }
 
 

selfdrive/ui/qt/offroad/software_settings.cc

@@ -110,6 +110,14 @@ SoftwarePanel::SoftwarePanel(QWidget* parent) : ListWidget(parent) {
   });
   addItem(uninstallBtn);
 
+  // error log button
+  auto errorLogBtn = new ButtonControl(tr("Error Log"), tr("VIEW"), tr("View the error log for openpilot crashes."));
+  connect(errorLogBtn, &ButtonControl::clicked, [=]() {
+    std::string txt = util::read_file("/data/error_logs/error.txt");
+    ConfirmationDialog::rich(QString::fromStdString(txt), this);
+  });
+  addItem(errorLogBtn);
+
   fs_watch = new ParamWatcher(this);
   QObject::connect(fs_watch, &ParamWatcher::paramChanged, [=](const QString &param_name, const QString &param_value) {
     updateLabels();

selfdrive/ui/qt/onroad/alerts.cc

@@ -32,7 +32,15 @@ OnroadAlerts::Alert OnroadAlerts::getAlert(const SubMaster &sm, const SubMaster
   const cereal::FrogPilotSelfdriveState::Reader &fpss = fpsm["frogpilotSelfdriveState"].getFrogpilotSelfdriveState();
 
   Alert a = {};
-  if (selfdrive_frame >= started_frame) {  // Don't get old alert.
+  static QString crash_log_path = "/data/error_logs/error.txt";
+  if (QFile::exists(crash_log_path)) {
+    a = {tr("openpilot crashed"),
+         tr("Please post the \"Error Log\" in the FrogPilot Discord!"),
+         "openpilotCrashed",
+         cereal::SelfdriveState::AlertSize::MID,
+         cereal::SelfdriveState::AlertStatus::CRITICAL};
+    return a;
+  } else if (selfdrive_frame >= started_frame) {  // Don't get old alert.
     a = {ss.getAlertText1().cStr(), ss.getAlertText2().cStr(),
          ss.getAlertType().cStr(), ss.getAlertSize(), ss.getAlertStatus()};
 

selfdrive/ui/soundd.py

@@ -16,7 +16,7 @@ from openpilot.common.swaglog import cloudlog
 from openpilot.system import micd
 from openpilot.system.hardware import HARDWARE
 
-from openpilot.frogpilot.common.frogpilot_variables import ACTIVE_THEME_PATH, get_frogpilot_toggles
+from openpilot.frogpilot.common.frogpilot_variables import ACTIVE_THEME_PATH, ERROR_LOGS_PATH, get_frogpilot_toggles
 
 SAMPLE_RATE = 48000
 SAMPLE_BUFFER = 4096 # (approx 100ms)
@@ -84,10 +84,14 @@ class Soundd:
 
     self.frogpilot_toggles = get_frogpilot_toggles()
 
+    self.openpilot_crashed_played = False
+
     self.auto_volume = 0
 
     self.previous_sound_pack = None
 
+    self.error_log = ERROR_LOGS_PATH / "error.txt"
+
     self.update_frogpilot_sounds()
 
   def load_sounds(self):
@@ -154,6 +158,9 @@ class Soundd:
     if self.params_memory.get("TestAlert"):
       self.update_alert(getattr(AudibleAlert, self.params_memory.get("TestAlert")))
       self.params_memory.remove("TestAlert")
+    elif not self.openpilot_crashed_played and self.error_log.is_file():
+      self.update_alert(AudibleAlert.prompt)
+      self.openpilot_crashed_played = True
     elif sm.updated['selfdriveState']:
       new_alert = sm['selfdriveState'].alertSound.raw
 

system/sentry.py

@@ -2,6 +2,7 @@
 import os
 import sentry_sdk
 import traceback
+from datetime import datetime
 from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
@@ -11,6 +12,8 @@ from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.version import get_build_metadata, get_version
 
+from openpilot.frogpilot.common.frogpilot_variables import ERROR_LOGS_PATH
+
 
 class SentryProject(Enum):
   # python project
@@ -35,7 +38,7 @@ def capture_block() -> None:
     sentry_sdk.flush()
 
 
-def capture_exception(*args, **kwargs) -> None:
+def capture_exception(*args, crash_log=True, **kwargs) -> None:
   exc_text = traceback.format_exc()
 
   errors_to_ignore = [
@@ -44,6 +47,7 @@ def capture_exception(*args, **kwargs) -> None:
   if any(error in exc_text for error in errors_to_ignore):
     return
 
+  save_exception(exc_text, crash_log)
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
   try:
@@ -57,6 +61,20 @@ def set_tag(key: str, value: str) -> None:
   sentry_sdk.set_tag(key, value)
 
 
+def save_exception(exc_text: str, crash_log) -> None:
+  files = [
+    ERROR_LOGS_PATH / datetime.now().astimezone().strftime("%Y-%m-%d--%H-%M-%S.log"),
+    ERROR_LOGS_PATH / "error.txt"
+  ]
+
+  for file_path in files:
+    if file_path.name == "error.txt" and crash_log:
+      lines = exc_text.splitlines()[-10:]
+      file_path.write_text("\n".join(lines))
+    else:
+      file_path.write_text(exc_text)
+
+
 def init(project: SentryProject) -> bool:
   build_metadata = get_build_metadata()
   # forks like to mess with this, so double check