diff --git a/cereal/car.capnp b/cereal/car.capnp index f4ed3f4c7..c2fd7f4c1 100644 --- a/cereal/car.capnp +++ b/cereal/car.capnp @@ -125,6 +125,7 @@ struct CarEvent @0x9b1657f34caf3ad3 { laneChangeBlockedLoud @128; leadDeparting @129; noLaneAvailable @130; + openpilotCrashed @131; torqueNNLoad @135; radarCanErrorDEPRECATED @15; diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index a6bf29e74..d72296692 100644 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -7,6 +7,7 @@ from types import SimpleNamespace from typing import SupportsFloat import cereal.messaging as messaging +import openpilot.selfdrive.sentry as sentry from cereal import car, custom, log from cereal.visionipc import VisionIpcClient, VisionStreamType @@ -947,6 +948,11 @@ class Controls: if lead_departing: self.events.add(EventName.leadDeparting) + if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')) and not self.openpilot_crashed_triggered: + self.events.add(EventName.openpilotCrashed) + + self.openpilot_crashed_triggered = True + if self.sm.frame == 550 and self.CP.lateralTuning.which() == 'torque' and self.CI.use_nnff: self.events.add(EventName.torqueNNLoad) diff --git a/selfdrive/controls/lib/events.py b/selfdrive/controls/lib/events.py index c974ee544..bf56c4647 100644 --- a/selfdrive/controls/lib/events.py +++ b/selfdrive/controls/lib/events.py @@ -1056,6 +1056,14 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = { ET.PERMANENT: no_lane_available_alert, }, + EventName.openpilotCrashed: { + ET.PERMANENT: Alert( + "openpilot crashed", + "Please post the error log in the FrogPilot Discord!", + AlertStatus.normal, AlertSize.mid, + Priority.HIGHEST, VisualAlert.none, AudibleAlert.none, 10.), + }, + EventName.torqueNNLoad: { ET.PERMANENT: torque_nn_load_alert, }, diff --git a/selfdrive/frogpilot/controls/lib/frogpilot_functions.py b/selfdrive/frogpilot/controls/lib/frogpilot_functions.py index f2c0c584e..3cb899a2a 100644 --- a/selfdrive/frogpilot/controls/lib/frogpilot_functions.py +++ b/selfdrive/frogpilot/controls/lib/frogpilot_functions.py @@ -5,6 +5,7 @@ import numpy as np import os import shutil import subprocess +import time from openpilot.common.basedir import BASEDIR from openpilot.common.numpy_fast import interp @@ -116,6 +117,22 @@ class FrogPilotFunctions: cmd = ['sudo', 'cp', '-a', '/data/params/.', f"{backup_path}/"] cls.run_cmd(cmd, f"Successfully backed up toggles to {backup_folder_name}.", f"Failed to backup toggles to {backup_folder_name}.") + @classmethod + def delete_logs(cls): + directories_to_check = [] + for root, dirs, files in os.walk('/data/media/0/realdata/', topdown=False): + for name in files: + filepath = os.path.join(root, name) + if time.time() - max(os.path.getctime(filepath), os.path.getmtime(filepath)) < 3600: + os.remove(filepath) + if root not in directories_to_check: + directories_to_check.append(root) + for directory in directories_to_check: + try: + os.rmdir(directory) + except OSError: + pass + @classmethod def setup_frogpilot(cls): remount_cmd = ['sudo', 'mount', '-o', 'remount,rw', '/persist'] diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py old mode 100644 new mode 100755 index 3512dbda5..f7fc3bd4d --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -360,7 +360,7 @@ def manager_cleanup() -> None: cloudlog.info("everything is dead") -def manager_thread() -> None: +def manager_thread(frogpilot_functions) -> None: cloudlog.bind(daemon="manager") cloudlog.info("manager start") cloudlog.info({"environ": os.environ}) @@ -386,10 +386,18 @@ def manager_thread() -> None: while True: sm.update(1000) + openpilot_crashed = os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + if openpilot_crashed: + frogpilot_functions.delete_logs() + started = sm['deviceState'].started if started and not started_prev: params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION) + + if openpilot_crashed: + os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + elif not started and started_prev: params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION) params_memory.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION) @@ -441,7 +449,7 @@ def main() -> None: signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(1)) try: - manager_thread() + manager_thread(frogpilot_functions) except Exception: traceback.print_exc() sentry.capture_exception() diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 5b63a9fe2..45c8f99a6 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -1,5 +1,9 @@ """Install exception handler for process crash.""" +import os import sentry_sdk +import traceback + +from datetime import datetime from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration @@ -10,6 +14,7 @@ from openpilot.common.swaglog import cloudlog from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \ is_comma_remote, is_dirty, is_tested_branch +CRASHES_DIR = "/data/community/crashes/" class SentryProject(Enum): # python project @@ -29,6 +34,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None: def capture_exception(*args, **kwargs) -> None: + save_exception(traceback.format_exc()) cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: @@ -38,6 +44,26 @@ def capture_exception(*args, **kwargs) -> None: cloudlog.exception("sentry exception") +def save_exception(exc_text: str) -> None: + if not os.path.exists(CRASHES_DIR): + os.makedirs(CRASHES_DIR) + + files = [ + os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')), + os.path.join(CRASHES_DIR, 'error.txt') + ] + + for file in files: + with open(file, 'w') as f: + if file.endswith("error.txt"): + lines = exc_text.splitlines()[-10:] + f.write("\n".join(lines)) + else: + f.write(exc_text) + + print('Logged current crash to {}'.format(files)) + + def set_tag(key: str, value: str) -> None: sentry_sdk.set_tag(key, value) diff --git a/selfdrive/ui/ui.h b/selfdrive/ui/ui.h index f32bdd97a..a192538cc 100644 --- a/selfdrive/ui/ui.h +++ b/selfdrive/ui/ui.h @@ -76,7 +76,12 @@ struct Alert { const int controls_missing = (nanos_since_boot() - sm.rcv_time("controlsState")) / 1e9; // Handle controls timeout - if (controls_frame < started_frame) { + if (std::ifstream("/data/community/crashes/error.txt")) { + alert = {"openpilot crashed", "Please post the error log in the FrogPilot Discord!", + "controlsWaiting", cereal::ControlsState::AlertSize::MID, + cereal::ControlsState::AlertStatus::NORMAL, + AudibleAlert::NONE}; + } else if (controls_frame < started_frame) { // car is started, but controlsState hasn't been seen at all alert = {"openpilot Unavailable", "Waiting for controls to start", "controlsWaiting", cereal::ControlsState::AlertSize::MID, diff --git a/system/loggerd/uploader.py b/system/loggerd/uploader.py index fdf6cd804..cbe40692c 100755 --- a/system/loggerd/uploader.py +++ b/system/loggerd/uploader.py @@ -14,6 +14,7 @@ from collections.abc import Iterator from cereal import log import cereal.messaging as messaging +import openpilot.selfdrive.sentry as sentry from openpilot.common.api import Api from openpilot.common.params import Params from openpilot.common.realtime import set_core_affinity @@ -251,7 +252,8 @@ def main(exit_event: threading.Event = None) -> None: offroad = params.get_bool("IsOffroad") network_type = sm['deviceState'].networkType if not force_wifi else NetworkType.wifi at_home = not params.get_bool("DisableOnroadUploads") or offroad and network_type in (NetworkType.ethernet, NetworkType.wifi) - if network_type == NetworkType.none or not at_home: + openpilot_crashed = os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + if network_type == NetworkType.none or not at_home or openpilot_crashed: if allow_sleep: time.sleep(60 if offroad else 5) continue