#!/usr/bin/env python3 import datetime import os import re import signal import subprocess import time import glob from pathlib import Path from typing import NoReturn from openpilot.common.hardware import PC from openpilot.common.hardware.hw import Paths from openpilot.common.swaglog import cloudlog from openpilot.common.version import get_build_metadata from openpilot.system import sentry from openpilot.system.crash_storage import writer, copy_report MAX_SIZE = 1_000_000 * 100 # allow up to 100M MAX_TOMBSTONE_FN_LEN = 62 # 85 - 23 ("/crash/") TOMBSTONE_DIR = "/data/tombstones/" APPORT_DIR = "/var/crash/" def safe_fn(s): extra = ['_'] return "".join(c for c in s if c.isalnum() or c in extra).rstrip() def clear_apport_folder(): for f in glob.glob(APPORT_DIR + '*'): try: os.remove(f) except Exception: pass def get_apport_stacktrace(fn): try: cmd = f'apport-retrace -s <(cat <(echo "Package: openpilot") "{fn}")' return subprocess.check_output(cmd, shell=True, encoding='utf8', timeout=30, executable='/bin/bash') except subprocess.CalledProcessError: return "Error getting stacktrace" except subprocess.TimeoutExpired: return "Timeout getting stacktrace" def get_tombstones(): """Returns list of (filename, ctime) for all crashlogs""" files = [] if os.path.exists(APPORT_DIR): with os.scandir(APPORT_DIR) as d: # Loop over first 1000 directory entries for _, f in zip(range(1000), d, strict=False): if f.name.startswith("tombstone"): files.append((f.path, int(f.stat().st_ctime))) elif f.name.endswith(".crash") and f.stat().st_mode == 0o100640: files.append((f.path, int(f.stat().st_ctime))) return files def report_tombstone_apport(fn): f_size = os.path.getsize(fn) if f_size > MAX_SIZE: cloudlog.error(f"Tombstone {fn} too big, {f_size}. Skipping...") return message = "" # One line description of the crash path = "" # File path relative to openpilot directory contents = "" with open(fn) as f: for line in f: if "CoreDump" in line: break contents += line if "ExecutablePath" in line: path = line.strip().split(': ')[-1] path = path.replace('/data/openpilot/', '') message += path elif "Signal" in line: message += " - " + line.strip() try: sig_num = int(line.strip().split(': ')[-1]) message += " (" + signal.Signals(sig_num).name + ")" except ValueError: pass stacktrace = get_apport_stacktrace(fn) stacktrace_s = stacktrace.split('\n') crash_function = "No stacktrace" if len(stacktrace_s) > 2: found = False # Try to find first entry in openpilot, fall back to first line for line in stacktrace_s: if "at openpilot/selfdrive/" in line or "at selfdrive/" in line: crash_function = line found = True break if not found: crash_function = stacktrace_s[1] # Remove arguments that can contain pointers from the crash summary crash_function = " ".join(x for x in crash_function.split(' ')[1:] if not x.startswith('0x')) crash_function = re.sub(r'\(.*?\)', '', crash_function) message = message + " - " + crash_function cloudlog.error({'tombstone': message}) contents += "\n" + stacktrace # Copy crashlog to upload folder clean_path = path.replace('/', '_') date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S") build_metadata = get_build_metadata() new_fn = f"{date}_{(build_metadata.openpilot.git_commit or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] crashlog_dir = os.path.join(Paths.log_root(), "crash") with writer(Path(crashlog_dir)) as directory: copy_report(directory, new_fn, fn) sentry.report_tombstone(fn, message, contents) try: os.remove(fn) except PermissionError: pass def main() -> NoReturn: sentry.init(sentry.SentryProject.SELFDRIVE_NATIVE) # Preserve the local native report even when the SDK or network is unavailable. should_report = not PC # Clear apport folder on start, otherwise duplicate crashes won't register clear_apport_folder() initial_tombstones = set(get_tombstones()) while True: now_tombstones = set(get_tombstones()) for fn, _ in (now_tombstones - initial_tombstones): # clear logs if we're not interested in them if not should_report: try: os.remove(fn) except Exception: pass continue try: cloudlog.info(f"reporting new tombstone {fn}") if fn.endswith(".crash"): report_tombstone_apport(fn) else: cloudlog.error(f"unknown crash type: {fn}") except Exception: cloudlog.exception(f"Error reporting tombstone {fn}") initial_tombstones = now_tombstones time.sleep(5) if __name__ == "__main__": main()