diff --git a/selfdrive/car/card.py b/selfdrive/car/card.py index 567a3c1e90..c41c9cc4f0 100755 --- a/selfdrive/car/card.py +++ b/selfdrive/car/card.py @@ -23,7 +23,7 @@ from openpilot.selfdrive.car.cruise import VCruiseHelper from openpilot.selfdrive.car.car_specific import MockCarState from openpilot.sunnypilot.mads.mads import MadsParams -from openpilot.sunnypilot.selfdrive.car.interfaces import setup_car_interface_sp, initialize_car_interface_sp +from openpilot.sunnypilot.selfdrive.car import interfaces REPLAY = "REPLAY" in os.environ @@ -102,7 +102,7 @@ class Car: cached_params = _cached_params self.CI = get_car(*self.can_callbacks, obd_callback(self.params), experimental_long_allowed, num_pandas, cached_params) - setup_car_interface_sp(self.CI.CP, self.params) + interfaces.setup_car_interface_sp(self.CI.CP, self.params) self.RI = get_radar_interface(self.CI.CP) self.CP = self.CI.CP @@ -167,6 +167,9 @@ class Car: # card is driven by can recv, expected at 100Hz self.rk = Ratekeeper(100, print_delay_threshold=None) + # log fingerprint in sentry + interfaces.log_fingerprint(self.CP) + def state_update(self) -> tuple[car.CarState, structs.RadarDataT | None]: """carState update loop, driven by can""" @@ -240,7 +243,7 @@ class Car: # Initialize CarInterface, once controls are ready # TODO: this can make us miss at least a few cycles when doing an ECU knockout self.CI.init(self.CP, *self.can_callbacks) - initialize_car_interface_sp(self.CP, self.params, *self.can_callbacks) + interfaces.initialize_car_interface_sp(self.CP, self.params, *self.can_callbacks) # signal pandad to switch to car safety mode self.params.put_bool_nonblocking("ControlsReady", True) diff --git a/sunnypilot/selfdrive/car/interfaces.py b/sunnypilot/selfdrive/car/interfaces.py index a92f8f60be..ff541ea9ed 100644 --- a/sunnypilot/selfdrive/car/interfaces.py +++ b/sunnypilot/selfdrive/car/interfaces.py @@ -12,6 +12,15 @@ from opendbc.car.hyundai.radar_interface import RADAR_START_ADDR from opendbc.car.hyundai.values import HyundaiFlags, DBC as HYUNDAI_DBC from opendbc.sunnypilot.car.hyundai.values import HyundaiFlagsSP +import openpilot.system.sentry as sentry + + +def log_fingerprint(CP: structs.CarParams) -> None: + if CP.carFingerprint == "MOCK": + sentry.capture_fingerprint_mock() + else: + sentry.capture_fingerprint(CP.carFingerprint, CP.carName) + def setup_car_interface_sp(CP: structs.CarParams, params): if CP.carName == 'hyundai': diff --git a/system/sentry.py b/system/sentry.py index 63bf789b6f..02a02c7e0e 100644 --- a/system/sentry.py +++ b/system/sentry.py @@ -1,26 +1,35 @@ """Install exception handler for process crash.""" +import os +import traceback +from datetime import datetime import sentry_sdk from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.system.athena.registration import is_registered_device -from openpilot.system.hardware import HARDWARE, PC +from openpilot.system.athena.registration import UNREGISTERED_DONGLE_ID +from openpilot.system.hardware import HARDWARE +from openpilot.system.hardware.hw import Paths from openpilot.common.swaglog import cloudlog from openpilot.system.version import get_build_metadata, get_version +from openpilot.sunnypilot.sunnylink.api import UNREGISTERED_SUNNYLINK_DONGLE_ID + +CRASHES_DIR = Paths.crash_log_root() + class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://7e3be9bfcfe04c9abe58bd25fe290d1a@o1138119.ingest.sentry.io/6191481" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://7e3be9bfcfe04c9abe58bd25fe290d1a@o1138119.ingest.sentry.io/6191481" def report_tombstone(fn: str, message: str, contents: str) -> None: cloudlog.error({'tombstone': message}) with sentry_sdk.configure_scope() as scope: + set_user() scope.set_extra("tombstone_fn", fn) scope.set_extra("tombstone", contents) sentry_sdk.capture_message(message=message) @@ -31,25 +40,83 @@ def capture_exception(*args, **kwargs) -> None: cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: + save_exception(traceback.format_exc()) + + set_user() sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 except Exception: cloudlog.exception("sentry exception") +def save_exception(content: str) -> None: + try: + if not os.path.exists(CRASHES_DIR): + os.makedirs(CRASHES_DIR) + + files = [ + os.path.join(CRASHES_DIR, datetime.now().strftime("%Y-%m-%d--%H-%M-%S.log")), + os.path.join(CRASHES_DIR, "error.log") + ] + + for fn in files: + with open(fn, 'w') as f: + if fn == "error.log": + lines = content.splitlines()[-3:] + f.write("\n".join(lines)) + else: + f.write(content) + + cloudlog.error(f"logged crash to {files}") + except Exception: + cloudlog.exception("error when attemping to save exception") + + +def capture_fingerprint_mock() -> None: + set_user() + message = "car doesn't match any fingerprints" + sentry_sdk.capture_message(message=message, level="error") + sentry_sdk.flush() + + +def capture_fingerprint(candidate: str, car_name: str) -> None: + with sentry_sdk.configure_scope() as scope: + set_user() + scope.set_extra("carFingerprint", candidate) + scope.set_extra("carName", car_name) + message = f"Fingerprinted {candidate}" + sentry_sdk.capture_message(message=message, level="info") + sentry_sdk.flush() + + def set_tag(key: str, value: str) -> None: sentry_sdk.set_tag(key, value) +def set_user() -> None: + dongle_id, git_username, _ = get_properties() + sentry_sdk.set_user({"id": dongle_id, "name": git_username}) + + +def get_properties() -> tuple[str, str, str]: + params = Params() + hardware_serial: str = params.get("HardwareSerial", encoding='utf-8') or "" + git_username: str = params.get("GithubUsername", encoding='utf-8') or "" + dongle_id: str = params.get("DongleId", encoding='utf-8') or f"{UNREGISTERED_DONGLE_ID}-{hardware_serial}" + sunnylink_dongle_id: str = params.get("SunnylinkDongleId", encoding='utf-8') or UNREGISTERED_SUNNYLINK_DONGLE_ID + + return dongle_id, git_username, sunnylink_dongle_id + + def init(project: SentryProject) -> bool: build_metadata = get_build_metadata() # forks like to mess with this, so double check - comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin - if not comma_remote or not is_registered_device() or PC: - return False + # comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin + # if not comma_remote or not is_registered_device() or PC: + # return False env = "release" if build_metadata.tested_channel else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + dongle_id, git_username, sunnylink_dongle_id = get_properties() integrations = [] if project == SentryProject.SELFDRIVE: @@ -61,13 +128,15 @@ def init(project: SentryProject) -> bool: integrations=integrations, traces_sample_rate=1.0, max_value_length=8192, - environment=env) + environment=env, + ) - sentry_sdk.set_user({"id": dongle_id}) + sentry_sdk.set_user({"id": dongle_id, "name": git_username}) sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty) sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin) sentry_sdk.set_tag("branch", build_metadata.channel) sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit) sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("sunnylink_dongle_id", sunnylink_dongle_id) return True