From cb1240846b1c00b95f596ff6e2ca938ae2282810 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Fri, 17 Jan 2025 01:11:07 -0500 Subject: [PATCH 1/7] sentry: log fingerprints and save exceptions --- selfdrive/car/card.py | 11 +++- sunnypilot/selfdrive/car/interfaces.py | 9 +++ system/sentry.py | 85 +++++++++++++++++++++++--- 3 files changed, 94 insertions(+), 11 deletions(-) diff --git a/selfdrive/car/card.py b/selfdrive/car/card.py index eecdbf6729..9a3d632c68 100755 --- a/selfdrive/car/card.py +++ b/selfdrive/car/card.py @@ -12,6 +12,7 @@ from panda import ALTERNATIVE_EXPERIENCE from openpilot.common.params import Params from openpilot.common.realtime import config_realtime_process, Priority, Ratekeeper from openpilot.common.swaglog import cloudlog, ForwardingHandler +from openpilot.system import sentry from opendbc.car import DT_CTRL, carlog, structs from opendbc.car.can_definitions import CanData, CanRecvCallable, CanSendCallable @@ -23,7 +24,7 @@ from openpilot.selfdrive.car.cruise import VCruiseHelper from openpilot.selfdrive.car.car_specific import MockCarState from openpilot.sunnypilot.mads.mads import MadsParams -from openpilot.sunnypilot.selfdrive.car.interfaces import setup_car_interface_sp, initialize_car_interface_sp +from openpilot.sunnypilot.selfdrive.car import interfaces REPLAY = "REPLAY" in os.environ @@ -102,7 +103,7 @@ class Car: cached_params = _cached_params self.CI = get_car(*self.can_callbacks, obd_callback(self.params), experimental_long_allowed, num_pandas, cached_params) - setup_car_interface_sp(self.CI.CP, self.params) + interfaces.setup_car_interface_sp(self.CI.CP, self.params) self.RI = get_radar_interface(self.CI.CP) self.CP = self.CI.CP @@ -164,6 +165,10 @@ class Car: # card is driven by can recv, expected at 100Hz self.rk = Ratekeeper(100, print_delay_threshold=None) + # log fingerprint in sentry + sentry.set_tag("daemon", "selfdrive.car.card") + interfaces.log_fingerprint(self.CP) + def state_update(self) -> tuple[car.CarState, structs.RadarDataT | None]: """carState update loop, driven by can""" @@ -237,7 +242,7 @@ class Car: # Initialize CarInterface, once controls are ready # TODO: this can make us miss at least a few cycles when doing an ECU knockout self.CI.init(self.CP, *self.can_callbacks) - initialize_car_interface_sp(self.CP, self.params, *self.can_callbacks) + interfaces.initialize_car_interface_sp(self.CP, self.params, *self.can_callbacks) # signal pandad to switch to car safety mode self.params.put_bool_nonblocking("ControlsReady", True) diff --git a/sunnypilot/selfdrive/car/interfaces.py b/sunnypilot/selfdrive/car/interfaces.py index a92f8f60be..1969e33c0a 100644 --- a/sunnypilot/selfdrive/car/interfaces.py +++ b/sunnypilot/selfdrive/car/interfaces.py @@ -12,6 +12,15 @@ from opendbc.car.hyundai.radar_interface import RADAR_START_ADDR from opendbc.car.hyundai.values import HyundaiFlags, DBC as HYUNDAI_DBC from opendbc.sunnypilot.car.hyundai.values import HyundaiFlagsSP +from openpilot.system import sentry + + +def log_fingerprint(CP: structs.CarParams) -> None: + if CP.carFingerprint == "MOCK": + sentry.capture_fingerprint_mock() + else: + sentry.capture_fingerprint(CP.carFingerprint, CP.carName) + def setup_car_interface_sp(CP: structs.CarParams, params): if CP.carName == 'hyundai': diff --git a/system/sentry.py b/system/sentry.py index 63bf789b6f..2c231e1e2d 100644 --- a/system/sentry.py +++ b/system/sentry.py @@ -1,26 +1,36 @@ """Install exception handler for process crash.""" +import os +import traceback +from datetime import datetime import sentry_sdk from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.system.athena.registration import is_registered_device +from openpilot.system.athena.registration import UNREGISTERED_DONGLE_ID, is_registered_device from openpilot.system.hardware import HARDWARE, PC +from openpilot.system.hardware.hw import Paths from openpilot.common.swaglog import cloudlog from openpilot.system.version import get_build_metadata, get_version +from openpilot.sunnypilot.sunnylink.api import UNREGISTERED_SUNNYLINK_DONGLE_ID + +CRASHES_DIR = Paths.crash_log_root() +IP_ADDRESS = "{{auto}}" + class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://7e3be9bfcfe04c9abe58bd25fe290d1a@o1138119.ingest.sentry.io/6191481" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://7e3be9bfcfe04c9abe58bd25fe290d1a@o1138119.ingest.sentry.io/6191481" def report_tombstone(fn: str, message: str, contents: str) -> None: cloudlog.error({'tombstone': message}) with sentry_sdk.configure_scope() as scope: + set_user() scope.set_extra("tombstone_fn", fn) scope.set_extra("tombstone", contents) sentry_sdk.capture_message(message=message) @@ -31,25 +41,83 @@ def capture_exception(*args, **kwargs) -> None: cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: + save_exception(traceback.format_exc()) + + set_user() sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 except Exception: cloudlog.exception("sentry exception") +def save_exception(content: str) -> None: + try: + if not os.path.exists(CRASHES_DIR): + os.makedirs(CRASHES_DIR) + + files = [ + os.path.join(CRASHES_DIR, datetime.now().strftime("%Y-%m-%d--%H-%M-%S.log")), + os.path.join(CRASHES_DIR, "error.log") + ] + + for fn in files: + with open(fn, 'w') as f: + if fn == "error.log": + lines = content.splitlines()[-3:] + f.write("\n".join(lines)) + else: + f.write(content) + + cloudlog.error(f"logged crash to {files}") + except Exception: + cloudlog.exception("error when attemping to save exception") + + +def capture_fingerprint_mock() -> None: + set_user() + message = "car doesn't match any fingerprints" + sentry_sdk.capture_message(message=message, level="error") + sentry_sdk.flush() + + +def capture_fingerprint(candidate: str, car_name: str) -> None: + with sentry_sdk.get_current_scope() as scope: + set_user() + scope.set_extra("carFingerprint", candidate) + scope.set_extra("carName", car_name) + message = f"fingerprinted {candidate}" + sentry_sdk.capture_message(message=message, level="info") + sentry_sdk.flush() + + def set_tag(key: str, value: str) -> None: sentry_sdk.set_tag(key, value) +def set_user() -> None: + dongle_id, git_username, _ = get_properties() + sentry_sdk.set_user({"id": dongle_id, "ip_address": IP_ADDRESS, "name": git_username}) + + +def get_properties() -> tuple[str, str, str]: + params = Params() + hardware_serial: str = params.get("HardwareSerial", encoding='utf-8') or "" + git_username: str = params.get("GithubUsername", encoding='utf-8') or "" + dongle_id: str = params.get("DongleId", encoding='utf-8') or f"{UNREGISTERED_DONGLE_ID}-{hardware_serial}" + sunnylink_dongle_id: str = params.get("SunnylinkDongleId", encoding='utf-8') or UNREGISTERED_SUNNYLINK_DONGLE_ID + + return dongle_id, git_username, sunnylink_dongle_id + + def init(project: SentryProject) -> bool: build_metadata = get_build_metadata() # forks like to mess with this, so double check - comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin - if not comma_remote or not is_registered_device() or PC: - return False + # comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin + # if not comma_remote or not is_registered_device() or PC: + # return False env = "release" if build_metadata.tested_channel else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + dongle_id, git_username, sunnylink_dongle_id = get_properties() integrations = [] if project == SentryProject.SELFDRIVE: @@ -63,11 +131,12 @@ def init(project: SentryProject) -> bool: max_value_length=8192, environment=env) - sentry_sdk.set_user({"id": dongle_id}) + sentry_sdk.set_user({"id": dongle_id, "ip_address": IP_ADDRESS, "name": git_username}) sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty) sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin) sentry_sdk.set_tag("branch", build_metadata.channel) sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit) sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("sunnylink_dongle_id", sunnylink_dongle_id) return True From 189e883116d93cfebccca2f3536424ca6e4bcc3d Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Fri, 17 Jan 2025 01:23:04 -0500 Subject: [PATCH 2/7] no point --- selfdrive/car/card.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/selfdrive/car/card.py b/selfdrive/car/card.py index 9a3d632c68..5cb3b6f598 100755 --- a/selfdrive/car/card.py +++ b/selfdrive/car/card.py @@ -12,7 +12,6 @@ from panda import ALTERNATIVE_EXPERIENCE from openpilot.common.params import Params from openpilot.common.realtime import config_realtime_process, Priority, Ratekeeper from openpilot.common.swaglog import cloudlog, ForwardingHandler -from openpilot.system import sentry from opendbc.car import DT_CTRL, carlog, structs from opendbc.car.can_definitions import CanData, CanRecvCallable, CanSendCallable @@ -166,7 +165,6 @@ class Car: self.rk = Ratekeeper(100, print_delay_threshold=None) # log fingerprint in sentry - sentry.set_tag("daemon", "selfdrive.car.card") interfaces.log_fingerprint(self.CP) def state_update(self) -> tuple[car.CarState, structs.RadarDataT | None]: From 378eb8a4d298b35968b6630b74063cfb52431579 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Fri, 17 Jan 2025 01:39:15 -0500 Subject: [PATCH 3/7] remove unused imports --- system/sentry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system/sentry.py b/system/sentry.py index 2c231e1e2d..52b642c0ad 100644 --- a/system/sentry.py +++ b/system/sentry.py @@ -7,8 +7,8 @@ from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.system.athena.registration import UNREGISTERED_DONGLE_ID, is_registered_device -from openpilot.system.hardware import HARDWARE, PC +from openpilot.system.athena.registration import UNREGISTERED_DONGLE_ID +from openpilot.system.hardware import HARDWARE from openpilot.system.hardware.hw import Paths from openpilot.common.swaglog import cloudlog from openpilot.system.version import get_build_metadata, get_version From df1ef23280417783eec42ba47e198533128cb610 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Fri, 17 Jan 2025 01:41:54 -0500 Subject: [PATCH 4/7] use deprecated --- system/sentry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/sentry.py b/system/sentry.py index 52b642c0ad..16a5d8a66a 100644 --- a/system/sentry.py +++ b/system/sentry.py @@ -81,7 +81,7 @@ def capture_fingerprint_mock() -> None: def capture_fingerprint(candidate: str, car_name: str) -> None: - with sentry_sdk.get_current_scope() as scope: + with sentry_sdk.configure_scope() as scope: set_user() scope.set_extra("carFingerprint", candidate) scope.set_extra("carName", car_name) From 7fde68ecf9c922a9447ef5572d271ccbdf170992 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Fri, 17 Jan 2025 01:48:43 -0500 Subject: [PATCH 5/7] have to do this --- sunnypilot/selfdrive/car/interfaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sunnypilot/selfdrive/car/interfaces.py b/sunnypilot/selfdrive/car/interfaces.py index 1969e33c0a..ff541ea9ed 100644 --- a/sunnypilot/selfdrive/car/interfaces.py +++ b/sunnypilot/selfdrive/car/interfaces.py @@ -12,7 +12,7 @@ from opendbc.car.hyundai.radar_interface import RADAR_START_ADDR from opendbc.car.hyundai.values import HyundaiFlags, DBC as HYUNDAI_DBC from opendbc.sunnypilot.car.hyundai.values import HyundaiFlagsSP -from openpilot.system import sentry +import openpilot.system.sentry as sentry def log_fingerprint(CP: structs.CarParams) -> None: From 3f4e372c34c0167d23d712a76a6db96999be22d4 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Fri, 17 Jan 2025 01:55:15 -0500 Subject: [PATCH 6/7] remove --- system/sentry.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/system/sentry.py b/system/sentry.py index 16a5d8a66a..1122e57750 100644 --- a/system/sentry.py +++ b/system/sentry.py @@ -16,7 +16,6 @@ from openpilot.system.version import get_build_metadata, get_version from openpilot.sunnypilot.sunnylink.api import UNREGISTERED_SUNNYLINK_DONGLE_ID CRASHES_DIR = Paths.crash_log_root() -IP_ADDRESS = "{{auto}}" class SentryProject(Enum): @@ -96,7 +95,7 @@ def set_tag(key: str, value: str) -> None: def set_user() -> None: dongle_id, git_username, _ = get_properties() - sentry_sdk.set_user({"id": dongle_id, "ip_address": IP_ADDRESS, "name": git_username}) + sentry_sdk.set_user({"id": dongle_id, "name": git_username}) def get_properties() -> tuple[str, str, str]: @@ -129,9 +128,10 @@ def init(project: SentryProject) -> bool: integrations=integrations, traces_sample_rate=1.0, max_value_length=8192, - environment=env) + environment=env, + ) - sentry_sdk.set_user({"id": dongle_id, "ip_address": IP_ADDRESS, "name": git_username}) + sentry_sdk.set_user({"id": dongle_id, "name": git_username}) sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty) sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin) sentry_sdk.set_tag("branch", build_metadata.channel) From 5ad8b461585a2e8f60983ad5ea04b1fe9e96e570 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Fri, 17 Jan 2025 02:04:59 -0500 Subject: [PATCH 7/7] cap --- system/sentry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/sentry.py b/system/sentry.py index 1122e57750..02a02c7e0e 100644 --- a/system/sentry.py +++ b/system/sentry.py @@ -84,7 +84,7 @@ def capture_fingerprint(candidate: str, car_name: str) -> None: set_user() scope.set_extra("carFingerprint", candidate) scope.set_extra("carName", car_name) - message = f"fingerprinted {candidate}" + message = f"Fingerprinted {candidate}" sentry_sdk.capture_message(message=message, level="info") sentry_sdk.flush()