From ffa46a747be410d0b81ed80ee727b3df0c85e29b Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Sun, 29 Jan 2023 21:33:26 -0500 Subject: [PATCH] General: Sentry logging --- selfdrive/car/car_helpers.py | 44 ++++++++++++++++++++ selfdrive/sentry.py | 79 +++++++++++++++++++++++++++++++++--- 2 files changed, 117 insertions(+), 6 deletions(-) diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 88e8c72153..850625aa07 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,7 @@ import os +import requests +import threading +import time from typing import Dict, List from cereal import car @@ -9,6 +12,7 @@ from selfdrive.car.interfaces import get_interface_attr from selfdrive.car.fingerprints import eliminate_incompatible_cars, all_legacy_fingerprint_cars from selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN from selfdrive.car.fw_versions import get_fw_versions_ordered, match_fw_to_car, get_present_ecus +import selfdrive.sentry as sentry from system.swaglog import cloudlog import cereal.messaging as messaging from selfdrive.car import gen_empty_fingerprint @@ -173,12 +177,52 @@ def fingerprint(logcan, sendcan, num_pandas): return car_fingerprint, finger, vin, car_fw, source, exact_match +def is_connected_to_internet(timeout=5): + try: + requests.get("https://sentry.io", timeout=timeout) + return True + except Exception: + return False + + +def crash_log(candidate): + no_internet = 0 + while True: + if is_connected_to_internet(): + sentry.capture_warning("fingerprinted %s" % candidate) + break + else: + no_internet += 1 + if no_internet >= 2: + break + time.sleep(600) + + +def crash_log2(fingerprints, fw): + no_internet = 0 + while True: + if is_connected_to_internet(): + sentry.capture_warning("car doesn't match any fingerprints: %s" % fingerprints) + sentry.capture_warning("car doesn't match any fw: %s" % fw) + break + else: + no_internet += 1 + if no_internet >= 2: + break + time.sleep(600) + + def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): candidate, fingerprints, vin, car_fw, source, exact_match = fingerprint(logcan, sendcan, num_pandas) if candidate is None: cloudlog.warning("car doesn't match any fingerprints: %r", fingerprints) candidate = "mock" + y = threading.Thread(target=crash_log2, args=(fingerprints, car_fw,)) + y.start() + + x = threading.Thread(target=crash_log, args=(candidate,)) + x.start() CarInterface, CarController, CarState = interfaces[candidate] CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed) diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index aa409ea394..573ce7ea6e 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -4,20 +4,51 @@ from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from common.params import Params -from selfdrive.athena.registration import is_registered_device +#from selfdrive.athena.registration import is_registered_device from system.hardware import HARDWARE, PC from system.swaglog import cloudlog from system.version import get_branch, get_commit, get_origin, get_version, \ is_comma_remote, is_dirty, is_tested_branch +import os +import traceback +import requests +from cereal import car +from datetime import datetime class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://7e3be9bfcfe04c9abe58bd25fe290d1a@o1138119.ingest.sentry.io/6191481" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://7e3be9bfcfe04c9abe58bd25fe290d1a@o1138119.ingest.sentry.io/6191481" +CRASHES_DIR = os.path.join('/data/community/crashes') +ret = car.CarParams.new_message() +candidate = ret.carFingerprint +params = Params() +#uniqueID = op_params.get('uniqueID') +try: + dongle_id = params.get("DongleId").decode('utf8') +except AttributeError: + dongle_id = "None" +try: + gitname = params.get("GithubUsername", encoding='utf-8') +except Exception: + gitname = "" +try: + ip = requests.get('https://checkip.amazonaws.com/').text.strip() +except Exception: + ip = "255.255.255.255" +error_tags = { + 'dirty': is_dirty(), + 'dongle_id': dongle_id, + 'branch': get_branch(), + 'remote': get_origin(), + 'fingerprintedAs': candidate, + 'gitname': gitname +} + def report_tombstone(fn: str, message: str, contents: str) -> None: cloudlog.error({'tombstone': message}) @@ -29,6 +60,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None: def capture_exception(*args, **kwargs) -> None: + save_exception(traceback.format_exc()) cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: @@ -38,18 +70,52 @@ def capture_exception(*args, **kwargs) -> None: cloudlog.exception("sentry exception") +def save_exception(exc_text): + if not os.path.exists(CRASHES_DIR): + os.makedirs(CRASHES_DIR) + + log_file = '{}/{}'.format(CRASHES_DIR, datetime.now().strftime('%m-%d-%Y--%I:%M.%S-%p.log')) + log_file_2 = f'{CRASHES_DIR}/error.txt' + with open(log_file, 'w') as f: + f.write(exc_text) + f.close() + with open(log_file_2, 'w') as f2: + f2.write(exc_text) + f2.close() + print('Logged current crash to {}'.format(log_file)) + + +def bind_user(**kwargs) -> None: + sentry_sdk.set_user(kwargs) + sentry_sdk.flush() + + +def capture_warning(warning_string): + bind_user(id=dongle_id, ip_address=ip, name=gitname) + sentry_sdk.capture_message(warning_string, level='warning') + sentry_sdk.flush() + + +def capture_info(info_string): + bind_user(id=dongle_id, ip_address=ip, name=gitname) + sentry_sdk.capture_message(info_string, level='info') + sentry_sdk.flush() + + def set_tag(key: str, value: str) -> None: sentry_sdk.set_tag(key, value) + sentry_sdk.flush() def init(project: SentryProject) -> None: # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin(default="") - if not comma_remote or not is_registered_device() or PC: - return + #comma_remote = is_comma_remote() and "commaai" in get_origin(default="") + #if not comma_remote or not is_registered_device() or PC: + # return env = "release" if is_tested_branch() else "master" dongle_id = Params().get("DongleId", encoding='utf-8') + gitname = Params().get("GithubUsername", encoding='utf-8') integrations = [] if project == SentryProject.SELFDRIVE: @@ -65,6 +131,7 @@ def init(project: SentryProject) -> None: environment=env) sentry_sdk.set_user({"id": dongle_id}) + sentry_sdk.set_user({"gitname": gitname}) sentry_sdk.set_tag("dirty", is_dirty()) sentry_sdk.set_tag("origin", get_origin()) sentry_sdk.set_tag("branch", get_branch())