diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 4c1d1e75d..8cd4942f7 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,5 @@ import os +import threading import time from collections.abc import Callable @@ -11,6 +12,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre from openpilot.selfdrive.car.mock.values import CAR as MOCK from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging +import openpilot.system.sentry as sentry from openpilot.selfdrive.car import gen_empty_fingerprint from openpilot.system.version import get_build_metadata @@ -198,6 +200,9 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): if get_build_metadata().channel == "FrogPilot-Development" and params.get("DongleId", encoding='utf-8') != "FrogsGoMoo": candidate = "MOCK" + threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params, True,)).start() + elif False: + threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params,)).start() CarInterface, _, _ = interfaces[candidate] CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed, docs=False) diff --git a/system/sentry.py b/system/sentry.py index 5f4772fa7..71cd4dfad 100644 --- a/system/sentry.py +++ b/system/sentry.py @@ -1,36 +1,124 @@ """Install exception handler for process crash.""" import sentry_sdk +import time + from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration -from openpilot.common.params import Params +from openpilot.common.params import Params, ParamKeyType from openpilot.system.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog from openpilot.system.version import get_build_metadata, get_version +from openpilot.selfdrive.frogpilot.controls.lib.frogpilot_functions import is_url_pingable class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" + + +def bind_user() -> None: + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) def report_tombstone(fn: str, message: str, contents: str) -> None: - cloudlog.error({'tombstone': message}) + FrogPilot = "frogai" in get_build_metadata().openpilot.git_origin.lower() + if not FrogPilot or PC: + return - with sentry_sdk.configure_scope() as scope: - scope.set_extra("tombstone_fn", fn) - scope.set_extra("tombstone", contents) - sentry_sdk.capture_message(message=message) - sentry_sdk.flush() + no_internet = 0 + while True: + if is_url_pingable("https://sentry.io"): + cloudlog.error({'tombstone': message}) + + with sentry_sdk.configure_scope() as scope: + bind_user() + scope.set_extra("tombstone_fn", fn) + scope.set_extra("tombstone", contents) + sentry_sdk.capture_message(message=message) + sentry_sdk.flush() + break + elif no_internet > 10: + break + else: + no_internet += 1 + time.sleep(no_internet * 60) + + +def capture_fingerprint(candidate, params, blocked=False): + bind_user() + + params_tracking = Params("/persist/tracking") + + param_types = { + "FrogPilot Controls": ParamKeyType.FROGPILOT_CONTROLS, + "FrogPilot Vehicles": ParamKeyType.FROGPILOT_VEHICLES, + "FrogPilot Visuals": ParamKeyType.FROGPILOT_VISUALS, + "FrogPilot Other": ParamKeyType.FROGPILOT_OTHER, + "FrogPilot Tracking": ParamKeyType.FROGPILOT_TRACKING + } + + matched_params = {label: {} for label in param_types} + for key in params.all_keys(): + for label, key_type in param_types.items(): + if params.get_key_type(key) & key_type: + if key_type == ParamKeyType.FROGPILOT_TRACKING: + try: + value = params_tracking.get_int(key) + except Exception: + value = "0" + else: + try: + value = params.get(key) + if isinstance(value, bytes): + value = value.decode('utf-8') + if isinstance(value, str) and value.replace('.', '', 1).isdigit(): + value = float(value) if '.' in value else int(value) + except Exception: + value = "0" + matched_params[label][key.decode('utf-8')] = value + + for label, key_values in matched_params.items(): + if label == "FrogPilot Tracking": + matched_params[label] = {k: f"{v:,}" for k, v in key_values.items()} + else: + matched_params[label] = {k: int(v) if isinstance(v, float) and v.is_integer() else v for k, v in sorted(key_values.items())} + + no_internet = 0 + while True: + if is_url_pingable("https://sentry.io"): + with sentry_sdk.configure_scope() as scope: + scope.fingerprint = [candidate, HARDWARE.get_serial()] + for label, key_values in matched_params.items(): + scope.set_extra(label, "\n".join([f"{k}: {v}" for k, v in key_values.items()])) + + if blocked: + sentry_sdk.capture_message("Blocked user from using the development branch", level='error') + else: + sentry_sdk.capture_message(f"Fingerprinted {candidate}", level='info') + params.put_bool_nonblocking("FingerprintLogged", True) + + sentry_sdk.flush() + break + elif no_internet > 10: + break + else: + no_internet += 1 + time.sleep(no_internet * 60) def capture_exception(*args, **kwargs) -> None: cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) + FrogPilot = "frogai" in get_build_metadata().openpilot.git_origin.lower() + if not FrogPilot or PC: + return + try: + bind_user() sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 except Exception: @@ -43,13 +131,23 @@ def set_tag(key: str, value: str) -> None: def init(project: SentryProject) -> bool: build_metadata = get_build_metadata() - # forks like to mess with this, so double check - comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin - if not comma_remote or not is_registered_device() or PC: + if PC: return False - env = "release" if build_metadata.tested_channel else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + params = Params() + installed = params.get("InstallDate", encoding='utf-8') + updated = params.get("Updated", encoding='utf-8') + + short_branch = build_metadata.channel + + if short_branch == "FrogPilot-Development": + env = "Development" + elif build_metadata.tested_channel: + env = "Staging" + elif build_metadata.release_channel: + env = "Release" + else: + env = short_branch integrations = [] if project == SentryProject.SELFDRIVE: @@ -65,12 +163,12 @@ def init(project: SentryProject) -> bool: build_metadata = get_build_metadata() - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty) + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin) - sentry_sdk.set_tag("branch", build_metadata.channel) + sentry_sdk.set_tag("branch", short_branch) sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("updated", updated) + sentry_sdk.set_tag("installed", installed) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session() diff --git a/system/updated/updated.py b/system/updated/updated.py index 68207bb48..c4dc99f17 100755 --- a/system/updated/updated.py +++ b/system/updated/updated.py @@ -11,6 +11,7 @@ import time import threading from collections import defaultdict from pathlib import Path +from zoneinfo import ZoneInfo from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params @@ -405,6 +406,8 @@ class Updater: finalize_update() cloudlog.info("finalize success!") + # Format "Updated" to Phoenix time zone + self.params.put_nonblocking("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) def main() -> None: params = Params() @@ -428,10 +431,6 @@ def main() -> None: if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") - if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() - params.put("InstallDate", t.encode('utf8')) - updater = Updater() update_failed_count = 0 # TODO: Load from param? wait_helper = WaitTimeHelper() @@ -444,6 +443,8 @@ def main() -> None: # Run the update loop first_run = True + install_date_set = params.get("InstallDate", encoding='utf-8') is not None and params.get("Updated", encoding='utf-8') is not None + while True: wait_helper.ready_event.clear() @@ -461,6 +462,11 @@ def main() -> None: wait_helper.sleep(60) continue + # Format "InstallDate" to Phoenix time zone + if not install_date_set: + params.put_nonblocking("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) + install_date_set = True + update_failed_count += 1 # check for update