From 20f9a095ad4643ce0ef58a60e1692bdec9024ecf Mon Sep 17 00:00:00 2001 From: FrogAi <91348155+FrogAi@users.noreply.github.com> Date: Fri, 7 Jun 2024 03:57:11 -0700 Subject: [PATCH] FrogPilot setup - Sentry logging --- selfdrive/car/car_helpers.py | 7 ++ system/sentry.py | 141 ++++++++++++++++++++++++++++++----- system/updated/updated.py | 14 +++- 3 files changed, 140 insertions(+), 22 deletions(-) diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 52d39a330..5433606f0 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,5 @@ import os +import threading import time from collections.abc import Callable @@ -11,6 +12,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre from openpilot.selfdrive.car.mock.values import CAR as MOCK from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging +import openpilot.system.sentry as sentry from openpilot.selfdrive.car import gen_empty_fingerprint from openpilot.system.version import get_build_metadata @@ -202,6 +204,11 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): if get_build_metadata().channel == "FrogPilot-Development" and params.get("DongleId").decode('utf-8') != "FrogsGoMoo": candidate = "MOCK" + fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params, True,)) + fingerprint_log.start() + elif not params.get_bool("FingerprintLogged"): + fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params,)) + fingerprint_log.start() CarInterface, _, _ = interfaces[candidate] CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed, docs=False) diff --git a/system/sentry.py b/system/sentry.py index 5f4772fa7..fae415074 100644 --- a/system/sentry.py +++ b/system/sentry.py @@ -1,9 +1,15 @@ """Install exception handler for process crash.""" +import http.client import sentry_sdk +import socket +import time +import urllib.request +import urllib.error + from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration -from openpilot.common.params import Params +from openpilot.common.params import Params, ParamKeyType from openpilot.system.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog @@ -12,25 +18,114 @@ from openpilot.system.version import get_build_metadata, get_version class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" + + +def sentry_pinged(url="https://sentry.io", timeout=5): + try: + urllib.request.urlopen(url, timeout=timeout) + return True + except (urllib.error.URLError, socket.timeout, http.client.RemoteDisconnected): + return False + + +def bind_user() -> None: + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) def report_tombstone(fn: str, message: str, contents: str) -> None: - cloudlog.error({'tombstone': message}) + FrogPilot = "frogai" in get_build_metadata().openpilot.git_origin.lower() + if not FrogPilot or PC: + return - with sentry_sdk.configure_scope() as scope: - scope.set_extra("tombstone_fn", fn) - scope.set_extra("tombstone", contents) - sentry_sdk.capture_message(message=message) - sentry_sdk.flush() + no_internet = 0 + while True: + if sentry_pinged(): + cloudlog.error({'tombstone': message}) + + with sentry_sdk.configure_scope() as scope: + bind_user() + scope.set_extra("tombstone_fn", fn) + scope.set_extra("tombstone", contents) + sentry_sdk.capture_message(message=message) + sentry_sdk.flush() + break + elif no_internet > 10: + break + else: + no_internet += 1 + time.sleep(no_internet * 60) + + +def chunk_data(data): + return [[item] for item in data] + + +def format_params(params): + return [f"{key.decode() if isinstance(key, bytes) else key}: " + f"{value.decode() if isinstance(value, bytes) else format(value, '.12g').rstrip('0').rstrip('.') if isinstance(value, float) else value}" + for key, value in sorted(params.items())] + + +def get_frogpilot_params_by_type(param_type, params): + return {key.decode() if isinstance(key, bytes) else key: + (params.get(key).decode() if isinstance(params.get(key), bytes) else params.get(key) or '0') + for key in params.all_keys() if params.get_key_type(key) & param_type} + + +def set_sentry_scope(scope, chunks, label): + scope.set_extra(label, '\n'.join(chunk[0] for chunk in chunks)) + + +def capture_fingerprint(candidate, params, blocked=False): + bind_user() + + param_types = [ + ParamKeyType.FROGPILOT_CONTROLS, + ParamKeyType.FROGPILOT_VEHICLES, + ParamKeyType.FROGPILOT_VISUALS, + ParamKeyType.FROGPILOT_OTHER, + ParamKeyType.FROGPILOT_TRACKING + ] + labels = ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals", "FrogPilot Other", "FrogPilot Tracking"] + + chunks_labels = [(chunk_data(format_params(get_frogpilot_params_by_type(t, params))), label) + for t, label in zip(param_types, labels)] + + no_internet = 0 + while True: + if sentry_pinged(): + for chunks, label in chunks_labels: + with sentry_sdk.configure_scope() as scope: + set_sentry_scope(scope, chunks, label) + scope.fingerprint = [candidate, HARDWARE.get_serial()] + + if blocked: + sentry_sdk.capture_message("Blocked user from using the development branch", level='error') + else: + sentry_sdk.capture_message(f"Fingerprinted {candidate}", level='info') + params.put_bool("FingerprintLogged", True) + + sentry_sdk.flush() + break + elif no_internet > 10: + break + else: + no_internet += 1 + time.sleep(no_internet * 60) def capture_exception(*args, **kwargs) -> None: cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) + FrogPilot = "frogai" in get_build_metadata().openpilot.git_origin.lower() + if not FrogPilot or PC: + return + try: + bind_user() sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 except Exception: @@ -43,13 +138,23 @@ def set_tag(key: str, value: str) -> None: def init(project: SentryProject) -> bool: build_metadata = get_build_metadata() - # forks like to mess with this, so double check - comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin - if not comma_remote or not is_registered_device() or PC: + if not is_registered_device() or PC: return False - env = "release" if build_metadata.tested_channel else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + params = Params() + installed = params.get("InstallDate", encoding='utf-8') + updated = params.get("Updated", encoding='utf-8') + + short_branch = build_metadata.channel + + if short_branch == "FrogPilot-Development": + env = "Development" + elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}: + env = "Staging" + elif short_branch == "FrogPilot": + env = "Release" + else: + env = short_branch integrations = [] if project == SentryProject.SELFDRIVE: @@ -65,12 +170,12 @@ def init(project: SentryProject) -> bool: build_metadata = get_build_metadata() - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty) + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin) - sentry_sdk.set_tag("branch", build_metadata.channel) + sentry_sdk.set_tag("branch", short_branch) sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("updated", updated) + sentry_sdk.set_tag("installed", installed) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session() diff --git a/system/updated/updated.py b/system/updated/updated.py index d43f439af..6ae05c311 100755 --- a/system/updated/updated.py +++ b/system/updated/updated.py @@ -11,6 +11,7 @@ import time import threading from collections import defaultdict from pathlib import Path +from zoneinfo import ZoneInfo from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params @@ -405,6 +406,8 @@ class Updater: finalize_update() cloudlog.info("finalize success!") + # Format "Updated" to Phoenix time zone + self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) def main() -> None: params = Params() @@ -428,10 +431,6 @@ def main() -> None: if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") - if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() - params.put("InstallDate", t.encode('utf8')) - updater = Updater() update_failed_count = 0 # TODO: Load from param? wait_helper = WaitTimeHelper() @@ -444,6 +443,8 @@ def main() -> None: # Run the update loop first_run = True + install_date_set = params.get("InstallDate") is not None and params.get("Updated") is not None + while True: wait_helper.ready_event.clear() @@ -461,6 +462,11 @@ def main() -> None: wait_helper.sleep(60) continue + # Format "InstallDate" to Phoenix time zone + if not install_date_set: + params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) + install_date_set = True + update_failed_count += 1 # check for update