From 331219fe70847cebfb4506f7b0cd21dd070fca73 Mon Sep 17 00:00:00 2001 From: FrogAi <91348155+FrogAi@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:24:28 -0700 Subject: [PATCH] Sentry logging Logging for my Sentry server that tracks the values of the serial number, car fingerprint, user set parameters, and the date and time for when FrogPilot was installed and last updated for debugging and support. --- common/params.cc | 2 + selfdrive/car/car_helpers.py | 7 ++ selfdrive/sentry.py | 168 ++++++++++++++++++++++++++++++----- selfdrive/updated/updated.py | 15 +++- 4 files changed, 166 insertions(+), 26 deletions(-) mode change 100755 => 100644 selfdrive/updated/updated.py diff --git a/common/params.cc b/common/params.cc index fc9b7bfd1..c3b9a7887 100644 --- a/common/params.cc +++ b/common/params.cc @@ -278,6 +278,7 @@ std::unordered_map keys = { {"ExperimentalModeViaLKAS", PERSISTENT}, {"ExperimentalModeViaTap", PERSISTENT}, {"Fahrenheit", PERSISTENT}, + {"FingerprintLogged", CLEAR_ON_MANAGER_START}, {"ForceAutoTune", PERSISTENT}, {"ForceFingerprint", PERSISTENT}, {"FPSCounter", PERSISTENT}, @@ -388,6 +389,7 @@ std::unordered_map keys = { {"ToyotaDoors", PERSISTENT}, {"UnlimitedLength", PERSISTENT}, {"UnlockDoors", PERSISTENT}, + {"Updated", PERSISTENT}, {"UseSI", PERSISTENT}, {"WarningImmediateVolume", PERSISTENT}, {"WarningSoftVolume", PERSISTENT}, diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 6d32f100e..a98910c6b 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,5 @@ import os +import threading import time from collections.abc import Callable @@ -13,6 +14,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre from openpilot.selfdrive.car.mock.values import CAR as MOCK from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging +import openpilot.selfdrive.sentry as sentry from openpilot.selfdrive.car import gen_empty_fingerprint FRAME_FINGERPRINT = 100 # 1s @@ -221,6 +223,11 @@ def get_car(params, logcan, sendcan, disable_openpilot_long, experimental_long_a if get_short_branch() == "FrogPilot-Development" and not Params("/persist/params").get_bool("FrogsGoMoo"): cloudlog.event("Blocked user from using the 'FrogPilot-Development' branch", fingerprints=repr(fingerprints), error=True) candidate = "mock" + fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(params, candidate, True,)) + fingerprint_log.start() + elif not params.get_bool("FingerprintLogged"): + fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(params, candidate,)) + fingerprint_log.start() CarInterface, _, _ = interfaces[candidate] CP = CarInterface.get_params(params, candidate, fingerprints, car_fw, disable_openpilot_long, experimental_long_allowed, docs=False) diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 45c8f99a6..1c5ebd635 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -1,43 +1,160 @@ """Install exception handler for process crash.""" import os import sentry_sdk +import socket +import time import traceback +import urllib.request +import urllib.error from datetime import datetime from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.selfdrive.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \ - is_comma_remote, is_dirty, is_tested_branch +from openpilot.system.version import get_commit, get_short_branch, get_origin, get_version CRASHES_DIR = "/data/community/crashes/" class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" + + +def sentry_pinged(url="https://sentry.io", timeout=5): + try: + urllib.request.urlopen(url, timeout=timeout) + return True + except (urllib.error.URLError, socket.timeout): + return False + + +def bind_user() -> None: + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) def report_tombstone(fn: str, message: str, contents: str) -> None: - cloudlog.error({'tombstone': message}) + FrogPilot = "frogai" in get_origin().lower() + if not FrogPilot or PC: + return - with sentry_sdk.configure_scope() as scope: - scope.set_extra("tombstone_fn", fn) - scope.set_extra("tombstone", contents) - sentry_sdk.capture_message(message=message) - sentry_sdk.flush() + no_internet = 0 + while True: + if sentry_pinged(): + cloudlog.error({'tombstone': message}) + + with sentry_sdk.configure_scope() as scope: + bind_user() + scope.set_extra("tombstone_fn", fn) + scope.set_extra("tombstone", contents) + sentry_sdk.capture_message(message=message) + sentry_sdk.flush() + break + else: + if no_internet > 5: + break + no_internet += 1 + time.sleep(600) + + +def chunk_data(data, size): + return [data[i:i+size] for i in range(0, len(data), size)] + + +def format_params(params): + formatted_params = [] + for k, v in params.items(): + if isinstance(v, bytes): + param_value = format(float(v), '.12g') if v.replace(b'.', b'').isdigit() else v.decode() + elif isinstance(v, float): + param_value = format(v, '.12g') + else: + param_value = v + formatted_params.append(f"{k}: {param_value}") + return formatted_params + + +def get_frogpilot_params(params, keys): + return {key: params.get(key) or '0' for key in keys} + + +def set_sentry_scope(scope, chunks, label): + scope.set_extra(label, '\n'.join(['\n'.join(chunk) for chunk in chunks])) + + +def capture_fingerprint(params, candidate, blocked=False): + bind_user() + + control_keys, vehicle_keys, visual_keys, other_keys, tracking_keys = [ + "AlwaysOnLateral", "AlwaysOnLateralMain", "HideAOLStatusBar", "ConditionalExperimental", "CESpeed", "CESpeedLead", "CECurves", "CECurvesLead", + "CENavigation", "CENavigationIntersections", "CENavigationTurns", "CENavigationLead", "CESlowerLead", "CEStopLights", "CEStopLightsLead", + "CESignal", "HideCEMStatusBar", "CustomPersonalities", "TrafficFollow", "TrafficJerk", "AggressiveFollow", "AggressiveJerk", "StandardFollow", + "StandardJerk", "RelaxedFollow", "RelaxedJerk", "DeviceManagement", "IncreaseThermalLimits", "DeviceShutdown", "NoLogging", "NoUploads", "LowVoltageShutdown", + "OfflineMode", "ExperimentalModeActivation", "ExperimentalModeViaLKAS", "ExperimentalModeViaTap", "ExperimentalModeViaDistance", "LateralTune", + "ForceAutoTune", "NNFF", "NNFFLite", "SteerRatio", "TacoTune", "TurnDesires", "SteerRatio", "LongitudinalTune", "AccelerationProfile", "DecelerationProfile", + "AggressiveAcceleration", "StoppingDistance", "LeadDetectionThreshold", "SmoothBraking", "SmoothBrakingFarLead", "SmoothBrakingJerk", "TrafficMode", + "MTSCEnabled", "DisableMTSCSmoothing", "MTSCCurvatureCheck", "MTSCAggressiveness", "ModelSelector", "Model", "NudgelessLaneChange", "LaneChangeTime", + "LaneDetectionWidth", "OneLaneChange", "LaneDetectionWidth", "QOLControls", "CustomCruise", "CustomCruiseLong", "DisableOnroadUploads", "HigherBitrate", + "OnroadDistanceButton", "KaofuiIcons", "PauseLateralSpeed", "PauseLateralOnSignal", "ReverseCruise", "SetSpeedOffset", "SpeedLimitController", "Offset1", + "Offset2", "Offset3", "Offset4", "SLCFallback", "SLCOverride", "SLCPriority", "SLCConfirmation", "SLCConfirmationLower", "SLCConfirmationHigher", + "ForceMPHDashboard", "SetSpeedLimit", "ShowSLCOffset", "ShowSLCOffsetUI", "UseVienna", "VisionTurnControl", "DisableVTSCSmoothing", "CurveSensitivity", + "TurnAggressiveness", + ], [ + "ForceFingerprint", "DisableOpenpilotLongitudinal", "EVTable", "LongPitch", "GasRegenCmd", "CrosstrekTorque", "LockDoors", "StockTune", "CydiaTune", + "DragonPilotTune", "FrogsGoMooTune", "LockDoors", "SNGHack", + ], [ + "AlertVolumeControl", "DisengageVolume", "EngageVolume", "PromptVolume", "PromptDistractedVolume", "RefuseVolume", "WarningSoftVolume", + "WarningImmediateVolume", "CustomAlerts", "GreenLightAlert", "LeadDepartingAlert", "LoudBlindspotAlert", "SpeedLimitChangedAlert", "CustomUI", + "Compass", "DeveloperUI", "ShowJerk", "LeadInfo", "ShowTuning", "UseSI", "FPSCounter", "CustomPaths", "AccelerationPath", "AdjacentPath", "BlindSpotPath", + "AdjacentPathMetrics", "PedalsOnUI", "RoadNameUI", "WheelIcon", "RotatingWheel", "CustomTheme", "CustomColors", "CustomIcons", "CustomSignals", "CustomSounds", + "GoatScream", "HolidayThemes", "RandomEvents", "ModelUI", "DynamicPathWidth", "HideLeadMarker", "LaneLinesWidth", "PathEdgeWidth", "PathWidth", "RoadEdgesWidth", + "UnlimitedLength", "QOLVisuals", "BigMap", "FullMap", "CameraView", "DriverCamera", "HideSpeed", "HideSpeedUI", "MapStyle", "NumericalTemp", "Fahrenheit", + "WheelSpeed", "ScreenManagement", "HideUIElements", "HideAlerts", "HideMapIcon", "HideMaxSpeed", "ScreenBrightness", "ScreenBrightnessOnroad", "ScreenRecorder", + "ScreenTimeout", "ScreenTimeoutOnroad", "StandbyMode", + ], [ + "AutomaticUpdates", "ShowCPU", "ShowGPU", "ShowIP", "ShowMemoryUsage", "ShowStorageLeft", "ShowStorageUsed", "Sidebar", "TetheringEnabled", + ], [ + "FrogPilotDrives", "FrogPilotKilometers", "FrogPilotMinutes" + ] + + control_params, vehicle_params, visual_params, other_params, tracking_params = map(lambda keys: get_frogpilot_params(params, keys), [control_keys, vehicle_keys, visual_keys, other_keys, tracking_keys]) + control_values, vehicle_values, visual_values, other_values, tracking_values = map(format_params, [control_params, vehicle_params, visual_params, other_params, tracking_params]) + control_chunks, vehicle_chunks, visual_chunks, other_chunks, tracking_chunks = map(lambda data: chunk_data(data, 50), [control_values, vehicle_values, visual_values, other_values, tracking_values]) + + no_internet = 0 + while True: + if sentry_pinged(): + for chunks, label in zip([control_chunks, vehicle_chunks, visual_chunks, other_chunks, tracking_chunks], ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals", "Other Toggles", "FrogPilot Tracking"]): + with sentry_sdk.configure_scope() as scope: + set_sentry_scope(scope, chunks, label) + if blocked: + sentry_sdk.capture_message("Blocked user from using the development branch", level='error') + else: + sentry_sdk.capture_message("Fingerprinted %s" % candidate, level='info') + params.put_bool("FingerprintLogged", True) + sentry_sdk.flush() + break + else: + if no_internet > 5: + break + no_internet += 1 + time.sleep(600) def capture_exception(*args, **kwargs) -> None: save_exception(traceback.format_exc()) cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) + FrogPilot = "frogai" in get_origin().lower() + if not FrogPilot or PC: + return + try: + bind_user() sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 except Exception: @@ -69,13 +186,20 @@ def set_tag(key: str, value: str) -> None: def init(project: SentryProject) -> bool: - # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin() - if not comma_remote or not is_registered_device() or PC: - return False + params = Params() + installed = params.get("InstallDate", encoding='utf-8') + updated = params.get("Updated", encoding='utf-8') - env = "release" if is_tested_branch() else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + short_branch = get_short_branch() + + if short_branch == "FrogPilot-Development": + env = "Development" + elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}: + env = "Staging" + elif short_branch == "FrogPilot": + env = "Release" + else: + env = short_branch integrations = [] if project == SentryProject.SELFDRIVE: @@ -89,12 +213,12 @@ def init(project: SentryProject) -> bool: max_value_length=8192, environment=env) - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) - sentry_sdk.set_tag("branch", get_branch()) + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) + sentry_sdk.set_tag("branch", short_branch) sentry_sdk.set_tag("commit", get_commit()) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("updated", updated) + sentry_sdk.set_tag("installed", installed) + sentry_sdk.set_tag("repo", get_origin()) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session() diff --git a/selfdrive/updated/updated.py b/selfdrive/updated/updated.py old mode 100755 new mode 100644 index f17be3f41..9084e205d --- a/selfdrive/updated/updated.py +++ b/selfdrive/updated/updated.py @@ -12,6 +12,7 @@ import threading from collections import defaultdict from pathlib import Path from markdown_it import MarkdownIt +from zoneinfo import ZoneInfo from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params @@ -409,6 +410,8 @@ class Updater: finalize_update() cloudlog.info("finalize success!") + # Format "Updated" to Phoenix time zone + self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) def main() -> None: params = Params() @@ -433,10 +436,6 @@ def main() -> None: if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") - if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() - params.put("InstallDate", t.encode('utf8')) - updater = Updater() update_failed_count = 0 # TODO: Load from param? wait_helper = WaitTimeHelper() @@ -450,6 +449,9 @@ def main() -> None: # Run the update loop first_run = True branches_set = "FrogPilot" in (params.get("UpdaterAvailableBranches", encoding='utf-8') or "").split(',') + install_date_set = params.get("InstallDate") is not None and params.get("Updated") is not None + install_date_set &= params.get("InstallDate") != "November 21, 2023 - 02:10PM" # Remove this on the June 1st update + while True: wait_helper.ready_event.clear() @@ -467,6 +469,11 @@ def main() -> None: wait_helper.sleep(60) continue + # Format "InstallDate" to Phoenix time zone + if not install_date_set: + params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) + install_date_set = True + if not (params.get_bool("AutomaticUpdates") or params_memory.get_bool("ManualUpdateInitiated") or not branches_set): wait_helper.sleep(60*60*24*365*100) continue