diff --git a/common/params.cc b/common/params.cc index fc9b7bfd1..c3b9a7887 100644 --- a/common/params.cc +++ b/common/params.cc @@ -278,6 +278,7 @@ std::unordered_map keys = { {"ExperimentalModeViaLKAS", PERSISTENT}, {"ExperimentalModeViaTap", PERSISTENT}, {"Fahrenheit", PERSISTENT}, + {"FingerprintLogged", CLEAR_ON_MANAGER_START}, {"ForceAutoTune", PERSISTENT}, {"ForceFingerprint", PERSISTENT}, {"FPSCounter", PERSISTENT}, @@ -388,6 +389,7 @@ std::unordered_map keys = { {"ToyotaDoors", PERSISTENT}, {"UnlimitedLength", PERSISTENT}, {"UnlockDoors", PERSISTENT}, + {"Updated", PERSISTENT}, {"UseSI", PERSISTENT}, {"WarningImmediateVolume", PERSISTENT}, {"WarningSoftVolume", PERSISTENT}, diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 6d32f100e..a98910c6b 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,5 @@ import os +import threading import time from collections.abc import Callable @@ -13,6 +14,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre from openpilot.selfdrive.car.mock.values import CAR as MOCK from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging +import openpilot.selfdrive.sentry as sentry from openpilot.selfdrive.car import gen_empty_fingerprint FRAME_FINGERPRINT = 100 # 1s @@ -221,6 +223,11 @@ def get_car(params, logcan, sendcan, disable_openpilot_long, experimental_long_a if get_short_branch() == "FrogPilot-Development" and not Params("/persist/params").get_bool("FrogsGoMoo"): cloudlog.event("Blocked user from using the 'FrogPilot-Development' branch", fingerprints=repr(fingerprints), error=True) candidate = "mock" + fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(params, candidate, True,)) + fingerprint_log.start() + elif not params.get_bool("FingerprintLogged"): + fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(params, candidate,)) + fingerprint_log.start() CarInterface, _, _ = interfaces[candidate] CP = CarInterface.get_params(params, candidate, fingerprints, car_fw, disable_openpilot_long, experimental_long_allowed, docs=False) diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 45c8f99a6..1c5ebd635 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -1,43 +1,160 @@ """Install exception handler for process crash.""" import os import sentry_sdk +import socket +import time import traceback +import urllib.request +import urllib.error from datetime import datetime from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.selfdrive.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \ - is_comma_remote, is_dirty, is_tested_branch +from openpilot.system.version import get_commit, get_short_branch, get_origin, get_version CRASHES_DIR = "/data/community/crashes/" class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" + + +def sentry_pinged(url="https://sentry.io", timeout=5): + try: + urllib.request.urlopen(url, timeout=timeout) + return True + except (urllib.error.URLError, socket.timeout): + return False + + +def bind_user() -> None: + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) def report_tombstone(fn: str, message: str, contents: str) -> None: - cloudlog.error({'tombstone': message}) + FrogPilot = "frogai" in get_origin().lower() + if not FrogPilot or PC: + return - with sentry_sdk.configure_scope() as scope: - scope.set_extra("tombstone_fn", fn) - scope.set_extra("tombstone", contents) - sentry_sdk.capture_message(message=message) - sentry_sdk.flush() + no_internet = 0 + while True: + if sentry_pinged(): + cloudlog.error({'tombstone': message}) + + with sentry_sdk.configure_scope() as scope: + bind_user() + scope.set_extra("tombstone_fn", fn) + scope.set_extra("tombstone", contents) + sentry_sdk.capture_message(message=message) + sentry_sdk.flush() + break + else: + if no_internet > 5: + break + no_internet += 1 + time.sleep(600) + + +def chunk_data(data, size): + return [data[i:i+size] for i in range(0, len(data), size)] + + +def format_params(params): + formatted_params = [] + for k, v in params.items(): + if isinstance(v, bytes): + param_value = format(float(v), '.12g') if v.replace(b'.', b'').isdigit() else v.decode() + elif isinstance(v, float): + param_value = format(v, '.12g') + else: + param_value = v + formatted_params.append(f"{k}: {param_value}") + return formatted_params + + +def get_frogpilot_params(params, keys): + return {key: params.get(key) or '0' for key in keys} + + +def set_sentry_scope(scope, chunks, label): + scope.set_extra(label, '\n'.join(['\n'.join(chunk) for chunk in chunks])) + + +def capture_fingerprint(params, candidate, blocked=False): + bind_user() + + control_keys, vehicle_keys, visual_keys, other_keys, tracking_keys = [ + "AlwaysOnLateral", "AlwaysOnLateralMain", "HideAOLStatusBar", "ConditionalExperimental", "CESpeed", "CESpeedLead", "CECurves", "CECurvesLead", + "CENavigation", "CENavigationIntersections", "CENavigationTurns", "CENavigationLead", "CESlowerLead", "CEStopLights", "CEStopLightsLead", + "CESignal", "HideCEMStatusBar", "CustomPersonalities", "TrafficFollow", "TrafficJerk", "AggressiveFollow", "AggressiveJerk", "StandardFollow", + "StandardJerk", "RelaxedFollow", "RelaxedJerk", "DeviceManagement", "IncreaseThermalLimits", "DeviceShutdown", "NoLogging", "NoUploads", "LowVoltageShutdown", + "OfflineMode", "ExperimentalModeActivation", "ExperimentalModeViaLKAS", "ExperimentalModeViaTap", "ExperimentalModeViaDistance", "LateralTune", + "ForceAutoTune", "NNFF", "NNFFLite", "SteerRatio", "TacoTune", "TurnDesires", "SteerRatio", "LongitudinalTune", "AccelerationProfile", "DecelerationProfile", + "AggressiveAcceleration", "StoppingDistance", "LeadDetectionThreshold", "SmoothBraking", "SmoothBrakingFarLead", "SmoothBrakingJerk", "TrafficMode", + "MTSCEnabled", "DisableMTSCSmoothing", "MTSCCurvatureCheck", "MTSCAggressiveness", "ModelSelector", "Model", "NudgelessLaneChange", "LaneChangeTime", + "LaneDetectionWidth", "OneLaneChange", "LaneDetectionWidth", "QOLControls", "CustomCruise", "CustomCruiseLong", "DisableOnroadUploads", "HigherBitrate", + "OnroadDistanceButton", "KaofuiIcons", "PauseLateralSpeed", "PauseLateralOnSignal", "ReverseCruise", "SetSpeedOffset", "SpeedLimitController", "Offset1", + "Offset2", "Offset3", "Offset4", "SLCFallback", "SLCOverride", "SLCPriority", "SLCConfirmation", "SLCConfirmationLower", "SLCConfirmationHigher", + "ForceMPHDashboard", "SetSpeedLimit", "ShowSLCOffset", "ShowSLCOffsetUI", "UseVienna", "VisionTurnControl", "DisableVTSCSmoothing", "CurveSensitivity", + "TurnAggressiveness", + ], [ + "ForceFingerprint", "DisableOpenpilotLongitudinal", "EVTable", "LongPitch", "GasRegenCmd", "CrosstrekTorque", "LockDoors", "StockTune", "CydiaTune", + "DragonPilotTune", "FrogsGoMooTune", "LockDoors", "SNGHack", + ], [ + "AlertVolumeControl", "DisengageVolume", "EngageVolume", "PromptVolume", "PromptDistractedVolume", "RefuseVolume", "WarningSoftVolume", + "WarningImmediateVolume", "CustomAlerts", "GreenLightAlert", "LeadDepartingAlert", "LoudBlindspotAlert", "SpeedLimitChangedAlert", "CustomUI", + "Compass", "DeveloperUI", "ShowJerk", "LeadInfo", "ShowTuning", "UseSI", "FPSCounter", "CustomPaths", "AccelerationPath", "AdjacentPath", "BlindSpotPath", + "AdjacentPathMetrics", "PedalsOnUI", "RoadNameUI", "WheelIcon", "RotatingWheel", "CustomTheme", "CustomColors", "CustomIcons", "CustomSignals", "CustomSounds", + "GoatScream", "HolidayThemes", "RandomEvents", "ModelUI", "DynamicPathWidth", "HideLeadMarker", "LaneLinesWidth", "PathEdgeWidth", "PathWidth", "RoadEdgesWidth", + "UnlimitedLength", "QOLVisuals", "BigMap", "FullMap", "CameraView", "DriverCamera", "HideSpeed", "HideSpeedUI", "MapStyle", "NumericalTemp", "Fahrenheit", + "WheelSpeed", "ScreenManagement", "HideUIElements", "HideAlerts", "HideMapIcon", "HideMaxSpeed", "ScreenBrightness", "ScreenBrightnessOnroad", "ScreenRecorder", + "ScreenTimeout", "ScreenTimeoutOnroad", "StandbyMode", + ], [ + "AutomaticUpdates", "ShowCPU", "ShowGPU", "ShowIP", "ShowMemoryUsage", "ShowStorageLeft", "ShowStorageUsed", "Sidebar", "TetheringEnabled", + ], [ + "FrogPilotDrives", "FrogPilotKilometers", "FrogPilotMinutes" + ] + + control_params, vehicle_params, visual_params, other_params, tracking_params = map(lambda keys: get_frogpilot_params(params, keys), [control_keys, vehicle_keys, visual_keys, other_keys, tracking_keys]) + control_values, vehicle_values, visual_values, other_values, tracking_values = map(format_params, [control_params, vehicle_params, visual_params, other_params, tracking_params]) + control_chunks, vehicle_chunks, visual_chunks, other_chunks, tracking_chunks = map(lambda data: chunk_data(data, 50), [control_values, vehicle_values, visual_values, other_values, tracking_values]) + + no_internet = 0 + while True: + if sentry_pinged(): + for chunks, label in zip([control_chunks, vehicle_chunks, visual_chunks, other_chunks, tracking_chunks], ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals", "Other Toggles", "FrogPilot Tracking"]): + with sentry_sdk.configure_scope() as scope: + set_sentry_scope(scope, chunks, label) + if blocked: + sentry_sdk.capture_message("Blocked user from using the development branch", level='error') + else: + sentry_sdk.capture_message("Fingerprinted %s" % candidate, level='info') + params.put_bool("FingerprintLogged", True) + sentry_sdk.flush() + break + else: + if no_internet > 5: + break + no_internet += 1 + time.sleep(600) def capture_exception(*args, **kwargs) -> None: save_exception(traceback.format_exc()) cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) + FrogPilot = "frogai" in get_origin().lower() + if not FrogPilot or PC: + return + try: + bind_user() sentry_sdk.capture_exception(*args, **kwargs) sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 except Exception: @@ -69,13 +186,20 @@ def set_tag(key: str, value: str) -> None: def init(project: SentryProject) -> bool: - # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin() - if not comma_remote or not is_registered_device() or PC: - return False + params = Params() + installed = params.get("InstallDate", encoding='utf-8') + updated = params.get("Updated", encoding='utf-8') - env = "release" if is_tested_branch() else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + short_branch = get_short_branch() + + if short_branch == "FrogPilot-Development": + env = "Development" + elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}: + env = "Staging" + elif short_branch == "FrogPilot": + env = "Release" + else: + env = short_branch integrations = [] if project == SentryProject.SELFDRIVE: @@ -89,12 +213,12 @@ def init(project: SentryProject) -> bool: max_value_length=8192, environment=env) - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) - sentry_sdk.set_tag("branch", get_branch()) + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) + sentry_sdk.set_tag("branch", short_branch) sentry_sdk.set_tag("commit", get_commit()) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("updated", updated) + sentry_sdk.set_tag("installed", installed) + sentry_sdk.set_tag("repo", get_origin()) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session() diff --git a/selfdrive/updated/updated.py b/selfdrive/updated/updated.py old mode 100755 new mode 100644 index f17be3f41..9084e205d --- a/selfdrive/updated/updated.py +++ b/selfdrive/updated/updated.py @@ -12,6 +12,7 @@ import threading from collections import defaultdict from pathlib import Path from markdown_it import MarkdownIt +from zoneinfo import ZoneInfo from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params @@ -409,6 +410,8 @@ class Updater: finalize_update() cloudlog.info("finalize success!") + # Format "Updated" to Phoenix time zone + self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) def main() -> None: params = Params() @@ -433,10 +436,6 @@ def main() -> None: if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") - if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() - params.put("InstallDate", t.encode('utf8')) - updater = Updater() update_failed_count = 0 # TODO: Load from param? wait_helper = WaitTimeHelper() @@ -450,6 +449,9 @@ def main() -> None: # Run the update loop first_run = True branches_set = "FrogPilot" in (params.get("UpdaterAvailableBranches", encoding='utf-8') or "").split(',') + install_date_set = params.get("InstallDate") is not None and params.get("Updated") is not None + install_date_set &= params.get("InstallDate") != "November 21, 2023 - 02:10PM" # Remove this on the June 1st update + while True: wait_helper.ready_event.clear() @@ -467,6 +469,11 @@ def main() -> None: wait_helper.sleep(60) continue + # Format "InstallDate" to Phoenix time zone + if not install_date_set: + params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) + install_date_set = True + if not (params.get_bool("AutomaticUpdates") or params_memory.get_bool("ManualUpdateInitiated") or not branches_set): wait_helper.sleep(60*60*24*365*100) continue