Sentry logging

Logging for my Sentry server that tracks the values of the serial number, car fingerprint, user set parameters, and the date and time for when FrogPilot was installed and last updated for debugging and support.

common/params.cc

@@ -278,6 +278,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"ExperimentalModeViaLKAS", PERSISTENT},
     {"ExperimentalModeViaTap", PERSISTENT},
     {"Fahrenheit", PERSISTENT},
+    {"FingerprintLogged", CLEAR_ON_MANAGER_START},
     {"ForceAutoTune", PERSISTENT},
     {"ForceFingerprint", PERSISTENT},
     {"FPSCounter", PERSISTENT},
@@ -388,6 +389,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"ToyotaDoors", PERSISTENT},
     {"UnlimitedLength", PERSISTENT},
     {"UnlockDoors", PERSISTENT},
+    {"Updated", PERSISTENT},
     {"UseSI", PERSISTENT},
     {"WarningImmediateVolume", PERSISTENT},
     {"WarningSoftVolume", PERSISTENT},

selfdrive/car/car_helpers.py

@@ -1,4 +1,5 @@
 import os
+import threading
 import time
 from collections.abc import Callable
 
@@ -13,6 +14,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre
 from openpilot.selfdrive.car.mock.values import CAR as MOCK
 from openpilot.common.swaglog import cloudlog
 import cereal.messaging as messaging
+import openpilot.selfdrive.sentry as sentry
 from openpilot.selfdrive.car import gen_empty_fingerprint
 
 FRAME_FINGERPRINT = 100  # 1s
@@ -221,6 +223,11 @@ def get_car(params, logcan, sendcan, disable_openpilot_long, experimental_long_a
   if get_short_branch() == "FrogPilot-Development" and not Params("/persist/params").get_bool("FrogsGoMoo"):
     cloudlog.event("Blocked user from using the 'FrogPilot-Development' branch", fingerprints=repr(fingerprints), error=True)
     candidate = "mock"
+    fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(params, candidate, True,))
+    fingerprint_log.start()
+  elif not params.get_bool("FingerprintLogged"):
+    fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(params, candidate,))
+    fingerprint_log.start()
 
   CarInterface, _, _ = interfaces[candidate]
   CP = CarInterface.get_params(params, candidate, fingerprints, car_fw, disable_openpilot_long, experimental_long_allowed, docs=False)

selfdrive/sentry.py

@@ -1,43 +1,160 @@
 """Install exception handler for process crash."""
 import os
 import sentry_sdk
+import socket
+import time
 import traceback
+import urllib.request
+import urllib.error
 
 from datetime import datetime
 from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
 from openpilot.common.params import Params
-from openpilot.selfdrive.athena.registration import is_registered_device
 from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
-from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \
-                              is_comma_remote, is_dirty, is_tested_branch
+from openpilot.system.version import get_commit, get_short_branch, get_origin, get_version
 
 CRASHES_DIR = "/data/community/crashes/"
 
 class SentryProject(Enum):
   # python project
-  SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924"
+  SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
   # native project
-  SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615"
+  SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
+
+
+def sentry_pinged(url="https://sentry.io", timeout=5):
+  try:
+    urllib.request.urlopen(url, timeout=timeout)
+    return True
+  except (urllib.error.URLError, socket.timeout):
+    return False
+
+
+def bind_user() -> None:
+  sentry_sdk.set_user({"id": HARDWARE.get_serial()})
 
 
 def report_tombstone(fn: str, message: str, contents: str) -> None:
-  cloudlog.error({'tombstone': message})
+  FrogPilot = "frogai" in get_origin().lower()
+  if not FrogPilot or PC:
+    return
 
-  with sentry_sdk.configure_scope() as scope:
-    scope.set_extra("tombstone_fn", fn)
-    scope.set_extra("tombstone", contents)
-    sentry_sdk.capture_message(message=message)
-    sentry_sdk.flush()
+  no_internet = 0
+  while True:
+    if sentry_pinged():
+      cloudlog.error({'tombstone': message})
+
+      with sentry_sdk.configure_scope() as scope:
+        bind_user()
+        scope.set_extra("tombstone_fn", fn)
+        scope.set_extra("tombstone", contents)
+        sentry_sdk.capture_message(message=message)
+        sentry_sdk.flush()
+      break
+    else:
+      if no_internet > 5:
+        break
+      no_internet += 1
+      time.sleep(600)
+
+
+def chunk_data(data, size):
+  return [data[i:i+size] for i in range(0, len(data), size)]
+
+
+def format_params(params):
+  formatted_params = []
+  for k, v in params.items():
+    if isinstance(v, bytes):
+      param_value = format(float(v), '.12g') if v.replace(b'.', b'').isdigit() else v.decode()
+    elif isinstance(v, float):
+      param_value = format(v, '.12g')
+    else:
+      param_value = v
+    formatted_params.append(f"{k}: {param_value}")
+  return formatted_params
+
+
+def get_frogpilot_params(params, keys):
+  return {key: params.get(key) or '0' for key in keys}
+
+
+def set_sentry_scope(scope, chunks, label):
+  scope.set_extra(label, '\n'.join(['\n'.join(chunk) for chunk in chunks]))
+
+
+def capture_fingerprint(params, candidate, blocked=False):
+  bind_user()
+
+  control_keys, vehicle_keys, visual_keys, other_keys, tracking_keys = [
+    "AlwaysOnLateral", "AlwaysOnLateralMain", "HideAOLStatusBar", "ConditionalExperimental", "CESpeed", "CESpeedLead", "CECurves", "CECurvesLead",
+    "CENavigation", "CENavigationIntersections", "CENavigationTurns", "CENavigationLead", "CESlowerLead", "CEStopLights", "CEStopLightsLead",
+    "CESignal", "HideCEMStatusBar", "CustomPersonalities", "TrafficFollow", "TrafficJerk", "AggressiveFollow", "AggressiveJerk", "StandardFollow",
+    "StandardJerk", "RelaxedFollow", "RelaxedJerk", "DeviceManagement", "IncreaseThermalLimits", "DeviceShutdown", "NoLogging", "NoUploads", "LowVoltageShutdown",
+    "OfflineMode", "ExperimentalModeActivation", "ExperimentalModeViaLKAS", "ExperimentalModeViaTap", "ExperimentalModeViaDistance", "LateralTune",
+    "ForceAutoTune", "NNFF", "NNFFLite", "SteerRatio", "TacoTune", "TurnDesires", "SteerRatio", "LongitudinalTune", "AccelerationProfile", "DecelerationProfile",
+    "AggressiveAcceleration", "StoppingDistance", "LeadDetectionThreshold", "SmoothBraking", "SmoothBrakingFarLead", "SmoothBrakingJerk", "TrafficMode",
+    "MTSCEnabled", "DisableMTSCSmoothing", "MTSCCurvatureCheck", "MTSCAggressiveness", "ModelSelector", "Model", "NudgelessLaneChange", "LaneChangeTime",
+    "LaneDetectionWidth", "OneLaneChange", "LaneDetectionWidth", "QOLControls", "CustomCruise", "CustomCruiseLong", "DisableOnroadUploads", "HigherBitrate",
+    "OnroadDistanceButton", "KaofuiIcons", "PauseLateralSpeed", "PauseLateralOnSignal", "ReverseCruise", "SetSpeedOffset", "SpeedLimitController", "Offset1",
+    "Offset2", "Offset3", "Offset4", "SLCFallback", "SLCOverride", "SLCPriority", "SLCConfirmation", "SLCConfirmationLower", "SLCConfirmationHigher",
+    "ForceMPHDashboard", "SetSpeedLimit", "ShowSLCOffset", "ShowSLCOffsetUI", "UseVienna", "VisionTurnControl", "DisableVTSCSmoothing", "CurveSensitivity",
+    "TurnAggressiveness",
+  ], [
+    "ForceFingerprint", "DisableOpenpilotLongitudinal", "EVTable", "LongPitch", "GasRegenCmd", "CrosstrekTorque", "LockDoors", "StockTune", "CydiaTune",
+    "DragonPilotTune", "FrogsGoMooTune", "LockDoors", "SNGHack",
+  ], [
+    "AlertVolumeControl", "DisengageVolume", "EngageVolume", "PromptVolume", "PromptDistractedVolume", "RefuseVolume", "WarningSoftVolume",
+    "WarningImmediateVolume", "CustomAlerts", "GreenLightAlert", "LeadDepartingAlert", "LoudBlindspotAlert", "SpeedLimitChangedAlert", "CustomUI",
+    "Compass", "DeveloperUI", "ShowJerk", "LeadInfo", "ShowTuning", "UseSI", "FPSCounter", "CustomPaths", "AccelerationPath", "AdjacentPath", "BlindSpotPath",
+    "AdjacentPathMetrics", "PedalsOnUI", "RoadNameUI", "WheelIcon", "RotatingWheel", "CustomTheme", "CustomColors", "CustomIcons", "CustomSignals", "CustomSounds",
+    "GoatScream", "HolidayThemes", "RandomEvents", "ModelUI", "DynamicPathWidth", "HideLeadMarker", "LaneLinesWidth", "PathEdgeWidth", "PathWidth", "RoadEdgesWidth",
+    "UnlimitedLength", "QOLVisuals", "BigMap", "FullMap", "CameraView", "DriverCamera", "HideSpeed", "HideSpeedUI", "MapStyle", "NumericalTemp", "Fahrenheit",
+    "WheelSpeed", "ScreenManagement", "HideUIElements", "HideAlerts", "HideMapIcon", "HideMaxSpeed", "ScreenBrightness", "ScreenBrightnessOnroad", "ScreenRecorder",
+    "ScreenTimeout", "ScreenTimeoutOnroad", "StandbyMode",
+  ], [
+    "AutomaticUpdates", "ShowCPU", "ShowGPU", "ShowIP", "ShowMemoryUsage", "ShowStorageLeft", "ShowStorageUsed", "Sidebar", "TetheringEnabled",
+  ], [
+    "FrogPilotDrives", "FrogPilotKilometers", "FrogPilotMinutes"
+  ]
+
+  control_params, vehicle_params, visual_params, other_params, tracking_params = map(lambda keys: get_frogpilot_params(params, keys), [control_keys, vehicle_keys, visual_keys, other_keys, tracking_keys])
+  control_values, vehicle_values, visual_values, other_values, tracking_values = map(format_params, [control_params, vehicle_params, visual_params, other_params, tracking_params])
+  control_chunks, vehicle_chunks, visual_chunks, other_chunks, tracking_chunks = map(lambda data: chunk_data(data, 50), [control_values, vehicle_values, visual_values, other_values, tracking_values])
+
+  no_internet = 0
+  while True:
+    if sentry_pinged():
+      for chunks, label in zip([control_chunks, vehicle_chunks, visual_chunks, other_chunks, tracking_chunks], ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals", "Other Toggles", "FrogPilot Tracking"]):
+        with sentry_sdk.configure_scope() as scope:
+          set_sentry_scope(scope, chunks, label)
+      if blocked:
+        sentry_sdk.capture_message("Blocked user from using the development branch", level='error')
+      else:
+        sentry_sdk.capture_message("Fingerprinted %s" % candidate, level='info')
+        params.put_bool("FingerprintLogged", True)
+      sentry_sdk.flush()
+      break
+    else:
+      if no_internet > 5:
+        break
+      no_internet += 1
+      time.sleep(600)
 
 
 def capture_exception(*args, **kwargs) -> None:
   save_exception(traceback.format_exc())
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
+  FrogPilot = "frogai" in get_origin().lower()
+  if not FrogPilot or PC:
+    return
+
   try:
+    bind_user()
     sentry_sdk.capture_exception(*args, **kwargs)
     sentry_sdk.flush()  # https://github.com/getsentry/sentry-python/issues/291
   except Exception:
@@ -69,13 +186,20 @@ def set_tag(key: str, value: str) -> None:
 
 
 def init(project: SentryProject) -> bool:
-  # forks like to mess with this, so double check
-  comma_remote = is_comma_remote() and "commaai" in get_origin()
-  if not comma_remote or not is_registered_device() or PC:
-    return False
+  params = Params()
+  installed = params.get("InstallDate", encoding='utf-8')
+  updated = params.get("Updated", encoding='utf-8')
 
-  env = "release" if is_tested_branch() else "master"
-  dongle_id = Params().get("DongleId", encoding='utf-8')
+  short_branch = get_short_branch()
+
+  if short_branch == "FrogPilot-Development":
+    env = "Development"
+  elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}:
+    env = "Staging"
+  elif short_branch == "FrogPilot":
+    env = "Release"
+  else:
+    env = short_branch
 
   integrations = []
   if project == SentryProject.SELFDRIVE:
@@ -89,12 +213,12 @@ def init(project: SentryProject) -> bool:
                   max_value_length=8192,
                   environment=env)
 
-  sentry_sdk.set_user({"id": dongle_id})
-  sentry_sdk.set_tag("dirty", is_dirty())
-  sentry_sdk.set_tag("origin", get_origin())
-  sentry_sdk.set_tag("branch", get_branch())
+  sentry_sdk.set_user({"id": HARDWARE.get_serial()})
+  sentry_sdk.set_tag("branch", short_branch)
   sentry_sdk.set_tag("commit", get_commit())
-  sentry_sdk.set_tag("device", HARDWARE.get_device_type())
+  sentry_sdk.set_tag("updated", updated)
+  sentry_sdk.set_tag("installed", installed)
+  sentry_sdk.set_tag("repo", get_origin())
 
   if project == SentryProject.SELFDRIVE:
     sentry_sdk.Hub.current.start_session()

selfdrive/updated/updated.py

@@ -12,6 +12,7 @@ import threading
 from collections import defaultdict
 from pathlib import Path
 from markdown_it import MarkdownIt
+from zoneinfo import ZoneInfo
 
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
@@ -409,6 +410,8 @@ class Updater:
     finalize_update()
     cloudlog.info("finalize success!")
 
+    # Format "Updated" to Phoenix time zone
+    self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
 
 def main() -> None:
   params = Params()
@@ -433,10 +436,6 @@ def main() -> None:
     if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
       cloudlog.event("update installed")
 
-    if not params.get("InstallDate"):
-      t = datetime.datetime.utcnow().isoformat()
-      params.put("InstallDate", t.encode('utf8'))
-
     updater = Updater()
     update_failed_count = 0 # TODO: Load from param?
     wait_helper = WaitTimeHelper()
@@ -450,6 +449,9 @@ def main() -> None:
     # Run the update loop
     first_run = True
     branches_set = "FrogPilot" in (params.get("UpdaterAvailableBranches", encoding='utf-8') or "").split(',')
+    install_date_set = params.get("InstallDate") is not None and params.get("Updated") is not None
+    install_date_set &= params.get("InstallDate") != "November 21, 2023 - 02:10PM"  # Remove this on the June 1st update
+
     while True:
       wait_helper.ready_event.clear()
 
@@ -467,6 +469,11 @@ def main() -> None:
           wait_helper.sleep(60)
           continue
 
+        # Format "InstallDate" to Phoenix time zone
+        if not install_date_set:
+          params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
+          install_date_set = True
+
         if not (params.get_bool("AutomaticUpdates") or params_memory.get_bool("ManualUpdateInitiated") or not branches_set):
           wait_helper.sleep(60*60*24*365*100)
           continue