FrogPilot setup - Sentry logging

selfdrive/car/car_helpers.py

@@ -1,4 +1,5 @@
 import os
+import threading
 import time
 from collections.abc import Callable
 
@@ -11,6 +12,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre
 from openpilot.selfdrive.car.mock.values import CAR as MOCK
 from openpilot.common.swaglog import cloudlog
 import cereal.messaging as messaging
+import openpilot.system.sentry as sentry
 from openpilot.selfdrive.car import gen_empty_fingerprint
 from openpilot.system.version import get_build_metadata
 
@@ -198,6 +200,9 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1):
 
   if get_build_metadata().channel == "FrogPilot-Development" and params.get("DongleId", encoding='utf-8') != "FrogsGoMoo":
     candidate = "MOCK"
+    threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params, True,)).start()
+  elif False:
+    threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params,)).start()
 
   CarInterface, _, _ = interfaces[candidate]
   CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed, docs=False)

system/sentry.py

@@ -1,36 +1,124 @@
 """Install exception handler for process crash."""
 import sentry_sdk
+import time
+
 from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
-from openpilot.common.params import Params
+from openpilot.common.params import Params, ParamKeyType
 from openpilot.system.athena.registration import is_registered_device
 from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.version import get_build_metadata, get_version
 
+from openpilot.selfdrive.frogpilot.controls.lib.frogpilot_functions import is_url_pingable
 
 class SentryProject(Enum):
   # python project
-  SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924"
+  SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
   # native project
-  SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615"
+  SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
+
+
+def bind_user() -> None:
+  sentry_sdk.set_user({"id": HARDWARE.get_serial()})
 
 
 def report_tombstone(fn: str, message: str, contents: str) -> None:
-  cloudlog.error({'tombstone': message})
+  FrogPilot = "frogai" in get_build_metadata().openpilot.git_origin.lower()
+  if not FrogPilot or PC:
+    return
 
-  with sentry_sdk.configure_scope() as scope:
-    scope.set_extra("tombstone_fn", fn)
-    scope.set_extra("tombstone", contents)
-    sentry_sdk.capture_message(message=message)
-    sentry_sdk.flush()
+  no_internet = 0
+  while True:
+    if is_url_pingable("https://sentry.io"):
+      cloudlog.error({'tombstone': message})
+
+      with sentry_sdk.configure_scope() as scope:
+        bind_user()
+        scope.set_extra("tombstone_fn", fn)
+        scope.set_extra("tombstone", contents)
+        sentry_sdk.capture_message(message=message)
+        sentry_sdk.flush()
+      break
+    elif no_internet > 10:
+      break
+    else:
+      no_internet += 1
+      time.sleep(no_internet * 60)
+
+
+def capture_fingerprint(candidate, params, blocked=False):
+  bind_user()
+
+  params_tracking = Params("/persist/tracking")
+
+  param_types = {
+    "FrogPilot Controls": ParamKeyType.FROGPILOT_CONTROLS,
+    "FrogPilot Vehicles": ParamKeyType.FROGPILOT_VEHICLES,
+    "FrogPilot Visuals": ParamKeyType.FROGPILOT_VISUALS,
+    "FrogPilot Other": ParamKeyType.FROGPILOT_OTHER,
+    "FrogPilot Tracking": ParamKeyType.FROGPILOT_TRACKING
+  }
+
+  matched_params = {label: {} for label in param_types}
+  for key in params.all_keys():
+    for label, key_type in param_types.items():
+      if params.get_key_type(key) & key_type:
+        if key_type == ParamKeyType.FROGPILOT_TRACKING:
+          try:
+            value = params_tracking.get_int(key)
+          except Exception:
+            value = "0"
+        else:
+          try:
+            value = params.get(key)
+            if isinstance(value, bytes):
+              value = value.decode('utf-8')
+            if isinstance(value, str) and value.replace('.', '', 1).isdigit():
+              value = float(value) if '.' in value else int(value)
+          except Exception:
+            value = "0"
+        matched_params[label][key.decode('utf-8')] = value
+
+  for label, key_values in matched_params.items():
+    if label == "FrogPilot Tracking":
+      matched_params[label] = {k: f"{v:,}" for k, v in key_values.items()}
+    else:
+      matched_params[label] = {k: int(v) if isinstance(v, float) and v.is_integer() else v for k, v in sorted(key_values.items())}
+
+  no_internet = 0
+  while True:
+    if is_url_pingable("https://sentry.io"):
+      with sentry_sdk.configure_scope() as scope:
+        scope.fingerprint = [candidate, HARDWARE.get_serial()]
+        for label, key_values in matched_params.items():
+          scope.set_extra(label, "\n".join([f"{k}: {v}" for k, v in key_values.items()]))
+
+      if blocked:
+        sentry_sdk.capture_message("Blocked user from using the development branch", level='error')
+      else:
+        sentry_sdk.capture_message(f"Fingerprinted {candidate}", level='info')
+        params.put_bool_nonblocking("FingerprintLogged", True)
+
+      sentry_sdk.flush()
+      break
+    elif no_internet > 10:
+      break
+    else:
+      no_internet += 1
+      time.sleep(no_internet * 60)
 
 
 def capture_exception(*args, **kwargs) -> None:
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
+  FrogPilot = "frogai" in get_build_metadata().openpilot.git_origin.lower()
+  if not FrogPilot or PC:
+    return
+
   try:
+    bind_user()
     sentry_sdk.capture_exception(*args, **kwargs)
     sentry_sdk.flush()  # https://github.com/getsentry/sentry-python/issues/291
   except Exception:
@@ -43,13 +131,23 @@ def set_tag(key: str, value: str) -> None:
 
 def init(project: SentryProject) -> bool:
   build_metadata = get_build_metadata()
-  # forks like to mess with this, so double check
-  comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin
-  if not comma_remote or not is_registered_device() or PC:
+  if PC:
     return False
 
-  env = "release" if build_metadata.tested_channel else "master"
-  dongle_id = Params().get("DongleId", encoding='utf-8')
+  params = Params()
+  installed = params.get("InstallDate", encoding='utf-8')
+  updated = params.get("Updated", encoding='utf-8')
+
+  short_branch = build_metadata.channel
+
+  if short_branch == "FrogPilot-Development":
+    env = "Development"
+  elif build_metadata.tested_channel:
+    env = "Staging"
+  elif build_metadata.release_channel:
+    env = "Release"
+  else:
+    env = short_branch
 
   integrations = []
   if project == SentryProject.SELFDRIVE:
@@ -65,12 +163,12 @@ def init(project: SentryProject) -> bool:
 
   build_metadata = get_build_metadata()
 
-  sentry_sdk.set_user({"id": dongle_id})
-  sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty)
+  sentry_sdk.set_user({"id": HARDWARE.get_serial()})
   sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin)
-  sentry_sdk.set_tag("branch", build_metadata.channel)
+  sentry_sdk.set_tag("branch", short_branch)
   sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit)
-  sentry_sdk.set_tag("device", HARDWARE.get_device_type())
+  sentry_sdk.set_tag("updated", updated)
+  sentry_sdk.set_tag("installed", installed)
 
   if project == SentryProject.SELFDRIVE:
     sentry_sdk.Hub.current.start_session()

system/updated/updated.py

@@ -11,6 +11,7 @@ import time
 import threading
 from collections import defaultdict
 from pathlib import Path
+from zoneinfo import ZoneInfo
 
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
@@ -405,6 +406,8 @@ class Updater:
     finalize_update()
     cloudlog.info("finalize success!")
 
+    # Format "Updated" to Phoenix time zone
+    self.params.put_nonblocking("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
 
 def main() -> None:
   params = Params()
@@ -428,10 +431,6 @@ def main() -> None:
     if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
       cloudlog.event("update installed")
 
-    if not params.get("InstallDate"):
-      t = datetime.datetime.utcnow().isoformat()
-      params.put("InstallDate", t.encode('utf8'))
-
     updater = Updater()
     update_failed_count = 0 # TODO: Load from param?
     wait_helper = WaitTimeHelper()
@@ -444,6 +443,8 @@ def main() -> None:
 
     # Run the update loop
     first_run = True
+    install_date_set = params.get("InstallDate", encoding='utf-8') is not None and params.get("Updated", encoding='utf-8') is not None
+
     while True:
       wait_helper.ready_event.clear()
 
@@ -461,6 +462,11 @@ def main() -> None:
           wait_helper.sleep(60)
           continue
 
+        # Format "InstallDate" to Phoenix time zone
+        if not install_date_set:
+          params.put_nonblocking("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
+          install_date_set = True
+
         update_failed_count += 1
 
         # check for update