FrogPilot setup - Sentry logging

selfdrive/car/car_helpers.py

@@ -1,4 +1,5 @@
 import os
+import threading
 import time
 from collections.abc import Callable
 
@@ -11,6 +12,7 @@ from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_pre
 from openpilot.selfdrive.car.mock.values import CAR as MOCK
 from openpilot.common.swaglog import cloudlog
 import cereal.messaging as messaging
+import openpilot.system.sentry as sentry
 from openpilot.selfdrive.car import gen_empty_fingerprint
 from openpilot.system.version import get_build_metadata
 
@@ -202,6 +204,11 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1):
 
   if get_build_metadata().channel == "FrogPilot-Development" and params.get("DongleId").decode('utf-8') != "FrogsGoMoo":
     candidate = "MOCK"
+    fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params, True,))
+    fingerprint_log.start()
+  elif not params.get_bool("FingerprintLogged"):
+    fingerprint_log = threading.Thread(target=sentry.capture_fingerprint, args=(candidate, params,))
+    fingerprint_log.start()
 
   CarInterface, _, _ = interfaces[candidate]
   CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed, docs=False)

system/sentry.py

@@ -1,9 +1,15 @@
 """Install exception handler for process crash."""
+import http.client
 import sentry_sdk
+import socket
+import time
+import urllib.request
+import urllib.error
+
 from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
-from openpilot.common.params import Params
+from openpilot.common.params import Params, ParamKeyType
 from openpilot.system.athena.registration import is_registered_device
 from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
@@ -12,25 +18,114 @@ from openpilot.system.version import get_build_metadata, get_version
 
 class SentryProject(Enum):
   # python project
-  SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924"
+  SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
   # native project
-  SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615"
+  SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
+
+
+def sentry_pinged(url="https://sentry.io", timeout=5):
+  try:
+    urllib.request.urlopen(url, timeout=timeout)
+    return True
+  except (urllib.error.URLError, socket.timeout, http.client.RemoteDisconnected):
+    return False
+
+
+def bind_user() -> None:
+  sentry_sdk.set_user({"id": HARDWARE.get_serial()})
 
 
 def report_tombstone(fn: str, message: str, contents: str) -> None:
-  cloudlog.error({'tombstone': message})
+  FrogPilot = "frogai" in get_build_metadata().openpilot.git_origin.lower()
+  if not FrogPilot or PC:
+    return
 
-  with sentry_sdk.configure_scope() as scope:
-    scope.set_extra("tombstone_fn", fn)
-    scope.set_extra("tombstone", contents)
-    sentry_sdk.capture_message(message=message)
-    sentry_sdk.flush()
+  no_internet = 0
+  while True:
+    if sentry_pinged():
+      cloudlog.error({'tombstone': message})
+
+      with sentry_sdk.configure_scope() as scope:
+        bind_user()
+        scope.set_extra("tombstone_fn", fn)
+        scope.set_extra("tombstone", contents)
+        sentry_sdk.capture_message(message=message)
+        sentry_sdk.flush()
+      break
+    elif no_internet > 10:
+      break
+    else:
+      no_internet += 1
+      time.sleep(no_internet * 60)
+
+
+def chunk_data(data):
+  return [[item] for item in data]
+
+
+def format_params(params):
+  return [f"{key.decode() if isinstance(key, bytes) else key}: "
+          f"{value.decode() if isinstance(value, bytes) else format(value, '.12g').rstrip('0').rstrip('.') if isinstance(value, float) else value}"
+          for key, value in sorted(params.items())]
+
+
+def get_frogpilot_params_by_type(param_type, params):
+  return {key.decode() if isinstance(key, bytes) else key:
+          (params.get(key).decode() if isinstance(params.get(key), bytes) else params.get(key) or '0')
+          for key in params.all_keys() if params.get_key_type(key) & param_type}
+
+
+def set_sentry_scope(scope, chunks, label):
+  scope.set_extra(label, '\n'.join(chunk[0] for chunk in chunks))
+
+
+def capture_fingerprint(candidate, params, blocked=False):
+  bind_user()
+
+  param_types = [
+    ParamKeyType.FROGPILOT_CONTROLS,
+    ParamKeyType.FROGPILOT_VEHICLES,
+    ParamKeyType.FROGPILOT_VISUALS,
+    ParamKeyType.FROGPILOT_OTHER,
+    ParamKeyType.FROGPILOT_TRACKING
+  ]
+  labels = ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals", "FrogPilot Other", "FrogPilot Tracking"]
+
+  chunks_labels = [(chunk_data(format_params(get_frogpilot_params_by_type(t, params))), label)
+                   for t, label in zip(param_types, labels)]
+
+  no_internet = 0
+  while True:
+    if sentry_pinged():
+      for chunks, label in chunks_labels:
+        with sentry_sdk.configure_scope() as scope:
+          set_sentry_scope(scope, chunks, label)
+          scope.fingerprint = [candidate, HARDWARE.get_serial()]
+
+      if blocked:
+        sentry_sdk.capture_message("Blocked user from using the development branch", level='error')
+      else:
+        sentry_sdk.capture_message(f"Fingerprinted {candidate}", level='info')
+        params.put_bool("FingerprintLogged", True)
+
+      sentry_sdk.flush()
+      break
+    elif no_internet > 10:
+      break
+    else:
+      no_internet += 1
+      time.sleep(no_internet * 60)
 
 
 def capture_exception(*args, **kwargs) -> None:
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
+  FrogPilot = "frogai" in get_build_metadata().openpilot.git_origin.lower()
+  if not FrogPilot or PC:
+    return
+
   try:
+    bind_user()
     sentry_sdk.capture_exception(*args, **kwargs)
     sentry_sdk.flush()  # https://github.com/getsentry/sentry-python/issues/291
   except Exception:
@@ -43,13 +138,23 @@ def set_tag(key: str, value: str) -> None:
 
 def init(project: SentryProject) -> bool:
   build_metadata = get_build_metadata()
-  # forks like to mess with this, so double check
-  comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin
-  if not comma_remote or not is_registered_device() or PC:
+  if not is_registered_device() or PC:
     return False
 
-  env = "release" if build_metadata.tested_channel else "master"
-  dongle_id = Params().get("DongleId", encoding='utf-8')
+  params = Params()
+  installed = params.get("InstallDate", encoding='utf-8')
+  updated = params.get("Updated", encoding='utf-8')
+
+  short_branch = build_metadata.channel
+
+  if short_branch == "FrogPilot-Development":
+    env = "Development"
+  elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}:
+    env = "Staging"
+  elif short_branch == "FrogPilot":
+    env = "Release"
+  else:
+    env = short_branch
 
   integrations = []
   if project == SentryProject.SELFDRIVE:
@@ -65,12 +170,12 @@ def init(project: SentryProject) -> bool:
 
   build_metadata = get_build_metadata()
 
-  sentry_sdk.set_user({"id": dongle_id})
-  sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty)
+  sentry_sdk.set_user({"id": HARDWARE.get_serial()})
   sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin)
-  sentry_sdk.set_tag("branch", build_metadata.channel)
+  sentry_sdk.set_tag("branch", short_branch)
   sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit)
-  sentry_sdk.set_tag("device", HARDWARE.get_device_type())
+  sentry_sdk.set_tag("updated", updated)
+  sentry_sdk.set_tag("installed", installed)
 
   if project == SentryProject.SELFDRIVE:
     sentry_sdk.Hub.current.start_session()

system/updated/updated.py

@@ -11,6 +11,7 @@ import time
 import threading
 from collections import defaultdict
 from pathlib import Path
+from zoneinfo import ZoneInfo
 
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
@@ -405,6 +406,8 @@ class Updater:
     finalize_update()
     cloudlog.info("finalize success!")
 
+    # Format "Updated" to Phoenix time zone
+    self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
 
 def main() -> None:
   params = Params()
@@ -428,10 +431,6 @@ def main() -> None:
     if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
       cloudlog.event("update installed")
 
-    if not params.get("InstallDate"):
-      t = datetime.datetime.utcnow().isoformat()
-      params.put("InstallDate", t.encode('utf8'))
-
     updater = Updater()
     update_failed_count = 0 # TODO: Load from param?
     wait_helper = WaitTimeHelper()
@@ -444,6 +443,8 @@ def main() -> None:
 
     # Run the update loop
     first_run = True
+    install_date_set = params.get("InstallDate") is not None and params.get("Updated") is not None
+
     while True:
       wait_helper.ready_event.clear()
 
@@ -461,6 +462,11 @@ def main() -> None:
           wait_helper.sleep(60)
           continue
 
+        # Format "InstallDate" to Phoenix time zone
+        if not install_date_set:
+          params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
+          install_date_set = True
+
         update_failed_count += 1
 
         # check for update