Merge branch 'new-logging' into dev-priv/master

selfdrive/car/car_helpers.py

@@ -1,4 +1,7 @@
 import os
+import requests
+import threading
+import time
 from typing import Dict, List
 
 from cereal import car
@@ -9,6 +12,7 @@ from selfdrive.car.interfaces import get_interface_attr
 from selfdrive.car.fingerprints import eliminate_incompatible_cars, all_legacy_fingerprint_cars
 from selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN
 from selfdrive.car.fw_versions import get_fw_versions_ordered, match_fw_to_car, get_present_ecus
+import selfdrive.sentry as sentry
 from system.swaglog import cloudlog
 import cereal.messaging as messaging
 from selfdrive.car import gen_empty_fingerprint
@@ -173,12 +177,52 @@ def fingerprint(logcan, sendcan, num_pandas):
   return car_fingerprint, finger, vin, car_fw, source, exact_match
 
 
+def is_connected_to_internet(timeout=5):
+  try:
+    requests.get("https://sentry.io", timeout=timeout)
+    return True
+  except Exception:
+    return False
+
+
+def crash_log(candidate):
+  no_internet = 0
+  while True:
+    if is_connected_to_internet():
+      sentry.capture_warning("fingerprinted %s" % candidate)
+      break
+    else:
+      no_internet += 1
+      if no_internet >= 2:
+        break
+      time.sleep(600)
+
+
+def crash_log2(fingerprints, fw):
+  no_internet = 0
+  while True:
+    if is_connected_to_internet():
+      sentry.capture_warning("car doesn't match any fingerprints: %s" % fingerprints)
+      sentry.capture_warning("car doesn't match any fw: %s" % fw)
+      break
+    else:
+      no_internet += 1
+      if no_internet >= 2:
+        break
+      time.sleep(600)
+
+
 def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1):
   candidate, fingerprints, vin, car_fw, source, exact_match = fingerprint(logcan, sendcan, num_pandas)
 
   if candidate is None:
     cloudlog.warning("car doesn't match any fingerprints: %r", fingerprints)
     candidate = "mock"
+    y = threading.Thread(target=crash_log2, args=(fingerprints, car_fw,))
+    y.start()
+
+  x = threading.Thread(target=crash_log, args=(candidate,))
+  x.start()
 
   CarInterface, CarController, CarState = interfaces[candidate]
   CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed)

selfdrive/sentry.py

@@ -4,20 +4,51 @@ from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
 from common.params import Params
-from selfdrive.athena.registration import is_registered_device
+#from selfdrive.athena.registration import is_registered_device
 from system.hardware import HARDWARE, PC
 from system.swaglog import cloudlog
 from system.version import get_branch, get_commit, get_origin, get_version, \
                               is_comma_remote, is_dirty, is_tested_branch
 
+import os
+import traceback
+import requests
+from cereal import car
+from datetime import datetime
 
 class SentryProject(Enum):
   # python project
-  SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924"
+  SELFDRIVE = "https://7e3be9bfcfe04c9abe58bd25fe290d1a@o1138119.ingest.sentry.io/6191481"
   # native project
-  SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615"
+  SELFDRIVE_NATIVE = "https://7e3be9bfcfe04c9abe58bd25fe290d1a@o1138119.ingest.sentry.io/6191481"
 
 
+CRASHES_DIR = os.path.join('/data/community/crashes')
+ret = car.CarParams.new_message()
+candidate = ret.carFingerprint
+params = Params()
+#uniqueID = op_params.get('uniqueID')
+try:
+  dongle_id = params.get("DongleId").decode('utf8')
+except AttributeError:
+  dongle_id = "None"
+try:
+  gitname = params.get("GithubUsername", encoding='utf-8')
+except Exception:
+  gitname = ""
+try:
+  ip = requests.get('https://checkip.amazonaws.com/').text.strip()
+except Exception:
+  ip = "255.255.255.255"
+error_tags = {
+  'dirty': is_dirty(),
+  'dongle_id': dongle_id,
+  'branch': get_branch(),
+  'remote': get_origin(),
+  'fingerprintedAs': candidate,
+  'gitname': gitname
+}
+
 def report_tombstone(fn: str, message: str, contents: str) -> None:
   cloudlog.error({'tombstone': message})
 
@@ -29,6 +60,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None:
 
 
 def capture_exception(*args, **kwargs) -> None:
+  save_exception(traceback.format_exc())
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
   try:
@@ -38,18 +70,52 @@ def capture_exception(*args, **kwargs) -> None:
     cloudlog.exception("sentry exception")
 
 
+def save_exception(exc_text):
+  if not os.path.exists(CRASHES_DIR):
+    os.makedirs(CRASHES_DIR)
+
+  log_file = '{}/{}'.format(CRASHES_DIR, datetime.now().strftime('%m-%d-%Y--%I:%M.%S-%p.log'))
+  log_file_2 = f'{CRASHES_DIR}/error.txt'
+  with open(log_file, 'w') as f:
+    f.write(exc_text)
+    f.close()
+  with open(log_file_2, 'w') as f2:
+    f2.write(exc_text)
+    f2.close()
+  print('Logged current crash to {}'.format(log_file))
+
+
+def bind_user(**kwargs) -> None:
+  sentry_sdk.set_user(kwargs)
+  sentry_sdk.flush()
+
+
+def capture_warning(warning_string):
+  bind_user(id=dongle_id, ip_address=ip, name=gitname)
+  sentry_sdk.capture_message(warning_string, level='warning')
+  sentry_sdk.flush()
+
+
+def capture_info(info_string):
+  bind_user(id=dongle_id, ip_address=ip, name=gitname)
+  sentry_sdk.capture_message(info_string, level='info')
+  sentry_sdk.flush()
+
+
 def set_tag(key: str, value: str) -> None:
   sentry_sdk.set_tag(key, value)
+  sentry_sdk.flush()
 
 
 def init(project: SentryProject) -> None:
   # forks like to mess with this, so double check
-  comma_remote = is_comma_remote() and "commaai" in get_origin(default="")
-  if not comma_remote or not is_registered_device() or PC:
-    return
+  #comma_remote = is_comma_remote() and "commaai" in get_origin(default="")
+  #if not comma_remote or not is_registered_device() or PC:
+  #  return
 
   env = "release" if is_tested_branch() else "master"
   dongle_id = Params().get("DongleId", encoding='utf-8')
+  gitname = Params().get("GithubUsername", encoding='utf-8')
 
   integrations = []
   if project == SentryProject.SELFDRIVE:
@@ -65,6 +131,7 @@ def init(project: SentryProject) -> None:
                   environment=env)
 
   sentry_sdk.set_user({"id": dongle_id})
+  sentry_sdk.set_user({"gitname": gitname})
   sentry_sdk.set_tag("dirty", is_dirty())
   sentry_sdk.set_tag("origin", get_origin())
   sentry_sdk.set_tag("branch", get_branch())