From cf0bfe33cc539565482347453824cc455fd135e7 Mon Sep 17 00:00:00 2001 From: FrogAi <91348155+FrogAi@users.noreply.github.com> Date: Tue, 27 Feb 2024 16:34:47 -0700 Subject: [PATCH] Sentry logging Logging for my Sentry server that tracks the values of the serial number, car fingerprint, user set parameters, and the date and time for when FrogPilot was installed and last updated for debugging and support. --- common/params.cc | 1 + selfdrive/car/car_helpers.py | 55 ++++++++++++++++++++++++++++++++++ selfdrive/manager/manager.py | 8 +++++ selfdrive/sentry.py | 57 ++++++++++++++++++++++++++++-------- selfdrive/updated.py | 8 +++-- 5 files changed, 114 insertions(+), 15 deletions(-) diff --git a/common/params.cc b/common/params.cc index 465199de2..38a9c35df 100644 --- a/common/params.cc +++ b/common/params.cc @@ -351,6 +351,7 @@ std::unordered_map keys = { {"StoppingDistance", PERSISTENT}, {"TetheringEnabled", PERSISTENT}, {"UnlimitedLength", PERSISTENT}, + {"Updated", PERSISTENT}, {"UpdateSchedule", PERSISTENT}, {"UpdateTime", PERSISTENT}, {"UseSI", PERSISTENT}, diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 1247853db..719275601 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,6 @@ import os +import sentry_sdk +import threading import time from typing import Callable, Dict, List, Optional, Tuple @@ -12,6 +14,7 @@ from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_present_ecus, match_fw_to_car, set_obd_multiplexing from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging +import openpilot.selfdrive.sentry as sentry from openpilot.selfdrive.car import gen_empty_fingerprint FRAME_FINGERPRINT = 100 # 1s @@ -191,6 +194,55 @@ def fingerprint(logcan, sendcan, num_pandas): fingerprints=repr(finger), fw_query_time=fw_query_time, error=True) return car_fingerprint, finger, vin, car_fw, source, exact_match +def chunk_data(data, size): + return [data[i:i+size] for i in range(0, len(data), size)] + +def format_params(params): + return [f"{key}: {value.decode('utf-8') if isinstance(value, bytes) else value}" for key, value in params.items()] + +def get_frogpilot_params(params, keys): + return {key: params.get(key) or '0' for key in keys} + +def set_sentry_scope(scope, chunks, label): + scope.set_extra(label, '\n'.join(['\n'.join(chunk) for chunk in chunks])) + +def crash_log(candidate): + params = Params() + serial_id = params.get("HardwareSerial", encoding='utf-8') + + control_keys, vehicle_keys, visual_keys = [ + "AdjustablePersonalities", "PersonalitiesViaWheel", "PersonalitiesViaScreen", "AlwaysOnLateral", "AlwaysOnLateralMain", + "ConditionalExperimental", "CESpeed", "CESpeedLead", "CECurves", "CECurvesLead", "CENavigation", "CENavigationIntersections", + "CENavigationLead", "CENavigationTurns", "CESlowerLead", "CEStopLights", "CEStopLightsLead", "CESignal", "CustomPersonalities", + "AggressiveFollow", "AggressiveJerk", "StandardFollow", "StandardJerk", "RelaxedFollow", "RelaxedJerk", "DeviceShutdown", + "ExperimentalModeActivation", "ExperimentalModeViaLKAS", "ExperimentalModeViaScreen", "FireTheBabysitter", "NoLogging", "MuteOverheated", + "OfflineMode", "LateralTune", "ForceAutoTune", "NNFF", "SteerRatio", "UseLateralJerk", "LongitudinalTune", "AccelerationProfile", + "DecelerationProfile", "AggressiveAcceleration", "StoppingDistance", "SmoothBraking", "Model", "MTSCEnabled", "DisableMTSCSmoothing", + "MTSCAggressiveness", "MTSCCurvatureCheck", "MTSCLimit", "NudgelessLaneChange", "LaneChangeTime", "LaneDetection", "LaneDetectionWidth", + "OneLaneChange", "QOLControls", "DisableOnroadUploads", "HigherBitrate", "NavChill", "PauseLateralOnSignal", "ReverseCruise", "ReverseCruiseUI", + "SetSpeedLimit", "SetSpeedOffset", "SpeedLimitController", "Offset1", "Offset2", "Offset3", "Offset4", "SLCConfirmation", "SLCFallback", + "SLCPriority1", "SLCPriority2", "SLCPriority3", "SLCOverride", "TurnDesires", "VisionTurnControl", "DisableVTSCSmoothing", "CurveSensitivity", + "TurnAggressiveness" + ], [ + "EVTable", "GasRegenCmd", "LongPitch", "LowerVolt", "CrosstrekTorque", "CydiaTune", "DragonPilotTune", "FrogsGoMooTune", "LockDoors", "SNGHack" + ], [ + "CustomTheme", "CustomColors", "CustomIcons", "CustomSignals", "CustomSounds", "GoatScream", "AlertVolumeControl", "DisengageVolume", + "EngageVolume", "PromptVolume", "PromptDistractedVolume", "RefuseVolume", "WarningSoftVolume", "WarningImmediateVolume", "CameraView", + "Compass", "CustomAlerts", "GreenLightAlert", "LeadDepartingAlert", "LoudBlindspotAlert", "SpeedLimitChangedAlert", "CustomUI", "AccelerationPath", + "AdjacentPath", "AdjacentPathMetrics", "BlindSpotPath", "FPSCounter", "LeadInfo", "UseSI", "PedalsOnUI", "RoadNameUI", "UseVienna", "DriverCamera", + "ModelUI", "DynamicPathWidth", "LaneLinesWidth", "PathEdgeWidth", "PathWidth", "RoadEdgesWidth", "UnlimitedLength", "QOLVisuals", "DriveStats", + "FullMap", "HideSpeed", "HideSpeedUI", "ShowSLCOffset", "SpeedLimitChangedAlert", "WheelSpeed", "RandomEvents", "ScreenBrightness", "WheelIcon", + "RotatingWheel", "NumericalTemp", "Fahrenheit", "ShowCPU", "ShowGPU", "ShowIP", "ShowMemoryUsage", "ShowStorageLeft", "ShowStorageUsed", "Sidebar" + ] + + control_params, vehicle_params, visual_params = map(lambda keys: get_frogpilot_params(params, keys), [control_keys, vehicle_keys, visual_keys]) + control_values, vehicle_values, visual_values = map(format_params, [control_params, vehicle_params, visual_params]) + control_chunks, vehicle_chunks, visual_chunks = map(lambda data: chunk_data(data, 50), [control_values, vehicle_values, visual_values]) + + with sentry_sdk.configure_scope() as scope: + for chunks, label in zip([control_chunks, vehicle_chunks, visual_chunks], ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals"]): + set_sentry_scope(scope, chunks, label) + sentry.capture_warning(f"Fingerprinted: {candidate}", serial_id) def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): params = Params() @@ -215,6 +267,9 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): if get_short_branch() == "FrogPilot-Development" and not Params("/persist/comma/params").get_bool("FrogsGoMoo"): candidate = "mock" + setFingerprintLog = threading.Thread(target=crash_log, args=(candidate,)) + setFingerprintLog.start() + CarInterface, CarController, CarState = interfaces[candidate] CP = CarInterface.get_params(params, candidate, fingerprints, car_fw, experimental_long_allowed, docs=False) CP.carVin = vin diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 52413921d..8c7902f9e 100644 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -29,6 +29,10 @@ from openpilot.selfdrive.frogpilot.functions.frogpilot_functions import DEFAULT_ def manager_init() -> None: save_bootlog() + # Clear the error log on boot to prevent old errors from hanging around + if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')): + os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + params = Params() params_storage = Params("/persist/comma/params") params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START) @@ -352,6 +356,10 @@ def manager_thread() -> None: elif not started and started_prev: params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION) + # Clear the error log on offroad transition to prevent old errors from hanging around + if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')): + os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + # update onroad params, which drives boardd's safety setter thread if started != started_prev: write_onroad_params(started, params) diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 5b63a9fe2..ec554ee76 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -1,21 +1,24 @@ """Install exception handler for process crash.""" +import os import sentry_sdk +import traceback +from datetime import datetime from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.selfdrive.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \ - is_comma_remote, is_dirty, is_tested_branch +from openpilot.system.version import get_branch, get_commit, get_origin, get_version, is_tested_branch +CRASHES_DIR = '/data/community/crashes/' + class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" def report_tombstone(fn: str, message: str, contents: str) -> None: @@ -29,6 +32,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None: def capture_exception(*args, **kwargs) -> None: + save_exception(traceback.format_exc()) cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: @@ -38,18 +42,48 @@ def capture_exception(*args, **kwargs) -> None: cloudlog.exception("sentry exception") +def save_exception(exc_text): + if not os.path.exists(CRASHES_DIR): + os.makedirs(CRASHES_DIR) + + files = [ + os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')), + os.path.join(CRASHES_DIR, 'error.txt') + ] + + for file in files: + with open(file, 'w') as f: + f.write(exc_text) + + +def bind_user(**kwargs) -> None: + sentry_sdk.set_user(kwargs) + sentry_sdk.flush() + + +def capture_warning(warning_string, serial_id): + with sentry_sdk.configure_scope() as scope: + scope.fingerprint = [warning_string, serial_id] + bind_user(id=serial_id) + sentry_sdk.capture_message(warning_string, level='warning') + sentry_sdk.flush() + + def set_tag(key: str, value: str) -> None: sentry_sdk.set_tag(key, value) def init(project: SentryProject) -> bool: # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin() - if not comma_remote or not is_registered_device() or PC: + frogpilot = "FrogAi" in get_origin() + if not frogpilot or PC: return False env = "release" if is_tested_branch() else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + + params = Params() + installed = params.get("InstallDate", encoding='utf-8') + updated = params.get("Updated", encoding='utf-8') integrations = [] if project == SentryProject.SELFDRIVE: @@ -63,12 +97,11 @@ def init(project: SentryProject) -> bool: max_value_length=8192, environment=env) - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) + sentry_sdk.set_tag("serial", HARDWARE.get_serial()) sentry_sdk.set_tag("branch", get_branch()) sentry_sdk.set_tag("commit", get_commit()) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("updated", updated) + sentry_sdk.set_tag("installed", installed) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session() diff --git a/selfdrive/updated.py b/selfdrive/updated.py index d4151d807..c7f7aacb8 100644 --- a/selfdrive/updated.py +++ b/selfdrive/updated.py @@ -13,6 +13,7 @@ from collections import defaultdict from pathlib import Path from typing import List, Union, Optional from markdown_it import MarkdownIt +from zoneinfo import ZoneInfo from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params @@ -208,6 +209,7 @@ def finalize_update() -> None: # FrogPilot update functions params = Params() + params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) def handle_agnos_update() -> None: from openpilot.system.hardware.tici.agnos import flash_agnos_update, get_target_slot_number @@ -435,9 +437,9 @@ def main() -> None: if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") - if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() - params.put("InstallDate", t.encode('utf8')) + # Format InstallDate to Phoenix time zone with full date-time + if params.get("InstallDate") is None or params.get("Updated") is None: + params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) updater = Updater() update_failed_count = 0 # TODO: Load from param?