log

selfdrive/ui/stall_monitor.py

@@ -9,6 +9,11 @@ from pathlib import Path
 from openpilot.common.swaglog import cloudlog
 
 
+def _capture_message(*args, **kwargs) -> None:
+  from openpilot.system import sentry
+  sentry.capture_message(*args, **kwargs)
+
+
 def _default_dump_dir() -> Path:
   for candidate in ("/data/log", "/tmp"):
     if os.path.isdir(candidate) and os.access(candidate, os.W_OK):
@@ -87,9 +92,29 @@ class UIStallMonitor:
         self._stalled_since = now
         self._stalled_phase = phase
 
-      preview = self._main_thread_preview()
-      path_s = str(dump_path) if dump_path is not None else "<write_failed>"
-      cloudlog.error(f"{self._name} main loop stalled for {stalled_for_s:.1f}s in phase={phase} (phase_for={phase_for_s:.1f}s) dump={path_s}\n{preview}")
+      self._report_stall(dump, dump_path, phase, stalled_for_s, phase_for_s)
+
+  def _report_stall(self, dump: str, dump_path: Path | None, phase: str, stalled_for_s: float, phase_for_s: float) -> None:
+    preview = self._main_thread_preview()
+    path_s = str(dump_path) if dump_path is not None else "<write_failed>"
+    cloudlog.error(f"{self._name} main loop stalled for {stalled_for_s:.1f}s in phase={phase} (phase_for={phase_for_s:.1f}s) dump={path_s}\n{preview}")
+    _capture_message(
+      "raylib UI main loop stalled",
+      tags={
+        "ui_stall_name": self._name,
+        "ui_stall_phase": phase,
+      },
+      extras={
+        "pid": os.getpid(),
+        "stalled_for_s": round(stalled_for_s, 3),
+        "phase_for_s": round(phase_for_s, 3),
+        "dump_path": path_s,
+        "main_thread_stack": preview,
+        "thread_dump": dump,
+      },
+      attachment_path=dump_path,
+      flush_timeout=2.0,
+    )
 
   def _build_dump(self, now: float, phase: str, stalled_for_s: float, phase_for_s: float) -> str:
     frames = sys._current_frames()

selfdrive/ui/tests/test_stall_monitor.py

@@ -0,0 +1,25 @@
+from openpilot.selfdrive.ui import stall_monitor
+
+
+def test_stall_report_is_sent_to_bugsink(monkeypatch, tmp_path):
+  report = {}
+
+  def capture_message(message, **kwargs):
+    report["message"] = message
+    report.update(kwargs)
+
+  monkeypatch.setattr(stall_monitor, "_capture_message", capture_message)
+
+  monitor = stall_monitor.UIStallMonitor("raylib_ui")
+  monitor._dump_dir = tmp_path
+  dump = "full thread dump"
+  dump_path = monitor._write_dump(dump)
+  monitor._report_stall(dump, dump_path, "gui_app.present_end", 5.001, 4.999)
+
+  assert report["message"] == "raylib UI main loop stalled"
+  assert report["tags"] == {"ui_stall_name": "raylib_ui", "ui_stall_phase": "gui_app.present_end"}
+  assert report["extras"]["stalled_for_s"] == 5.001
+  assert report["extras"]["phase_for_s"] == 4.999
+  assert "test_stall_report_is_sent_to_bugsink" in report["extras"]["main_thread_stack"]
+  assert report["extras"]["thread_dump"] == dump
+  assert report["attachment_path"] == dump_path

system/sentry.py

@@ -5,9 +5,10 @@ import sentry_sdk
 import traceback
 from datetime import datetime
 from enum import Enum
+from pathlib import Path
+from typing import Any
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
-from openpilot.common.params import Params
 from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.hardware.hw import Paths
@@ -19,7 +20,7 @@ class SentryProject(Enum):
   # python project
   SELFDRIVE = "https://7305139359a548fcb348ec09497dc389@bugsink.firestar.link/1"
   # native project
-  SELFDRIVE_NATIVE = "https://7305139359a548fcb348ec09497dc389@bugsink.firestar.link/1"
+  SELFDRIVE_NATIVE = "https://7305139359a548fcb348ec09497dc389@bugsink.firestar.link/1"  # noqa: PIE796
 
 
 def report_tombstone(fn: str, message: str, contents: str) -> None:
@@ -39,11 +40,32 @@ def report_tombstone(fn: str, message: str, contents: str) -> None:
 
 
 def capture_block():
-  with sentry_sdk.push_scope() as scope:
+  with sentry_sdk.push_scope():
     sentry_sdk.capture_message("Blocked user from using the development branch", level='info')
     sentry_sdk.flush()
 
 
+def capture_message(message: str, *, level: str = "error", tags: dict[str, str] | None = None,
+                    extras: dict[str, Any] | None = None, attachment_path: Path | None = None,
+                    flush_timeout: float | None = None) -> None:
+  try:
+    with sentry_sdk.push_scope() as scope:
+      for key, value in (tags or {}).items():
+        scope.set_tag(key, value)
+      for key, value in (extras or {}).items():
+        scope.set_extra(key, value)
+      if attachment_path is not None and attachment_path.is_file():
+        scope.add_attachment(path=str(attachment_path), filename=attachment_path.name)
+
+      sentry_sdk.capture_message(message, level=level)
+      if flush_timeout is None:
+        sentry_sdk.flush()
+      else:
+        sentry_sdk.flush(timeout=flush_timeout)
+  except Exception:
+    cloudlog.exception("sentry message")
+
+
 def capture_exception(*args, crash_log=True, **kwargs) -> None:
   exc_text = traceback.format_exc()
 
@@ -136,6 +158,7 @@ def init(project: SentryProject) -> bool:
   sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit)
   sentry_sdk.set_tag("updated", updated)
   sentry_sdk.set_tag("installed", installed)
+  sentry_sdk.set_tag("device", HARDWARE.get_device_type())
 
   if project == SentryProject.SELFDRIVE:
     sentry_sdk.Hub.current.start_session()