StarPilot

scripts/debug_ui_freeze.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Collects high-signal diagnostics for UI freeze incidents on-device.
+# Usage:
+#   ./scripts/debug_ui_freeze.sh
+# Optional env:
+#   OUT_DIR=/data/media/0/realdata/ui_freeze_debug
+
+OUT_DIR="${OUT_DIR:-/data/media/0/realdata/ui_freeze_debug}"
+TS="$(date +%Y%m%d_%H%M%S)"
+OUT_FILE="${OUT_DIR}/ui_freeze_${TS}.log"
+
+mkdir -p "${OUT_DIR}"
+
+{
+  echo "=== UI Freeze Debug ==="
+  echo "timestamp: $(date -Ins)"
+  echo "hostname: $(hostname)"
+  echo "kernel: $(uname -a)"
+  echo
+
+  echo "=== Process Snapshot ==="
+  ps -eo pid,ppid,stat,ni,etime,comm,args | grep -E "(manager.py|selfdrive.ui.ui|/selfdrive/ui/ui|weston|loggerd|encoderd|modeld|dmonitoringmodeld)" || true
+  echo
+
+  echo "=== Manager State (tmux service output) ==="
+  if command -v tmux >/dev/null 2>&1; then
+    tmux capture-pane -pt comma:0 -S -220 2>/dev/null || true
+  fi
+  echo
+
+  echo "=== UI Watchdog Files (/dev/shm/wd_*) ==="
+  ls -l /dev/shm/wd_* 2>/dev/null || true
+  echo
+
+  echo "=== UI PID + Watchdog Timestamp Delta ==="
+  for ui_pid in $(pgrep -f "selfdrive.ui.ui|/selfdrive/ui/ui" 2>/dev/null || true); do
+    echo "ui_pid=${ui_pid}"
+    wd_file="/dev/shm/wd_${ui_pid}"
+    if [[ -f "${wd_file}" ]]; then
+      now_ns="$(python3 - <<'PY'
+import time
+print(int(time.monotonic_ns()))
+PY
+)"
+      last_ns="$(python3 - "${wd_file}" <<'PY'
+import struct, sys
+with open(sys.argv[1], "rb") as f:
+  print(struct.unpack("Q", f.read())[0])
+PY
+)"
+      python3 - "${now_ns}" "${last_ns}" <<'PY'
+import sys
+now_ns = int(sys.argv[1]); last_ns = int(sys.argv[2])
+print(f"watchdog_age_s={(now_ns-last_ns)/1e9:.3f}")
+PY
+    else
+      echo "watchdog_file_missing=${wd_file}"
+    fi
+    echo
+  done
+
+  echo "=== /data/log (recent ui/watchdog/weston/gpu errors) ==="
+  grep -RinE "Watchdog timeout for ui|Process ui crashed|ui.*exit|weston|Wayland|EGL|gbm|drm|GPU|segfault|SIGSEGV|killed process|OOM|fatal" /data/log/swaglog* 2>/dev/null | tail -n 400 || true
+  echo
+
+  echo "=== dmesg (recent high-signal lines) ==="
+  dmesg 2>/dev/null | grep -Ei "drm|gpu|msm|kgsl|weston|wayland|segfault|oom|killed process|hang" | tail -n 300 || true
+  echo
+
+  echo "=== systemd journal (weston / ui if available) ==="
+  journalctl -n 300 --no-pager 2>/dev/null | grep -Ei "weston|wayland|ui|kgsl|drm|gpu|segfault|oom" || true
+  echo
+
+  echo "=== open files for UI process(es) ==="
+  for ui_pid in $(pgrep -f "selfdrive.ui.ui|/selfdrive/ui/ui" 2>/dev/null || true); do
+    echo "--- lsof for pid ${ui_pid} ---"
+    lsof -p "${ui_pid}" 2>/dev/null | tail -n 200 || true
+  done
+  echo
+
+  echo "=== End ==="
+} > "${OUT_FILE}" 2>&1
+
+echo "Wrote: ${OUT_FILE}"