diff --git a/system/manager/process.py b/system/manager/process.py index 6d7b89775..6d808c224 100644 --- a/system/manager/process.py +++ b/system/manager/process.py @@ -3,6 +3,7 @@ import os import signal import stat as statmod import struct +import threading import time import subprocess from pathlib import Path @@ -335,6 +336,83 @@ def _append_ui_watchdog_context(lines: list[str], ui_proc_dir: Path) -> None: ]) +def _capture_watchdog_debug_dump(name: str, pid: int, reason: str, dt: float) -> None: + try: + proc_dir = Path(f"/proc/{pid}") + if not proc_dir.exists(): + cloudlog.error(f"watchdog debug dump skipped for {name}: /proc/{pid} no longer exists") + return + + dump_path = _debug_dump_dir() / f"{name}_watchdog_dump_{pid}_{time.monotonic_ns()}.log" + lines = [ + f"name={name}", + f"pid={pid}", + f"dt={dt:.3f}", + f"reason={reason}", + f"wall_time={time.strftime('%Y-%m-%dT%H:%M:%S%z')}", + f"watchdog_file={WATCHDOG_FN}{pid}", + "", + "== /proc/status ==", + _read_text_file(proc_dir / "status"), + "", + "== /proc/wchan ==", + _read_text_file(proc_dir / "wchan", 1024), + "", + "== /proc/syscall ==", + _read_text_file(proc_dir / "syscall", 2048), + "", + ] + + watchdog_file = Path(f"{WATCHDOG_FN}{pid}") + lines.extend([ + "== watchdog file ==", + _describe_path(watchdog_file), + "", + ]) + + if name == "ui": + _append_ui_watchdog_context(lines, proc_dir) + + task_dir = proc_dir / "task" + try: + task_entries = sorted(task_dir.iterdir(), key=lambda p: int(p.name)) + except OSError as e: + task_entries = [] + lines.extend([ + "== /proc/task ==", + f"", + "", + ]) + + for entry in task_entries: + lines.extend([ + f"== thread {entry.name} ==", + "-- comm --", + _read_text_file(entry / "comm", 1024), + "", + "-- wchan --", + _read_text_file(entry / "wchan", 1024), + "", + "-- syscall --", + _read_text_file(entry / "syscall", 2048), + "", + "-- status --", + _read_text_file(entry / "status"), + "", + "-- stack --", + _read_text_file(entry / "stack"), + "", + ]) + + try: + dump_path.write_text("\n".join(lines)) + cloudlog.error(f"Wrote watchdog debug dump for {name} to {dump_path}") + except OSError as e: + cloudlog.error(f"failed to write watchdog debug dump for {name} to {dump_path}: {e}") + except Exception: + cloudlog.exception(f"failed to capture watchdog debug dump for {name} pid={pid}") + + def launcher(proc: str, name: str, nice: int | None = None) -> None: try: if nice is not None: @@ -412,78 +490,20 @@ class ManagerProcess(ABC): if self.proc is None or self.proc.pid is None: return - pid = self.proc.pid - proc_dir = Path(f"/proc/{pid}") - if not proc_dir.exists(): - cloudlog.error(f"watchdog debug dump skipped for {self.name}: /proc/{pid} no longer exists") + _capture_watchdog_debug_dump(self.name, self.proc.pid, reason, dt) + + def capture_watchdog_debug_dump_async(self, reason: str, dt: float) -> None: + if self.proc is None or self.proc.pid is None: return - dump_path = _debug_dump_dir() / f"{self.name}_watchdog_dump_{pid}_{time.monotonic_ns()}.log" - lines = [ - f"name={self.name}", - f"pid={pid}", - f"dt={dt:.3f}", - f"reason={reason}", - f"wall_time={time.strftime('%Y-%m-%dT%H:%M:%S%z')}", - f"watchdog_file={WATCHDOG_FN}{pid}", - "", - "== /proc/status ==", - _read_text_file(proc_dir / "status"), - "", - "== /proc/wchan ==", - _read_text_file(proc_dir / "wchan", 1024), - "", - "== /proc/syscall ==", - _read_text_file(proc_dir / "syscall", 2048), - "", - ] - - watchdog_file = Path(f"{WATCHDOG_FN}{pid}") - lines.extend([ - "== watchdog file ==", - _describe_path(watchdog_file), - "", - ]) - - if self.name == "ui": - _append_ui_watchdog_context(lines, proc_dir) - - task_dir = proc_dir / "task" - try: - task_entries = sorted(task_dir.iterdir(), key=lambda p: int(p.name)) - except OSError as e: - task_entries = [] - lines.extend([ - "== /proc/task ==", - f"", - "", - ]) - - for entry in task_entries: - lines.extend([ - f"== thread {entry.name} ==", - "-- comm --", - _read_text_file(entry / "comm", 1024), - "", - "-- wchan --", - _read_text_file(entry / "wchan", 1024), - "", - "-- syscall --", - _read_text_file(entry / "syscall", 2048), - "", - "-- status --", - _read_text_file(entry / "status"), - "", - "-- stack --", - _read_text_file(entry / "stack"), - "", - ]) - - try: - dump_path.write_text("\n".join(lines)) - cloudlog.error(f"Wrote watchdog debug dump for {self.name} to {dump_path}") - except OSError as e: - cloudlog.error(f"failed to write watchdog debug dump for {self.name} to {dump_path}: {e}") + # Capture watchdog evidence in the background so managerState stays responsive. + thread = threading.Thread( + name=f"{self.name}_watchdog_dump", + target=_capture_watchdog_debug_dump, + args=(self.name, self.proc.pid, reason, dt), + daemon=True, + ) + thread.start() def check_watchdog(self, started: bool) -> None: if self.watchdog_max_dt is None or self.proc is None: @@ -500,7 +520,7 @@ class ManagerProcess(ABC): dt = time.monotonic() - self.last_watchdog_time / 1e9 if dt > self.watchdog_max_dt and ENABLE_WATCHDOG: - self.capture_watchdog_debug_dump(f"watchdog_timeout started={started}", dt) + self.capture_watchdog_debug_dump_async(f"watchdog_timeout started={started}", dt) cloudlog.error(f"Watchdog timeout for {self.name} (exitcode {self.proc.exitcode}) restarting ({started=})") self.restart()