stop UI log

system/manager/process.py

@@ -3,6 +3,7 @@ import os
 import signal
 import stat as statmod
 import struct
+import threading
 import time
 import subprocess
 from pathlib import Path
@@ -335,6 +336,83 @@ def _append_ui_watchdog_context(lines: list[str], ui_proc_dir: Path) -> None:
     ])
 
 
+def _capture_watchdog_debug_dump(name: str, pid: int, reason: str, dt: float) -> None:
+  try:
+    proc_dir = Path(f"/proc/{pid}")
+    if not proc_dir.exists():
+      cloudlog.error(f"watchdog debug dump skipped for {name}: /proc/{pid} no longer exists")
+      return
+
+    dump_path = _debug_dump_dir() / f"{name}_watchdog_dump_{pid}_{time.monotonic_ns()}.log"
+    lines = [
+      f"name={name}",
+      f"pid={pid}",
+      f"dt={dt:.3f}",
+      f"reason={reason}",
+      f"wall_time={time.strftime('%Y-%m-%dT%H:%M:%S%z')}",
+      f"watchdog_file={WATCHDOG_FN}{pid}",
+      "",
+      "== /proc/status ==",
+      _read_text_file(proc_dir / "status"),
+      "",
+      "== /proc/wchan ==",
+      _read_text_file(proc_dir / "wchan", 1024),
+      "",
+      "== /proc/syscall ==",
+      _read_text_file(proc_dir / "syscall", 2048),
+      "",
+    ]
+
+    watchdog_file = Path(f"{WATCHDOG_FN}{pid}")
+    lines.extend([
+      "== watchdog file ==",
+      _describe_path(watchdog_file),
+      "",
+    ])
+
+    if name == "ui":
+      _append_ui_watchdog_context(lines, proc_dir)
+
+    task_dir = proc_dir / "task"
+    try:
+      task_entries = sorted(task_dir.iterdir(), key=lambda p: int(p.name))
+    except OSError as e:
+      task_entries = []
+      lines.extend([
+        "== /proc/task ==",
+        f"<read failed: {e}>",
+        "",
+      ])
+
+    for entry in task_entries:
+      lines.extend([
+        f"== thread {entry.name} ==",
+        "-- comm --",
+        _read_text_file(entry / "comm", 1024),
+        "",
+        "-- wchan --",
+        _read_text_file(entry / "wchan", 1024),
+        "",
+        "-- syscall --",
+        _read_text_file(entry / "syscall", 2048),
+        "",
+        "-- status --",
+        _read_text_file(entry / "status"),
+        "",
+        "-- stack --",
+        _read_text_file(entry / "stack"),
+        "",
+      ])
+
+    try:
+      dump_path.write_text("\n".join(lines))
+      cloudlog.error(f"Wrote watchdog debug dump for {name} to {dump_path}")
+    except OSError as e:
+      cloudlog.error(f"failed to write watchdog debug dump for {name} to {dump_path}: {e}")
+  except Exception:
+    cloudlog.exception(f"failed to capture watchdog debug dump for {name} pid={pid}")
+
+
 def launcher(proc: str, name: str, nice: int | None = None) -> None:
   try:
     if nice is not None:
@@ -412,78 +490,20 @@ class ManagerProcess(ABC):
     if self.proc is None or self.proc.pid is None:
       return
 
-    pid = self.proc.pid
-    proc_dir = Path(f"/proc/{pid}")
-    if not proc_dir.exists():
-      cloudlog.error(f"watchdog debug dump skipped for {self.name}: /proc/{pid} no longer exists")
+    _capture_watchdog_debug_dump(self.name, self.proc.pid, reason, dt)
+
+  def capture_watchdog_debug_dump_async(self, reason: str, dt: float) -> None:
+    if self.proc is None or self.proc.pid is None:
       return
 
-    dump_path = _debug_dump_dir() / f"{self.name}_watchdog_dump_{pid}_{time.monotonic_ns()}.log"
-    lines = [
-      f"name={self.name}",
-      f"pid={pid}",
-      f"dt={dt:.3f}",
-      f"reason={reason}",
-      f"wall_time={time.strftime('%Y-%m-%dT%H:%M:%S%z')}",
-      f"watchdog_file={WATCHDOG_FN}{pid}",
-      "",
-      "== /proc/status ==",
-      _read_text_file(proc_dir / "status"),
-      "",
-      "== /proc/wchan ==",
-      _read_text_file(proc_dir / "wchan", 1024),
-      "",
-      "== /proc/syscall ==",
-      _read_text_file(proc_dir / "syscall", 2048),
-      "",
-    ]
-
-    watchdog_file = Path(f"{WATCHDOG_FN}{pid}")
-    lines.extend([
-      "== watchdog file ==",
-      _describe_path(watchdog_file),
-      "",
-    ])
-
-    if self.name == "ui":
-      _append_ui_watchdog_context(lines, proc_dir)
-
-    task_dir = proc_dir / "task"
-    try:
-      task_entries = sorted(task_dir.iterdir(), key=lambda p: int(p.name))
-    except OSError as e:
-      task_entries = []
-      lines.extend([
-        "== /proc/task ==",
-        f"<read failed: {e}>",
-        "",
-      ])
-
-    for entry in task_entries:
-      lines.extend([
-        f"== thread {entry.name} ==",
-        "-- comm --",
-        _read_text_file(entry / "comm", 1024),
-        "",
-        "-- wchan --",
-        _read_text_file(entry / "wchan", 1024),
-        "",
-        "-- syscall --",
-        _read_text_file(entry / "syscall", 2048),
-        "",
-        "-- status --",
-        _read_text_file(entry / "status"),
-        "",
-        "-- stack --",
-        _read_text_file(entry / "stack"),
-        "",
-      ])
-
-    try:
-      dump_path.write_text("\n".join(lines))
-      cloudlog.error(f"Wrote watchdog debug dump for {self.name} to {dump_path}")
-    except OSError as e:
-      cloudlog.error(f"failed to write watchdog debug dump for {self.name} to {dump_path}: {e}")
+    # Capture watchdog evidence in the background so managerState stays responsive.
+    thread = threading.Thread(
+      name=f"{self.name}_watchdog_dump",
+      target=_capture_watchdog_debug_dump,
+      args=(self.name, self.proc.pid, reason, dt),
+      daemon=True,
+    )
+    thread.start()
 
   def check_watchdog(self, started: bool) -> None:
     if self.watchdog_max_dt is None or self.proc is None:
@@ -500,7 +520,7 @@ class ManagerProcess(ABC):
 
     dt = time.monotonic() - self.last_watchdog_time / 1e9
     if dt > self.watchdog_max_dt and ENABLE_WATCHDOG:
-      self.capture_watchdog_debug_dump(f"watchdog_timeout started={started}", dt)
+      self.capture_watchdog_debug_dump_async(f"watchdog_timeout started={started}", dt)
       cloudlog.error(f"Watchdog timeout for {self.name} (exitcode {self.proc.exitcode}) restarting ({started=})")
       self.restart()