From 5b3229576af8741e56c8c8072c5630fe16382a4a Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Wed, 8 Apr 2026 19:41:01 -0500 Subject: [PATCH] more ui logging --- system/manager/process.py | 303 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 303 insertions(+) diff --git a/system/manager/process.py b/system/manager/process.py index c40710111..6d7b89775 100644 --- a/system/manager/process.py +++ b/system/manager/process.py @@ -1,6 +1,7 @@ import importlib import os import signal +import stat as statmod import struct import time import subprocess @@ -22,6 +23,15 @@ from openpilot.common.watchdog import WATCHDOG_FN ENABLE_WATCHDOG = os.getenv("NO_WATCHDOG") is None +DEBUG_ENV_KEYS = ( + "XDG_RUNTIME_DIR", + "WAYLAND_DISPLAY", + "WAYLAND_SOCKET", + "QT_QPA_PLATFORM", + "QT_WAYLAND_SHELL_INTEGRATION", + "QT_WAYLAND_DISABLE_WINDOWDECORATION", +) + def _debug_dump_dir() -> Path: for candidate in (Path("/data/log"), Path("/tmp")): @@ -30,6 +40,12 @@ def _debug_dump_dir() -> Path: return Path.cwd() +def _truncate_text(text: str, max_chars: int = 32768) -> str: + if len(text) <= max_chars: + return text + return text[:max_chars] + "\n\n" + + def _read_text_file(path: Path, max_bytes: int = 16384) -> str: try: data = path.read_bytes() @@ -42,6 +58,283 @@ def _read_text_file(path: Path, max_bytes: int = 16384) -> str: return data.decode("utf-8", errors="replace") +def _read_proc_cmdline(proc_dir: Path, max_bytes: int = 4096) -> str: + try: + data = (proc_dir / "cmdline").read_bytes() + except OSError as e: + return f"" + + cmdline = data.replace(b"\0", b" ").decode("utf-8", errors="replace").strip() + return _truncate_text(cmdline or "", max_bytes) + + +def _read_proc_environ(proc_dir: Path) -> dict[str, str]: + try: + data = (proc_dir / "environ").read_bytes() + except OSError: + return {} + + env = {} + for item in data.split(b"\0"): + if not item or b"=" not in item: + continue + key, value = item.split(b"=", 1) + env[key.decode("utf-8", errors="replace")] = value.decode("utf-8", errors="replace") + return env + + +def _format_env_subset(env: dict[str, str]) -> str: + if not env: + return "" + + lines = [f"{key}={env[key]}" for key in DEBUG_ENV_KEYS if key in env] + return "\n".join(lines) if lines else "" + + +def _coerce_subprocess_output(data) -> str: + if data is None: + return "" + if isinstance(data, bytes): + return data.decode("utf-8", errors="replace") + return data + + +def _run_debug_command(cmd: str, timeout: float = 1.5, max_chars: int = 32768) -> str: + try: + result = subprocess.run(["bash", "-lc", cmd], capture_output=True, text=True, timeout=timeout) + parts = [f"$ {cmd}", f""] + if result.stdout: + parts.append(result.stdout.rstrip()) + if result.stderr: + parts.extend(["[stderr]", result.stderr.rstrip()]) + if len(parts) == 2: + parts.append("") + return _truncate_text("\n".join(parts), max_chars) + except subprocess.TimeoutExpired as e: + stdout = _coerce_subprocess_output(e.stdout).rstrip() + stderr = _coerce_subprocess_output(e.stderr).rstrip() + parts = [f"$ {cmd}", f""] + if stdout: + parts.append(stdout) + if stderr: + parts.extend(["[stderr]", stderr]) + if len(parts) == 2: + parts.append("") + return _truncate_text("\n".join(parts), max_chars) + + +def _read_proc_start_wall_time(proc_dir: Path) -> str: + try: + stat_text = (proc_dir / "stat").read_text() + rparen = stat_text.rfind(")") + fields = stat_text[rparen + 2:].split() + start_ticks = int(fields[19]) + uptime_s = float(Path("/proc/uptime").read_text().split()[0]) + hz = os.sysconf("SC_CLK_TCK") + start_time_s = time.time() - uptime_s + (start_ticks / hz) + return time.strftime("%Y-%m-%dT%H:%M:%S%z", time.localtime(start_time_s)) + except Exception as e: + return f"" + + +def _find_process_dirs(match_terms: tuple[str, ...]) -> list[Path]: + proc_root = Path("/proc") + try: + proc_dirs = sorted((p for p in proc_root.iterdir() if p.name.isdigit()), key=lambda p: int(p.name)) + except OSError: + return [] + + matches = [] + for proc_dir in proc_dirs: + comm = _read_text_file(proc_dir / "comm", 256).strip() + cmdline = _read_proc_cmdline(proc_dir, 2048) + haystacks = (comm, cmdline) + if any(term in haystack for term in match_terms for haystack in haystacks): + matches.append(proc_dir) + return matches + + +def _read_symlink(path: Path) -> str: + try: + return os.readlink(path) + except OSError as e: + return f"" + + +def _describe_path(path: Path) -> str: + try: + st = path.stat() + except OSError as e: + return f"path={path}\n" + + mode = statmod.filemode(st.st_mode) + if statmod.S_ISSOCK(st.st_mode): + path_type = "socket" + elif statmod.S_ISDIR(st.st_mode): + path_type = "dir" + elif statmod.S_ISREG(st.st_mode): + path_type = "file" + else: + path_type = "other" + + return "\n".join([ + f"path={path}", + f"type={path_type}", + f"mode={mode}", + f"inode={st.st_ino}", + f"size={st.st_size}", + f"mtime={time.strftime('%Y-%m-%dT%H:%M:%S%z', time.localtime(st.st_mtime))}", + f"mtime_ns={st.st_mtime_ns}", + f"ctime_ns={st.st_ctime_ns}", + ]) + + +def _proc_net_unix_matches(path: Path) -> str: + path_str = str(path) + try: + rows = [line for line in Path("/proc/net/unix").read_text().splitlines() if path_str in line] + except OSError as e: + return f"" + + return "\n".join(rows) if rows else "" + + +def _resolve_wayland_socket(runtime_dir: Path | None, display: str | None) -> Path | None: + if not display: + return runtime_dir / "wayland-0" if runtime_dir is not None else None + + socket_path = Path(display) + if socket_path.is_absolute(): + return socket_path + if runtime_dir is None: + return None + return runtime_dir / socket_path + + +def _append_ui_watchdog_context(lines: list[str], ui_proc_dir: Path) -> None: + ui_env = _read_proc_environ(ui_proc_dir) + weston_procs = _find_process_dirs(("weston",)) + weston_envs = [_read_proc_environ(proc_dir) for proc_dir in weston_procs] + + lines.extend([ + "== ui proc cmdline ==", + _read_proc_cmdline(ui_proc_dir), + "", + "== ui proc exe ==", + _read_symlink(ui_proc_dir / "exe"), + "", + "== ui proc env ==", + _format_env_subset(ui_env), + "", + ]) + + for proc_dir, env in zip(weston_procs, weston_envs): + lines.extend([ + f"== weston process {proc_dir.name} ==", + "-- cmdline --", + _read_proc_cmdline(proc_dir), + "", + "-- exe --", + _read_symlink(proc_dir / "exe"), + "", + "-- start wall time --", + _read_proc_start_wall_time(proc_dir), + "", + "-- status --", + _read_text_file(proc_dir / "status"), + "", + "-- wchan --", + _read_text_file(proc_dir / "wchan", 1024), + "", + "-- syscall --", + _read_text_file(proc_dir / "syscall", 2048), + "", + "-- env --", + _format_env_subset(env), + "", + ]) + + runtime_dirs: set[Path] = set() + socket_paths: set[Path] = set() + + for env in (ui_env, *weston_envs): + runtime_dir = Path(env["XDG_RUNTIME_DIR"]) if env.get("XDG_RUNTIME_DIR") else None + if runtime_dir is not None: + runtime_dirs.add(runtime_dir) + socket_path = _resolve_wayland_socket(runtime_dir, env.get("WAYLAND_DISPLAY")) + if socket_path is not None: + socket_paths.add(socket_path) + + if not runtime_dirs: + for candidate in (Path("/run/user/1000"), Path("/tmp")): + if candidate.exists(): + runtime_dirs.add(candidate) + + for runtime_dir in sorted(runtime_dirs): + lines.extend([ + f"== runtime dir {runtime_dir} ==", + _describe_path(runtime_dir), + "", + ]) + + candidates = {path for path in socket_paths if path.parent == runtime_dir} + if runtime_dir.exists(): + candidates.update(runtime_dir.glob("wayland-*")) + candidates.update(runtime_dir.glob("weston*")) + + for candidate in sorted(candidates): + lines.extend([ + f"-- wayland path {candidate.name} --", + _describe_path(candidate), + "-- /proc/net/unix --", + _proc_net_unix_matches(candidate), + "", + ]) + + for socket_path in sorted(socket_paths): + if socket_path.parent not in runtime_dirs: + lines.extend([ + f"== wayland path {socket_path} ==", + _describe_path(socket_path), + "-- /proc/net/unix --", + _proc_net_unix_matches(socket_path), + "", + ]) + + commands = [ + ( + "== systemctl weston ==", + "systemctl show weston.service weston-ready.service " + "--property=Id,ActiveState,SubState,MainPID,ExecMainStartTimestamp,ExecMainStartTimestampMonotonic,ExecMainStatus " + "--no-pager 2>/dev/null || true", + 1.5, + ), + ( + "== journalctl weston ==", + "journalctl -u weston.service -u weston-ready.service -n 120 --no-pager 2>&1 || true", + 1.5, + ), + ( + "== recent swaglog weston/ui ==", + "grep -RinE 'Watchdog timeout for ui|weston|Wayland|EGL|gbm|drm|GPU|segfault|SIGSEGV|OOM|fatal' " + "/data/log/swaglog* 2>/dev/null | tail -n 160 || true", + 1.5, + ), + ( + "== dmesg weston/gpu ==", + "dmesg 2>/dev/null | grep -Ei 'drm|gpu|msm|kgsl|weston|wayland|segfault|oom|hang' | tail -n 160 || true", + 1.5, + ), + ] + + for title, cmd, timeout in commands: + lines.extend([ + title, + _run_debug_command(cmd, timeout=timeout), + "", + ]) + + def launcher(proc: str, name: str, nice: int | None = None) -> None: try: if nice is not None: @@ -145,6 +438,16 @@ class ManagerProcess(ABC): "", ] + watchdog_file = Path(f"{WATCHDOG_FN}{pid}") + lines.extend([ + "== watchdog file ==", + _describe_path(watchdog_file), + "", + ]) + + if self.name == "ui": + _append_ui_watchdog_context(lines, proc_dir) + task_dir = proc_dir / "task" try: task_entries = sorted(task_dir.iterdir(), key=lambda p: int(p.name))