Here a we go

This commit is contained in:
firestar5683
2026-08-17 06:36:55 -05:00
parent b567d9fc90
commit 161ecbc1ab
16 changed files with 311 additions and 29 deletions
+18 -1
View File
@@ -522,6 +522,14 @@ class ManagerProcess(ABC):
if dt > self.watchdog_max_dt and ENABLE_WATCHDOG:
self.capture_watchdog_debug_dump_async(f"watchdog_timeout started={started}", dt)
cloudlog.error(f"Watchdog timeout for {self.name} (exitcode {self.proc.exitcode}) restarting ({started=})")
if isinstance(self, NativeProcess):
sentry.capture_message(
f"Native process watchdog timeout: {self.name}",
level="fatal",
tags={"process": self.name, "process_kind": "native", "failure": "watchdog"},
extras={"pid": self.proc.pid, "watchdog_dt": dt, "started": started},
flush_timeout=0.5,
)
self.restart()
def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals = None) -> int | None:
@@ -715,7 +723,16 @@ def ensure_running(procs: ValuesView[ManagerProcess], started: bool, params=None
for p in procs:
# Reap crashed processes so they can be cleanly restarted below.
if p.proc is not None and p.proc.exitcode is not None and not p.shutting_down:
cloudlog.error(f"Process {p.name} crashed with exitcode {p.proc.exitcode}, restarting")
exitcode = p.proc.exitcode
cloudlog.error(f"Process {p.name} crashed with exitcode {exitcode}, restarting")
if isinstance(p, NativeProcess):
sentry.capture_message(
f"Native process crashed: {p.name}",
level="fatal",
tags={"process": p.name, "process_kind": "native"},
extras={"exitcode": exitcode, "started": started},
flush_timeout=0.5,
)
p.stop(retry=False)
if p.enabled and p.name not in not_run and p.should_run(started, params, CP, starpilot_toggles):
+3 -1
View File
@@ -12,6 +12,7 @@ from openpilot.system.manager.process import PythonProcess, NativeProcess, Daemo
WEBCAM = os.getenv("USE_WEBCAM") is not None
UI_WATCHDOG_MAX_DT = int(os.getenv("UI_WATCHDOG_MAX_DT", "10"))
CAMERAD_WATCHDOG_MAX_DT = int(os.getenv("CAMERAD_WATCHDOG_MAX_DT", "5"))
def driverview(started: bool, params: Params, CP: car.CarParams, starpilot_toggles: SimpleNamespace) -> bool:
return started or params.get_bool("IsDriverViewEnabled")
@@ -123,7 +124,8 @@ procs = [
NativeProcess("stream_encoderd", "system/loggerd", ["./encoderd", "--stream"], or_(and_(livestream, not_(iscar)), notcar)),
PythonProcess("logmessaged", "system.logmessaged", always_run),
NativeProcess("camerad", "system/camerad", ["./camerad"], or_(camera_run, livestream), enabled=not WEBCAM),
NativeProcess("camerad", "system/camerad", ["./camerad"], or_(camera_run, livestream), enabled=not WEBCAM,
watchdog_max_dt=CAMERAD_WATCHDOG_MAX_DT),
PythonProcess("webcamerad", "tools.webcam.camerad", driverview, enabled=WEBCAM),
PythonProcess("proclogd", "system.proclogd", and_(allow_logging, only_onroad), enabled=platform.system() != "Darwin"),
PythonProcess("journald", "system.journald", and_(allow_logging, only_onroad), platform.system() != "Darwin"),