Add bounded restart logic and output logging for crashed or stalled car UI renderers

openpilot/starpilot/system/android_auto/tests/test_view_unavailable.py

@@ -36,3 +36,71 @@ def test_failed_car_renderer_withdraws_frames_and_reports_unavailable():
   assert source.view == 'unavailable' and source.source.latest() is None
   assert 'renderer exited' in source.label
   source.close()
+
+
+def crashing_view(tmp_path, events, script="import sys; print('Traceback: boom', flush=True); sys.exit(1)"):
+  import sys
+  return ViewSource('car', request(), lambda event, **fields: events.append((event, fields)),
+                    car_path=str(tmp_path / 'frame'), touch_path=str(tmp_path / 'touch'),
+                    renderer_command=[sys.executable, '-c', script], renderer_log=tmp_path / 'car_ui.log')
+
+
+def test_crashed_renderer_is_restarted_and_its_output_is_logged(tmp_path):
+  events = []
+  source = crashing_view(tmp_path, events)
+  try:
+    source.process.wait(timeout=10)
+    source.check(100.0)
+    assert source.view == 'car' and not source.fallback_reason
+    assert source.waiting_for_first_frame
+    output = [fields for event, fields in events if event == 'car_view_output']
+    assert output == [{'reason': 'renderer exited with 1', 'lines': ['Traceback: boom']}]
+    assert ('car_view_restart', {'reason': 'renderer exited with 1', 'attempt': 1}) in events
+    assert sum(event == 'car_view_started' for event, _ in events) == 2
+  finally:
+    source.close()
+
+
+def test_renderer_restarts_are_bounded_then_fall_back(tmp_path):
+  from openpilot.starpilot.system.android_auto import view
+  events = []
+  source = crashing_view(tmp_path, events)
+  try:
+    for attempt in range(view.MAX_RESTARTS + 1):
+      source.process.wait(timeout=10)
+      source.check(100.0 + attempt)
+    assert source.view == 'unavailable' and 'renderer exited with 1' in source.label
+    assert [event for event, _ in events].count('car_view_restart') == view.MAX_RESTARTS
+    assert [event for event, _ in events].count('car_view_output') == view.MAX_RESTARTS + 1
+  finally:
+    source.close()
+
+
+def test_restart_budget_recovers_after_the_window(tmp_path):
+  from openpilot.starpilot.system.android_auto import view
+  events = []
+  source = crashing_view(tmp_path, events)
+  try:
+    for attempt in range(view.MAX_RESTARTS):
+      source.process.wait(timeout=10)
+      source.check(100.0 + attempt)
+    source.process.wait(timeout=10)
+    source.check(100.0 + view.RESTART_WINDOW + 1)
+    assert source.view == 'car'
+  finally:
+    source.close()
+
+
+def test_stalled_renderer_output_is_captured_before_restart(tmp_path):
+  events = []
+  source = crashing_view(tmp_path, events, "import time; print('stuck', flush=True); time.sleep(60)")
+  try:
+    import time
+    deadline = time.monotonic() + 10
+    while not (tmp_path / 'car_ui.log').read_text() and time.monotonic() < deadline:
+      time.sleep(0.05)
+    source.check(source.started_at + 30.0)
+    assert ('car_view_output', {'reason': 'no frames after 25 s', 'lines': ['stuck']}) in events
+    assert source.view == 'car'
+  finally:
+    source.close()

openpilot/starpilot/system/android_auto/view.py

@@ -1,8 +1,9 @@
 """What the car shows: a separate StarPilot view or a test pattern.
 
 The car view runs ``current_car_ui`` as a child process and reads its frames from a
-dedicated shared-memory slot. An unavailable renderer sends no stale frames;
-the session displays an unavailable message until it ends.
+dedicated shared-memory slot. An unavailable renderer sends no stale frames; a failed
+renderer is restarted a bounded number of times before the session shows an
+unavailable message until it ends.
 """
 
 from __future__ import annotations
@@ -19,6 +20,10 @@ from openpilot.starpilot.system.android_auto.touch import DEFAULT_TOUCH_SOCKET,
 CAR_FRAME_PATH = "/dev/shm/starpilot_android_auto_car_frame"
 CAR_STARTUP_TIMEOUT = 25.0   # loading the full UI (fonts, textures, layouts) takes a few seconds on device
 CAR_STALL_TIMEOUT = 5.0
+MAX_RESTARTS = 2
+RESTART_WINDOW = 300.0
+OUTPUT_TAIL_LINES = 40
+OUTPUT_TAIL_BYTES = 8192
 
 
 class UnavailableFrames:
@@ -45,6 +50,7 @@ class ViewSource:
     self.last_frame_at = 0.0
     self.unfocused_since: float | None = None
     self.frames = 0
+    self.restarts: list[float] = []
     if synthetic:
       self.view = "synthetic"
       self.source = SyntheticFrames()
@@ -140,7 +146,41 @@ class ViewSource:
     elif self.frames and now - self.last_frame_at > CAR_STALL_TIMEOUT:
       reason = f"frames stopped for {CAR_STALL_TIMEOUT:.0f} s"
     if reason:
-      self.fallback(reason)
+      self._log_output(reason)
+      self.restarts = [t for t in self.restarts if now - t < RESTART_WINDOW]
+      if len(self.restarts) < MAX_RESTARTS:
+        self.restarts.append(now)
+        self.log("car_view_restart", reason=reason, attempt=len(self.restarts))
+        self._stop_car()
+        self.frames = 0
+        self.unfocused_since = None
+        self._start_car()
+      else:
+        self.fallback(reason)
+
+  def _output_tail(self) -> list[str]:
+    if self.renderer_log is None:
+      return []
+    try:
+      with open(self.renderer_log, "rb") as f:
+        f.seek(max(0, f.seek(0, os.SEEK_END) - OUTPUT_TAIL_BYTES))
+        text = f.read().decode("utf-8", errors="replace")
+    except OSError:
+      return []
+    return [line[:300] for line in text.splitlines()[-OUTPUT_TAIL_LINES:]]
+
+  def _log_output(self, reason: str) -> None:
+    process = self.process
+    if process is not None and process.poll() is None:
+      process.terminate()
+      try:
+        process.wait(timeout=3.0)
+      except subprocess.TimeoutExpired:
+        process.kill()
+        process.wait(timeout=2.0)
+    lines = self._output_tail()
+    if lines:
+      self.log("car_view_output", reason=reason, lines=lines)
 
   def fallback(self, reason: str) -> None:
     self.log("car_view_fallback", reason=reason)