diff --git a/openpilot/starpilot/system/android_auto/tests/test_view_unavailable.py b/openpilot/starpilot/system/android_auto/tests/test_view_unavailable.py index 6c345b55e5..fd646836a6 100644 --- a/openpilot/starpilot/system/android_auto/tests/test_view_unavailable.py +++ b/openpilot/starpilot/system/android_auto/tests/test_view_unavailable.py @@ -36,3 +36,71 @@ def test_failed_car_renderer_withdraws_frames_and_reports_unavailable(): assert source.view == 'unavailable' and source.source.latest() is None assert 'renderer exited' in source.label source.close() + + +def crashing_view(tmp_path, events, script="import sys; print('Traceback: boom', flush=True); sys.exit(1)"): + import sys + return ViewSource('car', request(), lambda event, **fields: events.append((event, fields)), + car_path=str(tmp_path / 'frame'), touch_path=str(tmp_path / 'touch'), + renderer_command=[sys.executable, '-c', script], renderer_log=tmp_path / 'car_ui.log') + + +def test_crashed_renderer_is_restarted_and_its_output_is_logged(tmp_path): + events = [] + source = crashing_view(tmp_path, events) + try: + source.process.wait(timeout=10) + source.check(100.0) + assert source.view == 'car' and not source.fallback_reason + assert source.waiting_for_first_frame + output = [fields for event, fields in events if event == 'car_view_output'] + assert output == [{'reason': 'renderer exited with 1', 'lines': ['Traceback: boom']}] + assert ('car_view_restart', {'reason': 'renderer exited with 1', 'attempt': 1}) in events + assert sum(event == 'car_view_started' for event, _ in events) == 2 + finally: + source.close() + + +def test_renderer_restarts_are_bounded_then_fall_back(tmp_path): + from openpilot.starpilot.system.android_auto import view + events = [] + source = crashing_view(tmp_path, events) + try: + for attempt in range(view.MAX_RESTARTS + 1): + source.process.wait(timeout=10) + source.check(100.0 + attempt) + assert source.view == 'unavailable' and 'renderer exited with 1' in source.label + assert [event for event, _ in events].count('car_view_restart') == view.MAX_RESTARTS + assert [event for event, _ in events].count('car_view_output') == view.MAX_RESTARTS + 1 + finally: + source.close() + + +def test_restart_budget_recovers_after_the_window(tmp_path): + from openpilot.starpilot.system.android_auto import view + events = [] + source = crashing_view(tmp_path, events) + try: + for attempt in range(view.MAX_RESTARTS): + source.process.wait(timeout=10) + source.check(100.0 + attempt) + source.process.wait(timeout=10) + source.check(100.0 + view.RESTART_WINDOW + 1) + assert source.view == 'car' + finally: + source.close() + + +def test_stalled_renderer_output_is_captured_before_restart(tmp_path): + events = [] + source = crashing_view(tmp_path, events, "import time; print('stuck', flush=True); time.sleep(60)") + try: + import time + deadline = time.monotonic() + 10 + while not (tmp_path / 'car_ui.log').read_text() and time.monotonic() < deadline: + time.sleep(0.05) + source.check(source.started_at + 30.0) + assert ('car_view_output', {'reason': 'no frames after 25 s', 'lines': ['stuck']}) in events + assert source.view == 'car' + finally: + source.close() diff --git a/openpilot/starpilot/system/android_auto/view.py b/openpilot/starpilot/system/android_auto/view.py index 19771f08c0..8c3c96af5d 100644 --- a/openpilot/starpilot/system/android_auto/view.py +++ b/openpilot/starpilot/system/android_auto/view.py @@ -1,8 +1,9 @@ """What the car shows: a separate StarPilot view or a test pattern. The car view runs ``current_car_ui`` as a child process and reads its frames from a -dedicated shared-memory slot. An unavailable renderer sends no stale frames; -the session displays an unavailable message until it ends. +dedicated shared-memory slot. An unavailable renderer sends no stale frames; a failed +renderer is restarted a bounded number of times before the session shows an +unavailable message until it ends. """ from __future__ import annotations @@ -19,6 +20,10 @@ from openpilot.starpilot.system.android_auto.touch import DEFAULT_TOUCH_SOCKET, CAR_FRAME_PATH = "/dev/shm/starpilot_android_auto_car_frame" CAR_STARTUP_TIMEOUT = 25.0 # loading the full UI (fonts, textures, layouts) takes a few seconds on device CAR_STALL_TIMEOUT = 5.0 +MAX_RESTARTS = 2 +RESTART_WINDOW = 300.0 +OUTPUT_TAIL_LINES = 40 +OUTPUT_TAIL_BYTES = 8192 class UnavailableFrames: @@ -45,6 +50,7 @@ class ViewSource: self.last_frame_at = 0.0 self.unfocused_since: float | None = None self.frames = 0 + self.restarts: list[float] = [] if synthetic: self.view = "synthetic" self.source = SyntheticFrames() @@ -140,7 +146,41 @@ class ViewSource: elif self.frames and now - self.last_frame_at > CAR_STALL_TIMEOUT: reason = f"frames stopped for {CAR_STALL_TIMEOUT:.0f} s" if reason: - self.fallback(reason) + self._log_output(reason) + self.restarts = [t for t in self.restarts if now - t < RESTART_WINDOW] + if len(self.restarts) < MAX_RESTARTS: + self.restarts.append(now) + self.log("car_view_restart", reason=reason, attempt=len(self.restarts)) + self._stop_car() + self.frames = 0 + self.unfocused_since = None + self._start_car() + else: + self.fallback(reason) + + def _output_tail(self) -> list[str]: + if self.renderer_log is None: + return [] + try: + with open(self.renderer_log, "rb") as f: + f.seek(max(0, f.seek(0, os.SEEK_END) - OUTPUT_TAIL_BYTES)) + text = f.read().decode("utf-8", errors="replace") + except OSError: + return [] + return [line[:300] for line in text.splitlines()[-OUTPUT_TAIL_LINES:]] + + def _log_output(self, reason: str) -> None: + process = self.process + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=3.0) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=2.0) + lines = self._output_tail() + if lines: + self.log("car_view_output", reason=reason, lines=lines) def fallback(self, reason: str) -> None: self.log("car_view_fallback", reason=reason)