From da82092bc9d059bf677906c35511df34393fb559 Mon Sep 17 00:00:00 2001 From: dirwin31 <83434411+dirwin31@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:56:24 -0700 Subject: [PATCH] Car view: drop a frame on a failed async readback instead of crashing A failed GPU wait (GL_WAIT_FAILED as openpilot went onroad) raised out of the renderer, three crashes in a row exhausted its restart budget and the car showed 'display unavailable'. Now the frame is dropped, the GL error is logged with it, and after three consecutive failures the session switches to synchronous readback. --- .../system/android_auto/current_car_ui.py | 28 +++++++++++++++---- .../system/android_auto/headless_egl.py | 13 ++++++++- .../tests/test_current_display_source.py | 1 + .../android_auto/tests/test_gpu_nv12.py | 18 ++++++++++++ 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/openpilot/starpilot/system/android_auto/current_car_ui.py b/openpilot/starpilot/system/android_auto/current_car_ui.py index 3f1267f1bc..bdd67c39e2 100644 --- a/openpilot/starpilot/system/android_auto/current_car_ui.py +++ b/openpilot/starpilot/system/android_auto/current_car_ui.py @@ -23,6 +23,7 @@ from openpilot.starpilot.system.android_auto.projection_geometry import projecti STARTUP_WAIT_SECONDS = 15.0 CAMERA_WAIT_STEP = 0.01 # s; keeps demand/stop checks responsive while waiting for the camera CAMERA_MAX_GAP = 0.1 # s; a shown camera silent this long stops pacing (the encoder rate applies) +ASYNC_READBACK_FAILURES = 3 # consecutive failed asynchronous readbacks before the session reads back synchronously class CameraPacer: @@ -183,7 +184,8 @@ def run(frames_path: str) -> int: asynchronous=bool(request.flags & FLAG_ASYNC_READBACK)) resources.callback(lambda: readback.close()) rgba_regions = [(output.id, request.width, request.height, 0)] - pipeline = f"{'nv12' if converter is not None else 'rgba'}, {'async' if readback.asynchronous else 'sync'} readback" + pixel_format_name = 'nv12' if converter is not None else 'rgba' + pipeline = f"{pixel_format_name}, {'async' if readback.asynchronous else 'sync'} readback" print(f"car view pipeline: {pipeline}", flush=True) from openpilot.starpilot.system.android_auto import identity as identity_store from openpilot.starpilot.system.android_auto.render_profile import RenderSampler, RenderSummary @@ -197,13 +199,27 @@ def run(frames_path: str) -> int: camera_pacer = CameraPacer() in_flight_ns = 0 + async_failures = 0 + def publish_readback() -> None: - """Hand the frame read back last to android_autod.""" - nonlocal readback - readback, pixels = finish_readback(FrameReadback, readback) - if pixels is None: - return + """Hand the frame read back last to android_autod. + + A failed asynchronous wait drops only that frame; repeated failures switch + the session to synchronous readback instead of crashing the renderer.""" + nonlocal async_failures try: + try: + pixels = readback.finish() + except RuntimeError as error: + if not readback.asynchronous: + raise + async_failures += 1 + print(f"car view dropped a frame: {error}", flush=True) + if async_failures >= ASYNC_READBACK_FAILURES: + readback.fall_back_to_sync() + print(f"car view pipeline: {pixel_format_name}, sync readback (async failed {async_failures} times in a row)", flush=True) + return + async_failures = 0 producer.publish(request, pixels, in_flight_ns, pixel_format, advance=False) finally: readback.release() diff --git a/openpilot/starpilot/system/android_auto/headless_egl.py b/openpilot/starpilot/system/android_auto/headless_egl.py index 9439d73a86..26e0d4a7e5 100644 --- a/openpilot/starpilot/system/android_auto/headless_egl.py +++ b/openpilot/starpilot/system/android_auto/headless_egl.py @@ -70,6 +70,7 @@ class FrameReadback: "glDeleteSync": (None, [C.c_void_p]), "glFlush": (None, []), "glFinish": (None, []), + "glGetError": (C.c_uint, []), } for name, (result, args) in signatures.items(): function = getattr(gl, name) @@ -125,7 +126,7 @@ class FrameReadback: gl.glDeleteSync(fence) if result in (GL_TIMEOUT_EXPIRED, GL_WAIT_FAILED): self.pending = False - raise RuntimeError(f"GPU readback did not complete ({result:#x})") + raise RuntimeError(f"GPU readback did not complete ({result:#x}, GL error {gl.glGetError():#x})") gl.glBindBuffer(GL_PIXEL_PACK_BUFFER, self._buffer) address = gl.glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, self.size, GL_MAP_READ_BIT) if not address: @@ -147,6 +148,16 @@ class FrameReadback: self._fence = None self.pending = False + def fall_back_to_sync(self) -> None: + """Read back synchronously from now on, dropping any frame in flight.""" + self.release() + if self._buffer.value: + self.gl.glDeleteBuffers(1, C.byref(self._buffer)) + self._buffer = C.c_uint(0) + self.asynchronous = False + self._storage = (C.c_ubyte * self.size)() + self.pixels = memoryview(self._storage).cast("B") + def close(self) -> None: self.release() if self._buffer.value: diff --git a/openpilot/starpilot/system/android_auto/tests/test_current_display_source.py b/openpilot/starpilot/system/android_auto/tests/test_current_display_source.py index 6132b3e454..ad399a15fa 100644 --- a/openpilot/starpilot/system/android_auto/tests/test_current_display_source.py +++ b/openpilot/starpilot/system/android_auto/tests/test_current_display_source.py @@ -42,6 +42,7 @@ class TestCurrentDisplaySource(unittest.TestCase): self.assertNotIn("openpilot.starpilot.ui.runtime_app", imports) self.assertIn("openpilot.starpilot.system.android_auto.frame_source", imports) self.assertNotIn("openpilot.starpilot.system.android_auto.ui.main", imports) + self.assertIn("pixels = readback.finish()", source) self.assertIn("producer.publish(request, pixels, in_flight_ns, pixel_format, advance=False)", source) def test_view_uses_current_renderer(self): diff --git a/openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py b/openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py index 504d97ea60..6f07ed5be9 100644 --- a/openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py +++ b/openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py @@ -12,6 +12,7 @@ from unittest import SkipTest import numpy as np import pytest +from unittest.mock import Mock from openpilot.starpilot.system.android_auto import gpu_nv12, headless_egl @@ -97,3 +98,20 @@ def test_composed_car_frame_converts_to_the_same_picture(gpu, width, height, mar converter.close() rl.unload_render_texture(output) rl.rl_unload_texture(texture_id) + + +def test_failed_async_readback_reports_gl_error_and_falls_back_to_sync(monkeypatch): + gl = Mock() + gl.glFenceSync.return_value = 1 + gl.glClientWaitSync.return_value = headless_egl.GL_WAIT_FAILED + gl.glGetError.return_value = 0x502 + monkeypatch.setattr(headless_egl.C, "CDLL", lambda _name: gl) + readback = headless_egl.FrameReadback(16, asynchronous=True) + readback.start([(7, 2, 2, 0)]) + with pytest.raises(RuntimeError, match="0x911d, GL error 0x502"): + readback.finish() + readback.release() + readback.fall_back_to_sync() + readback.start([(7, 2, 2, 0)]) + assert not readback.asynchronous and readback.finish() is readback.pixels + readback.close()