diff --git a/openpilot/starpilot/system/android_auto/current_car_ui.py b/openpilot/starpilot/system/android_auto/current_car_ui.py index 3f1267f1bc..bdd67c39e2 100644 --- a/openpilot/starpilot/system/android_auto/current_car_ui.py +++ b/openpilot/starpilot/system/android_auto/current_car_ui.py @@ -23,6 +23,7 @@ from openpilot.starpilot.system.android_auto.projection_geometry import projecti STARTUP_WAIT_SECONDS = 15.0 CAMERA_WAIT_STEP = 0.01 # s; keeps demand/stop checks responsive while waiting for the camera CAMERA_MAX_GAP = 0.1 # s; a shown camera silent this long stops pacing (the encoder rate applies) +ASYNC_READBACK_FAILURES = 3 # consecutive failed asynchronous readbacks before the session reads back synchronously class CameraPacer: @@ -183,7 +184,8 @@ def run(frames_path: str) -> int: asynchronous=bool(request.flags & FLAG_ASYNC_READBACK)) resources.callback(lambda: readback.close()) rgba_regions = [(output.id, request.width, request.height, 0)] - pipeline = f"{'nv12' if converter is not None else 'rgba'}, {'async' if readback.asynchronous else 'sync'} readback" + pixel_format_name = 'nv12' if converter is not None else 'rgba' + pipeline = f"{pixel_format_name}, {'async' if readback.asynchronous else 'sync'} readback" print(f"car view pipeline: {pipeline}", flush=True) from openpilot.starpilot.system.android_auto import identity as identity_store from openpilot.starpilot.system.android_auto.render_profile import RenderSampler, RenderSummary @@ -197,13 +199,27 @@ def run(frames_path: str) -> int: camera_pacer = CameraPacer() in_flight_ns = 0 + async_failures = 0 + def publish_readback() -> None: - """Hand the frame read back last to android_autod.""" - nonlocal readback - readback, pixels = finish_readback(FrameReadback, readback) - if pixels is None: - return + """Hand the frame read back last to android_autod. + + A failed asynchronous wait drops only that frame; repeated failures switch + the session to synchronous readback instead of crashing the renderer.""" + nonlocal async_failures try: + try: + pixels = readback.finish() + except RuntimeError as error: + if not readback.asynchronous: + raise + async_failures += 1 + print(f"car view dropped a frame: {error}", flush=True) + if async_failures >= ASYNC_READBACK_FAILURES: + readback.fall_back_to_sync() + print(f"car view pipeline: {pixel_format_name}, sync readback (async failed {async_failures} times in a row)", flush=True) + return + async_failures = 0 producer.publish(request, pixels, in_flight_ns, pixel_format, advance=False) finally: readback.release() diff --git a/openpilot/starpilot/system/android_auto/headless_egl.py b/openpilot/starpilot/system/android_auto/headless_egl.py index 9439d73a86..26e0d4a7e5 100644 --- a/openpilot/starpilot/system/android_auto/headless_egl.py +++ b/openpilot/starpilot/system/android_auto/headless_egl.py @@ -70,6 +70,7 @@ class FrameReadback: "glDeleteSync": (None, [C.c_void_p]), "glFlush": (None, []), "glFinish": (None, []), + "glGetError": (C.c_uint, []), } for name, (result, args) in signatures.items(): function = getattr(gl, name) @@ -125,7 +126,7 @@ class FrameReadback: gl.glDeleteSync(fence) if result in (GL_TIMEOUT_EXPIRED, GL_WAIT_FAILED): self.pending = False - raise RuntimeError(f"GPU readback did not complete ({result:#x})") + raise RuntimeError(f"GPU readback did not complete ({result:#x}, GL error {gl.glGetError():#x})") gl.glBindBuffer(GL_PIXEL_PACK_BUFFER, self._buffer) address = gl.glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, self.size, GL_MAP_READ_BIT) if not address: @@ -147,6 +148,16 @@ class FrameReadback: self._fence = None self.pending = False + def fall_back_to_sync(self) -> None: + """Read back synchronously from now on, dropping any frame in flight.""" + self.release() + if self._buffer.value: + self.gl.glDeleteBuffers(1, C.byref(self._buffer)) + self._buffer = C.c_uint(0) + self.asynchronous = False + self._storage = (C.c_ubyte * self.size)() + self.pixels = memoryview(self._storage).cast("B") + def close(self) -> None: self.release() if self._buffer.value: diff --git a/openpilot/starpilot/system/android_auto/tests/test_current_display_source.py b/openpilot/starpilot/system/android_auto/tests/test_current_display_source.py index 6132b3e454..ad399a15fa 100644 --- a/openpilot/starpilot/system/android_auto/tests/test_current_display_source.py +++ b/openpilot/starpilot/system/android_auto/tests/test_current_display_source.py @@ -42,6 +42,7 @@ class TestCurrentDisplaySource(unittest.TestCase): self.assertNotIn("openpilot.starpilot.ui.runtime_app", imports) self.assertIn("openpilot.starpilot.system.android_auto.frame_source", imports) self.assertNotIn("openpilot.starpilot.system.android_auto.ui.main", imports) + self.assertIn("pixels = readback.finish()", source) self.assertIn("producer.publish(request, pixels, in_flight_ns, pixel_format, advance=False)", source) def test_view_uses_current_renderer(self): diff --git a/openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py b/openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py index 504d97ea60..6f07ed5be9 100644 --- a/openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py +++ b/openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py @@ -12,6 +12,7 @@ from unittest import SkipTest import numpy as np import pytest +from unittest.mock import Mock from openpilot.starpilot.system.android_auto import gpu_nv12, headless_egl @@ -97,3 +98,20 @@ def test_composed_car_frame_converts_to_the_same_picture(gpu, width, height, mar converter.close() rl.unload_render_texture(output) rl.rl_unload_texture(texture_id) + + +def test_failed_async_readback_reports_gl_error_and_falls_back_to_sync(monkeypatch): + gl = Mock() + gl.glFenceSync.return_value = 1 + gl.glClientWaitSync.return_value = headless_egl.GL_WAIT_FAILED + gl.glGetError.return_value = 0x502 + monkeypatch.setattr(headless_egl.C, "CDLL", lambda _name: gl) + readback = headless_egl.FrameReadback(16, asynchronous=True) + readback.start([(7, 2, 2, 0)]) + with pytest.raises(RuntimeError, match="0x911d, GL error 0x502"): + readback.finish() + readback.release() + readback.fall_back_to_sync() + readback.start([(7, 2, 2, 0)]) + assert not readback.asynchronous and readback.finish() is readback.pixels + readback.close()