Car view: drop a frame on a failed async readback instead of crashing

A failed GPU wait (GL_WAIT_FAILED as openpilot went onroad) raised out of the
renderer, three crashes in a row exhausted its restart budget and the car
showed 'display unavailable'. Now the frame is dropped, the GL error is
logged with it, and after three consecutive failures the session switches to
synchronous readback.

openpilot/starpilot/system/android_auto/current_car_ui.py

@@ -23,6 +23,7 @@ from openpilot.starpilot.system.android_auto.projection_geometry import projecti
 STARTUP_WAIT_SECONDS = 15.0
 CAMERA_WAIT_STEP = 0.01    # s; keeps demand/stop checks responsive while waiting for the camera
 CAMERA_MAX_GAP = 0.1       # s; a shown camera silent this long stops pacing (the encoder rate applies)
+ASYNC_READBACK_FAILURES = 3  # consecutive failed asynchronous readbacks before the session reads back synchronously
 
 
 class CameraPacer:
@@ -183,7 +184,8 @@ def run(frames_path: str) -> int:
                              asynchronous=bool(request.flags & FLAG_ASYNC_READBACK))
     resources.callback(lambda: readback.close())
     rgba_regions = [(output.id, request.width, request.height, 0)]
-    pipeline = f"{'nv12' if converter is not None else 'rgba'}, {'async' if readback.asynchronous else 'sync'} readback"
+    pixel_format_name = 'nv12' if converter is not None else 'rgba'
+    pipeline = f"{pixel_format_name}, {'async' if readback.asynchronous else 'sync'} readback"
     print(f"car view pipeline: {pipeline}", flush=True)
     from openpilot.starpilot.system.android_auto import identity as identity_store
     from openpilot.starpilot.system.android_auto.render_profile import RenderSampler, RenderSummary
@@ -197,13 +199,27 @@ def run(frames_path: str) -> int:
     camera_pacer = CameraPacer()
     in_flight_ns = 0
 
+    async_failures = 0
+
     def publish_readback() -> None:
-      """Hand the frame read back last to android_autod."""
-      nonlocal readback
-      readback, pixels = finish_readback(FrameReadback, readback)
-      if pixels is None:
-        return
+      """Hand the frame read back last to android_autod.
+
+      A failed asynchronous wait drops only that frame; repeated failures switch
+      the session to synchronous readback instead of crashing the renderer."""
+      nonlocal async_failures
       try:
+        try:
+          pixels = readback.finish()
+        except RuntimeError as error:
+          if not readback.asynchronous:
+            raise
+          async_failures += 1
+          print(f"car view dropped a frame: {error}", flush=True)
+          if async_failures >= ASYNC_READBACK_FAILURES:
+            readback.fall_back_to_sync()
+            print(f"car view pipeline: {pixel_format_name}, sync readback (async failed {async_failures} times in a row)", flush=True)
+          return
+        async_failures = 0
         producer.publish(request, pixels, in_flight_ns, pixel_format, advance=False)
       finally:
         readback.release()

openpilot/starpilot/system/android_auto/headless_egl.py

@@ -70,6 +70,7 @@ class FrameReadback:
       "glDeleteSync": (None, [C.c_void_p]),
       "glFlush": (None, []),
       "glFinish": (None, []),
+      "glGetError": (C.c_uint, []),
     }
     for name, (result, args) in signatures.items():
       function = getattr(gl, name)
@@ -125,7 +126,7 @@ class FrameReadback:
     gl.glDeleteSync(fence)
     if result in (GL_TIMEOUT_EXPIRED, GL_WAIT_FAILED):
       self.pending = False
-      raise RuntimeError(f"GPU readback did not complete ({result:#x})")
+      raise RuntimeError(f"GPU readback did not complete ({result:#x}, GL error {gl.glGetError():#x})")
     gl.glBindBuffer(GL_PIXEL_PACK_BUFFER, self._buffer)
     address = gl.glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, self.size, GL_MAP_READ_BIT)
     if not address:
@@ -147,6 +148,16 @@ class FrameReadback:
       self._fence = None
     self.pending = False
 
+  def fall_back_to_sync(self) -> None:
+    """Read back synchronously from now on, dropping any frame in flight."""
+    self.release()
+    if self._buffer.value:
+      self.gl.glDeleteBuffers(1, C.byref(self._buffer))
+      self._buffer = C.c_uint(0)
+    self.asynchronous = False
+    self._storage = (C.c_ubyte * self.size)()
+    self.pixels = memoryview(self._storage).cast("B")
+
   def close(self) -> None:
     self.release()
     if self._buffer.value:

openpilot/starpilot/system/android_auto/tests/test_current_display_source.py

@@ -42,6 +42,7 @@ class TestCurrentDisplaySource(unittest.TestCase):
     self.assertNotIn("openpilot.starpilot.ui.runtime_app", imports)
     self.assertIn("openpilot.starpilot.system.android_auto.frame_source", imports)
     self.assertNotIn("openpilot.starpilot.system.android_auto.ui.main", imports)
+    self.assertIn("pixels = readback.finish()", source)
     self.assertIn("producer.publish(request, pixels, in_flight_ns, pixel_format, advance=False)", source)
 
   def test_view_uses_current_renderer(self):

openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py

@@ -12,6 +12,7 @@ from unittest import SkipTest
 
 import numpy as np
 import pytest
+from unittest.mock import Mock
 
 from openpilot.starpilot.system.android_auto import gpu_nv12, headless_egl
 
@@ -97,3 +98,20 @@ def test_composed_car_frame_converts_to_the_same_picture(gpu, width, height, mar
     converter.close()
     rl.unload_render_texture(output)
     rl.rl_unload_texture(texture_id)
+
+
+def test_failed_async_readback_reports_gl_error_and_falls_back_to_sync(monkeypatch):
+  gl = Mock()
+  gl.glFenceSync.return_value = 1
+  gl.glClientWaitSync.return_value = headless_egl.GL_WAIT_FAILED
+  gl.glGetError.return_value = 0x502
+  monkeypatch.setattr(headless_egl.C, "CDLL", lambda _name: gl)
+  readback = headless_egl.FrameReadback(16, asynchronous=True)
+  readback.start([(7, 2, 2, 0)])
+  with pytest.raises(RuntimeError, match="0x911d, GL error 0x502"):
+    readback.finish()
+  readback.release()
+  readback.fall_back_to_sync()
+  readback.start([(7, 2, 2, 0)])
+  assert not readback.asynchronous and readback.finish() is readback.pixels
+  readback.close()