From dda425f7c6a9523b2f5a89b3a94c6a65141213bc Mon Sep 17 00:00:00 2001
From: firestar5683 <168790843+firestar5683@users.noreply.github.com>
Date: Sun, 20 Sep 2026 13:28:17 -0700
Subject: [PATCH] Keep native demo playing through Galaxy status timeouts

---
 roadscore/MAC_SHOWCASE.md                     |  2 +-
 roadscore/prototype/paired_showcase.py        | 63 +++++++++++++-
 .../prototype/test_paired_audio_first.py      | 83 +++++++++++++++++--
 3 files changed, 136 insertions(+), 12 deletions(-)

diff --git a/roadscore/MAC_SHOWCASE.md b/roadscore/MAC_SHOWCASE.md
index b8f70fc42b..afdd147374 100644
--- a/roadscore/MAC_SHOWCASE.md
+++ b/roadscore/MAC_SHOWCASE.md
@@ -48,7 +48,7 @@ Compatible archives contain `dry.wav`, `launch.json`, `audio_blocks.jsonl`, `rep
 
 Both saved launchers support `--muted`, `--duration SECONDS` and `--no-browser`. Paired `--check` contacts Galaxy to verify the target is offroad; independent `--check` validates local prerequisites only. The controls server binds to loopback and refuses an occupied port. The paired target must be an explicit private IPv4 address on Galaxy port 8082.
 
-The independent mode can follow Galaxy controls with `--paired-comma URL`, but controls alone do not synchronize playback. The paired `--demo` launcher additionally holds both starts and enables playhead following. The Mac follows the comma; it never adjusts the comma's audio clock to match its screen. If the Mac replay fails after playback begins, the launcher records a degraded display and lets healthy comma audio finish. Startup failure, native failure, ownership changes and Ctrl+C still stop the owned pair. `--demo --screen-mirror` explicitly enables JPEG capture for the earlier browser mirror. Normal paired playback does not capture or encode a video feed.
+The independent mode can follow Galaxy controls with `--paired-comma URL`, but controls alone do not synchronize playback. The paired `--demo` launcher additionally holds both starts and enables playhead following. The Mac follows the comma; it never adjusts the comma's audio clock to match its screen. If the Mac replay fails after playback begins, the launcher records a degraded display and lets healthy comma audio finish. Temporary Galaxy status failures during playback retry without restarting either replay. After four consecutive failures, the Mac closes and leaves comma playback running to its local end; the terminal reports that its status is unknown. Reconnect to Galaxy to check or stop it. Retry and recovery evidence is saved in `peer_monitor.json`. Startup failure, verified native failure, ownership changes and Ctrl+C still attempt to stop the owned pair. `--demo --screen-mirror` explicitly enables JPEG capture for the earlier browser mirror. Normal paired playback does not capture or encode a video feed.
 
 Prepared native Bluetooth playback requests250ms of output buffering while retaining20ms callbacks. The backend may round this request; both requested and actual latency are recorded in `prepared_summary.json`, along with callback gaps, processing times and underflow flags. DAC timestamps continue to drive presentation timing, so no second manual delay is added for this queue. Mac, system-speaker and calibration defaults are unchanged.
 
diff --git a/roadscore/prototype/paired_showcase.py b/roadscore/prototype/paired_showcase.py
index 336d91f789..fdecf73cda 100644
--- a/roadscore/prototype/paired_showcase.py
+++ b/roadscore/prototype/paired_showcase.py
@@ -11,6 +11,8 @@ import threading
 import time
 import uuid
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from http.client import IncompleteRead
+from urllib.error import HTTPError, URLError
 from urllib.parse import urlencode
 from urllib.request import Request, build_opener, ProxyHandler
 
@@ -195,6 +197,48 @@ def stop_mac(process):
       process.wait(timeout=5)
 
 
+def retryable_status_error(error):
+  if isinstance(error, HTTPError):
+    return error.code in (408, 429, 500, 502, 503, 504)
+  return isinstance(error, (URLError, TimeoutError, ConnectionError, IncompleteRead))
+
+
+class PlaybackMonitor:
+  """Retry only status reads, without restarting or stopping native playback."""
+  def __init__(self, peer, out, request_id):
+    self.peer, self.out, self.request_id = peer, out, request_id
+    self.failures = self.total_failures = 0
+    self.handoff = False
+
+  def record(self, state, error=None):
+    report = dict(state=state,request_id=self.request_id,wall=time.monotonic(),
+                  consecutive_failures=self.failures,total_failures=self.total_failures,
+                  error=repr(error) if error else None,native_stop_requested=False)
+    try:(self.out/'peer_monitor.json').write_text(json.dumps(report,indent=2))
+    except OSError as error:print('Could not record connection status: '+str(error),flush=True)
+
+  def read(self):
+    try:return self.peer.call('demo_ready',{})
+    except Exception as error:
+      if not retryable_status_error(error):raise
+      self.failures += 1;self.total_failures += 1
+      self.handoff = self.failures >= 4
+      self.record('UNREACHABLE' if self.handoff else 'RETRYING',error)
+      if self.failures == 1:
+        print('Galaxy status unavailable; leaving playback running and retrying.',flush=True)
+      if self.handoff:
+        print('Galaxy monitoring lost after four attempts. Comma playback has NOT been stopped; '
+              'it can finish locally. Reconnect to Galaxy to check or stop it.',flush=True)
+      else:time.sleep((.5,1.,2.)[self.failures-1])
+      return None
+
+  def recovered(self):
+    if self.failures:
+      self.failures = 0
+      self.record('CONNECTED')
+      print('Galaxy monitoring restored; playback was not restarted.',flush=True)
+
+
 def native_pair_main():
   parser=argparse.ArgumentParser(description=__doc__)
   parser.add_argument('--roadscore',action='store_true');parser.add_argument('--demo',action='store_true')
@@ -225,7 +269,7 @@ def native_pair_main():
   command=mac_command(project,args.route,selected,peer.peer.base_url,mac_out,args.duration,fullscreen=args.fullscreen)
   (out/'launch.json').write_text(json.dumps(dict(request_id=request_id,route=selected['route'],
       command=command,archive=selected['archive'],generation_invoked=False,screen_mirror=False,muted=args.muted),indent=2))
-  process=None;launched=False;began=time.monotonic()
+  process=None;launched=False;leave_native_running=False;began=time.monotonic()
   def interrupt(*_):raise KeyboardInterrupt
   signal.signal(signal.SIGTERM,interrupt)
   try:
@@ -261,10 +305,17 @@ def native_pair_main():
     print('Session evidence: '+str(out),flush=True)
     started=time.monotonic()
     mac_finished=False
+    monitor=PlaybackMonitor(peer,out,request_id)
     while time.monotonic()-started<args.duration:
-      current=peer.call('demo_ready',{})
+      current=monitor.read()
+      if current is None:
+        if monitor.handoff:
+          leave_native_running=True
+          break
+        continue
       if current.get('request_id')!=request_id:raise RuntimeError('Device demo ownership changed')
       if current.get('failure'):raise RuntimeError(current['failure'])
+      monitor.recovered()
       if not current.get('running'):
         if current.get('complete'):break
         raise RuntimeError('Device demo stopped before completion')
@@ -277,15 +328,19 @@ def native_pair_main():
         except OSError as error:print('Could not record Mac display status: '+str(error),flush=True)
         print('Mac replay '+('failed' if code else 'ended')+'; comma audio continues to its own end. See '+str(out/'mac.log'),flush=True)
       time.sleep(.5)
+  except KeyboardInterrupt:
+    leave_native_running=False
+    raise
   finally:
     # A second interrupt must not abandon the native windows or owned peer.
     signal.signal(signal.SIGTERM,signal.SIG_IGN);signal.signal(signal.SIGINT,signal.SIG_IGN)
     try:stop_mac(process)
     finally:
-      if launched:
+      if launched and not leave_native_running:
         try:peer.call('demo_stop',{'request_id':request_id})
         except Exception as error:print('Could not confirm device demo stop: '+str(error),flush=True)
-    print('Paired native demo stopped. Elapsed %.1f seconds.'%(time.monotonic()-began),flush=True)
+    print(('Mac demo closed; comma playback left running because its status is unknown.' if leave_native_running
+           else 'Paired native demo stopped.')+' Elapsed %.1f seconds.'%(time.monotonic()-began),flush=True)
 
 
 def main():
diff --git a/roadscore/prototype/test_paired_audio_first.py b/roadscore/prototype/test_paired_audio_first.py
index 2b1bc5b6ad..34c6804f57 100644
--- a/roadscore/prototype/test_paired_audio_first.py
+++ b/roadscore/prototype/test_paired_audio_first.py
@@ -1,15 +1,18 @@
 import json
+from http.client import IncompleteRead
 from pathlib import Path
 import tempfile
 from types import SimpleNamespace
 import unittest
 from unittest.mock import Mock, patch
+from urllib.error import HTTPError, URLError
 
-from paired_showcase import native_pair_main
+from paired_showcase import native_pair_main, retryable_status_error
 
 
 class AudioFirstPairTests(unittest.TestCase):
-  def run_pair(self, playback, *, mac_code=1, preparation_code=None):
+  def run_pair(self, playback, *, mac_code=1, preparation_code=None,
+               preparation_error=None, expected_stop=True, interrupt_sleep=False):
     calls=[];clock=[1000.];seen=[];ready_calls=0
     events=iter(playback)
     owned=dict(request_id='audio-first-test',route='private/route',running=True,
@@ -20,7 +23,9 @@ class AudioFirstPairTests(unittest.TestCase):
       if action=='status':return {'offroad':True}
       if action=='demo_ready':
         ready_calls+=1
-        if ready_calls==1:return owned
+        if ready_calls==1:
+          if preparation_error:raise preparation_error
+          return owned
         event=next(events)
         if isinstance(event,BaseException):raise event
         seen.append(event)
@@ -32,10 +37,12 @@ class AudioFirstPairTests(unittest.TestCase):
     def local_ready(path,route):
       path.parent.mkdir(exist_ok=True)
       return {'session_id':'mac-session'}
-    def sleep(seconds):clock[0]+=seconds
+    def sleep(seconds):
+      if interrupt_sleep:raise KeyboardInterrupt()
+      clock[0]+=seconds
     peer=SimpleNamespace(call=call,peer=SimpleNamespace(base_url='http://192.168.8.156:8082'))
     process=Mock();process.poll.side_effect=[preparation_code,mac_code]
-    failure=None;report=None
+    failure=None;report=None;monitor=None
     with tempfile.TemporaryDirectory() as folder:
       root=Path(folder)/'roadscore';(root/'assets').mkdir(parents=True);(root/'prototype').mkdir()
       (root/'assets/demo_catalog.json').write_text(json.dumps({'paired_comma':peer.peer.base_url}))
@@ -51,10 +58,13 @@ class AudioFirstPairTests(unittest.TestCase):
         except BaseException as error:failure=error
       path=root/'results/paired_showcase_audio-first-test/mac_display.json'
       if path.exists():report=json.loads(path.read_text())
+      path=path.with_name('peer_monitor.json')
+      if path.exists():monitor=json.loads(path.read_text())
     stop_mac.assert_called_once_with(process)
-    self.assertEqual(calls[-1][:2],('demo_stop',{'request_id':'audio-first-test'}))
+    stops=[(action,data) for action,data,_ in calls if action=='demo_stop']
+    self.assertEqual(stops,[('demo_stop',{'request_id':'audio-first-test'})] if expected_stop else [])
     return SimpleNamespace(failure=failure,report=report,calls=calls,seen=seen,
-                           elapsed=clock[0]-1000.,output=output)
+                           elapsed=clock[0]-1000.,output=output,monitor=monitor)
 
   def test_failed_mac_does_not_stop_healthy_native_audio_before_its_eof(self):
     result=self.run_pair([{}]*20+[{'running':False,'complete':True}])
@@ -93,5 +103,64 @@ class AudioFirstPairTests(unittest.TestCase):
     self.assertIsInstance(result.failure,KeyboardInterrupt)
     self.assertEqual(result.report['native_audio'],'continuing')
 
+  def test_status_timeout_recovers_without_restart_or_early_stop(self):
+    result=self.run_pair([URLError(TimeoutError('timed out')),{},
+                          {'running':False,'complete':True}])
+    self.assertIsNone(result.failure)
+    self.assertEqual(result.monitor['state'],'CONNECTED')
+    self.assertEqual(result.monitor['total_failures'],1)
+    self.assertEqual(result.calls[-1][0],'demo_stop')
+    for action in ('demo_start','demo_play'):
+      self.assertEqual(sum(c[0]==action for c in result.calls),1)
+    self.assertTrue(result.seen[-1]['complete'])
+
+  def test_recovery_resets_consecutive_failures(self):
+    result=self.run_pair([TimeoutError()]*3+[{}]+[URLError('connection reset')]*3+
+                         [{'running':False,'complete':True}])
+    self.assertIsNone(result.failure)
+    self.assertEqual(result.monitor['state'],'CONNECTED')
+    self.assertEqual(result.monitor['total_failures'],6)
+    self.assertEqual(result.monitor['consecutive_failures'],0)
+
+  def test_persistent_monitor_failure_leaves_native_playback_running(self):
+    result=self.run_pair([URLError('timed out')]*4,expected_stop=False)
+    self.assertIsNone(result.failure)
+    self.assertEqual(result.monitor['state'],'UNREACHABLE')
+    self.assertEqual(result.monitor['consecutive_failures'],4)
+    self.assertFalse(result.monitor['native_stop_requested'])
+    self.assertEqual(result.elapsed,3.5)
+    self.assertTrue(any('status is unknown' in str(c) for c in result.output.call_args_list))
+
+  def test_ctrl_c_during_network_retry_still_attempts_owned_stop(self):
+    result=self.run_pair([TimeoutError()],interrupt_sleep=True)
+    self.assertIsInstance(result.failure,KeyboardInterrupt)
+
+  def test_verified_failure_after_timeout_is_not_hidden(self):
+    for event,reason in [({'failure':'audio failed'},'audio failed'),
+                         ({'request_id':'replacement-owner'},'ownership changed')]:
+      with self.subTest(event=event):
+        result=self.run_pair([TimeoutError(),event])
+        self.assertIsInstance(result.failure,RuntimeError)
+        self.assertIn(reason,str(result.failure))
+
+  def test_protocol_and_permission_errors_do_not_retry(self):
+    for error in (ValueError('not JSON'),HTTPError('http://peer',403,'Forbidden',{},None)):
+      with self.subTest(error=error):
+        result=self.run_pair([error])
+        self.assertIs(result.failure,error)
+        self.assertIsNone(result.monitor)
+
+  def test_preparation_network_failure_still_cleans_up_without_release(self):
+    error=URLError('timed out')
+    result=self.run_pair([],preparation_error=error)
+    self.assertIs(result.failure,error)
+    self.assertFalse(any(c[0]=='demo_play' for c in result.calls))
+
+  def test_transient_http_and_connection_errors_are_retryable(self):
+    for error in (HTTPError('http://peer',503,'Unavailable',{},None),
+                  IncompleteRead(b'partial'),ConnectionResetError(),TimeoutError(),
+                  URLError('timed out')):
+      with self.subTest(error=error):self.assertTrue(retryable_status_error(error))
+
 
 if __name__=='__main__':unittest.main()