diff --git a/tools/radar/ui.py b/tools/radar/ui.py index 26f88d89d5..1d2f1da38b 100755 --- a/tools/radar/ui.py +++ b/tools/radar/ui.py @@ -122,7 +122,10 @@ RADAR_DETAIL_SIGNALS = { } TABLE_MODES = ("motion", "kinematics", "object", "signals") PLAYBACK_SPEEDS = (0.2, 0.5, 1.0, 2.0, 4.0, 8.0) -AUDIO_MAX_BUFFER_SECONDS = 0.5 +AUDIO_PACKET_SECONDS = 0.05 +AUDIO_PREBUFFER_SECONDS = 0.15 +AUDIO_FADE_SECONDS = 0.005 +AUDIO_MAX_BUFFER_SECONDS = 0.75 TRACK_COUNT_FIELD_WIDTH = 3 SOURCE_CIRCLE_RADIUS_SCALE = 0.8 @@ -235,17 +238,53 @@ class ReplayAudioPlayer: self._last_log_mono_time = 0 self._buffer = bytearray() self._lock = threading.Lock() + self._prebuffer_size = 0 + self._fade_samples = 0 + self._rebuffering = True + self._fade_out_pending = False + self._last_output_sample = 0 def _callback(self, outdata, frames, time_info, status) -> None: - del frames, time_info, status + del frames, time_info + output = np.frombuffer(outdata, dtype=np.int16) + output.fill(0) output_size = len(outdata) + chunk = b"" + fade_in = False + fade_out_from = 0 with self._lock: - chunk_size = min(output_size, len(self._buffer)) - chunk = bytes(self._buffer[:chunk_size]) - del self._buffer[:chunk_size] - outdata[:] = bytes(output_size) + if status: + self._rebuffering = True + self._fade_out_pending = True + + if self._fade_out_pending: + fade_out_from = self._last_output_sample + self._last_output_sample = 0 + self._fade_out_pending = False + else: + if self._rebuffering and len(self._buffer) >= self._prebuffer_size: + self._rebuffering = False + fade_in = True + if not self._rebuffering: + if len(self._buffer) >= output_size: + chunk = bytes(self._buffer[:output_size]) + del self._buffer[:output_size] + self._last_output_sample = int(np.frombuffer(chunk, dtype=np.int16)[-1]) + else: + self._rebuffering = True + fade_out_from = self._last_output_sample + self._last_output_sample = 0 + if chunk: - outdata[:chunk_size] = chunk + output[:] = np.frombuffer(chunk, dtype=np.int16) + if fade_in: + fade_samples = min(self._fade_samples, output.size) + output[:fade_samples] = ( + output[:fade_samples].astype(np.float64) * np.linspace(0.0, 1.0, fade_samples) + ).astype(np.int16) + elif fade_out_from: + fade_samples = min(self._fade_samples, output.size) + output[:fade_samples] = np.linspace(fade_out_from, 0, fade_samples).astype(np.int16) def _close_stream(self) -> None: if self._stream is not None: @@ -263,10 +302,16 @@ class ReplayAudioPlayer: self._close_stream() try: import sounddevice as sd + with self._lock: + self._prebuffer_size = round(sample_rate * np.dtype(np.int16).itemsize * AUDIO_PREBUFFER_SECONDS) + self._fade_samples = max(1, round(sample_rate * AUDIO_FADE_SECONDS)) + self._rebuffering = True self._stream = sd.RawOutputStream( samplerate=sample_rate, channels=1, dtype="int16", + blocksize=round(sample_rate * AUDIO_PACKET_SECONDS), + latency="high", callback=self._callback, ) self._stream.start() @@ -281,11 +326,16 @@ class ReplayAudioPlayer: def clear(self) -> None: with self._lock: self._buffer.clear() + self._rebuffering = True + self._fade_out_pending = self._stream is not None self._last_log_mono_time = 0 def reset(self) -> None: self.clear() self._close_stream() + with self._lock: + self._fade_out_pending = False + self._last_output_sample = 0 self.available = False self.error = "" @@ -309,14 +359,26 @@ class ReplayAudioPlayer: or log_mono_time - self._last_log_mono_time > 500_000_000)): self.clear() self._last_log_mono_time = log_mono_time - if not self._open_stream(sample_rate): - return + if self._sample_rate not in (0, sample_rate): + self.reset() + self.available = True + if self._sample_rate == 0: + self._sample_rate = sample_rate audio = resample_audio_for_speed(data, playback_speed) max_buffer_size = round(sample_rate * np.dtype(np.int16).itemsize * AUDIO_MAX_BUFFER_SECONDS) + open_stream = False with self._lock: self._buffer.extend(audio) if len(self._buffer) > max_buffer_size: - del self._buffer[:len(self._buffer) - max_buffer_size] + overflow = len(self._buffer) - max_buffer_size + overflow += overflow % np.dtype(np.int16).itemsize + del self._buffer[:overflow] + self._rebuffering = True + self._fade_out_pending = self._stream is not None + prebuffer_size = round(sample_rate * np.dtype(np.int16).itemsize * AUDIO_PREBUFFER_SECONDS) + open_stream = self._stream is None and len(self._buffer) >= prebuffer_size + if open_stream: + self._open_stream(sample_rate) @dataclass(frozen=True) @@ -2454,10 +2516,11 @@ def ui_thread(addr: str, start_wide: bool = False, start_fused: bool = False, st "liveTracks", "modelV2", "radarState", - "rawAudioData", "roadCameraState", "wideRoadCameraState", ], addr=messaging_addr) + # SubMaster conflates messages, but every 50 ms audio packet is needed for gapless playback. + audio_socket = messaging.sub_sock("rawAudioData", addr=messaging_addr, conflate=False) process_context = multiprocessing.get_context("spawn") decoder_output = process_context.Queue(maxsize=1) @@ -2548,6 +2611,7 @@ def ui_thread(addr: str, start_wide: bool = False, start_fused: bool = False, st replay_timeline = ReplayTimeline() replay_scrub = ReplayScrubState() audio_player.reset() + messaging.drain_sock(audio_socket) replay_seek_input = "" replay_seek_active = False route_loading = False @@ -2709,14 +2773,15 @@ def ui_thread(addr: str, start_wide: bool = False, start_fused: bool = False, st route_select_all = False sm.update(0) - if replay_process is not None and sm.updated["rawAudioData"]: - raw_audio = sm["rawAudioData"] - audio_player.push( - bytes(raw_audio.data), - int(raw_audio.sampleRate), - sm.logMonoTime["rawAudioData"], - replay_process.playback_speed, - ) + for audio_event in messaging.drain_sock(audio_socket): + if replay_process is not None: + raw_audio = audio_event.rawAudioData + audio_player.push( + bytes(raw_audio.data), + int(raw_audio.sampleRate), + int(audio_event.logMonoTime), + replay_process.playback_speed, + ) if remote_address is not None and any(sm.updated.values()): remote_bridge_last_message = time.monotonic() if sm.updated["carParams"]: