IQ.Pilot Release Commit @ f82ff4d

This commit is contained in:
IQ.Lvbs CI [bot]
2026-07-21 13:43:46 -05:00
parent 7b20edda67
commit b712a31728
717 changed files with 4347 additions and 3060 deletions
-20
View File
@@ -21,7 +21,6 @@ from openpilot.common.realtime import DT_HW
from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
from openpilot.system.hardware import HARDWARE, TICI, AGNOS
from openpilot.system.loggerd.config import get_available_percent
from openpilot.system.statsd import statlog
from openpilot.common.swaglog import cloudlog
from openpilot.system.hardware.power_monitoring import PowerMonitoring, VBATT_LOW_POWER_EXIT
from openpilot.system.hardware.fan_controller import FanController
@@ -479,11 +478,9 @@ def hardware_thread(end_event, hw_queue) -> None:
msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used()
msg.deviceState.carBatteryCapacityUwh = max(0, power_monitor.get_car_battery_capacity())
current_power_draw = HARDWARE.get_current_power_draw()
statlog.sample("power_draw", current_power_draw)
msg.deviceState.powerDrawW = current_power_draw
som_power_draw = HARDWARE.get_som_power_draw()
statlog.sample("som_power_draw", som_power_draw)
msg.deviceState.somPowerDrawW = som_power_draw
# FastSleep deep standby: shed heavy processes at low battery instead of shutting down,
@@ -523,23 +520,6 @@ def hardware_thread(end_event, hw_queue) -> None:
msg.deviceState.thermalStatus = thermal_status
pm.send("deviceState", msg)
# Log to statsd
statlog.gauge("free_space_percent", msg.deviceState.freeSpacePercent)
statlog.gauge("gpu_usage_percent", msg.deviceState.gpuUsagePercent)
statlog.gauge("memory_usage_percent", msg.deviceState.memoryUsagePercent)
for i, usage in enumerate(msg.deviceState.cpuUsagePercent):
statlog.gauge(f"cpu{i}_usage_percent", usage)
for i, temp in enumerate(msg.deviceState.cpuTempC):
statlog.gauge(f"cpu{i}_temperature", temp)
for i, temp in enumerate(msg.deviceState.gpuTempC):
statlog.gauge(f"gpu{i}_temperature", temp)
statlog.gauge("memory_temperature", msg.deviceState.memoryTempC)
for i, temp in enumerate(msg.deviceState.pmicTempC):
statlog.gauge(f"pmic{i}_temperature", temp)
for i, temp in enumerate(last_hw_state.modem_temps):
statlog.gauge(f"modem_temperature{i}", temp)
statlog.gauge("fan_speed_percent_desired", msg.deviceState.fanSpeedPercentDesired)
statlog.gauge("screen_brightness_percent", msg.deviceState.screenBrightnessPercent)
# report to server once every 10 minutes
rising_edge_started = should_start and not should_start_prev
-2
View File
@@ -4,7 +4,6 @@ import threading
from openpilot.common.params import Params
from openpilot.system.hardware import HARDWARE
from openpilot.common.swaglog import cloudlog
from openpilot.system.statsd import statlog
CAR_VOLTAGE_LOW_PASS_K = 0.011 # LPF gain for 45s tau (dt/tau / (dt/tau + 1))
@@ -56,7 +55,6 @@ class PowerMonitoring:
# Low-pass battery voltage
self.car_voltage_instant_mV = voltage
self.car_voltage_mV = ((voltage * CAR_VOLTAGE_LOW_PASS_K) + (self.car_voltage_mV * (1 - CAR_VOLTAGE_LOW_PASS_K)))
statlog.gauge("car_voltage", self.car_voltage_mV / 1e3)
# Cap the car battery power and save it in a param every 10-ish seconds
self.car_battery_capacity_uWh = max(self.car_battery_capacity_uWh, 0)
@@ -7,7 +7,7 @@ from tabulate import tabulate
import cereal.messaging as messaging
from cereal.services import SERVICE_LIST
from opendbc.car.car_helpers import get_demo_car_params
from iqdbc.car.car_helpers import get_demo_car_params
from openpilot.common.mock import mock_messages
from openpilot.common.params import Params
from openpilot.system.hardware.tici.power_monitor import get_power
-3
View File
@@ -5,9 +5,6 @@ from openpilot.system.hardware.hw import Paths
CAMERA_FPS = 20
SEGMENT_LENGTH = 60
STATS_DIR_FILE_LIMIT = 10000
STATS_SOCKET = "ipc:///tmp/stats"
STATS_FLUSH_TIME_S = 60
PATH_DICT = {
"internal": Paths.log_root(),
-1
View File
@@ -174,7 +174,6 @@ procs = [
PythonProcess("tombstoned", "system.tombstoned", always_run, enabled=not PC),
PythonProcess("updated", "system.updated.updated", and_(only_offroad, not_low_power), enabled=not PC),
BundleProcess("iquploaderd", "iqpilot_hephaestusd_private", "iqpilot_private.konn3kt.uploaderd.iquploaderd", and_(iquploaderd_ready, not_low_power), restart_if_crash=True),
PythonProcess("statsd", "system.statsd", always_run),
PythonProcess("feedbackd", "selfdrive.ui.feedback.feedbackd", and_(only_onroad, not_lat_maneuver)),
# debug procs
-234
View File
@@ -1,234 +0,0 @@
#!/usr/bin/env python3
import base64
import json
import os
from decimal import Decimal
import zmq
import time
import uuid
from pathlib import Path
from collections import defaultdict
from datetime import datetime, UTC, date
from typing import NoReturn
from openpilot.common.params import Params
from cereal.messaging import SubMaster
from openpilot.system.hardware.hw import Paths
from openpilot.common.swaglog import cloudlog
from openpilot.system.hardware import HARDWARE
from openpilot.common.utils import atomic_write
from openpilot.system.version import get_build_metadata
from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
class METRIC_TYPE:
GAUGE = 'g'
SAMPLE = 'sa'
RAW = 'r'
class StatLog:
def __init__(self):
self.pid = None
self.zctx = None
self.sock = None
self.stats_socket = STATS_SOCKET
def connect(self) -> None:
self.zctx = zmq.Context.instance() or zmq.Context()
self.sock = self.zctx.socket(zmq.PUSH)
self.sock.setsockopt(zmq.LINGER, 10)
self.sock.connect(self.stats_socket)
self.pid = os.getpid()
def __del__(self):
if self.sock is not None:
self.sock.close()
if self.zctx is not None:
self.zctx.term()
def _send(self, metric: str) -> None:
if os.getpid() != self.pid:
self.connect()
try:
self.sock.send_string(metric, zmq.NOBLOCK)
except zmq.error.Again:
# drop :/
pass
def gauge(self, name: str, value: float) -> None:
self._send(f"{name}:{value}|{METRIC_TYPE.GAUGE}")
# Samples will be recorded in a buffer and at aggregation time,
# statistical properties will be logged (mean, count, percentiles, ...)
def sample(self, name: str, value: float):
self._send(f"{name}:{value}|{METRIC_TYPE.SAMPLE}")
class StatLogIQ(StatLog):
def __init__(self, intercept=True):
"""
Initializes the class instance with an optional parameter to determine
if statistical logging should be configured or not.
:param intercept: A boolean flag that indicates whether to initialize
the `comma_statlog`. If True, the `comma_statlog` attribute is
instantiated as a `StatLog` object. Defaults to True.
"""
super().__init__()
self.comma_statlog = StatLog() if intercept else None
self.stats_socket = f"{STATS_SOCKET}_iq"
def connect(self) -> None:
super().connect()
if self.comma_statlog:
self.comma_statlog.connect()
def __del__(self):
super().__del__()
if self.comma_statlog:
self.comma_statlog.__del__()
def _send(self, metric: str) -> None:
super()._send(metric)
if self.comma_statlog:
self.comma_statlog._send(metric)
@staticmethod
def default_converter(obj):
if isinstance(obj, (datetime, date)):
return obj.isoformat()
if isinstance(obj, set):
return list(obj)
if isinstance(obj, Decimal):
return float(obj)
return str(obj) # fallback for unknown types
def raw(self, name: str, value: dict) -> None:
encoded_dict = base64.b64encode(json.dumps(value, default=self.default_converter).encode("utf-8")).decode("utf-8")
self._send(f"{name}:{encoded_dict}|{METRIC_TYPE.RAW}")
def main() -> NoReturn:
dongle_id = Params().get("DongleId")
def get_influxdb_line(measurement: str, value: float | dict[str, float], timestamp: datetime, tags: dict) -> str:
res = f"{measurement}"
for k, v in tags.items():
res += f",{k}={str(v)}"
res += " "
if isinstance(value, float):
value = {'value': value}
for k, v in value.items():
res += f"{k}={v},"
res += f"dongle_id=\"{dongle_id}\" {int(timestamp.timestamp() * 1e9)}\n"
return res
# open statistics socket
ctx = zmq.Context.instance()
sock = ctx.socket(zmq.PULL)
sock.bind(STATS_SOCKET)
STATS_DIR = Paths.stats_root()
# initialize stats directory
Path(STATS_DIR).mkdir(parents=True, exist_ok=True)
build_metadata = get_build_metadata()
# initialize tags
tags = {
'started': False,
'version': build_metadata.openpilot.version,
'branch': build_metadata.channel,
'dirty': build_metadata.openpilot.is_dirty,
'origin': build_metadata.openpilot.git_normalized_origin,
'deviceType': HARDWARE.get_device_type(),
}
# subscribe to deviceState for started state
sm = SubMaster(['deviceState'])
idx = 0
boot_uid = str(uuid.uuid4())[:8]
last_flush_time = time.monotonic()
gauges = {}
samples: dict[str, list[float]] = defaultdict(list)
try:
while True:
started_prev = sm['deviceState'].started
sm.update()
# Update metrics
while True:
try:
metric = sock.recv_string(zmq.NOBLOCK)
try:
metric_type = metric.split('|')[1]
metric_name = metric.split(':')[0]
metric_value = float(metric.split('|')[0].split(':')[1])
if metric_type == METRIC_TYPE.GAUGE:
gauges[metric_name] = metric_value
elif metric_type == METRIC_TYPE.SAMPLE:
samples[metric_name].append(metric_value)
else:
cloudlog.event("unknown metric type", metric_type=metric_type)
except Exception:
cloudlog.event("malformed metric", metric=metric)
except zmq.error.Again:
break
# flush when started state changes or after FLUSH_TIME_S
if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev):
result = ""
current_time = datetime.now(UTC)
tags['started'] = sm['deviceState'].started
for key, value in gauges.items():
result += get_influxdb_line(f"gauge.{key}", value, current_time, tags)
for key, values in samples.items():
values.sort()
sample_count = len(values)
sample_sum = sum(values)
stats = {
'count': sample_count,
'min': values[0],
'max': values[-1],
'mean': sample_sum / sample_count,
}
for percentile in [0.05, 0.5, 0.95]:
value = values[int(round(percentile * (sample_count - 1)))]
stats[f"p{int(percentile * 100)}"] = value
result += get_influxdb_line(f"sample.{key}", stats, current_time, tags)
# clear intermediate data
gauges.clear()
samples.clear()
last_flush_time = time.monotonic()
# check that we aren't filling up the drive
if len(os.listdir(STATS_DIR)) < STATS_DIR_FILE_LIMIT:
if len(result) > 0:
stats_path = os.path.join(STATS_DIR, f"{boot_uid}_{idx}")
with atomic_write(stats_path) as f:
f.write(result)
idx += 1
else:
cloudlog.error("stats dir full")
finally:
sock.close()
ctx.term()
if __name__ == "__main__":
main()
else:
statlog = StatLogIQ(intercept=True)
+4 -6
View File
@@ -41,12 +41,10 @@ class GuiScrollPanel:
if DEBUG:
rl.draw_rectangle_lines(0, 0, abs(int(self._velocity_filter_y.x)), 10, rl.RED)
# Handle mouse wheel only when the mouse cursor is over this panel
mouse_wheel = rl.get_mouse_wheel_move()
if mouse_wheel != 0:
mouse_pos = rl.get_mouse_position()
if rl.check_collision_point_rec(mouse_pos, bounds):
self._offset_filter_y.x += mouse_wheel * MOUSE_WHEEL_SCROLL_SPEED
# wheel scrolls this panel only while the cursor sits inside its bounds
wheel = rl.get_mouse_wheel_move()
if wheel and rl.check_collision_point_rec(rl.get_mouse_position(), bounds):
self._offset_filter_y.x += wheel * MOUSE_WHEEL_SCROLL_SPEED
max_scroll_distance = max(0, content.height - bounds.height)
if self._scroll_state == ScrollState.IDLE:
+124
View File
@@ -0,0 +1,124 @@
import asyncio
import contextlib
from collections import deque
from fractions import Fraction
import av
from cereal import messaging
from openpilot.selfdrive.ui.soundd import SAMPLE_RATE as SOUND_SAMPLE_RATE
from openpilot.system.webrtc.device.audio import WEBRTC_AUDIO_PTIME, WEBRTC_AUDIO_SERVICE
class AudioInputOpusProducer:
"""Micd PCM -> 48 kHz Opus payloads for libdatachannel's RTP packetizer."""
def __init__(self) -> None:
self._sock = messaging.sub_sock("rawAudioData", conflate=False)
self._pcm = bytearray()
self._source_rate = 16_000
self._next_pts = 0
self._pending: deque[tuple[bytes, int]] = deque()
self._enabled = True
self._resampler = av.AudioResampler(format="fltp", layout="mono", rate=48_000)
self._encoder = av.CodecContext.create("libopus", "w")
self._encoder.sample_rate = 48_000
self._encoder.layout = "mono"
self._encoder.format = "fltp"
self._encoder.open()
def enable(self, enabled: bool) -> None:
self._enabled = enabled
async def _read_pcm_frame(self) -> av.AudioFrame:
while True:
samples = max(1, int(WEBRTC_AUDIO_PTIME * self._source_rate))
target_bytes = samples * 2
while len(self._pcm) < target_bytes:
msg = messaging.recv_one_or_none(self._sock)
if msg is None:
await asyncio.sleep(0.002)
continue
audio = msg.rawAudioData
rate = int(audio.sampleRate) or self._source_rate
if rate != self._source_rate:
self._source_rate = rate
self._pcm.clear()
continue
self._pcm.extend(bytes(audio.data))
data = bytes(self._pcm[:target_bytes])
del self._pcm[:target_bytes]
frame = av.AudioFrame(format="s16", layout="mono", samples=samples)
frame.planes[0].update(data)
frame.sample_rate = self._source_rate
return frame
async def recv(self) -> tuple[bytes, int] | None:
if not self._enabled:
await asyncio.sleep(WEBRTC_AUDIO_PTIME)
return None
while not self._pending:
source_frame = await self._read_pcm_frame()
for frame in self._resampler.resample(source_frame):
frame.pts = self._next_pts
frame.time_base = Fraction(1, 48_000)
self._next_pts += frame.samples
for packet in self._encoder.encode(frame):
self._pending.append((bytes(packet), int(packet.pts or 0)))
return self._pending.popleft()
class IncomingOpusCerealProxy:
"""libdatachannel Opus payloads -> soundd-compatible PCM cereal messages."""
def __init__(self, track) -> None:
self._loop = asyncio.get_running_loop()
self._queue: asyncio.Queue[bytes] = asyncio.Queue(maxsize=32)
self._pm = messaging.PubMaster([WEBRTC_AUDIO_SERVICE])
self._decoder = av.CodecContext.create("opus", "r")
self._resampler = av.AudioResampler(format="s16", layout="mono", rate=SOUND_SAMPLE_RATE)
self._task: asyncio.Task | None = None
track.on_frame(self._on_frame)
def _on_frame(self, payload: bytes, _info) -> None:
def enqueue() -> None:
if self._queue.full():
with contextlib.suppress(asyncio.QueueEmpty):
self._queue.get_nowait()
self._queue.put_nowait(bytes(payload))
self._loop.call_soon_threadsafe(enqueue)
def start(self) -> None:
if self._task is None:
self._task = asyncio.create_task(self.run())
async def stop(self) -> None:
if self._task is None:
return
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
self._task = None
def _publish(self, frame: av.AudioFrame) -> None:
data = frame.to_ndarray().tobytes()
if not data:
return
msg = messaging.new_message(WEBRTC_AUDIO_SERVICE, valid=True)
msg.webrtcAudioData.data = data
msg.webrtcAudioData.sampleRate = frame.sample_rate
self._pm.send(WEBRTC_AUDIO_SERVICE, msg)
async def run(self) -> None:
while True:
payload = await self._queue.get()
try:
for decoded in self._decoder.decode(av.Packet(payload)):
for frame in self._resampler.resample(decoded):
self._publish(frame)
except Exception:
# A malformed or stale packet must not end the video/control session.
continue
+3
View File
@@ -86,6 +86,9 @@ class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
self._kf_requested = True
self._request_keyframe(True)
def request_keyframe(self) -> None:
self._mark_keyframe_needed()
def _mark_keyframe_received(self) -> None:
"""This track got its keyframe; only clear the global request once no track needs one."""
if self._kf_requested:
+274
View File
@@ -0,0 +1,274 @@
import asyncio
import logging
import struct
import time
import av
from openpilot.system.webrtc.teleoprtc_ldc.tracks import TiciVideoStreamTrack
from cereal import messaging
from openpilot.common.params import Params
from openpilot.common.realtime import DT_MDL, DT_DMON
# Arbitrary 16-byte UUID identifying konn3kt frame-timing SEI messages. When timing
# telemetry is enabled, each frame carries a user_data_unregistered SEI NAL with four
# big-endian doubles (ms): encode duration, IPC/queue delay, host transit, and the
# device wall clock. The client decodes these to compute true glass-to-glass latency.
TIMING_SEI_UUID = bytes([
0xa5, 0xe0, 0xc4, 0xa4, 0x5b, 0x6e, 0x4e, 0x1e,
0x9c, 0x7e, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc,
])
# Annex-B start code + SEI NAL (type 6) + user_data_unregistered (type 5) + payload size
# (0x30 = 48 bytes = 16 UUID + 32 data). Trailing 0x80 is the RBSP stop bit.
_SEI_PREFIX = b'\x00\x00\x00\x01\x06\x05\x30' + TIMING_SEI_UUID
class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
livestream_camera_to_sock_mapping = {
"driver": "livestreamDriverEncodeData",
"wideRoad": "livestreamWideRoadEncodeData",
"road": "livestreamRoadEncodeData",
}
main_camera_to_sock_mapping = {
"driver": "driverEncodeData",
"wideRoad": "wideRoadEncodeData",
"road": "roadEncodeData",
}
# Number of live tracks still waiting for their first keyframe. The on-demand
# keyframe request (LivestreamRequestKeyframe) is a single global param honored by
# every encoder, so with multiple concurrent tracks (dual-camera PiP) we must not
# clear it until *all* tracks have received an IDR — otherwise the first track to
# get its keyframe clears the request and starves the others (black feed).
_kf_pending_count = 0
def __init__(self, camera_type: str):
dt = DT_DMON if camera_type == "driver" else DT_MDL
super().__init__(camera_type, dt)
self._params = Params()
self._camera_type = camera_type
self._candidate_topics = [
self.main_camera_to_sock_mapping[camera_type],
self.livestream_camera_to_sock_mapping[camera_type],
]
self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
self._active_topic = self._preferred_topics()[0]
self._pts = 0
self._t0_ns = time.monotonic_ns()
self._cached_header: bytes = b""
self._sent_keyframe = False
self._kf_requested = False # whether this track counts toward _kf_pending_count
self._frame_count = 0
self._last_frame_time = 0.0
self._last_preference_refresh = 0.0
# Tracks how long the H264 livestream feed has been silent, to gate the last-resort main-feed
# fallback (see recv) without flapping between sources frame-by-frame.
self._live_silent_since: float | None = None
# Opt-in glass-to-glass latency telemetry (toggled by the client over the data channel).
self.timing_sei_enabled = False
self._logger = logging.getLogger("LiveStreamVideoStreamTrack")
# Ask the encoder for an immediate IDR so the stream starts fast instead of waiting up to a full
# GOP for the next periodic keyframe (encoderd honors LivestreamRequestKeyframe per-frame).
self._mark_keyframe_needed()
def _request_keyframe(self, enabled: bool) -> None:
try:
self._params.put_bool("LivestreamRequestKeyframe", enabled)
except Exception:
self._logger.exception("failed to set LivestreamRequestKeyframe")
def _mark_keyframe_needed(self) -> None:
"""This track needs (another) keyframe: keep the global request asserted."""
if not self._kf_requested:
LiveStreamVideoStreamTrack._kf_pending_count += 1
self._kf_requested = True
self._request_keyframe(True)
def request_keyframe(self) -> None:
"""RTCP PLI hook used by libdatachannel's native H.264 packetizer."""
self._mark_keyframe_needed()
def _mark_keyframe_received(self) -> None:
"""This track got its keyframe; only clear the global request once no track needs one."""
if self._kf_requested:
self._kf_requested = False
LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
if LiveStreamVideoStreamTrack._kf_pending_count == 0:
self._request_keyframe(False)
def stop(self):
# Release our pending-keyframe hold so a torn-down track that never received an
# IDR doesn't pin LivestreamRequestKeyframe True forever (continuous keyframes).
if getattr(self, "_kf_requested", False):
self._kf_requested = False
LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
try:
super().stop()
except Exception:
pass
def switch_camera(self, camera_type: str) -> None:
"""Repoint this track at a different camera without renegotiating the peer connection.
Lets a single video track back the whole Live View — the client flips cameras over the
data channel and we swap the source here, instead of uplinking every camera at once."""
if camera_type not in self.livestream_camera_to_sock_mapping:
self._logger.warning("[%s] ignoring switch to unknown camera %s", self._id, camera_type)
return
if camera_type == self._camera_type:
return
self._logger.info("[%s] switching camera %s -> %s", self._id, self._camera_type, camera_type)
self._camera_type = camera_type
self._candidate_topics = [
self.main_camera_to_sock_mapping[camera_type],
self.livestream_camera_to_sock_mapping[camera_type],
]
self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
self._active_topic = self._preferred_topics()[0]
# Force a fresh keyframe/header before emitting frames from the new source, and ask the encoder
# for an immediate IDR so the camera switch isn't stalled waiting for the next periodic keyframe.
self._cached_header = b""
self._sent_keyframe = False
self._last_preference_refresh = 0.0
self._live_silent_since = None
self._mark_keyframe_needed()
def _preferred_topics(self) -> list[str]:
# WebRTC currently forces H.264. The dedicated livestream topics are the H.264 feeds,
# while the main encode topics are the full-resolution HEVC recordings. Prefer the
# livestream feeds both onroad and offroad, and keep the main topics only as fallback.
return [
self.livestream_camera_to_sock_mapping[self._camera_type],
self.main_camera_to_sock_mapping[self._camera_type],
]
def _reset_decoder_state(self, topic: str) -> None:
if topic == self._active_topic:
return
self._logger.info("[%s] switching video source from %s to %s", self._id, self._active_topic, topic)
self._active_topic = topic
self._cached_header = b""
self._sent_keyframe = False
def _timing_sei(self, evta, log_mono_time: int) -> bytes:
"""Build a timing SEI NAL from encode metadata, or empty bytes when disabled."""
if not self.timing_sei_enabled:
return b""
idx = evta.idx
return _SEI_PREFIX + struct.pack(
'>4d',
(idx.timestampEof - idx.timestampSof) / 1e6, # encode duration (ms)
(log_mono_time - idx.timestampEof) / 1e6, # IPC/queue delay (ms)
(time.monotonic_ns() - log_mono_time) / 1e6, # host transit so far (ms)
time.time() * 1000, # device wall clock (ms) # noqa: TID251
) + b'\x80'
def _is_keyframe(self, data: bytes) -> bool:
"""Check if H.264 NAL unit contains an IDR keyframe (NAL type 5)."""
i = 0
while i < len(data) - 4:
# Look for Annex B start codes: 0x000001 or 0x00000001
if data[i:i+3] == b'\x00\x00\x01':
nal_type = data[i+3] & 0x1f
if nal_type == 5: # IDR slice
return True
i += 3
elif data[i:i+4] == b'\x00\x00\x00\x01':
nal_type = data[i+4] & 0x1f
if nal_type == 5: # IDR slice
return True
i += 4
else:
i += 1
return False
async def recv(self):
while True:
now = time.monotonic()
# Resolve topics each iteration: a camera switch (different async task) can rebuild self._socks
# across the await below, so a value cached before the loop would index a stale key (KeyError).
live_topic = self.livestream_camera_to_sock_mapping[self._camera_type]
main_topic = self.main_camera_to_sock_mapping[self._camera_type]
# Lock onto the dedicated H264 livestream feed. Onroad the HEVC main feed also publishes at
# 20fps; eagerly preferring whichever socket had a frame ready raced frame-by-frame, reset the
# decoder every frame, and (the track is negotiated H264) shoved HEVC garbage into the stream —
# the onroad choppiness. Only fall back to the main feed as a last resort after a long
# livestream silence (e.g. stream_encoderd still spinning up), and snap back when it returns.
msg = messaging.recv_one_or_none(self._socks[live_topic])
if msg is not None:
self._reset_decoder_state(live_topic)
self._last_frame_time = now
self._live_silent_since = None
break
if self._live_silent_since is None:
self._live_silent_since = now
elif now - self._live_silent_since > 3.0:
maybe_msg = messaging.recv_one_or_none(self._socks[main_topic])
if maybe_msg is not None:
self._reset_decoder_state(main_topic)
self._last_frame_time = now
msg = maybe_msg
break
await asyncio.sleep(0.005)
evta = getattr(msg, msg.which())
header = bytes(evta.header)
data = bytes(evta.data)
self._frame_count += 1
# Cache SPS/PPS header when it arrives
if header:
self._cached_header = header
self._logger.debug(f"[{self._id}] cached SPS/PPS header ({len(header)} bytes)")
# CRITICAL: Cannot decode without SPS/PPS. Wait for it.
if not self._cached_header:
self._logger.debug(f"[{self._id}] frame {self._frame_count}: no SPS/PPS yet, skipping")
return await self.recv()
is_keyframe = self._is_keyframe(data)
# Wait for first keyframe before sending any frames
# Browser decoder needs IDR to initialize properly
if not self._sent_keyframe:
if not is_keyframe:
self._logger.debug(f"[{self._id}] frame {self._frame_count}: waiting for keyframe")
return await self.recv()
self._sent_keyframe = True
# Got the IDR we asked for — stop nagging the encoder, but only once every
# concurrent track has its keyframe (multi-track PiP shares the global param).
self._mark_keyframe_received()
self._logger.info(f"[{self._id}] first keyframe received, starting stream")
# Optional timing SEI NAL, inserted before the slice data (and after SPS/PPS on keyframes).
sei_nal = self._timing_sei(evta, msg.logMonoTime)
# Prepend SPS/PPS header to keyframes (required by some decoders)
# For non-keyframes, header is optional but safe to include
if is_keyframe:
payload = self._cached_header + sei_nal + data
else:
payload = sei_nal + data
self._pts = ((time.monotonic_ns() - self._t0_ns) * self._clock_rate) // 1_000_000_000
packet = av.Packet(payload)
packet.time_base = self._time_base
packet.pts = int(self._pts)
packet.dts = int(self._pts)
packet.duration = int(self._dt * self._clock_rate)
if is_keyframe:
packet.is_keyframe = True
self.log_debug("track sending frame %s (keyframe=%s, size=%d)", self._pts, is_keyframe, len(payload))
return packet
def codec_preference(self) -> str | None:
return "H264"
+2
View File
@@ -0,0 +1,2 @@
from .builder import WebRTCOfferBuilder, WebRTCAnswerBuilder # noqa
from .stream import WebRTCBaseStream, StreamingOffer, ConnectionProvider, MessageHandler # noqa
+89
View File
@@ -0,0 +1,89 @@
# ruff: noqa: TID251, UP006, UP035
import abc
from typing import Dict, List, Optional
from .stream import RTCSessionDescription, WebRTCBaseStream, WebRTCOfferStream, WebRTCAnswerStream, ConnectionProvider
from .tracks import TiciVideoStreamTrack, TiciTrackWrapper
class WebRTCStreamBuilder(abc.ABC):
@abc.abstractmethod
def stream(self) -> WebRTCBaseStream:
raise NotImplementedError
class WebRTCOfferBuilder(WebRTCStreamBuilder):
def __init__(self, connection_provider: ConnectionProvider, bind_address: Optional[str] = None,
ice_servers: Optional[List[dict]] = None):
self.connection_provider = connection_provider
self.bind_address = bind_address
self.ice_servers = ice_servers
self.requested_camera_types: List[str] = []
self.requested_audio = False
self.audio_tracks: List[object] = []
self.messaging_enabled = False
def offer_to_receive_video_stream(self, camera_type: str):
assert camera_type in ["driver", "wideRoad", "road"]
self.requested_camera_types.append(camera_type)
def offer_to_receive_audio_stream(self):
self.requested_audio = True
def add_audio_stream(self, track: object):
assert len(self.audio_tracks) == 0
self.audio_tracks = [track]
def add_messaging(self):
self.messaging_enabled = True
def stream(self) -> WebRTCBaseStream:
return WebRTCOfferStream(
self.connection_provider,
consumed_camera_types=self.requested_camera_types,
consume_audio=self.requested_audio,
video_producer_tracks=[],
audio_producer_tracks=self.audio_tracks,
should_add_data_channel=self.messaging_enabled,
bind_address=self.bind_address,
ice_servers=self.ice_servers,
)
class WebRTCAnswerBuilder(WebRTCStreamBuilder):
def __init__(self, offer_sdp: str, bind_address: Optional[str] = None,
ice_servers: Optional[List[dict]] = None):
self.offer_sdp = offer_sdp
self.bind_address = bind_address
self.ice_servers = ice_servers
self.video_tracks: Dict[str, TiciVideoStreamTrack] = {}
self.requested_audio = False
self.audio_tracks: List[object] = []
def offer_to_receive_audio_stream(self):
self.requested_audio = True
def add_video_stream(self, camera_type: str, track: object):
assert camera_type not in self.video_tracks
assert camera_type in ["driver", "wideRoad", "road"]
if not isinstance(track, TiciVideoStreamTrack):
track = TiciTrackWrapper(camera_type, track)
self.video_tracks[camera_type] = track
def add_audio_stream(self, track: object):
assert len(self.audio_tracks) == 0
self.audio_tracks = [track]
def stream(self) -> WebRTCBaseStream:
description = RTCSessionDescription(sdp=self.offer_sdp, type="offer")
return WebRTCAnswerStream(
description,
consumed_camera_types=[],
consume_audio=self.requested_audio,
video_producer_tracks=list(self.video_tracks.values()),
audio_producer_tracks=self.audio_tracks,
should_add_data_channel=False,
bind_address=self.bind_address,
ice_servers=self.ice_servers,
)
+52
View File
@@ -0,0 +1,52 @@
# ruff: noqa: UP006, UP035
import dataclasses
import struct
from typing import List
@dataclasses.dataclass(frozen=True)
class RtcpReceiverReport:
ssrc: int
fraction_lost: int
packets_lost: int
highest_seq_no: int
jitter: int
lsr: int
dlsr: int
def _decode_receiver_reports(message: bytes) -> List[RtcpReceiverReport]:
reports: List[RtcpReceiverReport] = []
offset = 0
while offset + 4 <= len(message):
flags, packet_type, length_words = struct.unpack_from("!BBH", message, offset)
packet_end = offset + (length_words + 1) * 4
if flags >> 6 != 2 or packet_end > len(message):
break
report_count = flags & 0x1F
if packet_type == 200: # Sender Report
report_offset = offset + 28
elif packet_type == 201: # Receiver Report
report_offset = offset + 8
else:
offset = packet_end
continue
if report_offset + report_count * 24 > packet_end:
break
for i in range(report_count):
block_offset = report_offset + i * 24
ssrc, loss, highest_seq_no, jitter, lsr, dlsr = struct.unpack_from("!IIIIII", message, block_offset)
fraction_lost = loss >> 24
packets_lost = loss & 0xFFFFFF
if packets_lost & 0x800000:
packets_lost -= 1 << 24
reports.append(RtcpReceiverReport(ssrc, fraction_lost, packets_lost, highest_seq_no, jitter, lsr, dlsr))
offset = packet_end
return reports
+37
View File
@@ -0,0 +1,37 @@
import dataclasses
from libdatachannel import Description
@dataclasses.dataclass
class StreamingMediaInfo:
n_expected_camera_tracks: int
expected_audio_track: bool
incoming_audio_track: bool
incoming_datachannel: bool
def parse_info_from_offer(sdp: str) -> StreamingMediaInfo:
"""
helper function to parse info about outgoing and incoming streams from an offer sdp
"""
desc = Description(sdp, Description.Type.Offer)
n_video = 0
expected_audio_track = False
incoming_audio_track = False
incoming_datachannel = desc.has_application()
for i in range(desc.media_count()):
media = desc.media(i)
if media is None:
continue
direction = media.direction()
if media.type() == "video" and direction in (Description.Direction.RecvOnly, Description.Direction.SendRecv):
n_video += 1
elif media.type() == "audio":
if direction in (Description.Direction.RecvOnly, Description.Direction.SendRecv):
expected_audio_track = True
if direction in (Description.Direction.SendOnly, Description.Direction.SendRecv):
incoming_audio_track = True
return StreamingMediaInfo(n_video, expected_audio_track, incoming_audio_track, incoming_datachannel)
+530
View File
@@ -0,0 +1,530 @@
# ruff: noqa: TID251, UP006, UP035
import abc
import asyncio
import contextlib
import dataclasses
import logging
import random
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, Union
from libdatachannel import (
Configuration,
DataChannel,
Candidate,
Description,
FrameInfo,
H264RtpPacketizer,
IceServer,
NalUnit,
OpusRtpDepacketizer,
OpusRtpPacketizer,
PeerConnection,
PliHandler,
RtcpNackResponder,
RtcpSrReporter,
RtpPacketizationConfig,
Track,
)
from .decoder import RtcpReceiverReport, _decode_receiver_reports
from .tracks import TiciVideoStreamTrack, parse_video_track_id
@dataclasses.dataclass
class StreamingOffer:
sdp: str
video: List[str]
@dataclasses.dataclass
class RTCSessionDescription:
sdp: str
type: str
ConnectionProvider = Callable[[StreamingOffer], Awaitable[RTCSessionDescription]]
MessageHandler = Callable[[Union[bytes, str]], None]
class WebRTCBaseStream(abc.ABC):
def __init__(self,
consumed_camera_types: List[str],
consume_audio: bool,
video_producer_tracks: List[TiciVideoStreamTrack],
audio_producer_tracks: List[Any],
should_add_data_channel: bool,
bind_address: Optional[str] = None,
ice_servers: Optional[List[dict]] = None):
config = Configuration()
config.force_media_transport = True
config.disable_auto_negotiation = True
config.ice_servers = self._make_ice_servers(ice_servers)
if bind_address is not None:
config.bind_address = bind_address
self.peer_connection = PeerConnection(config)
self.expected_incoming_camera_types = consumed_camera_types
self.expected_incoming_audio = consume_audio
self.expected_number_of_incoming_media: Optional[int] = None
self.incoming_camera_tracks: Dict[str, Any] = {}
self.incoming_audio_tracks: List[Any] = []
self.outgoing_video_tracks = video_producer_tracks
self.outgoing_audio_tracks = audio_producer_tracks
self.should_add_data_channel = should_add_data_channel
self.messaging_channel: Optional[DataChannel] = None
self.incoming_message_handlers: List[MessageHandler] = []
self._consumer_tracks: List[Track] = []
self._sender_tasks: List[asyncio.Task] = []
self._track_state: List[Tuple[Track, TiciVideoStreamTrack, RtpPacketizationConfig]] = []
self._audio_track_state: List[Tuple[Track, Any, RtpPacketizationConfig]] = []
self._receiver_reports: Dict[str, RtcpReceiverReport] = {}
self._receiver_report_tracks: Dict[str, Tuple[Track, int]] = {}
self.incoming_media_ready_event = asyncio.Event()
self.messaging_channel_ready_event = asyncio.Event()
self.connection_attempted_event = asyncio.Event()
self.connection_stopped_event = asyncio.Event()
self.gathering_complete_event = asyncio.Event()
self._loop: Optional[asyncio.AbstractEventLoop] = None
self.peer_connection.on_state_change(self._on_connectionstatechange)
self.peer_connection.on_gathering_state_change(self._on_gatheringstatechange)
self.peer_connection.on_data_channel(self._on_incoming_datachannel)
if self.expected_incoming_camera_types or self.expected_incoming_audio:
self.peer_connection.on_track(self._on_incoming_track)
self.logger = logging.getLogger("WebRTCStream")
@staticmethod
def _make_ice_servers(servers: Optional[List[dict]]) -> List[IceServer]:
"""Preserve Konn3kt's authenticated STUN/TURN configuration in libdatachannel."""
parsed: List[IceServer] = []
for server in servers or []:
urls = server.get("urls", []) if isinstance(server, dict) else []
if isinstance(urls, str):
urls = [urls]
username = str(server.get("username") or "")
credential = str(server.get("credential") or "")
for url in urls:
if not isinstance(url, str) or not url:
continue
try:
ice_server = IceServer(url)
ice_server.username = username
ice_server.password = credential
parsed.append(ice_server)
except Exception:
logging.getLogger("WebRTCStream").warning("Ignoring invalid ICE server %r", url, exc_info=True)
# A supplied list is authoritative, including LAN-only sessions where relay is deliberately filtered.
return parsed or [IceServer("stun:stun.l.google.com:19302")]
def _log_debug(self, msg: Any, *args):
self.logger.debug(f"{type(self)}() {msg}", *args)
def _call_soon_threadsafe(self, fn: Callable, *args) -> None:
if self._loop is not None and self._loop.is_running():
self._loop.call_soon_threadsafe(fn, *args)
else:
fn(*args)
def _set_event(self, event: asyncio.Event) -> None:
self._call_soon_threadsafe(event.set)
@property
def _number_of_incoming_media(self) -> int:
media = len(self.incoming_camera_tracks) + len(self.incoming_audio_tracks)
media += int(self.messaging_channel is not None) if not self.should_add_data_channel else 0
return media
def _add_consumer_transceivers(self):
for camera_type in self.expected_incoming_camera_types:
media = Description.Video(camera_type, Description.Direction.RecvOnly)
media.add_h264_codec(96)
track = self.peer_connection.add_track(media)
track.set_media_handler(OpusRtpDepacketizer())
self._consumer_tracks.append(track)
self.incoming_camera_tracks[camera_type] = track
if self.expected_incoming_audio:
media = Description.Audio("audio", Description.Direction.RecvOnly)
media.add_opus_codec(111)
track = self.peer_connection.add_track(media)
self._consumer_tracks.append(track)
self.incoming_audio_tracks.append(track)
def _find_offer_video(self, remote_sdp: str) -> Tuple[str, int]:
desc = Description(remote_sdp, Description.Type.Offer)
for i in range(desc.media_count()):
media = desc.media(i)
if media is None or media.type() != "video":
continue
for payload_type in media.payload_types():
with contextlib.suppress(ValueError):
rtp_map = media.rtp_map(payload_type)
if rtp_map is not None and rtp_map.format.upper() == "H264":
return media.mid(), payload_type
raise ValueError("Remote SDP does not offer H264 video")
def _make_video_media(self, track: TiciVideoStreamTrack, remote_sdp: str) -> Tuple[Description.Video, int, int, str]:
mid, payload_type = self._find_offer_video(remote_sdp)
ssrc = random.randint(1, 0xFFFFFFFF)
cname = f"teleoprtc-{random.getrandbits(32):08x}"
stream_id = f"stream-{random.getrandbits(32):08x}"
media = Description.Video(mid, Description.Direction.SendOnly)
media.add_h264_codec(payload_type)
media.add_ssrc(ssrc, cname, stream_id, track.id)
return media, ssrc, payload_type, cname
def _find_offer_audio(self, remote_sdp: str) -> Tuple[str, int]:
desc = Description(remote_sdp, Description.Type.Offer)
for i in range(desc.media_count()):
media = desc.media(i)
if media is None or media.type() != "audio":
continue
for payload_type in media.payload_types():
with contextlib.suppress(ValueError):
rtp_map = media.rtp_map(payload_type)
if rtp_map is not None and rtp_map.format.upper() == "OPUS":
return media.mid(), payload_type
raise ValueError("Remote SDP does not offer Opus audio")
def _make_audio_media(self, remote_sdp: str) -> Tuple[Description.Audio, int, int, str]:
mid, payload_type = self._find_offer_audio(remote_sdp)
ssrc = random.randint(1, 0xFFFFFFFF)
cname = f"teleoprtc-audio-{random.getrandbits(32):08x}"
direction = Description.Direction.SendRecv if self.expected_incoming_audio else Description.Direction.SendOnly
media = Description.Audio(mid, direction)
media.add_opus_codec(payload_type)
media.add_ssrc(ssrc, cname, "audio", "audio")
return media, ssrc, payload_type, cname
def _add_producer_tracks(self, remote_sdp: Optional[str] = None):
for track in self.outgoing_video_tracks:
media, ssrc, payload_type, cname = self._make_video_media(track, remote_sdp or "")
rtc_track = self.peer_connection.add_track(media)
rtp_config = RtpPacketizationConfig(ssrc, cname, payload_type, H264RtpPacketizer.CLOCK_RATE)
rtp_config.start_timestamp = random.randint(0, 0xFFFFFFFF)
rtp_config.timestamp = rtp_config.start_timestamp
rtp_config.sequence_number = random.randint(0, 0xFFFF)
packetizer = H264RtpPacketizer(NalUnit.Separator.LongStartSequence, rtp_config, 1200)
packetizer.add_to_chain(RtcpSrReporter(rtp_config))
packetizer.add_to_chain(PliHandler(track.request_keyframe))
packetizer.add_to_chain(RtcpNackResponder())
rtc_track.set_media_handler(packetizer)
camera_type, _ = parse_video_track_id(track.id)
rtc_track.reset_callbacks()
self._receiver_report_tracks[camera_type] = (rtc_track, ssrc)
self._track_state.append((rtc_track, track, rtp_config))
for producer in self.outgoing_audio_tracks:
media, ssrc, payload_type, cname = self._make_audio_media(remote_sdp or "")
if self.expected_incoming_audio and self.incoming_audio_tracks:
# set_remote_description() creates the browser's sendrecv audio track. Reuse that
# negotiated m-line for our outbound Opus instead of adding a second track with the
# same MID, which leaves libdatachannel stuck in signalling/ICE negotiation.
rtc_track = self.incoming_audio_tracks[0]
negotiated_media = rtc_track.description()
negotiated_media.add_ssrc(ssrc, cname, "audio", "audio")
rtc_track.set_description(negotiated_media)
else:
rtc_track = self.peer_connection.add_track(media)
rtp_config = RtpPacketizationConfig(ssrc, cname, payload_type, OpusRtpPacketizer.DEFAULT_CLOCK_RATE)
rtp_config.start_timestamp = random.randint(0, 0xFFFFFFFF)
rtp_config.timestamp = rtp_config.start_timestamp
rtp_config.sequence_number = random.randint(0, 0xFFFF)
packetizer = OpusRtpPacketizer(rtp_config)
packetizer.add_to_chain(RtcpSrReporter(rtp_config))
packetizer.add_to_chain(RtcpNackResponder())
rtc_track.set_media_handler(packetizer)
self._audio_track_state.append((rtc_track, producer, rtp_config))
def _add_messaging_channel(self, channel: Optional[DataChannel] = None):
if channel is None:
channel = self.peer_connection.create_data_channel("data")
self.messaging_channel = channel
def on_message(message: Union[bytes, str]):
for handler in list(self.incoming_message_handlers):
self._call_soon_threadsafe(handler, message)
channel.on_message(on_message)
channel.on_open(lambda: self._set_event(self.messaging_channel_ready_event))
channel.on_closed(lambda: self._set_event(self.connection_stopped_event))
if channel.is_open():
self._set_event(self.messaging_channel_ready_event)
self._on_after_media()
def _on_connectionstatechange(self, state: PeerConnection.State):
self._log_debug("connection state is %s", state)
if state in (PeerConnection.State.Connected, PeerConnection.State.Failed):
self._set_event(self.connection_attempted_event)
if state in (PeerConnection.State.Disconnected, PeerConnection.State.Closed, PeerConnection.State.Failed):
self._set_event(self.connection_stopped_event)
def _on_gatheringstatechange(self, state: PeerConnection.GatheringState):
self._log_debug("gathering state is %s", state)
if state == PeerConnection.GatheringState.Complete:
self._set_event(self.gathering_complete_event)
def _on_incoming_track(self, track: Track):
self._log_debug("got track: %s", track.mid())
media_type = track.description().type()
if media_type == "audio":
if self.expected_incoming_audio:
self.incoming_audio_tracks.append(track)
self._on_after_media()
return
if media_type != "video":
self._on_after_media()
return
try:
camera_type, _ = parse_video_track_id(track.mid())
except ValueError:
camera_type = track.mid()
if camera_type in self.expected_incoming_camera_types:
self.incoming_camera_tracks[camera_type] = track
self._on_after_media()
def _on_incoming_datachannel(self, channel: DataChannel):
self._log_debug("got data channel: %s", channel.label())
if channel.label() == "data" and self.messaging_channel is None:
self._add_messaging_channel(channel)
def _update_receiver_report(self, camera_type: str, ssrc: int, message: bytes) -> None:
for report in _decode_receiver_reports(message):
if report.ssrc == ssrc:
self._receiver_reports[camera_type] = report
def _on_after_media(self):
if self.expected_number_of_incoming_media is not None and self._number_of_incoming_media >= self.expected_number_of_incoming_media:
self._set_event(self.incoming_media_ready_event)
def _parse_incoming_streams(self, remote_sdp: str):
desc = Description(remote_sdp, Description.Type.Offer)
media_count = 0
for i in range(desc.media_count()):
media = desc.media(i)
if media is None:
continue
direction = media.direction()
if media.type() == "video" and direction in (Description.Direction.SendOnly, Description.Direction.SendRecv):
media_count += 1
elif media.type() == "audio" and self.expected_incoming_audio and direction in (Description.Direction.SendOnly, Description.Direction.SendRecv):
media_count += 1
data_media_count = int(desc.has_application()) if not self.should_add_data_channel else 0
self.expected_number_of_incoming_media = media_count + data_media_count
if self.expected_number_of_incoming_media == 0:
self._set_event(self.incoming_media_ready_event)
def has_incoming_video_track(self, camera_type: str) -> bool:
return camera_type in self.incoming_camera_tracks
def has_incoming_audio_track(self) -> bool:
return len(self.incoming_audio_tracks) > 0
def has_messaging_channel(self) -> bool:
return self.messaging_channel is not None
def get_incoming_video_track(self, camera_type: str) -> Track:
assert camera_type in self.incoming_camera_tracks, "Video tracks are not enabled on this stream"
assert self.is_started, "Stream must be started"
return self.incoming_camera_tracks[camera_type]
def get_incoming_audio_track(self) -> Track:
assert len(self.incoming_audio_tracks) > 0, "Audio tracks are not enabled on this stream"
assert self.is_started, "Stream must be started"
return self.incoming_audio_tracks[0]
def get_messaging_channel(self) -> DataChannel:
assert self.messaging_channel is not None, "Messaging channel is not enabled on this stream"
assert self.is_started, "Stream must be started"
return self.messaging_channel
def get_receiver_report_stats(self) -> Dict[str, RtcpReceiverReport]:
return dict(self._receiver_reports)
def set_message_handler(self, message_handler: MessageHandler):
self.incoming_message_handlers.append(message_handler)
def add_ice_candidate(self, candidate: dict) -> None:
"""Accept post-offer browser candidates for Konn3kt's existing trickle endpoint."""
candidate_sdp = str(candidate.get("candidate") or "") if isinstance(candidate, dict) else ""
if not candidate_sdp:
return
try:
self.peer_connection.add_remote_candidate(Candidate(candidate_sdp, str(candidate.get("sdpMid") or "")))
except Exception:
self.logger.warning("Ignoring invalid trickle ICE candidate", exc_info=True)
@property
def is_started(self) -> bool:
return self.peer_connection is not None and \
self.peer_connection.local_description() is not None and \
self.peer_connection.remote_description() is not None and \
self.peer_connection.state() != PeerConnection.State.Closed
@property
def is_connected_and_ready(self) -> bool:
return self.peer_connection is not None and \
self.peer_connection.state() == PeerConnection.State.Connected and \
(self.expected_number_of_incoming_media == 0 or self.incoming_media_ready_event.is_set())
async def _wait_for_gathering_complete(self):
if self.peer_connection.gathering_state() != PeerConnection.GatheringState.Complete:
await self.gathering_complete_event.wait()
async def _send_track_loop(self, rtc_track: Track, producer_track: TiciVideoStreamTrack, rtp_config: RtpPacketizationConfig):
while True:
if not rtc_track.is_open():
await asyncio.sleep(0.01)
continue
try:
packet = await producer_track.recv()
data = bytes(packet)
if not data:
continue
pts = int(packet.pts or 0)
timestamp = (rtp_config.start_timestamp + pts) & 0xFFFFFFFF
rtc_track.send_frame(data, FrameInfo(timestamp))
except asyncio.CancelledError:
raise
except Exception:
self.logger.exception("Error in send track loop for track %s", producer_track.id)
self._set_event(self.connection_stopped_event)
break
async def _receiver_report_loop(self):
while True:
for camera_type, (rtc_track, ssrc) in self._receiver_report_tracks.items():
for _ in range(32):
try:
message = rtc_track.receive()
if message is None: # go until queue empty (bounded to 32)
break
if isinstance(message, bytes):
self._update_receiver_report(camera_type, ssrc, message)
except asyncio.CancelledError:
raise
except Exception:
self.logger.exception("Error receiving report for %s", camera_type)
break
await asyncio.sleep(0.05)
async def _send_audio_track_loop(self, rtc_track: Track, producer: Any, rtp_config: RtpPacketizationConfig):
while True:
if not rtc_track.is_open():
await asyncio.sleep(0.01)
continue
try:
packet = await producer.recv()
if packet is None:
continue
data, pts = packet
if data:
rtc_track.send_frame(data, FrameInfo((rtp_config.start_timestamp + int(pts)) & 0xFFFFFFFF))
except asyncio.CancelledError:
raise
except Exception:
self.logger.exception("Error in audio send loop")
self._set_event(self.connection_stopped_event)
break
def _start_sender_tasks(self):
for rtc_track, producer_track, rtp_config in self._track_state:
self._sender_tasks.append(asyncio.create_task(self._send_track_loop(rtc_track, producer_track, rtp_config)))
for rtc_track, producer, rtp_config in self._audio_track_state:
self._sender_tasks.append(asyncio.create_task(self._send_audio_track_loop(rtc_track, producer, rtp_config)))
if self._track_state:
self._sender_tasks.append(asyncio.create_task(self._receiver_report_loop()))
async def wait_for_connection(self):
assert self.is_started
await self.connection_attempted_event.wait()
if self.peer_connection.state() != PeerConnection.State.Connected:
raise ValueError("Connection failed.")
if self.expected_number_of_incoming_media:
await self.incoming_media_ready_event.wait()
if self.messaging_channel is not None:
await self.messaging_channel_ready_event.wait()
self._start_sender_tasks()
async def wait_for_disconnection(self):
assert self.is_connected_and_ready, "Stream is not connected/ready yet (make sure wait_for_connection was awaited)"
await self.connection_stopped_event.wait()
async def stop(self):
for task in self._sender_tasks:
task.cancel()
for task in self._sender_tasks:
with contextlib.suppress(asyncio.CancelledError):
await task
self._sender_tasks.clear()
self.peer_connection.close()
self.peer_connection.reset_callbacks()
self.messaging_channel = None
self.incoming_camera_tracks.clear()
self.incoming_audio_tracks.clear()
self._consumer_tracks.clear()
self._track_state.clear()
self._audio_track_state.clear()
self._receiver_reports.clear()
self._receiver_report_tracks.clear()
@abc.abstractmethod
async def start(self) -> RTCSessionDescription:
raise NotImplementedError
class WebRTCOfferStream(WebRTCBaseStream):
def __init__(self, session_provider: ConnectionProvider, *args, **kwargs):
super().__init__(*args, **kwargs)
self.session_provider = session_provider
async def start(self) -> RTCSessionDescription:
self._loop = asyncio.get_running_loop()
self._add_consumer_transceivers()
if self.should_add_data_channel:
self._add_messaging_channel()
self.peer_connection.set_local_description(Description.Type.Offer)
await self._wait_for_gathering_complete()
actual_offer = self.peer_connection.local_description()
streaming_offer = StreamingOffer(
sdp=str(actual_offer),
video=list(self.expected_incoming_camera_types),
)
remote_answer = await self.session_provider(streaming_offer)
self._parse_incoming_streams(remote_sdp=remote_answer.sdp)
self.peer_connection.set_remote_description(Description(remote_answer.sdp, Description.Type.Answer))
self._on_after_media()
actual_answer = self.peer_connection.remote_description()
return RTCSessionDescription(str(actual_answer), actual_answer.type_string())
class WebRTCAnswerStream(WebRTCBaseStream):
def __init__(self, session: RTCSessionDescription, *args, **kwargs):
super().__init__(*args, **kwargs)
self.session = session
async def start(self) -> RTCSessionDescription:
self._loop = asyncio.get_running_loop()
assert self.peer_connection.remote_description() is None, "Connection already established"
self._parse_incoming_streams(remote_sdp=self.session.sdp)
self.peer_connection.set_remote_description(Description(self.session.sdp, Description.Type.Offer))
self._add_producer_tracks(self.session.sdp)
self.peer_connection.set_local_description(Description.Type.Answer)
await self._wait_for_gathering_complete()
actual_answer = self.peer_connection.local_description()
return RTCSessionDescription(str(actual_answer), actual_answer.type_string())
+77
View File
@@ -0,0 +1,77 @@
# ruff: noqa: UP006, UP035
import fractions
import logging
import uuid
from typing import Any, Tuple
VIDEO_CLOCK_RATE = 90000
VIDEO_TIME_BASE = fractions.Fraction(1, VIDEO_CLOCK_RATE)
def video_track_id(camera_type: str, track_id: str) -> str:
return f"{camera_type}:{track_id}"
def parse_video_track_id(track_id: str) -> Tuple[str, str]:
parts = track_id.split(":")
if len(parts) != 2:
raise ValueError(f"Invalid video track id: {track_id}")
camera_type, track_id = parts
return camera_type, track_id
class TiciVideoStreamTrack:
"""
Abstract video track which associates video track with camera_type.
"""
kind = "video"
def __init__(self, camera_type: str, dt: float, time_base: fractions.Fraction = VIDEO_TIME_BASE, clock_rate: int = VIDEO_CLOCK_RATE):
assert camera_type in ["driver", "wideRoad", "road"]
self._id: str = video_track_id(camera_type, str(uuid.uuid4()))
self._time_base: fractions.Fraction = time_base
self._clock_rate: int = clock_rate
self._logger = logging.getLogger("WebRTCStream")
self.readyState = "live"
@property
def id(self) -> str:
return self._id
def stop(self) -> None:
self.readyState = "ended"
def log_debug(self, msg: Any, *args):
self._logger.debug(f"{type(self)}() {msg}", *args)
async def recv(self):
raise NotImplementedError()
def request_keyframe(self) -> None:
pass
class TiciTrackWrapper(TiciVideoStreamTrack):
"""
Associates a generic video track with camera_type.
"""
def __init__(self, camera_type: str, track: Any):
assert track.kind == "video"
super().__init__(camera_type, getattr(track, "_dt", 0.05))
try:
parsed_camera, _ = parse_video_track_id(track.id)
self._id = track.id if parsed_camera == camera_type else video_track_id(camera_type, track.id)
except ValueError:
self._id = video_track_id(camera_type, track.id)
self._track = track
async def recv(self):
return await self._track.recv()
def stop(self) -> None:
super().stop()
if hasattr(self._track, "stop"):
self._track.stop()
+18 -4
View File
@@ -542,6 +542,22 @@ class StreamRequestBody:
ui_stream: bool = False
def _new_stream_session(offer_sdp: str, body: StreamRequestBody, debug_mode: bool):
if Params().get_bool("Konn3ktLibdatachannelWebRTC"):
try:
from openpilot.system.webrtc.webrtcd_ldc import StreamSessionLibdatachannel
return StreamSessionLibdatachannel(
offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
ui_stream=body.ui_stream,
)
except Exception:
logging.getLogger("webrtcd").exception("libdatachannel unavailable; falling back to aiortc")
return StreamSession(
offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
ui_stream=body.ui_stream,
)
async def get_stream(request: 'web.Request'):
stream_dict, debug_mode = request.app['streams'], request.app['debug']
logger = logging.getLogger("webrtcd")
@@ -561,8 +577,7 @@ async def get_stream(request: 'web.Request'):
logger.exception("Failed to stop previous stream session")
stream_dict.clear()
session = StreamSession(offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
ui_stream=body.ui_stream)
session = _new_stream_session(offer_sdp, body, debug_mode)
# Creating an answer can occasionally stall (ICE gathering, codec negotiation, etc).
# Bound it so the HTTP request doesn't hang forever and athena can surface a useful error.
try:
@@ -579,8 +594,7 @@ async def get_stream(request: 'web.Request'):
else:
logger.info("Retrying with fresh session and original SDP (no mDNS host candidates removed)")
session = StreamSession(retry_offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
ui_stream=body.ui_stream)
session = _new_stream_session(retry_offer_sdp, body, debug_mode)
answer = await asyncio.wait_for(session.get_answer(), timeout=15.0)
session.start()
+244
View File
@@ -0,0 +1,244 @@
import asyncio
import contextlib
import json
import logging
import os
import socket
import uuid
from typing import Any
from openpilot.common.params import Params
from cereal import messaging
from openpilot.system.webrtc.webrtcd import CerealIncomingMessageProxy, CerealOutgoingMessageProxy, CerealProxyRunner, DynamicPubMaster
def _default_route_ip() -> str | None:
"""Use the interface the kernel will actually use for Internet/relay media."""
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
sock.connect(("8.8.8.8", 53))
return sock.getsockname()[0]
except OSError:
return None
finally:
sock.close()
class LibdatachannelBitrateController:
"""Loss-driven bitrate control using native RTCP receiver reports."""
bitrates = [500_000, 1_500_000, int(os.environ.get("STREAM_BITRATE", 5_000_000))]
label_to_bitrate = {"low": bitrates[0], "med": bitrates[1], "high": bitrates[2]}
sample_interval = 0.2
high_loss = 0.10
sustained_loss = 0.05
down_samples = 5
def __init__(self, get_stats, enabled: bool = True):
self._get_stats = get_stats
self._enabled = enabled
self._auto = True
self._level = 1
self._counter = 0
self._up_samples = 5
self._previous: tuple[Any, ...] | None = None
self._task: asyncio.Task | None = None
self.current_bitrate = self.bitrates[self._level]
self._publish(self.current_bitrate)
def start(self) -> None:
if self._task is None:
self._task = asyncio.create_task(self.run())
def stop(self) -> None:
if self._task is not None:
self._task.cancel()
self._task = None
def enable(self, enabled: bool) -> None:
self._enabled = enabled
def set_quality(self, quality: str) -> None:
if quality in self.label_to_bitrate:
self._auto = False
self._publish(self.label_to_bitrate[quality])
elif quality == "auto":
self._auto = True
def _publish(self, bitrate: int) -> None:
self.current_bitrate = int(bitrate)
Params().put("LivestreamEncoderBitrate", self.current_bitrate)
def _sample_loss(self) -> float | None:
report = next(iter(self._get_stats().values()), None)
if report is None:
return None
current = (report.ssrc, report.fraction_lost, report.packets_lost, report.highest_seq_no, report.jitter, report.lsr, report.dlsr)
if current == self._previous:
return None
self._previous = current
return report.fraction_lost / 256.0
async def run(self) -> None:
while True:
await asyncio.sleep(self.sample_interval)
if not self._enabled or not self._auto:
continue
loss = self._sample_loss()
if loss is None:
continue
if loss >= self.sustained_loss and self._level > 0:
self._counter += 1
if loss >= self.high_loss or self._counter >= self.down_samples:
self._level -= 1
self._up_samples *= 2
self._counter = 0
self._publish(self.bitrates[self._level])
elif loss <= 0 and self._level < len(self.bitrates) - 1:
self._counter -= 1
if -self._counter >= self._up_samples:
self._level += 1
self._counter = 0
self._publish(self.bitrates[self._level])
class StreamSessionLibdatachannel:
"""IQ.Pilot's libdatachannel stream path. It preserves Konn3kt signalling and controls."""
shared_pub_master = DynamicPubMaster([])
def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], outgoing_services: list[str],
ice_servers: list[dict[str, Any]] | None = None, debug_mode: bool = False, ui_stream: bool = False):
from openpilot.system.webrtc.device.video_ldc import LiveStreamVideoStreamTrack
from openpilot.system.webrtc.teleoprtc_ldc.builder import WebRTCAnswerBuilder
from openpilot.system.webrtc.teleoprtc_ldc.info import parse_info_from_offer
config = parse_info_from_offer(sdp)
if len(cameras) != config.n_expected_camera_tracks:
raise ValueError("Incoming stream has misconfigured number of video tracks")
if debug_mode:
raise ValueError("libdatachannel debug tracks are not supported")
builder = WebRTCAnswerBuilder(sdp, bind_address=_default_route_ip(), ice_servers=ice_servers or [])
self.video_tracks = [LiveStreamVideoStreamTrack(camera) for camera in cameras]
for camera, track in zip(cameras, self.video_tracks, strict=True):
builder.add_video_stream(camera, track)
# The browser uses a single sendrecv audio m-line. libdatachannel's Python binding
# currently cannot negotiate that bidirectional track reliably, so this experimental
# transport deliberately remains video/control-only. The default aiortc path keeps
# both audio directions until the native duplex path passes the same integration test.
self.audio_output = None
self.stream = builder.stream()
self.identifier = str(uuid.uuid4())
self.incoming_bridge_services = incoming_services
self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master) if incoming_services else None
self.outgoing_bridge = CerealOutgoingMessageProxy(messaging.SubMaster(outgoing_services)) if outgoing_services else None
self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge) if self.outgoing_bridge is not None else None
self.ui_stream_requested = ui_stream
self.ui_stream_runner: CerealProxyRunner | None = None
self.audio_input_proxy = None
self.audio_recv_requested = False
self.bitrate_controller = LibdatachannelBitrateController(self.stream.get_receiver_report_stats)
self.run_task: asyncio.Task | None = None
self._cleanup_lock = asyncio.Lock()
self._cleanup_done = False
self.logger = logging.getLogger("webrtcd")
def start(self) -> None:
self.run_task = asyncio.create_task(self.run())
async def get_answer(self):
return await self.stream.start()
async def stop_async(self) -> None:
if self.run_task is not None and not self.run_task.done() and self.run_task is not asyncio.current_task():
self.run_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self.run_task
self.run_task = None
await self.post_run_cleanup()
def add_ice_candidate(self, candidate: Any) -> None:
self.stream.add_ice_candidate(candidate)
def message_handler(self, message: bytes | str) -> None:
try:
payload = json.loads(message)
except (TypeError, ValueError):
return
if not isinstance(payload, dict):
return
message_type = payload.get("type")
if message_type == "timingSei":
for track in self.video_tracks:
track.timing_sei_enabled = bool(payload.get("enabled", False))
elif message_type == "setQuality":
self.bitrate_controller.set_quality(str(payload.get("quality", "auto")))
elif message_type == "setAudioEnabled" and self.audio_output is not None:
self.audio_output.enable(bool(payload.get("enabled", True)))
elif message_type == "switchCamera":
for track in self.video_tracks:
track.switch_camera(str(payload.get("camera", "")))
elif message_type == "setUiStream":
self.set_ui_stream(bool(payload.get("enabled", False)))
elif self.incoming_bridge is not None:
try:
self.incoming_bridge.send(message)
except Exception:
self.logger.exception("Cereal incoming proxy failure")
def set_ui_stream(self, enabled: bool) -> None:
if enabled:
if self.ui_stream_runner is not None or not self.stream.has_messaging_channel():
return
from openpilot.system.webrtc.ui_stream import UIStreamMessageProxy
proxy = UIStreamMessageProxy(bitrate_getter=lambda: self.bitrate_controller.current_bitrate)
proxy.add_channel(self.stream.get_messaging_channel())
self.ui_stream_runner = CerealProxyRunner(proxy)
self.ui_stream_runner.start()
elif self.ui_stream_runner is not None:
self.ui_stream_runner.stop()
self.ui_stream_runner = None
async def run(self) -> None:
try:
await self.stream.wait_for_connection()
if self.stream.has_messaging_channel():
self.stream.set_message_handler(self.message_handler)
if self.incoming_bridge is not None:
await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services)
if self.outgoing_bridge_runner is not None:
self.outgoing_bridge_runner.proxy.add_channel(self.stream.get_messaging_channel())
self.outgoing_bridge_runner.start()
if self.ui_stream_requested:
self.set_ui_stream(True)
if self.audio_recv_requested and self.stream.has_incoming_audio_track():
from openpilot.system.webrtc.device.audio_ldc import IncomingOpusCerealProxy
self.audio_input_proxy = IncomingOpusCerealProxy(self.stream.get_incoming_audio_track())
self.audio_input_proxy.start()
self.bitrate_controller.start()
await self.stream.wait_for_disconnection()
except Exception:
self.logger.exception("libdatachannel stream session failure")
finally:
await self.post_run_cleanup()
async def post_run_cleanup(self) -> None:
async with self._cleanup_lock:
if self._cleanup_done:
return
self._cleanup_done = True
self.bitrate_controller.stop()
if self.ui_stream_runner is not None:
self.ui_stream_runner.stop()
self.ui_stream_runner = None
if self.outgoing_bridge_runner is not None:
self.outgoing_bridge_runner.stop()
if self.audio_input_proxy is not None:
await self.audio_input_proxy.stop()
for track in self.video_tracks:
track.stop()
await self.stream.stop()