openpilot v0.11.1 release

date: 2026-06-04T09:49:56
master commit: c0ab3550eca2e9daf197c46b7e4b24aa9637cf2e

system/webrtc/device/video.py

@@ -0,0 +1,45 @@
+import asyncio
+import time
+
+import av
+from teleoprtc.tracks import TiciVideoStreamTrack
+
+from cereal import messaging
+from openpilot.common.realtime import DT_MDL, DT_DMON
+
+
+class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
+  camera_to_sock_mapping = {
+    "driver": "livestreamDriverEncodeData",
+    "wideRoad": "livestreamWideRoadEncodeData",
+    "road": "livestreamRoadEncodeData",
+  }
+
+  def __init__(self, camera_type: str):
+    dt = DT_DMON if camera_type == "driver" else DT_MDL
+    super().__init__(camera_type, dt)
+
+    self._sock = messaging.sub_sock(self.camera_to_sock_mapping[camera_type], conflate=True)
+    self._pts = 0
+    self._t0_ns = time.monotonic_ns()
+
+  async def recv(self):
+    while True:
+      msg = messaging.recv_one_or_none(self._sock)
+      if msg is not None:
+        break
+      await asyncio.sleep(0.005)
+
+    evta = getattr(msg, msg.which())
+
+    packet = av.Packet(evta.header + evta.data)
+    packet.time_base = self._time_base
+
+    self._pts =  ((time.monotonic_ns() - self._t0_ns) * self._clock_rate) // 1_000_000_000
+    packet.pts = self._pts
+    self.log_debug("track sending frame %d", self._pts)
+
+    return packet
+
+  def codec_preference(self) -> str | None:
+    return "H264"

system/webrtc/schema.py

@@ -0,0 +1,43 @@
+import capnp
+from typing import Any
+
+
+def generate_type(type_walker, schema_walker) -> str | list[Any] | dict[str, Any]:
+  data_type = next(type_walker)
+  if data_type.which() == 'struct':
+    return generate_struct(next(schema_walker))
+  elif data_type.which() == 'list':
+    _ = next(schema_walker)
+    return [generate_type(type_walker, schema_walker)]
+  elif data_type.which() == 'enum':
+    return "text"
+  else:
+    return str(data_type.which())
+
+
+def generate_struct(schema: capnp.lib.capnp._StructSchema) -> dict[str, Any]:
+  return {field: generate_field(schema.fields[field]) for field in schema.fields if not field.endswith("DEPRECATED") and field != "deprecated"}
+
+
+def generate_field(field: capnp.lib.capnp._StructSchemaField) -> str | list[Any] | dict[str, Any]:
+  def schema_walker(field):
+    yield field.schema
+
+    s = field.schema
+    while hasattr(s, 'elementType'):
+      s = s.elementType
+      yield s
+
+  def type_walker(field):
+    yield field.proto.slot.type
+
+    t = field.proto.slot.type
+    while hasattr(getattr(t, t.which()), 'elementType'):
+      t = getattr(t, t.which()).elementType
+      yield t
+
+  if field.proto.which() == "slot":
+    schema_gen, type_gen = schema_walker(field), type_walker(field)
+    return generate_type(type_gen, schema_gen)
+  else:
+    return generate_struct(field.schema)

system/webrtc/tests/test_stream_session.py

@@ -0,0 +1,87 @@
+import asyncio
+import json
+import time
+# for aiortc and its dependencies
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
+
+from aiortc import RTCDataChannel
+from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
+import capnp
+from cereal import messaging, log
+
+from openpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy
+from openpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
+
+
+class TestStreamSession:
+  def setup_method(self):
+    self.loop = asyncio.new_event_loop()
+
+  def teardown_method(self):
+    self.loop.stop()
+    self.loop.close()
+
+  def test_outgoing_proxy(self, mocker):
+    test_msg = log.Event.new_message()
+    test_msg.logMonoTime = 123
+    test_msg.valid = True
+    test_msg.customReservedRawData0 = b"test"
+    expected_dict = {"type": "customReservedRawData0", "logMonoTime": 123, "valid": True, "data": "test"}
+    expected_json = json.dumps(expected_dict).encode()
+
+    channel = mocker.Mock(spec=RTCDataChannel)
+    mocked_submaster = messaging.SubMaster(["customReservedRawData0"])
+    def mocked_update(t):
+      mocked_submaster.update_msgs(0, [test_msg])
+
+    mocker.patch.object(messaging.SubMaster, "update", side_effect=mocked_update)
+    proxy = CerealOutgoingMessageProxy(mocked_submaster)
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    channel.send.assert_called_once_with(expected_json)
+
+  def test_incoming_proxy(self, mocker):
+    tested_msgs = [
+      {"type": "customReservedRawData0", "data": "test"}, # primitive
+      {"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]}, # list
+      {"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}}, # dict
+    ]
+
+    mocked_pubmaster = mocker.MagicMock(spec=messaging.PubMaster)
+
+    proxy = CerealIncomingMessageProxy(mocked_pubmaster)
+
+    for msg in tested_msgs:
+      proxy.send(json.dumps(msg).encode())
+
+      mocked_pubmaster.send.assert_called_once()
+      mt, md = mocked_pubmaster.send.call_args.args
+      assert mt == msg["type"]
+      assert isinstance(md, capnp._DynamicStructBuilder)
+      assert hasattr(md, msg["type"])
+
+      mocked_pubmaster.reset_mock()
+
+  def test_livestream_track(self, mocker):
+    fake_msg = messaging.new_message("livestreamDriverEncodeData")
+
+    config = {"receive.return_value": fake_msg.to_bytes()}
+    mocker.patch("msgq.SubSocket", spec=True, **config)
+    track = LiveStreamVideoStreamTrack("driver")
+
+    assert track.id.startswith("driver")
+    assert track.codec_preference() == "H264"
+
+    for i in range(5):
+      packet = self.loop.run_until_complete(track.recv())
+      assert packet.time_base == VIDEO_TIME_BASE
+      if i == 0:
+        start_ns = time.monotonic_ns()
+        start_pts = packet.pts
+      assert abs(i + packet.pts - (start_pts + (((time.monotonic_ns() - start_ns) * VIDEO_CLOCK_RATE) // 1_000_000_000))) < 450 #5ms
+      assert packet.size == 0
+

system/webrtc/webrtcd.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import json
+import uuid
+import logging
+from dataclasses import dataclass, field
+from typing import Any, TYPE_CHECKING
+
+# aiortc and its dependencies have lots of internal warnings :(
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
+
+import capnp
+from aiohttp import web
+if TYPE_CHECKING:
+  from aiortc.rtcdatachannel import RTCDataChannel
+
+from openpilot.system.webrtc.schema import generate_field
+from cereal import messaging, log
+
+
+class CerealOutgoingMessageProxy:
+  def __init__(self, sm: messaging.SubMaster):
+    self.sm = sm
+    self.channels: list[RTCDataChannel] = []
+
+  def add_channel(self, channel: 'RTCDataChannel'):
+    self.channels.append(channel)
+
+  def to_json(self, msg_content: Any):
+    if isinstance(msg_content, capnp._DynamicStructReader):
+      msg_dict = msg_content.to_dict()
+    elif isinstance(msg_content, capnp._DynamicListReader):
+      msg_dict = [self.to_json(msg) for msg in msg_content]
+    elif isinstance(msg_content, bytes):
+      msg_dict = msg_content.decode()
+    else:
+      msg_dict = msg_content
+
+    return msg_dict
+
+  def update(self):
+    # this is blocking in async context...
+    self.sm.update(0)
+    for service, updated in self.sm.updated.items():
+      if not updated:
+        continue
+      msg_dict = self.to_json(self.sm[service])
+      mono_time, valid = self.sm.logMonoTime[service], self.sm.valid[service]
+      outgoing_msg = {"type": service, "logMonoTime": mono_time, "valid": valid, "data": msg_dict}
+      encoded_msg = json.dumps(outgoing_msg).encode()
+      for channel in self.channels:
+        channel.send(encoded_msg)
+
+
+class CerealIncomingMessageProxy:
+  def __init__(self, pm: messaging.PubMaster):
+    self.pm = pm
+
+  def send(self, message: bytes):
+    msg_json = json.loads(message)
+    msg_type, msg_data = msg_json["type"], msg_json["data"]
+    size = None
+    if not isinstance(msg_data, dict):
+      size = len(msg_data)
+
+    msg = messaging.new_message(msg_type, size=size)
+    setattr(msg, msg_type, msg_data)
+    self.pm.send(msg_type, msg)
+
+
+class CerealProxyRunner:
+  def __init__(self, proxy: CerealOutgoingMessageProxy):
+    self.proxy = proxy
+    self.is_running = False
+    self.task = None
+    self.logger = logging.getLogger("webrtcd")
+
+  def start(self):
+    assert self.task is None
+    self.task = asyncio.create_task(self.run())
+
+  def stop(self):
+    if self.task is None or self.task.done():
+      return
+    self.task.cancel()
+    self.task = None
+
+  async def run(self):
+    from aiortc.exceptions import InvalidStateError
+
+    while True:
+      try:
+        self.proxy.update()
+      except InvalidStateError:
+        self.logger.warning("Cereal outgoing proxy invalid state (connection closed)")
+        break
+      except Exception:
+        self.logger.exception("Cereal outgoing proxy failure")
+      await asyncio.sleep(0.01)
+
+
+class DynamicPubMaster(messaging.PubMaster):
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.lock = asyncio.Lock()
+
+  async def add_services_if_needed(self, services):
+    async with self.lock:
+      for service in services:
+        if service not in self.sock:
+          self.sock[service] = messaging.pub_sock(service)
+
+
+class StreamSession:
+  shared_pub_master = DynamicPubMaster([])
+
+  def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], outgoing_services: list[str], debug_mode: bool = False):
+    from aiortc.mediastreams import VideoStreamTrack
+    from openpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
+    from teleoprtc import WebRTCAnswerBuilder
+    from teleoprtc.info import parse_info_from_offer
+
+    config = parse_info_from_offer(sdp)
+    builder = WebRTCAnswerBuilder(sdp)
+
+    assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks"
+    for cam in cameras:
+      builder.add_video_stream(cam, LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack())
+
+    self.stream = builder.stream()
+    self.identifier = str(uuid.uuid4())
+
+    self.incoming_bridge: CerealIncomingMessageProxy | None = None
+    self.incoming_bridge_services = incoming_services
+    self.outgoing_bridge: CerealOutgoingMessageProxy | None = None
+    self.outgoing_bridge_runner: CerealProxyRunner | None = None
+    if len(incoming_services) > 0:
+      self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master)
+    if len(outgoing_services) > 0:
+      self.outgoing_bridge = CerealOutgoingMessageProxy(messaging.SubMaster(outgoing_services))
+      self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge)
+
+    self.run_task: asyncio.Task | None = None
+    self.logger = logging.getLogger("webrtcd")
+    self.logger.info("New stream session (%s), cameras %s, incoming services %s, outgoing services %s",
+                      self.identifier, cameras, incoming_services, outgoing_services)
+
+  def start(self):
+    self.run_task = asyncio.create_task(self.run())
+
+  def stop(self):
+    if self.run_task.done():
+      return
+    self.run_task.cancel()
+    self.run_task = None
+    asyncio.run(self.post_run_cleanup())
+
+  async def get_answer(self):
+    return await self.stream.start()
+
+  async def message_handler(self, message: bytes):
+    assert self.incoming_bridge is not None
+    try:
+      self.incoming_bridge.send(message)
+    except Exception:
+      self.logger.exception("Cereal incoming proxy failure")
+
+  async def run(self):
+    try:
+      await self.stream.wait_for_connection()
+      if self.stream.has_messaging_channel():
+        if self.incoming_bridge is not None:
+          await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services)
+          self.stream.set_message_handler(self.message_handler)
+        if self.outgoing_bridge_runner is not None:
+          channel = self.stream.get_messaging_channel()
+          self.outgoing_bridge_runner.proxy.add_channel(channel)
+          self.outgoing_bridge_runner.start()
+      self.logger.info("Stream session (%s) connected", self.identifier)
+
+      await self.stream.wait_for_disconnection()
+      await self.post_run_cleanup()
+
+      self.logger.info("Stream session (%s) ended", self.identifier)
+    except Exception:
+      self.logger.exception("Stream session failure")
+
+  async def post_run_cleanup(self):
+    await self.stream.stop()
+    if self.outgoing_bridge is not None:
+      self.outgoing_bridge_runner.stop()
+
+
+@dataclass
+class StreamRequestBody:
+  sdp: str
+  cameras: list[str]
+  bridge_services_in: list[str] = field(default_factory=list)
+  bridge_services_out: list[str] = field(default_factory=list)
+
+
+async def get_stream(request: 'web.Request'):
+  stream_dict, debug_mode = request.app['streams'], request.app['debug']
+  raw_body = await request.json()
+  body = StreamRequestBody(**raw_body)
+
+  session = StreamSession(body.sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, debug_mode)
+  answer = await session.get_answer()
+  session.start()
+
+  stream_dict[session.identifier] = session
+
+  return web.json_response({"sdp": answer.sdp, "type": answer.type})
+
+
+async def get_schema(request: 'web.Request'):
+  services = request.query["services"].split(",")
+  services = [s for s in services if s]
+  assert all(s in log.Event.schema.fields and not s.endswith("DEPRECATED") for s in services), "Invalid service name"
+  schema_dict = {s: generate_field(log.Event.schema.fields[s]) for s in services}
+  return web.json_response(schema_dict)
+
+async def post_notify(request: 'web.Request'):
+  try:
+    payload = await request.json()
+  except Exception as e:
+    raise web.HTTPBadRequest(text="Invalid JSON") from e
+
+  for session in list(request.app.get('streams', {}).values()):
+    try:
+      ch = session.stream.get_messaging_channel()
+      ch.send(json.dumps(payload))
+    except Exception:
+      continue
+
+  return web.Response(status=200, text="OK")
+
+async def on_shutdown(app: 'web.Application'):
+  for session in app['streams'].values():
+    session.stop()
+  del app['streams']
+
+
+def webrtcd_thread(host: str, port: int, debug: bool):
+  logging.basicConfig(level=logging.CRITICAL, handlers=[logging.StreamHandler()])
+  logging_level = logging.DEBUG if debug else logging.INFO
+  logging.getLogger("WebRTCStream").setLevel(logging_level)
+  logging.getLogger("webrtcd").setLevel(logging_level)
+
+  app = web.Application()
+
+  app['streams'] = dict()
+  app['debug'] = debug
+  app.on_shutdown.append(on_shutdown)
+  app.router.add_post("/stream", get_stream)
+  app.router.add_post("/notify", post_notify)
+  app.router.add_get("/schema", get_schema)
+
+  web.run_app(app, host=host, port=port)
+
+
+def main():
+  parser = argparse.ArgumentParser(description="WebRTC daemon")
+  parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to listen on")
+  parser.add_argument("--port", type=int, default=5001, help="Port to listen on")
+  parser.add_argument("--debug", action="store_true", help="Enable debug mode")
+  args = parser.parse_args()
+
+  webrtcd_thread(args.host, args.port, args.debug)
+
+
+if __name__=="__main__":
+  main()