Fix offline GPU firmware and Connect streaming

.gitignore

@@ -81,6 +81,7 @@ selfdrive/modeld/models/*.pkl
 # openpilot log files
 *.bz2
 *.zst
+!selfdrive/modeld/firmware/amdgpu/*.zst
 
 build/
 

selfdrive/modeld/compile_modeld.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 import argparse
 import atexit
+import hashlib
 import math
 import os
+import pathlib
 import pickle
 import shutil
 import tempfile
@@ -13,10 +15,26 @@ from functools import partial
 import numpy as np
 
 
-def _patch_tinygrad_fetch_fw():
-  import hashlib
-  import pathlib
+FIRMWARE_ROOTS = (
+  pathlib.Path(__file__).resolve().parent / "firmware",
+  pathlib.Path("/lib/firmware"),
+)
+FIRMWARE_CACHE_DIR = pathlib.Path("/data/tinygrad_fw_cache")
 
+
+def _read_firmware(path, sha256, compressed=False, zstandard_module=None):
+  if not path.is_file():
+    return None
+
+  blob = path.read_bytes()
+  if compressed:
+    if zstandard_module is None:
+      import zstandard as zstandard_module
+    blob = zstandard_module.ZstdDecompressor().stream_reader(blob).read()
+  return blob if hashlib.sha256(blob).hexdigest() == sha256 else None
+
+
+def _patch_tinygrad_fetch_fw():
   try:
     import zstandard
   except ImportError:
@@ -28,12 +46,41 @@ def _patch_tinygrad_fetch_fw():
     return
 
   def fetch_fw(path, name, sha256):
-    firmware_path = pathlib.Path(f"/lib/firmware/{path}/{name}.zst")
-    if firmware_path.is_file():
-      blob = zstandard.ZstdDecompressor().stream_reader(firmware_path.read_bytes()).read()
-      if hashlib.sha256(blob).hexdigest() == sha256:
+    for root in FIRMWARE_ROOTS:
+      if (blob := _read_firmware(root / path / f"{name}.zst", sha256, compressed=True, zstandard_module=zstandard)) is not None:
         return blob
-    return original_fetch_fw(path, name, sha256)
+
+    cached_path = FIRMWARE_CACHE_DIR / path / f"{name}.{sha256}"
+    if (blob := _read_firmware(cached_path, sha256)) is not None:
+      return blob
+
+    last_error = None
+    for attempt in range(3):
+      if attempt:
+        time.sleep(5)
+      try:
+        blob = original_fetch_fw(path, name, sha256)
+        break
+      except Exception as error:
+        last_error = error
+    else:
+      assert last_error is not None
+      raise last_error
+
+    cache_path = None
+    try:
+      cached_path.parent.mkdir(parents=True, exist_ok=True)
+      with tempfile.NamedTemporaryFile(dir=cached_path.parent, delete=False) as cache_file:
+        cache_file.write(blob)
+        cache_path = pathlib.Path(cache_file.name)
+      cache_path.replace(cached_path)
+    except OSError:
+      try:
+        if cache_path is not None:
+          cache_path.unlink(missing_ok=True)
+      except OSError:
+        pass
+    return blob
 
   helpers.fetch_fw = fetch_fw
 

selfdrive/modeld/tests/test_amd_firmware.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+import hashlib
+from pathlib import Path
+
+import pytest
+import zstandard
+
+from tinygrad.runtime.autogen.am import fw
+
+
+CHESTNUT_FIRMWARE = (
+  "gc_12_0_0_imu.bin",
+  "gc_12_0_0_me.bin",
+  "gc_12_0_0_mec.bin",
+  "gc_12_0_0_pfp.bin",
+  "gc_12_0_0_rlc.bin",
+  "psp_14_0_2_sos.bin",
+  "sdma_7_0_0.bin",
+  "smu_14_0_2.bin",
+)
+
+
+@pytest.mark.parametrize("name", CHESTNUT_FIRMWARE)
+def test_bundled_chestnut_firmware(name):
+  path = Path(__file__).parents[1] / "firmware" / "amdgpu" / f"{name}.zst"
+  blob = zstandard.ZstdDecompressor().stream_reader(path.read_bytes()).read()
+  assert hashlib.sha256(blob).hexdigest() == fw.hashes[name]

system/webrtc/webrtcd.py

@@ -21,10 +21,15 @@ from typing import Any
 from openpilot.system.webrtc.helpers import StreamRequestBody
 from openpilot.system.webrtc.schema import generate_field
 from openpilot.common.params import Params
+from openpilot.common.swaglog import cloudlog
 from cereal import messaging, log
 
 SESSION_TIMEOUT_SECONDS = 300
 
+
+def _ice_candidates(sdp: str) -> list[str]:
+  return [line.removeprefix("a=") for line in sdp.splitlines() if line.startswith("a=candidate:")]
+
 # socket trick: route lookup for 8.8.8.8 (nothing is sent or actually connected to)
 # return the source interfaces IP which is the default interface of the device
 def _default_route_ip() -> str | None:
@@ -252,7 +257,7 @@ class StreamSession:
     self.run_task: asyncio.Task | None = None
     self._cleanup_lock = asyncio.Lock()
     self._cleanup_done = False
-    self.logger = logging.getLogger("webrtcd")
+    self.logger = cloudlog
     self.logger.info(
       "New stream session (%s), video cameras %s, video enabled %s, incoming services %s, outgoing services %s",
       self.identifier, [t.id for t in self.video_tracks], body.enabled, body.bridge_services_in, body.bridge_services_out,
@@ -350,7 +355,10 @@ class StreamSession:
         await self.run_normal_session()
       self.logger.info("Stream session (%s) ended", self.identifier)
     except Exception:
-      self.logger.exception("Stream session failure")
+      pc = self.stream.peer_connection
+      with cloudlog.ctx(session_id=self.identifier, connection_state=str(pc.state()), ice_state=str(pc.ice_state()),
+                        gathering_state=str(pc.gathering_state())):
+        cloudlog.exception("webrtcd.session.failure")
     finally:
       await self.post_run_cleanup()
 
@@ -427,6 +435,8 @@ async def handle_get_stream(state: ServerState, raw_body: bytes, content_type: s
     stream_dict[session.identifier] = session
     try:
       answer = await asyncio.wait_for(session.get_answer(), timeout=30)
+      cloudlog.event("webrtcd.session.ice_candidates", session_id=session.identifier,
+                     offer_candidates=_ice_candidates(body.sdp), answer_candidates=_ice_candidates(answer.sdp))
     except TimeoutError:
       await session.stop()
       stream_dict.pop(session.identifier, None)

tools/webrtc/README.md

@@ -0,0 +1,58 @@
+# WebRTC ICE compatibility
+
+The August 24, 2026 AGNOS update (`05140149fb`) replaced the aiortc backend
+with libdatachannel. `libdatachannel-py==2026.1.0.dev2` embeds libdatachannel
+v0.24.0 and libjuice revision `5948a4162d37bc213d6051b67ee2876ccc5a99a6`.
+
+That libjuice revision uses a zero ICE tie-breaker to mean "role attribute
+absent". A present `ICE-CONTROLLING` attribute containing zero is consequently
+rejected with STUN `400 Bad Request`. This was captured on the device during
+Connect attempts: its outgoing checks succeed, but it rejects the browser's
+nominating checks. It remains at ICE Connected, never completes DTLS, and
+Connect eventually reports that no direct peer-to-peer routes were found.
+
+The patch records attribute presence separately from the 64-bit value. It does
+not change credentials, integrity checks, DTLS, camera handling, Sentry, media
+codecs, or data-channel behavior. It also continues to reject missing roles and
+rejects requests containing both role attributes.
+
+## Build and verify
+
+On the target architecture, with git, uv, a C/C++ compiler and Python 3.12:
+
+```sh
+bash tools/webrtc/build_libdatachannel.sh /absolute/path/to/wheels /usr/bin/python3.12
+```
+
+This builds `2026.1.0.dev2+starpilot.ice1` in an isolated temporary environment
+and runs real loopback UDP checks. Source and build directories are retained
+for inspection. It does not install into the device's runtime environment.
+
+To test an installed runtime independently:
+
+```sh
+/usr/local/venv/bin/python tools/webrtc/check_ice.py
+```
+
+The original wheel fails the zero-tie-breaker case; a corrected wheel must pass
+all cases, including invalid-authentication rejection. A successful loopback
+check is not proof that a Connect video session works; verify that separately.
+
+## Deployment
+
+The correction is in a compiled dependency, not just openpilot Python source.
+A git pull alone does not replace the affected AGNOS library. Install the
+validated, architecture-matching wheel into the image's managed Python
+environment and run the check above when building a release image. Existing
+devices need that runtime update too. Back up the original package and its
+distribution metadata before a temporary device-side installation; a later
+AGNOS image or dependency sync can otherwise overwrite the hotfix.
+
+AGNOS normally mounts the OS read-only. For a reversible live test, the
+corrected extension can be bind-mounted read-only over the original extension
+file. This leaves the OS partition unchanged, but the test hotfix is lost on
+reboot. Do not confuse that with deploying a corrected release image.
+
+Upstream sources: [Python bindings](https://github.com/shiguredo/libdatachannel-py/tree/989d29a32968046a002b5b9deb7a00f5012c530c),
+[libjuice](https://github.com/paullouisageneau/libjuice/tree/5948a4162d37bc213d6051b67ee2876ccc5a99a6).
+The libjuice source modification is covered by its MPL-2.0 license.

tools/webrtc/build_libdatachannel.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Build the existing WebRTC backend with the ICE role-presence correction.
+# Requires git, uv, a C/C++ compiler, and Python 3.12+. Does not install anything
+# into the running device environment. The output wheel is architecture-specific.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+PATCH_DIR="$SCRIPT_DIR/patches"
+OUTPUT_DIR="${1:?usage: bash tools/webrtc/build_libdatachannel.sh OUTPUT_DIR [PYTHON]}"
+BUILD_PYTHON="${2:-python3.12}"
+SOURCE_REV="989d29a32968046a002b5b9deb7a00f5012c530c"
+BUILD_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/starpilot-webrtc.XXXXXX")"
+echo "Build directory (retained for inspection): $BUILD_ROOT"
+mkdir -p -- "$OUTPUT_DIR"
+OUTPUT_DIR="$(cd -- "$OUTPUT_DIR" && pwd)"
+
+git clone --depth 1 --branch 2026.1.0.dev2 https://github.com/shiguredo/libdatachannel-py.git "$BUILD_ROOT/source"
+[[ "$(git -C "$BUILD_ROOT/source" rev-parse HEAD)" == "$SOURCE_REV" ]]
+git -C "$BUILD_ROOT/source" apply "$PATCH_DIR/libdatachannel-build.patch"
+cp "$PATCH_DIR/libjuice-zero-tiebreaker.patch" "$BUILD_ROOT/source/"
+
+uv venv --python "$BUILD_PYTHON" "$BUILD_ROOT/venv"
+uv pip install --python "$BUILD_ROOT/venv/bin/python" \
+  build==1.6.0 scikit-build-core==1.0.3 nanobind==3.0.1 cmake==4.4.3 ninja==1.13.2
+export PATH="$BUILD_ROOT/venv/bin:$PATH"
+export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-2}"
+cd -- "$BUILD_ROOT/source"
+python -m build --wheel --no-isolation --outdir "$OUTPUT_DIR"
+uv pip install --python "$BUILD_ROOT/venv/bin/python" --no-index --find-links "$OUTPUT_DIR" \
+  'libdatachannel-py==2026.1.0.dev2+starpilot.ice1'
+python "$SCRIPT_DIR/check_ice.py"
+echo "Validated wheel written to $OUTPUT_DIR; the running environment was not modified."

tools/webrtc/check_ice.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Check the installed WebRTC library's ICE role handling over real UDP sockets.
+
+No cameras, device parameters, external servers, or live sessions are used.
+Run this with the same Python environment as webrtcd when validating an image.
+"""
+
+import binascii
+import hashlib
+import hmac
+import secrets
+import socket
+import struct
+import time
+
+from libdatachannel import Configuration, Description, PeerConnection
+
+
+def attribute(kind: int, value: bytes) -> bytes:
+  return struct.pack("!HH", kind, len(value)) + value + bytes(-len(value) % 4)
+
+
+def binding_request(username: str, password: str, roles: list[tuple[int, int]]) -> tuple[bytes, bytes]:
+  transaction = secrets.token_bytes(12)
+  attrs = attribute(0x0006, username.encode())
+  # Match Chromium's attribute ordering, including its optional network info.
+  attrs += attribute(0xC057, bytes(4))
+  for role, tiebreaker in roles:
+    attrs += attribute(role, struct.pack("!Q", tiebreaker))
+  attrs += attribute(0x0025, b"") + attribute(0x0024, struct.pack("!I", 1853824767))
+  def header(size: int) -> bytes:
+    return struct.pack("!HHI12s", 1, size, 0x2112A442, transaction)
+
+  integrity = hmac.new(password.encode(), header(len(attrs) + 24) + attrs, hashlib.sha1).digest()
+  attrs += attribute(0x0008, integrity)
+  packet = header(len(attrs) + 8) + attrs
+  packet += attribute(0x8028, struct.pack("!I", binascii.crc32(packet) ^ 0x5354554E))
+  return packet, transaction
+
+
+def sdp_value(sdp: str, name: str) -> str:
+  prefix = f"a={name}:"
+  return next(line[len(prefix):] for line in sdp.splitlines() if line.startswith(prefix))
+
+
+def check_binding(roles: list[tuple[int, int]], *, invalid_password: bool = False) -> int | None:
+  config = Configuration()
+  config.bind_address = "127.0.0.1"
+  config.disable_auto_negotiation = True
+  offerer = PeerConnection(config)
+  answerer = PeerConnection(config)
+  channel = offerer.create_data_channel("data")
+  try:
+    offerer.set_local_description(Description.Type.Offer)
+    offer = str(offerer.local_description())
+    # Only our probe sends checks; do not start an independent connection.
+    offer = "\r\n".join(line for line in offer.splitlines() if not line.startswith("a=candidate:")) + "\r\n"
+    answerer.set_remote_description(Description(offer, Description.Type.Offer))
+    answerer.set_local_description(Description.Type.Answer)
+    deadline = time.monotonic() + 2
+    while answerer.gathering_state() != PeerConnection.GatheringState.Complete:
+      if time.monotonic() >= deadline:
+        raise TimeoutError("Loopback candidate gathering timed out")
+      time.sleep(0.01)
+    answer = str(answerer.local_description())
+    candidate = sdp_value(answer, "candidate").split()
+    username = f"{sdp_value(answer, 'ice-ufrag')}:{sdp_value(offer, 'ice-ufrag')}"
+    password = "invalid-test-password" if invalid_password else sdp_value(answer, "ice-pwd")
+    packet, transaction = binding_request(username, password, roles)
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
+      sock.bind(("127.0.0.1", 0))
+      sock.settimeout(1)
+      sock.sendto(packet, (candidate[4], int(candidate[5])))
+      deadline = time.monotonic() + 1
+      while time.monotonic() < deadline:
+        try:
+          response = sock.recv(2048)
+        except TimeoutError:
+          return None
+        if len(response) < 20 or response[8:20] != transaction:
+          continue
+        if response[:2] == b"\x01\x01":
+          return 200
+        offset = 20
+        while offset + 4 <= len(response):
+          kind, size = struct.unpack_from("!HH", response, offset)
+          value = response[offset + 4:offset + 4 + size]
+          if kind == 0x0009 and len(value) >= 4:
+            return value[2] * 100 + value[3]
+          offset += 4 + (size + 3) // 4 * 4
+        raise AssertionError("Unexpected ICE response")
+    return None
+  finally:
+    answerer.close()
+    offerer.close()
+    # Keep the wrapper alive until after its connection has closed.
+    assert channel is not None
+
+
+def main() -> None:
+  cases = [
+    ("nonzero controlling tiebreaker", [(0x802A, 123)], False, 200),
+    ("zero controlling tiebreaker", [(0x802A, 0)], False, 200),
+    ("missing role rejected", [], False, 400),
+    ("both roles rejected", [(0x802A, 123), (0x8029, 456)], False, 400),
+    ("invalid authentication rejected", [(0x802A, 0)], True, None),
+  ]
+  failed = False
+  for name, roles, invalid_password, expected in cases:
+    actual = check_binding(roles, invalid_password=invalid_password)
+    passed = actual == expected
+    print(f"{'PASS' if passed else 'FAIL'}: {name}: expected {expected}, got {actual}", flush=True)
+    failed |= not passed
+  raise SystemExit(int(failed))
+
+
+if __name__ == "__main__":
+  main()

tools/webrtc/patches/libdatachannel-build.patch

@@ -0,0 +1,20 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index ba0479b..04a5134 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -122,6 +122,7 @@ else()
+     GIT_SHALLOW TRUE
+     GIT_SUBMODULES_RECURSE TRUE
+     SOURCE_DIR ${LIBDATACHANNEL_SOURCE_DIR}
++    PATCH_COMMAND git -C ${LIBDATACHANNEL_SOURCE_DIR}/deps/libjuice apply ${PROJECT_ROOT}/libjuice-zero-tiebreaker.patch
+     BINARY_DIR ${LIBDATACHANNEL_BUILD_DIR}
+     STAMP_DIR ${LIBDATACHANNEL_BUILD_DIR}/stamp
+     TMP_DIR ${LIBDATACHANNEL_BUILD_DIR}/tmp
+diff --git a/VERSION b/VERSION
+index 7655152..2270f25 100644
+--- a/VERSION
++++ b/VERSION
+@@ -1 +1 @@
+-2026.1.0.dev2
+\ No newline at end of file
++2026.1.0.dev2+starpilot.ice1

tools/webrtc/patches/libjuice-zero-tiebreaker.patch

@@ -0,0 +1,99 @@
+diff --git a/src/agent.c b/src/agent.c
+index 00bbecb..40819eb 100644
+--- a/src/agent.c
++++ b/src/agent.c
+@@ -1431,7 +1431,7 @@ int agent_process_stun_binding(juice_agent_t *agent, const stun_message_t *msg,
+ 			return -1;
+ 
+ 		ice_candidate_pair_t *pair = entry->pair;
+-		if (msg->ice_controlling == msg->ice_controlled) {
++		if (msg->has_ice_controlling == msg->has_ice_controlled) {
+ 			JLOG_WARN("Controlling and controlled attributes mismatch in request");
+ 			agent_send_stun_binding(agent, entry, STUN_CLASS_RESP_ERROR, 400, msg->transaction_id,
+ 			                        NULL);
+@@ -1445,7 +1445,7 @@ int agent_process_stun_binding(juice_agent_t *agent, const stun_message_t *msg,
+ 		//  ERROR-CODE attribute with a value of 487 (Role Conflict) but retains its role.
+ 		//  * If the agent's tiebreaker value is less than the contents of the ICE-CONTROLLING
+ 		//  attribute, the agent switches to the controlled role.
+-		if (agent->mode == AGENT_MODE_CONTROLLING && msg->ice_controlling) {
++		if (agent->mode == AGENT_MODE_CONTROLLING && msg->has_ice_controlling) {
+ 			JLOG_WARN("ICE role conflict (both controlling)");
+ 			if (agent->ice_tiebreaker >= msg->ice_controlling) {
+ 				JLOG_DEBUG("Asking remote peer to switch roles");
+@@ -1465,7 +1465,7 @@ int agent_process_stun_binding(juice_agent_t *agent, const stun_message_t *msg,
+ 		//  * If the agent's tiebreaker value is less than the contents of the ICE-CONTROLLED
+ 		//  attribute, the agent generates a Binding error response and includes an ERROR-CODE
+ 		//  attribute with a value of 487 (Role Conflict) but retains its role.
+-		if (agent->mode == AGENT_MODE_CONTROLLED && msg->ice_controlled) {
++		if (agent->mode == AGENT_MODE_CONTROLLED && msg->has_ice_controlled) {
+ 			JLOG_WARN("ICE role conflict (both controlled)");
+ 			if (agent->ice_tiebreaker >= msg->ice_controlling) {
+ 				JLOG_DEBUG("Switching to controlling role");
+@@ -1479,7 +1479,7 @@ int agent_process_stun_binding(juice_agent_t *agent, const stun_message_t *msg,
+ 			break;
+ 		}
+ 		if (msg->use_candidate) {
+-			if (!msg->ice_controlling) {
++			if (!msg->has_ice_controlling) {
+ 				JLOG_WARN("STUN message use_candidate missing ice_controlling attribute");
+ 				agent_send_stun_binding(agent, entry, STUN_CLASS_RESP_ERROR, 400,
+ 				                        msg->transaction_id, NULL);
+@@ -1709,6 +1709,8 @@ int agent_send_stun_binding(juice_agent_t *agent, agent_stun_entry_t *entry, stu
+ 			password = agent->remote.ice_pwd;
+ 			msg.ice_controlling = agent->mode == AGENT_MODE_CONTROLLING ? agent->ice_tiebreaker : 0;
+ 			msg.ice_controlled = agent->mode == AGENT_MODE_CONTROLLED ? agent->ice_tiebreaker : 0;
++			msg.has_ice_controlling = agent->mode == AGENT_MODE_CONTROLLING;
++			msg.has_ice_controlled = agent->mode == AGENT_MODE_CONTROLLED;
+ 
+ 			// RFC 8445 7.1.1. PRIORITY
+ 			// The PRIORITY attribute MUST be included in a Binding request and be set to the value
+diff --git a/src/stun.c b/src/stun.c
+index 842b6c1..06e90ec 100644
+--- a/src/stun.c
++++ b/src/stun.c
+@@ -157,14 +157,14 @@ int stun_write(void *buf, size_t size, const stun_message_t *msg, const char *pa
+ 			goto overflow;
+ 		pos += len;
+ 	}
+-	if (msg->ice_controlling) {
++	if (msg->has_ice_controlling || msg->ice_controlling) {
+ 		uint64_t ice_controlling = htonll(msg->ice_controlling);
+ 		len = stun_write_attr(pos, end - pos, STUN_ATTR_ICE_CONTROLLING, &ice_controlling, 8);
+ 		if (len <= 0)
+ 			goto overflow;
+ 		pos += len;
+ 	}
+-	if (msg->ice_controlled) {
++	if (msg->has_ice_controlled || msg->ice_controlled) {
+ 		uint64_t ice_controlled = htonll(msg->ice_controlled);
+ 		len = stun_write_attr(pos, end - pos, STUN_ATTR_ICE_CONTROLLED, &ice_controlled, 8);
+ 		if (len <= 0)
+@@ -916,6 +916,7 @@ int stun_read_attr(const void *data, size_t size, stun_message_t *msg, uint8_t *
+ 		}
+ 		uint32_t *value32 = (uint32_t *)attr->value;
+ 		msg->ice_controlling = ((uint64_t)ntohl(value32[0]) << 32) | ntohl(value32[1]);
++		msg->has_ice_controlling = true;
+ 		break;
+ 	}
+ 	case STUN_ATTR_ICE_CONTROLLED: {
+@@ -926,6 +927,7 @@ int stun_read_attr(const void *data, size_t size, stun_message_t *msg, uint8_t *
+ 		}
+ 		uint32_t *value32 = (uint32_t *)attr->value;
+ 		msg->ice_controlled = ((uint64_t)ntohl(value32[0]) << 32) | ntohl(value32[1]);
++		msg->has_ice_controlled = true;
+ 		break;
+ 	}
+ 	case STUN_ATTR_CHANNEL_NUMBER: {
+diff --git a/src/stun.h b/src/stun.h
+index 398bfb0..61f4246 100644
+--- a/src/stun.h
++++ b/src/stun.h
+@@ -319,6 +319,8 @@ typedef struct stun_message {
+ 	uint32_t priority;
+ 	uint64_t ice_controlling;
+ 	uint64_t ice_controlled;
++	bool has_ice_controlling;
++	bool has_ice_controlled;
+ 	bool use_candidate;
+ 	addr_record_t mapped;
+