web work (#283)

2026-06-08 11:04:57 +08:00 · 2026-04-08 12:50:33 +09:00
parent 733871cb7e
commit d6cbdd6475
10 changed files with 609 additions and 95 deletions
--- a/selfdrive/carrot/realtime/transports/raw_ws.py
+++ b/selfdrive/carrot/realtime/transports/raw_ws.py
@@ -35,6 +35,13 @@ class RawWsHub:
  # 0 = no throttle (send every message)
  _THROTTLE_MAP = {
    "modelV2": 0,             # camera-synced, don't throttle
+    "carState": 0,            # plot 1,2,5,7,8 — needs full rate for dense graphs
+    "controlsState": 0,       # plot 6 — needs full rate for dense graphs
+    "longitudinalPlan": 0,    # plot 1,2,4 — needs full rate for dense graphs
+    "carControl": 0,          # plot 1,6,7,8 — needs full rate for dense graphs
+    "radarState": 0,          # plot 4,5 — needs full rate for dense graphs
+    "lateralPlan": 0,         # overlay path rendering
+    "carrotMan": 0,           # HUD status updates
    "roadCameraState": 0.25,  # metadata/debug only on web HUD
    "deviceState": 0.5,       # slow-changing HUD stats
    "peripheralState": 0.5,
--- a/selfdrive/carrot/server/core.py
+++ b/selfdrive/carrot/server/core.py
@@ -27,7 +27,7 @@ import traceback
 import numpy as np
 from typing import Dict, Any, Tuple, Optional, List

-from aiohttp import web, ClientSession, WSMsgType
+from aiohttp import web, ClientSession, ClientTimeout, WSMsgType
 from cereal import messaging
 from opendbc.car import structs
 import shlex
@@ -204,14 +204,16 @@ async def proxy_stream(request: web.Request) -> web.StreamResponse:
  sess: ClientSession = request.app["http"]

  try:
-    async with sess.post(WEBRTCD_URL, data=body, headers={"Content-Type": ct}) as resp:
+    async with sess.post(WEBRTCD_URL, data=body, headers={"Content-Type": ct},
+                         timeout=ClientTimeout(total=15)) as resp:
      resp_body = await resp.read()
-      # 그대로 전달
      out = web.Response(body=resp_body, status=resp.status)
      rct = resp.headers.get("Content-Type")
      if rct:
        out.headers["Content-Type"] = rct
      return out
+  except asyncio.TimeoutError:
+    return web.json_response({"ok": False, "error": "webrtcd timeout"}, status=504)
  except Exception as e:
    return web.json_response({"ok": False, "error": str(e)}, status=502)

--- a/selfdrive/carrot/web/css/app.css
+++ b/selfdrive/carrot/web/css/app.css
@@ -2152,16 +2152,13 @@ body[data-page="carrot"] #driveHudCard.driveHudCard--loading {
  border-radius: calc(var(--r-2xl) + 2px);
  overflow: hidden;
  border: 1px solid color-mix(in srgb, var(--md-outline-var) 82%, rgba(255,255,255,0.12));
-  background:
-    radial-gradient(circle at 16% 20%, rgba(255, 146, 77, 0.15), transparent 28%),
-    radial-gradient(circle at 80% 80%, rgba(104, 201, 255, 0.12), transparent 24%),
-    linear-gradient(180deg, rgba(255,255,255,0.03), rgba(255,255,255,0)),
-    #04070c;
+  background: #0b0e12;
  box-shadow: 0 20px 48px rgba(0, 0, 0, 0.24);
  contain: layout style;
 }

 .carrot-stage__video,
+.carrot-stage__videoHold,
 .carrot-stage__canvas,
 .carrot-stage__hud {
  position: absolute;
@@ -2212,6 +2209,10 @@ body[data-page="carrot"] #driveHudCard.driveHudCard--loading {
  opacity: 1;
 }

+.carrot-stage.is-video-held .carrot-stage__videoHold {
+  opacity: 1;
+}
+
 .carrot-stage.is-loading .carrot-stage__controls {
  opacity: 0;
  transform: translateY(8px);
@@ -2225,6 +2226,14 @@ body[data-page="carrot"] #driveHudCard.driveHudCard--loading {
  pointer-events: none;
 }

+.carrot-stage__videoHold {
+  transform-origin: 0 0;
+  will-change: transform;
+  background: #000;
+  pointer-events: none;
+  z-index: 1;
+}
+
 .carrot-stage__canvas {
  transform-origin: 0 0;
  will-change: transform;
--- a/selfdrive/carrot/web/index.html
+++ b/selfdrive/carrot/web/index.html
@@ -7,7 +7,7 @@
  <link rel="icon" type="image/png" href="/assets/img_chffr_wheel.png?v=2604-01" />
  <link rel="stylesheet" href="/css/tokens.css?v=2603-73" />
  <link rel="stylesheet" href="/css/hud_card.css?v=2604-09" />
-  <link rel="stylesheet" href="/css/app.css?v=2604-38" />
+  <link rel="stylesheet" href="/css/app.css?v=2604-40" />
 </head>

 <body>
@@ -243,6 +243,7 @@

      <div id="carrotStage" class="carrot-stage">
        <video id="carrotRoadVideo" class="carrot-stage__video" autoplay playsinline muted style="display:none;"></video>
+        <canvas id="carrotLastFrameCanvas" class="carrot-stage__videoHold" aria-hidden="true"></canvas>
        <canvas id="carrotOverlayCanvas" class="carrot-stage__canvas"></canvas>
        <canvas id="carrotHudCanvas" class="carrot-stage__hud"></canvas>
        <div id="carrotOnroadAlert" class="carrot-stage__alert" hidden aria-live="polite" aria-atomic="true">
@@ -441,7 +442,7 @@
  <script src="/js/app_core.js?v=2604-18"></script>
  <script src="/js/app_pages.js?v=2604-23"></script>
  <script src="/js/raw_capnp.js?v=2604-05"></script>
-  <script src="/js/app_realtime.js?v=2604-11"></script>
-  <script src="/js/home_drive.js?v=2604-26"></script>
+  <script src="/js/app_realtime.js?v=2604-16"></script>
+  <script src="/js/home_drive.js?v=2604-27"></script>
 </body>
 </html>
--- a/selfdrive/carrot/web/js/app_realtime.js
+++ b/selfdrive/carrot/web/js/app_realtime.js
@@ -49,7 +49,8 @@ let LIVE_RUNTIME_FETCH_IN_FLIGHT = null;
 let LIVE_RUNTIME_POLL_ACTIVE = false;
 let LAST_HUD_PAYLOAD_SIGNATURE = "";
 const RTC_STATS_POLL_MS = 1000;
-const RTC_FREEZE_MAX_STALL_SAMPLES = 3;
+const RTC_FREEZE_MAX_STALL_SAMPLES = 8;
+const RTC_INITIAL_FRAME_MAX_STALL_SAMPLES = 14;
 const RTC_FREEZE_CURRENT_TIME_EPSILON = 0.05;
 const RTC_FREEZE_RECOVERY_COOLDOWN_MS = 4000;
 const RTC_RESUME_PROGRESS_CHECK_MS = 900;
@@ -76,6 +77,8 @@ const RTC_FREEZE_STATE = {
  lastTotalVideoFrames: null,
  lastCurrentTime: null,
  lastRecoveredAtMs: 0,
+  consecutiveRecoveries: 0,
+  everDecodedFrame: false,
 };
 let RTC_FREEZE_RECOVER_T = null;
 let RTC_VIDEO_EVENTS_BOUND = false;
@@ -85,6 +88,49 @@ const RTC_VISIBILITY_STATE = {
  hiddenAtMs: 0,
  currentTimeAtHide: null,
 };
+const RTC_TRACE_ENABLED = false;
+let RTC_PC_SEQ = 0;
+
+function rtcPcLabel(pc) {
+  if (!pc) return "none";
+  if (!pc.__carrotRtcLabel) {
+    RTC_PC_SEQ += 1;
+    pc.__carrotRtcLabel = `pc${RTC_PC_SEQ}`;
+  }
+  return pc.__carrotRtcLabel;
+}
+
+function rtcBuildTraceSnapshot(pc = RTC_PC) {
+  const video = getRtcVideoElement();
+  const track = video?.srcObject?.getVideoTracks?.()?.[0] || null;
+  return {
+    conn: pc?.connectionState || "none",
+    ice: pc?.iceConnectionState || "none",
+    framesDecoded: RTC_PERF_STATE.inbound?.framesDecoded ?? null,
+    fps: RTC_PERF_STATE.inbound?.framesPerSecond ?? null,
+    currentTime: Number.isFinite(Number(video?.currentTime)) ? Number(video.currentTime).toFixed(2) : null,
+    readyState: Number.isFinite(Number(video?.readyState)) ? Number(video.readyState) : null,
+    trackState: track?.readyState || null,
+    trackMuted: typeof track?.muted === "boolean" ? track.muted : null,
+    hold: Boolean(getRtcStageElement()?.classList.contains("is-video-held")),
+  };
+}
+
+function rtcTrace(event, extra = {}, pc = RTC_PC) {
+  if (!RTC_TRACE_ENABLED) return;
+  console.log("[RTC TRACE]", {
+    ts: Date.now(),
+    iso: new Date().toISOString(),
+    event,
+    pc: rtcPcLabel(pc),
+    ...rtcBuildTraceSnapshot(pc),
+    ...extra,
+  });
+}
+
+function rtcPcSawTrack(pc) {
+  return Boolean(pc && pc.__carrotTrackSeen);
+}

 function getRtcVideoElement() {
  return document.getElementById("carrotRoadVideo") || document.getElementById("rtcVideo");
@@ -94,6 +140,55 @@ function getLegacyRtcVideoElement() {
  return document.getElementById("rtcVideo");
 }

+function getRtcVideoHoldElement() {
+  return document.getElementById("carrotLastFrameCanvas");
+}
+
+function getRtcStageElement() {
+  return document.getElementById("carrotStage");
+}
+
+function rtcShowVideoHold(show) {
+  const stage = getRtcStageElement();
+  if (!stage) return;
+  stage.classList.toggle("is-video-held", Boolean(show));
+}
+
+function rtcClearVideoHold() {
+  const hold = getRtcVideoHoldElement();
+  if (hold) {
+    const ctx = hold.getContext("2d");
+    if (ctx) {
+      ctx.clearRect(0, 0, hold.width || 0, hold.height || 0);
+    }
+  }
+  rtcShowVideoHold(false);
+}
+
+function rtcCaptureVideoHoldFrame() {
+  const video = getRtcVideoElement();
+  const hold = getRtcVideoHoldElement();
+  if (!video || !hold || Number(video.readyState || 0) < 2) return false;
+
+  const targetWidth = Math.max(1, Number(hold.width || video.videoWidth || 0));
+  const targetHeight = Math.max(1, Number(hold.height || video.videoHeight || 0));
+  if (!targetWidth || !targetHeight) return false;
+
+  const ctx = hold.getContext("2d");
+  if (!ctx) return false;
+
+  if (hold.width !== targetWidth) hold.width = targetWidth;
+  if (hold.height !== targetHeight) hold.height = targetHeight;
+  try {
+    ctx.clearRect(0, 0, targetWidth, targetHeight);
+    ctx.drawImage(video, 0, 0, targetWidth, targetHeight);
+    rtcShowVideoHold(true);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function isCarrotPageActive() {
  return document.body?.dataset?.page === "carrot";
 }
@@ -318,6 +413,7 @@ function rtcResetFreezeWatchdog() {
  RTC_FREEZE_STATE.lastFramesDecoded = null;
  RTC_FREEZE_STATE.lastTotalVideoFrames = null;
  RTC_FREEZE_STATE.lastCurrentTime = null;
+  RTC_FREEZE_STATE.everDecodedFrame = false;
 }

 function rtcCancelResumeCheck() {
@@ -505,6 +601,8 @@ function startRtcPerfPolling(force = false) {

 // ===== WebRTC (auto) =====
 let RTC_PC = null;
+let RTC_PENDING_PC = null;
+let RTC_STANDBY_PC = null;
 let RTC_RETRY_T = null;
 let RTC_WAIT_TRACK_T = null;
 let RTC_FAIL_COUNT = 0;
@@ -527,7 +625,18 @@ function rtcHasUsableTrack() {
  if (typeof stream.getVideoTracks !== "function") return true;
  const tracks = stream.getVideoTracks();
  if (!tracks.length) return true;
-  return tracks.some((track) => track && track.readyState !== "ended" && track.muted !== true);
+  return tracks.some((track) => track && track.readyState !== "ended");
+}
+
+function rtcClosePeer(pc) {
+  if (!pc) return;
+  try { pc.ontrack = null; } catch {}
+  try { pc.onconnectionstatechange = null; } catch {}
+  try { pc.oniceconnectionstatechange = null; } catch {}
+  try { pc.close(); } catch {}
+  if (RTC_PC === pc) RTC_PC = null;
+  if (RTC_PENDING_PC === pc) RTC_PENDING_PC = null;
+  if (RTC_STANDBY_PC === pc) RTC_STANDBY_PC = null;
 }

 function rtcStatusSet(s) {
@@ -542,26 +651,35 @@ function rtcCancelRetry() {
  }
 }

-async function rtcDisconnect() {
+async function rtcDisconnect(options = {}) {
+  const keepVideo = Boolean(options.keepVideo);
  rtcCancelRetry();
  rtcDisarmTrackTimeout();
  rtcCancelResumeCheck();
  rtcCancelFreezeRecovery();
  stopRtcPerfPolling();
-  try { if (RTC_PC) RTC_PC.close(); } catch {}
+  const activePc = RTC_PC;
+  const pendingPc = RTC_PENDING_PC;
+  const standbyPc = RTC_STANDBY_PC;
  RTC_PC = null;
+  RTC_PENDING_PC = null;
+  RTC_STANDBY_PC = null;
+  rtcClosePeer(pendingPc);
+  rtcClosePeer(activePc);
+  rtcClosePeer(standbyPc);
  resetRtcPerfState();
  rtcResetFreezeWatchdog();

+  if (!keepVideo) {
+    rtcClearVideoHold();
    const video = getRtcVideoElement();
    if (video) {
      video.srcObject = null;
-    video.style.display = "none";
    }
    const legacyVideo = getLegacyRtcVideoElement();
    if (legacyVideo && legacyVideo !== video) {
      legacyVideo.srcObject = null;
-    legacyVideo.style.display = "none";
+    }
  }
 }

@@ -574,6 +692,10 @@ function rtcIsWaitingForInitialTrack(pc = RTC_PC) {
  return Boolean(RTC_WAIT_TRACK_T && RTC_WAIT_TRACK_PC && RTC_WAIT_TRACK_PC === pc);
 }

+function rtcIsWaitingForInitialTrack(pc = RTC_PC) {
+  return Boolean(RTC_WAIT_TRACK_T && RTC_WAIT_TRACK_PC && RTC_WAIT_TRACK_PC === pc);
+}
+
 function rtcUpdateFreezeSnapshot(snapshot) {
  RTC_FREEZE_STATE.lastFramesDecoded = snapshot.framesDecoded;
  RTC_FREEZE_STATE.lastTotalVideoFrames = snapshot.totalVideoFrames;
@@ -582,15 +704,22 @@ function rtcUpdateFreezeSnapshot(snapshot) {

 function rtcScheduleFreezeRecovery(reason, options = {}) {
  const force = Boolean(options.force);
-  if (!shouldRunCarrotVisionRealtime() || _rtcConnecting || RTC_FREEZE_RECOVER_T) return;
+  if (!shouldRunCarrotVisionRealtime() || _rtcConnecting || RTC_PENDING_PC || RTC_FREEZE_RECOVER_T) return;
  const now = Date.now();
  if (!force && (now - RTC_FREEZE_STATE.lastRecoveredAtMs < RTC_FREEZE_RECOVERY_COOLDOWN_MS)) return;

+  RTC_FREEZE_STATE.consecutiveRecoveries++;
  RTC_FREEZE_STATE.lastRecoveredAtMs = now;
  RTC_FREEZE_STATE.stallSamples = 0;
  rtcStatusSet(reason);
+  rtcTrace("freeze_recovery_scheduled", {
+    reason,
+    force,
+    attempt: RTC_FREEZE_STATE.consecutiveRecoveries,
+  }, RTC_PC || RTC_PENDING_PC);
  console.warn("[RTC] road video stalled, reconnecting", {
    reason,
+    attempt: RTC_FREEZE_STATE.consecutiveRecoveries,
    connectionState: RTC_PERF_STATE.connectionState,
    iceConnectionState: RTC_PERF_STATE.iceConnectionState,
    inbound: RTC_PERF_STATE.inbound,
@@ -600,9 +729,9 @@ function rtcScheduleFreezeRecovery(reason, options = {}) {
  RTC_FREEZE_RECOVER_T = setTimeout(async () => {
    RTC_FREEZE_RECOVER_T = null;
    if (!shouldRunCarrotVisionRealtime()) return;
-    await rtcDisconnect();
+    rtcCaptureVideoHoldFrame();
    RTC_FAIL_COUNT = 0;
-    await rtcConnectOnce().catch(() => {});
+    await rtcConnectOnce({ force: true }).catch(() => {});
  }, 0);
 }

@@ -620,11 +749,10 @@ function rtcUpdateFreezeWatchdog(pc, video) {
  // PC connected but track dead/muted/inactive → force reconnect
  if (rtcConnectionLooksLive(pc) && !rtcHasUsableTrack() && video.srcObject) {
    rtcResetFreezeWatchdog();
-    rtcScheduleFreezeRecovery("track ended, reconnecting...");
    return;
  }

-  if (!rtcConnectionLooksLive(pc) || !rtcHasUsableTrack()) {
+  if (!rtcConnectionLooksLive(pc) || !rtcHasLiveTrack()) {
    rtcResetFreezeWatchdog();
    return;
  }
@@ -653,11 +781,20 @@ function rtcUpdateFreezeWatchdog(pc, video) {
    (snapshot.totalVideoFrames != null && RTC_FREEZE_STATE.lastTotalVideoFrames != null && snapshot.totalVideoFrames > RTC_FREEZE_STATE.lastTotalVideoFrames) ||
    (snapshot.currentTime != null && RTC_FREEZE_STATE.lastCurrentTime != null && snapshot.currentTime > RTC_FREEZE_STATE.lastCurrentTime + RTC_FREEZE_CURRENT_TIME_EPSILON);

-  RTC_FREEZE_STATE.stallSamples = hasProgress ? 0 : RTC_FREEZE_STATE.stallSamples + 1;
+  if (hasProgress) {
+    RTC_FREEZE_STATE.stallSamples = 0;
+    RTC_FREEZE_STATE.consecutiveRecoveries = 0;
+    if (!RTC_FREEZE_STATE.everDecodedFrame) {
+      RTC_FREEZE_STATE.everDecodedFrame = true;
+    }
+  } else {
+    RTC_FREEZE_STATE.stallSamples++;
+  }
  rtcUpdateFreezeSnapshot(snapshot);

-  if (RTC_FREEZE_STATE.stallSamples >= RTC_FREEZE_MAX_STALL_SAMPLES) {
-    rtcScheduleFreezeRecovery("video stalled, reconnecting...");
+  const stallLimit = RTC_FREEZE_STATE.everDecodedFrame ? RTC_FREEZE_MAX_STALL_SAMPLES : RTC_INITIAL_FRAME_MAX_STALL_SAMPLES;
+  if (RTC_FREEZE_STATE.stallSamples >= stallLimit) {
+    rtcScheduleFreezeRecovery(RTC_FREEZE_STATE.everDecodedFrame ? "video stalled, reconnecting..." : "no initial frame, reconnecting...");
  }
 }

@@ -675,6 +812,8 @@ function rtcBindVideoEvents() {

  video.addEventListener("playing", () => {
    RTC_FREEZE_STATE.stallSamples = 0;
+    RTC_FREEZE_STATE.everDecodedFrame = true;
+    rtcClearVideoHold();
    collectRtcPerfStats().catch(() => {});
  });
  ["waiting", "stalled", "suspend", "pause", "ended"].forEach((eventName) => {
@@ -695,14 +834,30 @@ function rtcScheduleRetry(ms = 2000) {
 }

 function rtcArmTrackTimeout(ms = 5000, expectedPc = RTC_PC) {
+  if (rtcPcSawTrack(expectedPc)) {
+    rtcTrace("track_timeout_arm_skipped", { timeoutMs: ms, reason: "track already seen" }, expectedPc);
+    return;
+  }
  if (RTC_WAIT_TRACK_T) clearTimeout(RTC_WAIT_TRACK_T);
  RTC_WAIT_TRACK_PC = expectedPc;
  RTC_WAIT_TRACK_T = setTimeout(async () => {
    RTC_WAIT_TRACK_T = null;
-    if (RTC_WAIT_TRACK_PC !== expectedPc || RTC_PC !== expectedPc) return;
+    if (RTC_WAIT_TRACK_PC !== expectedPc || (RTC_PC !== expectedPc && RTC_PENDING_PC !== expectedPc)) return;
+    if (rtcPcSawTrack(expectedPc)) {
      RTC_WAIT_TRACK_PC = null;
+      rtcTrace("track_timeout_ignored", { timeoutMs: ms, reason: "track arrived before timeout fired" }, expectedPc);
+      return;
+    }
+    RTC_WAIT_TRACK_PC = null;
+    rtcTrace("track_timeout", { timeoutMs: ms }, expectedPc);
    rtcStatusSet("no track, retry...");
-    await rtcDisconnect();
+    rtcCaptureVideoHoldFrame();
+    if (RTC_PENDING_PC === expectedPc) {
+      rtcClosePeer(expectedPc);
+      rtcScheduleRetry(2000);
+      return;
+    }
+    await rtcDisconnect({ keepVideo: true });
    rtcScheduleRetry(2000);
  }, ms);
 }
@@ -720,12 +875,8 @@ function rtcScheduleResumeHealthCheck(reason = "returned visible") {
  rtcCancelResumeCheck();
  RTC_RESUME_CHECK_T = setTimeout(async () => {
    RTC_RESUME_CHECK_T = null;
-    if (!shouldRunCarrotVisionRealtime() || _rtcConnecting || !RTC_PC) return;
-    const video = getRtcVideoElement();
-    const currentTime = Number(video?.currentTime || 0);
-    const hiddenTime = Number(RTC_VISIBILITY_STATE.currentTimeAtHide || 0);
-    const progressed = currentTime > hiddenTime + RTC_FREEZE_CURRENT_TIME_EPSILON;
-    if (!rtcConnectionLooksLive(RTC_PC) || !rtcHasUsableTrack() || !progressed) {
+    if (!shouldRunCarrotVisionRealtime() || _rtcConnecting || RTC_PENDING_PC || !RTC_PC) return;
+    if (!rtcConnectionLooksLive(RTC_PC) || !rtcHasLiveTrack()) {
      rtcScheduleFreezeRecovery(`${reason}, reconnecting...`, { force: true });
    }
  }, RTC_RESUME_PROGRESS_CHECK_MS);
@@ -748,23 +899,56 @@ async function waitIceComplete(pc, timeoutMs = 8000) {

 let _rtcConnecting = false;

-async function rtcConnectOnce() {
+async function rtcConnectOnce(options = {}) {
+  const force = Boolean(options.force);
  if (!shouldRunCarrotVisionRealtime()) return;
-  if (_rtcConnecting) return;
-  if (RTC_PC && (RTC_PC.connectionState === "connected" || RTC_PC.connectionState === "connecting")) return;
+  if (_rtcConnecting || RTC_PENDING_PC) return;
+  if (!force && RTC_PC && (RTC_PC.connectionState === "connected" || RTC_PC.connectionState === "connecting") && rtcHasLiveTrack()) return;

  _rtcConnecting = true;
+  let previousPc = RTC_PC || RTC_STANDBY_PC;
  try {
-    await rtcDisconnect();
+    rtcCancelRetry();
+    rtcDisarmTrackTimeout();
+    rtcCancelResumeCheck();
+    rtcCancelFreezeRecovery();
+    rtcTrace("connect_start", {
+      force,
+      hasPreviousPc: Boolean(previousPc),
+      hasLiveTrack: rtcHasLiveTrack(),
+    }, previousPc || RTC_PC);
+
+    const keepPreviousVisible = Boolean(previousPc && rtcHasLiveTrack());
+    if (keepPreviousVisible) {
+      if (RTC_STANDBY_PC && RTC_STANDBY_PC !== previousPc) {
+        rtcClosePeer(RTC_STANDBY_PC);
+      }
+      RTC_STANDBY_PC = previousPc;
+      if (RTC_PC === previousPc) {
+        RTC_PC = null;
+      }
+      stopRtcPerfPolling();
+      resetRtcPerfState();
+      rtcResetFreezeWatchdog();
+      rtcStatusSet("reconnecting...");
+    } else {
+      await rtcDisconnect({ keepVideo: true });
+      previousPc = null;
      rtcStatusSet("connecting...");
+    }

    const pc = new RTCPeerConnection({
      iceServers: [],
      sdpSemantics: "unified-plan",
      iceCandidatePoolSize: 1,
    });
-    RTC_PC = pc;
-    startRtcPerfPolling(true);
+    rtcPcLabel(pc);
+    pc.__carrotTrackSeen = false;
+    RTC_PENDING_PC = pc;
+    rtcTrace("pc_created", {
+      keepPreviousVisible,
+      hasPreviousPc: Boolean(previousPc),
+    }, pc);

    const video = getRtcVideoElement();
    if (video) {
@@ -775,9 +959,14 @@ async function rtcConnectOnce() {
    pc.addTransceiver("video", { direction: "recvonly" });

    pc.ontrack = async (ev) => {
-      if (RTC_PC !== pc) return;
+      if (RTC_PENDING_PC !== pc) return;
      const videoEl = getRtcVideoElement();
      if (!videoEl) return;
+      rtcTrace("track_received", {
+        kind: ev.track?.kind || null,
+        streamCount: Array.isArray(ev.streams) ? ev.streams.length : 0,
+      }, pc);
+      pc.__carrotTrackSeen = true;

      let stream = ev.streams && ev.streams[0];
      if (!stream) {
@@ -785,44 +974,83 @@ async function rtcConnectOnce() {
      }

      videoEl.srcObject = stream;
-      if (videoEl.id === "rtcVideo") {
-        videoEl.style.display = "block";
-      }
+      RTC_PENDING_PC = null;
+      RTC_PC = pc;
+      const retiringPc = RTC_STANDBY_PC && RTC_STANDBY_PC !== pc ? RTC_STANDBY_PC : null;
+      RTC_STANDBY_PC = null;
      try { await videoEl.play(); } catch (e) { console.log("[RTC] play() failed", e); }
      rtcStatusSet("track: " + ev.track.kind);
      rtcDisarmTrackTimeout(pc);
      RTC_FAIL_COUNT = 0;
      rtcResetFreezeWatchdog();
+      rtcClearVideoHold();
+      startRtcPerfPolling(true);
      collectRtcPerfStats().catch(() => {});
+      if (retiringPc) {
+        rtcClosePeer(retiringPc);
+      }

-      // Detect server-side track close → immediate recovery
+      // Detect server-side track close → immediate recovery (guarded by PC identity)
      ev.track.addEventListener("ended", () => {
+        rtcTrace("track_ended", {
+          kind: ev.track?.kind || null,
+          trackReadyState: ev.track?.readyState || null,
+        }, pc);
        console.warn("[RTC] remote track ended");
-        if (RTC_PC === pc && shouldRunCarrotVisionRealtime() && !_rtcConnecting) {
-          rtcScheduleFreezeRecovery("remote track ended");
+        if ((RTC_PC === pc || RTC_PENDING_PC === pc) && shouldRunCarrotVisionRealtime() && !_rtcConnecting) {
+          rtcCaptureVideoHoldFrame();
+          rtcScheduleFreezeRecovery("remote track ended", { force: true });
        }
      });
    };

    pc.onconnectionstatechange = () => {
-      if (RTC_PC !== pc) return;
+      const isPending = RTC_PENDING_PC === pc;
+      const isActive = RTC_PC === pc;
+      if (!isPending && !isActive) return;
      const state = pc.connectionState;
+      rtcTrace("connection_state_change", {
+        isPending,
+        isActive,
+        state,
+      }, pc);
      rtcStatusSet("conn: " + state);
      if (state === "connected") RTC_FAIL_COUNT = 0;
+      if (isActive) {
        collectRtcPerfStats().catch(() => {});
-      if (state === "failed" || state === "disconnected" || state === "closed") {
-        rtcDisconnect();
+      }
+      if (state === "failed" || state === "closed") {
+        rtcCaptureVideoHoldFrame();
+        if (isPending) {
+          rtcClosePeer(pc);
+        } else {
+          rtcDisconnect({ keepVideo: true }).catch(() => {});
+        }
        rtcScheduleRetry(2000);
      }
    };

    pc.oniceconnectionstatechange = () => {
-      if (RTC_PC !== pc) return;
+      const isPending = RTC_PENDING_PC === pc;
+      const isActive = RTC_PC === pc;
+      if (!isPending && !isActive) return;
      const state = pc.iceConnectionState;
+      rtcTrace("ice_state_change", {
+        isPending,
+        isActive,
+        state,
+      }, pc);
      rtcStatusSet("ice: " + state);
+      if (isActive) {
        collectRtcPerfStats().catch(() => {});
-      if (state === "failed" || state === "disconnected" || state === "closed") {
-        rtcDisconnect();
+      }
+      if (state === "failed" || state === "closed") {
+        rtcCaptureVideoHoldFrame();
+        if (isPending) {
+          rtcClosePeer(pc);
+        } else {
+          rtcDisconnect({ keepVideo: true }).catch(() => {});
+        }
        rtcScheduleRetry(2000);
      }
    };
@@ -830,6 +1058,9 @@ async function rtcConnectOnce() {
    const offer = await pc.createOffer();
    await pc.setLocalDescription(offer);
    await waitIceComplete(pc, 8000);
+    rtcTrace("offer_ready", {
+      localSdpBytes: pc.localDescription?.sdp?.length || 0,
+    }, pc);

    const body = {
      sdp: pc.localDescription.sdp,
@@ -851,13 +1082,27 @@ async function rtcConnectOnce() {

    const answer = await response.json();
    if (!answer || !answer.sdp) throw new Error("bad answer");
+    rtcTrace("answer_received", {
+      remoteSdpBytes: answer.sdp?.length || 0,
+      answerType: answer.type || "answer",
+    }, pc);

    await pc.setRemoteDescription({ type: answer.type || "answer", sdp: answer.sdp });
+    rtcTrace("answer_applied", {}, pc);
    rtcStatusSet("connected (waiting track...)");
    rtcArmTrackTimeout(6000, pc);
  } catch (e) {
+    rtcTrace("connect_error", {
+      message: e?.message || String(e),
+    }, RTC_PENDING_PC || previousPc || RTC_PC);
    rtcStatusSet("error: " + e.message);
-    await rtcDisconnect();
+    rtcCaptureVideoHoldFrame();
+    if (RTC_PENDING_PC) {
+      rtcClosePeer(RTC_PENDING_PC);
+    }
+    if (!RTC_PC && !RTC_STANDBY_PC) {
+      await rtcDisconnect({ keepVideo: true });
+    }
    rtcScheduleRetry(2000);
  } finally {
    _rtcConnecting = false;
@@ -1639,7 +1884,7 @@ function syncCarrotRealtimeLifecycle(forceFetch = false) {
  const nextVisionActive = shouldRunCarrotVisionRealtime();

  if (nextHudActive === _carrotHudRealtimeActive && nextVisionActive === _carrotVisionRealtimeActive && !forceFetch) {
-    if (nextVisionActive && !_rtcConnecting && !rtcHasLiveTrack()) {
+    if (nextVisionActive && !_rtcConnecting && (!RTC_PC || !rtcHasLiveTrack())) {
      rtcConnectOnce().catch(() => {});
    }
    if (nextVisionActive) scheduleRtcPerfPolling();
@@ -1669,7 +1914,7 @@ function syncCarrotRealtimeLifecycle(forceFetch = false) {
    ensureRawDecodeWorker();
    rawOverlayConnectAll();
    startRtcPerfPolling(true);
-    if (!_rtcConnecting && !rtcHasLiveTrack()) {
+    if (!_rtcConnecting && (!RTC_PC || !rtcHasLiveTrack())) {
      rtcCancelRetry();
      RTC_FAIL_COUNT = 0;
      rtcConnectOnce().catch(() => {});
--- a/selfdrive/carrot/web/js/home_drive.js
+++ b/selfdrive/carrot/web/js/home_drive.js
@@ -3,6 +3,7 @@
 window.HomeDrive = (() => {
  const stageEl = document.getElementById("carrotStage");
  const videoEl = document.getElementById("carrotRoadVideo");
+  const videoHoldEl = document.getElementById("carrotLastFrameCanvas");
  const canvasEl = document.getElementById("carrotOverlayCanvas");
  const hudCanvasEl = document.getElementById("carrotHudCanvas");
  const onroadAlertEl = document.getElementById("carrotOnroadAlert");
@@ -1614,6 +1615,12 @@ window.HomeDrive = (() => {
      overlaySizeSignature = nextOverlaySignature;
      videoEl.style.width = `${videoWidth}px`;
      videoEl.style.height = `${videoHeight}px`;
+      if (videoHoldEl) {
+        videoHoldEl.style.width = `${videoWidth}px`;
+        videoHoldEl.style.height = `${videoHeight}px`;
+        videoHoldEl.width = Math.max(1, Math.round(videoWidth * dpr));
+        videoHoldEl.height = Math.max(1, Math.round(videoHeight * dpr));
+      }
      canvasEl.style.width = `${videoWidth}px`;
      canvasEl.style.height = `${videoHeight}px`;
      canvasEl.width = Math.max(1, Math.round(videoWidth * dpr));
@@ -1639,6 +1646,7 @@ window.HomeDrive = (() => {
    transformSignature = nextSignature;
    const cssMatrix = `matrix(${transform.scale}, 0, 0, ${transform.scale}, ${transform.tx}, ${transform.ty})`;
    videoEl.style.transform = cssMatrix;
+    if (videoHoldEl) videoHoldEl.style.transform = cssMatrix;
    canvasEl.style.transform = cssMatrix;
  }

@@ -2075,32 +2083,31 @@ window.HomeDrive = (() => {

  function getLeadBoxClampMargins(videoWidth, videoHeight, stageWidth = videoWidth, stageHeight = videoHeight, transform = null, options = {}) {
    const visibleRect = getVisibleSourceRect(videoWidth, videoHeight, stageWidth, stageHeight, transform);
-    const baseTopMargin = Math.min(videoHeight * 0.28, 200.0);
-    const baseBottomMargin = Math.max(videoHeight * 0.14, 80.0);
-    const offsets = getLeadBadgeOffsets(videoWidth, videoHeight);
-    let bottomReserve = baseBottomMargin;
+    // C3 fixed margins: top=200, bottom=80, marginX=350
+    const topMargin = Math.max(200.0, visibleRect.top + 6);

+    // C3 base: maxCenterY = fb_h - 80
+    let maxCenterY = videoHeight - 80.0;
+
+    // In crop/fit modes, also keep badges inside visible area
+    const offsets = getLeadBadgeOffsets(videoWidth, videoHeight);
+    let badgeReserve = 0;
    if (options.includeDistanceBadge !== false) {
-      bottomReserve = Math.max(bottomReserve, offsets.rectTopOffset + offsets.badgeHeight + 8);
+      badgeReserve = Math.max(badgeReserve, offsets.rectTopOffset + offsets.badgeHeight + 8);
    }
    if (options.includeStateText) {
-      const stateBottomReserve = offsets.textBaselineOffset + Math.max(offsets.fontSize * 0.28, 8);
-      bottomReserve = Math.max(bottomReserve, stateBottomReserve);
+      badgeReserve = Math.max(badgeReserve, offsets.textBaselineOffset + Math.max(offsets.fontSize * 0.28, 8));
    }
-
-    const topMargin = Math.max(baseTopMargin, visibleRect.top + 6);
-    const maxCenterY = Math.max(
-      topMargin,
-      Math.min(videoHeight - baseBottomMargin, visibleRect.bottom - bottomReserve),
-    );
+    maxCenterY = Math.min(maxCenterY, visibleRect.bottom - Math.max(badgeReserve, 80.0));
+    maxCenterY = Math.max(topMargin, maxCenterY);

    return {
-      marginX: Math.min(videoWidth * 0.35, 350.0),
+      marginX: 350.0,
      topMargin,
-      bottomMargin: Math.max(baseBottomMargin, videoHeight - maxCenterY),
+      bottomMargin: Math.max(80.0, videoHeight - maxCenterY),
      maxCenterY,
      visibleRect,
-      bottomReserve,
+      bottomReserve: badgeReserve,
    };
  }

--- a/system/webrtc/device/audio.py
+++ b/system/webrtc/device/audio.py
@@ -0,0 +1,109 @@
+import asyncio
+import io
+
+import aiortc
+import av
+import numpy as np
+import pyaudio
+
+
+class AudioInputStreamTrack(aiortc.mediastreams.AudioStreamTrack):
+  PYAUDIO_TO_AV_FORMAT_MAP = {
+      pyaudio.paUInt8: 'u8',
+      pyaudio.paInt16: 's16',
+      pyaudio.paInt24: 's24',
+      pyaudio.paInt32: 's32',
+      pyaudio.paFloat32: 'flt',
+  }
+
+  def __init__(self, audio_format: int = pyaudio.paInt16, rate: int = 16000, channels: int = 1, packet_time: float = 0.020, device_index: int = None):
+    super().__init__()
+
+    self.p = pyaudio.PyAudio()
+    chunk_size = int(packet_time * rate)
+    self.stream = self.p.open(format=audio_format,
+                              channels=channels,
+                              rate=rate,
+                              frames_per_buffer=chunk_size,
+                              input=True,
+                              input_device_index=device_index)
+    self.format = audio_format
+    self.rate = rate
+    self.channels = channels
+    self.packet_time = packet_time
+    self.chunk_size = chunk_size
+    self.pts = 0
+
+  async def recv(self):
+    mic_data = self.stream.read(self.chunk_size)
+    mic_array = np.frombuffer(mic_data, dtype=np.int16)
+    mic_array = np.expand_dims(mic_array, axis=0)
+    layout = 'stereo' if self.channels > 1 else 'mono'
+    frame = av.AudioFrame.from_ndarray(mic_array, format=self.PYAUDIO_TO_AV_FORMAT_MAP[self.format], layout=layout)
+    frame.rate = self.rate
+    frame.pts = self.pts
+    self.pts += frame.samples
+
+    return frame
+
+
+class AudioOutputSpeaker:
+  def __init__(self, audio_format: int = pyaudio.paInt16, rate: int = 48000, channels: int = 2, packet_time: float = 0.2, device_index: int = None):
+
+    chunk_size = int(packet_time * rate)
+    self.p = pyaudio.PyAudio()
+    self.buffer = io.BytesIO()
+    self.channels = channels
+    self.stream = self.p.open(format=audio_format,
+                              channels=channels,
+                              rate=rate,
+                              frames_per_buffer=chunk_size,
+                              output=True,
+                              output_device_index=device_index,
+                              stream_callback=self.__pyaudio_callback)
+    self.tracks_and_tasks: list[tuple[aiortc.MediaStreamTrack, asyncio.Task | None]] = []
+
+  def __pyaudio_callback(self, in_data, frame_count, time_info, status):
+    if self.buffer.getbuffer().nbytes < frame_count * self.channels * 2:
+      buff = b'\x00\x00' * frame_count * self.channels
+    elif self.buffer.getbuffer().nbytes > 115200:  # 3x the usual read size
+      self.buffer.seek(0)
+      buff = self.buffer.read(frame_count * self.channels * 4)
+      buff = buff[:frame_count * self.channels * 2]
+      self.buffer.seek(2)
+    else:
+      self.buffer.seek(0)
+      buff = self.buffer.read(frame_count * self.channels * 2)
+      self.buffer.seek(2)
+    return (buff, pyaudio.paContinue)
+
+  async def __consume(self, track):
+    while True:
+      try:
+        frame = await track.recv()
+      except aiortc.MediaStreamError:
+        return
+
+      self.buffer.write(bytes(frame.planes[0]))
+
+  def hasTrack(self, track: aiortc.MediaStreamTrack) -> bool:
+    return any(t == track for t, _ in self.tracks_and_tasks)
+
+  def addTrack(self, track: aiortc.MediaStreamTrack):
+    if not self.hasTrack(track):
+      self.tracks_and_tasks.append((track, None))
+
+  def start(self):
+    for index, (track, task) in enumerate(self.tracks_and_tasks):
+      if task is None:
+        self.tracks_and_tasks[index] = (track, asyncio.create_task(self.__consume(track)))
+
+  def stop(self):
+    for _, task in self.tracks_and_tasks:
+      if task is not None:
+        task.cancel()
+
+    self.tracks_and_tasks = []
+    self.stream.stop_stream()
+    self.stream.close()
+    self.p.terminate()
--- a/system/webrtc/device/video.py
+++ b/system/webrtc/device/video.py
@@ -6,9 +6,12 @@ from teleoprtc.tracks import TiciVideoStreamTrack

 from cereal import messaging
 from openpilot.common.realtime import DT_MDL, DT_DMON
+from openpilot.common.swaglog import cloudlog


 class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
+  RECOVERY_LOG_THRESHOLD_SECONDS = 1.0
+
  camera_to_sock_mapping = {
    "driver": "livestreamDriverEncodeData",
    "wideRoad": "livestreamWideRoadEncodeData",
@@ -19,22 +22,37 @@ class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
    dt = DT_DMON if camera_type == "driver" else DT_MDL
    super().__init__(camera_type, dt)

-    self._sock = messaging.sub_sock(self.camera_to_sock_mapping[camera_type], conflate=True)
+    self._source_name = self.camera_to_sock_mapping[camera_type]
+    self._sock = messaging.sub_sock(self._source_name, conflate=True)
    self._pts = 0
-    self._t0_ns = time.monotonic_ns()
+    self._sock_closed = False
+    self._timeout_started_at = None

  async def recv(self):
+    waited = 0
    while True:
      try:
        msg = messaging.recv_one_or_none(self._sock)
        if msg is None:
          await asyncio.sleep(0.005)
          waited += 0.005
+          if self._timeout_started_at is None:
+            self._timeout_started_at = time.monotonic()
          if waited > 1.0:
-            self._logger.warning("%s frame recv timed out", self.camera_to_sock_mapping.get(self._id.split(':', 1)[0], "video"))
+            elapsed = time.monotonic() - self._timeout_started_at if self._timeout_started_at is not None else waited
+            self._logger.warning("%s frame recv timed out (elapsed=%.2fs pts=%s sock_closed=%s)",
+                                 self._source_name, elapsed, self._pts, self._sock_closed)
+            cloudlog.warning("[webrtcd-video] %s frame recv timed out (elapsed=%.2fs pts=%s sock_closed=%s)",
+                             self._source_name, elapsed, self._pts, self._sock_closed)
            waited = 0
          continue

+        if self._timeout_started_at is not None:
+          elapsed = time.monotonic() - self._timeout_started_at
+          if elapsed >= self.RECOVERY_LOG_THRESHOLD_SECONDS:
+            self._logger.info("%s frame recv recovered after %.2fs (pts=%s)", self._source_name, elapsed, self._pts)
+            cloudlog.info("[webrtcd-video] %s frame recv recovered after %.2fs (pts=%s)", self._source_name, elapsed, self._pts)
+          self._timeout_started_at = None
        waited = 0
        evta = getattr(msg, msg.which())

@@ -49,8 +67,17 @@ class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
      except asyncio.CancelledError:
        raise
      except Exception:
-        self._logger.exception("failed to build outgoing video packet")
+        self._logger.exception("failed to build outgoing video packet (%s pts=%s)", self._source_name, self._pts)
+        cloudlog.exception("[webrtcd-video] failed to build outgoing video packet (%s pts=%s)", self._source_name, self._pts)
        await asyncio.sleep(0.01)

+  def close_sock(self):
+    if not self._sock_closed and self._sock is not None:
+      try:
+        self._sock.close()
+      except Exception:
+        pass
+      self._sock_closed = True
+
  def codec_preference(self) -> str | None:
    return "H264"
--- a/system/webrtc/webrtcd.py
+++ b/system/webrtc/webrtcd.py
@@ -18,10 +18,17 @@ from aiohttp import web
 if TYPE_CHECKING:
  from aiortc.rtcdatachannel import RTCDataChannel

+from openpilot.common.swaglog import cloudlog
 from openpilot.system.webrtc.schema import generate_field
 from cereal import messaging, log


+def webrtcd_log(level: str, msg: str, *args):
+  logger = logging.getLogger("webrtcd")
+  getattr(logger, level)(msg, *args)
+  getattr(cloudlog, level)("[webrtcd] " + msg, *args)
+
+
 class CerealOutgoingMessageProxy:
  def __init__(self, sm: messaging.SubMaster):
    self.sm = sm
@@ -137,15 +144,26 @@ class StreamSession:
    from teleoprtc import WebRTCAnswerBuilder
    from teleoprtc.info import parse_info_from_offer

+    self.logger = logging.getLogger("webrtcd")
    config = parse_info_from_offer(sdp)
    builder = WebRTCAnswerBuilder(sdp)

+    self._video_tracks = []
    assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks"
    for cam in cameras:
-      builder.add_video_stream(cam, LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack())
+      try:
+        track = LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack()
+        builder.add_video_stream(cam, track)
+        if hasattr(track, 'close_sock'):
+          self._video_tracks.append(track)
+        self.logger.info("added camera track: %s", cam)
+      except Exception:
+        self.logger.exception("failed to create camera track: %s", cam)
+        raise

-    self.stream = builder.stream()
    self.identifier = str(uuid.uuid4())
+    self.stream = builder.stream()
+    setattr(self.stream, "session_identifier", self.identifier)

    self.incoming_bridge: CerealIncomingMessageProxy | None = None
    self.incoming_bridge_services = incoming_services
@@ -158,17 +176,22 @@ class StreamSession:
      self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge)

    self.run_task: asyncio.Task | None = None
-    self.logger = logging.getLogger("webrtcd")
+    self.connected_event = asyncio.Event()
    self.logger.info("New stream session (%s), cameras %s, incoming services %s, outgoing services %s",
                      self.identifier, cameras, incoming_services, outgoing_services)
+    self.logger.info("request cameras=%s", cameras)
+

  def start(self):
+    webrtcd_log("info", "Stream session (%s) start requested", self.identifier)
    self.run_task = asyncio.create_task(self.run())

  async def stop(self):
    if self.run_task is None:
      return

+    webrtcd_log("info", "Stream session (%s) stop requested (done=%s)", self.identifier, self.run_task.done())
+
    if not self.run_task.done():
      self.run_task.cancel()
      try:
@@ -178,6 +201,7 @@ class StreamSession:

    self.run_task = None
    await self.post_run_cleanup()
+    webrtcd_log("info", "Stream session (%s) stop cleanup complete", self.identifier)

    try:
      if hasattr(self, "stream_dict"):
@@ -187,7 +211,10 @@ class StreamSession:
    

  async def get_answer(self):
-    return await self.stream.start()
+    webrtcd_log("info", "Stream session (%s) building answer", self.identifier)
+    answer = await self.stream.start()
+    webrtcd_log("info", "Stream session (%s) answer ready", self.identifier)
+    return answer

  async def message_handler(self, message: bytes):
    assert self.incoming_bridge is not None
@@ -198,6 +225,7 @@ class StreamSession:

  async def run(self):
    try:
+      webrtcd_log("info", "Stream session (%s) waiting for peer connection", self.identifier)
      await self.stream.wait_for_connection()
      if self.stream.has_messaging_channel():
        if self.incoming_bridge is not None:
@@ -207,12 +235,14 @@ class StreamSession:
          channel = self.stream.get_messaging_channel()
          self.outgoing_bridge_runner.proxy.add_channel(channel)
          self.outgoing_bridge_runner.start()
-      self.logger.info("Stream session (%s) connected", self.identifier)
+      self.connected_event.set()
+      webrtcd_log("info", "Stream session (%s) connected", self.identifier)

+      webrtcd_log("info", "Stream session (%s) waiting for disconnection", self.identifier)
      await self.stream.wait_for_disconnection()
      await self.post_run_cleanup()

-      self.logger.info("Stream session (%s) ended", self.identifier)
+      webrtcd_log("info", "Stream session (%s) ended", self.identifier)
    except Exception:
      self.logger.exception("Stream session failure")
    finally:
@@ -227,6 +257,12 @@ class StreamSession:
    await self.stream.stop()
    if self.outgoing_bridge is not None:
      self.outgoing_bridge_runner.stop()
+    # Close ZMQ sockets held by video tracks
+    for track in getattr(self, '_video_tracks', []):
+      try:
+        track.close_sock()
+      except Exception:
+        self.logger.exception("Failed to close video track socket")


@dataclass
@@ -237,10 +273,36 @@ class StreamRequestBody:
  bridge_services_out: list[str] = field(default_factory=list)


+async def retire_old_sessions_after_handover(old_sessions: list[StreamSession], new_session: StreamSession, timeout_s: float = 12.0):
+  if not old_sessions:
+    return
+
+  old_ids = [getattr(old, "identifier", "?") for old in old_sessions]
+  webrtcd_log("info", "Handover: waiting to retire old sessions %s after new session %s", old_ids, new_session.identifier)
+  try:
+    await asyncio.wait_for(new_session.connected_event.wait(), timeout=timeout_s)
+    webrtcd_log("info", "New session %s connected, retiring old sessions %s", new_session.identifier, old_ids)
+  except TimeoutError:
+    webrtcd_log("warning", "New session %s did not connect within %.1fs, retiring old sessions %s anyway",
+                new_session.identifier, timeout_s, old_ids)
+
+  for old in old_sessions:
+    try:
+      webrtcd_log("info", "Handover: stopping old session %s", getattr(old, 'identifier', '?'))
+      await old.stop()
+      webrtcd_log("info", "Handover: stopped old session %s", getattr(old, 'identifier', '?'))
+    except Exception:
+      webrtcd_log("exception", "Failed to stop old session %s during handover", getattr(old, 'identifier', '?'))
+
+
 async def get_stream(request: 'web.Request'):
  stream_dict, debug_mode = request.app['streams'], request.app['debug']
+
+  old_sessions = list(stream_dict.values())
  raw_body = await request.json()
  body = StreamRequestBody(**raw_body)
+  webrtcd_log("info", "get_stream request from %s cameras=%s old_sessions=%s", request.remote, body.cameras,
+              [getattr(old, "identifier", "?") for old in old_sessions])

  session = StreamSession(body.sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, debug_mode)
  session.stream_dict = stream_dict
@@ -248,6 +310,9 @@ async def get_stream(request: 'web.Request'):
  session.start()

  stream_dict[session.identifier] = session
+  webrtcd_log("info", "get_stream created new session %s active_sessions=%d", session.identifier, len(stream_dict))
+  if old_sessions:
+    asyncio.create_task(retire_old_sessions_after_handover(old_sessions, session))

  return web.json_response({"sdp": answer.sdp, "type": answer.type}, headers={'Access-Control-Allow-Origin': '*'})

--- a/teleoprtc_repo/teleoprtc/stream.py
+++ b/teleoprtc_repo/teleoprtc/stream.py
@@ -9,6 +9,11 @@ from aiortc.contrib.media import MediaRelay

 from teleoprtc.tracks import parse_video_track_id

+try:
+  from openpilot.common.swaglog import cloudlog as _cloudlog
+except Exception:
+  _cloudlog = None
+

@dataclasses.dataclass
 class StreamingOffer:
@@ -49,8 +54,11 @@ class WebRTCBaseStream(abc.ABC):
    self.connection_attempted_event = asyncio.Event()
    self.connection_stopped_event = asyncio.Event()
    self._disconnect_grace_task: Optional[asyncio.Task] = None
+    self._last_connection_state = self.peer_connection.connectionState
+    self._last_ice_connection_state = getattr(self.peer_connection, "iceConnectionState", None)

    self.peer_connection.on("connectionstatechange", self._on_connectionstatechange)
+    self.peer_connection.on("iceconnectionstatechange", self._on_iceconnectionstatechange)
    self.peer_connection.on("datachannel", self._on_incoming_datachannel)
    self.peer_connection.on("track", self._on_incoming_track)

@@ -59,15 +67,36 @@ class WebRTCBaseStream(abc.ABC):
  def _log_debug(self, msg: Any, *args):
    self.logger.debug(f"{type(self)}() {msg}", *args)

-  def _cancel_disconnect_grace(self):
+  def _stream_label(self) -> str:
+    return getattr(self, "session_identifier", f"{type(self).__name__}:{id(self):x}")
+
+  def _log_info(self, msg: str, *args):
+    prefix = f"stream {self._stream_label()} "
+    self.logger.info(prefix + msg, *args)
+    if _cloudlog is not None:
+      _cloudlog.info("[webrtcd] " + prefix + msg, *args)
+
+  def _log_warning(self, msg: str, *args):
+    prefix = f"stream {self._stream_label()} "
+    self.logger.warning(prefix + msg, *args)
+    if _cloudlog is not None:
+      _cloudlog.warning("[webrtcd] " + prefix + msg, *args)
+
+  def _cancel_disconnect_grace(self, reason: Optional[str] = None):
    if self._disconnect_grace_task is not None:
      self._disconnect_grace_task.cancel()
      self._disconnect_grace_task = None
+      if reason is not None:
+        self._log_info("disconnect grace canceled (%s)", reason)

  def _schedule_disconnect_grace(self):
    if self._disconnect_grace_task is not None and not self._disconnect_grace_task.done():
      return

+    self._log_warning("disconnect grace scheduled for %.1fs (conn=%s ice=%s)",
+                      self.DISCONNECTED_GRACE_SECONDS,
+                      self.peer_connection.connectionState,
+                      getattr(self.peer_connection, "iceConnectionState", "unknown"))
    async def _grace_disconnect():
      try:
        await asyncio.sleep(self.DISCONNECTED_GRACE_SECONDS)
@@ -76,10 +105,19 @@ class WebRTCBaseStream(abc.ABC):

      if self.peer_connection.connectionState == "disconnected":
        self._log_debug("disconnect grace expired")
+        self._log_warning("disconnect grace expired (conn=%s ice=%s)",
+                          self.peer_connection.connectionState,
+                          getattr(self.peer_connection, "iceConnectionState", "unknown"))
        self.connection_stopped_event.set()

    self._disconnect_grace_task = asyncio.create_task(_grace_disconnect())

+  def _on_iceconnectionstatechange(self):
+    prev_state = self._last_ice_connection_state
+    state = getattr(self.peer_connection, "iceConnectionState", None)
+    self._last_ice_connection_state = state
+    self._log_info("ice state changed %s -> %s (conn=%s)",
+                   prev_state, state, self.peer_connection.connectionState)
  @property
  def _number_of_incoming_media(self) -> int:
    media = len(self.incoming_camera_tracks) + len(self.incoming_audio_tracks)
@@ -140,19 +178,23 @@ class WebRTCBaseStream(abc.ABC):
    transceiver.setCodecPreferences(rtp_codec)

  def _on_connectionstatechange(self):
+    prev_state = self._last_connection_state
    state = self.peer_connection.connectionState
+    self._last_connection_state = state
    self._log_debug("connection state is %s", state)
+    self._log_info("connection state changed %s -> %s (ice=%s)",
+                   prev_state, state, getattr(self.peer_connection, "iceConnectionState", "unknown"))
    if state in ['connected', 'failed']:
      self.connection_attempted_event.set()
    if state == 'connected':
-      self._cancel_disconnect_grace()
+      self._cancel_disconnect_grace(reason="connection recovered")
    elif state == 'disconnected':
      self._schedule_disconnect_grace()
    elif state in ['closed', 'failed']:
-      self._cancel_disconnect_grace()
+      self._cancel_disconnect_grace(reason=f"connection {state}")
      self.connection_stopped_event.set()
    else:
-      self._cancel_disconnect_grace()
+      self._cancel_disconnect_grace(reason=f"connection {state}")

  def _on_incoming_track(self, track: aiortc.MediaStreamTrack):
    self._log_debug("got track: %s %s", track.kind, track.id)