mawei/carrotpilot
1
0
Fork 0
mirror of https://github.com/ajouatom/openpilot.git synced 2026-06-08 11:04:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

web work (#283)

Browse Source
This commit is contained in:
jominki354
2026-04-08 12:50:33 +09:00
committed by ajouatom
parent 733871cb7e
commit d6cbdd6475
10 changed files with 609 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
selfdrive/carrot/realtime/transports/raw_ws.py
View File

@@ -35,6 +35,13 @@ class RawWsHub:
# 0 = no throttle (send every message) # 0 = no throttle (send every message)
_THROTTLE_MAP = { _THROTTLE_MAP = {
"modelV2": 0, # camera-synced, don't throttle "modelV2": 0, # camera-synced, don't throttle
"carState": 0, # plot 1,2,5,7,8 — needs full rate for dense graphs
"controlsState": 0, # plot 6 — needs full rate for dense graphs
"longitudinalPlan": 0, # plot 1,2,4 — needs full rate for dense graphs
"carControl": 0, # plot 1,6,7,8 — needs full rate for dense graphs
"radarState": 0, # plot 4,5 — needs full rate for dense graphs
"lateralPlan": 0, # overlay path rendering
"carrotMan": 0, # HUD status updates
"roadCameraState": 0.25, # metadata/debug only on web HUD "roadCameraState": 0.25, # metadata/debug only on web HUD
"deviceState": 0.5, # slow-changing HUD stats "deviceState": 0.5, # slow-changing HUD stats
"peripheralState": 0.5, "peripheralState": 0.5,

8
selfdrive/carrot/server/core.py
View File

@@ -27,7 +27,7 @@ import traceback
import numpy as np import numpy as np
from typing import Dict, Any, Tuple, Optional, List from typing import Dict, Any, Tuple, Optional, List
from aiohttp import web, ClientSession, WSMsgType from aiohttp import web, ClientSession, ClientTimeout, WSMsgType
from cereal import messaging from cereal import messaging
from opendbc.car import structs from opendbc.car import structs
import shlex import shlex
@@ -204,14 +204,16 @@ async def proxy_stream(request: web.Request) -> web.StreamResponse:
sess: ClientSession = request.app["http"] sess: ClientSession = request.app["http"]
try: try:
async with sess.post(WEBRTCD_URL, data=body, headers={"Content-Type": ct}) as resp: async with sess.post(WEBRTCD_URL, data=body, headers={"Content-Type": ct},
timeout=ClientTimeout(total=15)) as resp:
resp_body = await resp.read() resp_body = await resp.read()
# 그대로 전달
out = web.Response(body=resp_body, status=resp.status) out = web.Response(body=resp_body, status=resp.status)
rct = resp.headers.get("Content-Type") rct = resp.headers.get("Content-Type")
if rct: if rct:
out.headers["Content-Type"] = rct out.headers["Content-Type"] = rct
return out return out
except asyncio.TimeoutError:
return web.json_response({"ok": False, "error": "webrtcd timeout"}, status=504)
except Exception as e: except Exception as e:
return web.json_response({"ok": False, "error": str(e)}, status=502) return web.json_response({"ok": False, "error": str(e)}, status=502)

19
selfdrive/carrot/web/css/app.css
View File

@@ -2152,16 +2152,13 @@ body[data-page="carrot"] #driveHudCard.driveHudCard--loading {
border-radius: calc(var(--r-2xl) + 2px); border-radius: calc(var(--r-2xl) + 2px);
overflow: hidden; overflow: hidden;
border: 1px solid color-mix(in srgb, var(--md-outline-var) 82%, rgba(255,255,255,0.12)); border: 1px solid color-mix(in srgb, var(--md-outline-var) 82%, rgba(255,255,255,0.12));
background: background: #0b0e12;
radial-gradient(circle at 16% 20%, rgba(255, 146, 77, 0.15), transparent 28%),
radial-gradient(circle at 80% 80%, rgba(104, 201, 255, 0.12), transparent 24%),
linear-gradient(180deg, rgba(255,255,255,0.03), rgba(255,255,255,0)),
#04070c;
box-shadow: 0 20px 48px rgba(0, 0, 0, 0.24); box-shadow: 0 20px 48px rgba(0, 0, 0, 0.24);
contain: layout style; contain: layout style;
} }
.carrot-stage__video, .carrot-stage__video,
.carrot-stage__videoHold,
.carrot-stage__canvas, .carrot-stage__canvas,
.carrot-stage__hud { .carrot-stage__hud {
position: absolute; position: absolute;
@@ -2212,6 +2209,10 @@ body[data-page="carrot"] #driveHudCard.driveHudCard--loading {
opacity: 1; opacity: 1;
} }
.carrot-stage.is-video-held .carrot-stage__videoHold {
opacity: 1;
}
.carrot-stage.is-loading .carrot-stage__controls { .carrot-stage.is-loading .carrot-stage__controls {
opacity: 0; opacity: 0;
transform: translateY(8px); transform: translateY(8px);
@@ -2225,6 +2226,14 @@ body[data-page="carrot"] #driveHudCard.driveHudCard--loading {
pointer-events: none; pointer-events: none;
} }
.carrot-stage__videoHold {
transform-origin: 0 0;
will-change: transform;
background: #000;
pointer-events: none;
z-index: 1;
}
.carrot-stage__canvas { .carrot-stage__canvas {
transform-origin: 0 0; transform-origin: 0 0;
will-change: transform; will-change: transform;

7
selfdrive/carrot/web/index.html
View File

@@ -7,7 +7,7 @@
<link rel="icon" type="image/png" href="/assets/img_chffr_wheel.png?v=2604-01" /> <link rel="icon" type="image/png" href="/assets/img_chffr_wheel.png?v=2604-01" />
<link rel="stylesheet" href="/css/tokens.css?v=2603-73" /> <link rel="stylesheet" href="/css/tokens.css?v=2603-73" />
<link rel="stylesheet" href="/css/hud_card.css?v=2604-09" /> <link rel="stylesheet" href="/css/hud_card.css?v=2604-09" />
<link rel="stylesheet" href="/css/app.css?v=2604-38" /> <link rel="stylesheet" href="/css/app.css?v=2604-40" />
</head> </head>
<body> <body>
@@ -243,6 +243,7 @@
<div id="carrotStage" class="carrot-stage"> <div id="carrotStage" class="carrot-stage">
<video id="carrotRoadVideo" class="carrot-stage__video" autoplay playsinline muted style="display:none;"></video> <video id="carrotRoadVideo" class="carrot-stage__video" autoplay playsinline muted style="display:none;"></video>
<canvas id="carrotLastFrameCanvas" class="carrot-stage__videoHold" aria-hidden="true"></canvas>
<canvas id="carrotOverlayCanvas" class="carrot-stage__canvas"></canvas> <canvas id="carrotOverlayCanvas" class="carrot-stage__canvas"></canvas>
<canvas id="carrotHudCanvas" class="carrot-stage__hud"></canvas> <canvas id="carrotHudCanvas" class="carrot-stage__hud"></canvas>
<div id="carrotOnroadAlert" class="carrot-stage__alert" hidden aria-live="polite" aria-atomic="true"> <div id="carrotOnroadAlert" class="carrot-stage__alert" hidden aria-live="polite" aria-atomic="true">
@@ -441,7 +442,7 @@
<script src="/js/app_core.js?v=2604-18"></script> <script src="/js/app_core.js?v=2604-18"></script>
<script src="/js/app_pages.js?v=2604-23"></script> <script src="/js/app_pages.js?v=2604-23"></script>
<script src="/js/raw_capnp.js?v=2604-05"></script> <script src="/js/raw_capnp.js?v=2604-05"></script>
<script src="/js/app_realtime.js?v=2604-11"></script> <script src="/js/app_realtime.js?v=2604-16"></script>
<script src="/js/home_drive.js?v=2604-26"></script> <script src="/js/home_drive.js?v=2604-27"></script>
</body> </body>
</html> </html>

353
selfdrive/carrot/web/js/app_realtime.js
View File

@@ -49,7 +49,8 @@ let LIVE_RUNTIME_FETCH_IN_FLIGHT = null;
let LIVE_RUNTIME_POLL_ACTIVE = false; let LIVE_RUNTIME_POLL_ACTIVE = false;
let LAST_HUD_PAYLOAD_SIGNATURE = ""; let LAST_HUD_PAYLOAD_SIGNATURE = "";
const RTC_STATS_POLL_MS = 1000; const RTC_STATS_POLL_MS = 1000;
const RTC_FREEZE_MAX_STALL_SAMPLES = 3; const RTC_FREEZE_MAX_STALL_SAMPLES = 8;
const RTC_INITIAL_FRAME_MAX_STALL_SAMPLES = 14;
const RTC_FREEZE_CURRENT_TIME_EPSILON = 0.05; const RTC_FREEZE_CURRENT_TIME_EPSILON = 0.05;
const RTC_FREEZE_RECOVERY_COOLDOWN_MS = 4000; const RTC_FREEZE_RECOVERY_COOLDOWN_MS = 4000;
const RTC_RESUME_PROGRESS_CHECK_MS = 900; const RTC_RESUME_PROGRESS_CHECK_MS = 900;
@@ -76,6 +77,8 @@ const RTC_FREEZE_STATE = {
lastTotalVideoFrames: null, lastTotalVideoFrames: null,
lastCurrentTime: null, lastCurrentTime: null,
lastRecoveredAtMs: 0, lastRecoveredAtMs: 0,
consecutiveRecoveries: 0,
everDecodedFrame: false,
}; };
let RTC_FREEZE_RECOVER_T = null; let RTC_FREEZE_RECOVER_T = null;
let RTC_VIDEO_EVENTS_BOUND = false; let RTC_VIDEO_EVENTS_BOUND = false;
@@ -85,6 +88,49 @@ const RTC_VISIBILITY_STATE = {
hiddenAtMs: 0, hiddenAtMs: 0,
currentTimeAtHide: null, currentTimeAtHide: null,
}; };
const RTC_TRACE_ENABLED = false;
let RTC_PC_SEQ = 0;
function rtcPcLabel(pc) {
if (!pc) return "none";
if (!pc.__carrotRtcLabel) {
RTC_PC_SEQ += 1;
pc.__carrotRtcLabel = `pc${RTC_PC_SEQ}`;
}
return pc.__carrotRtcLabel;
}
function rtcBuildTraceSnapshot(pc = RTC_PC) {
const video = getRtcVideoElement();
const track = video?.srcObject?.getVideoTracks?.()?.[0] || null;
return {
conn: pc?.connectionState || "none",
ice: pc?.iceConnectionState || "none",
framesDecoded: RTC_PERF_STATE.inbound?.framesDecoded ?? null,
fps: RTC_PERF_STATE.inbound?.framesPerSecond ?? null,
currentTime: Number.isFinite(Number(video?.currentTime)) ? Number(video.currentTime).toFixed(2) : null,
readyState: Number.isFinite(Number(video?.readyState)) ? Number(video.readyState) : null,
trackState: track?.readyState || null,
trackMuted: typeof track?.muted === "boolean" ? track.muted : null,
hold: Boolean(getRtcStageElement()?.classList.contains("is-video-held")),
};
}
function rtcTrace(event, extra = {}, pc = RTC_PC) {
if (!RTC_TRACE_ENABLED) return;
console.log("[RTC TRACE]", {
ts: Date.now(),
iso: new Date().toISOString(),
event,
pc: rtcPcLabel(pc),
...rtcBuildTraceSnapshot(pc),
...extra,
});
}
function rtcPcSawTrack(pc) {
return Boolean(pc && pc.__carrotTrackSeen);
}
function getRtcVideoElement() { function getRtcVideoElement() {
return document.getElementById("carrotRoadVideo") || document.getElementById("rtcVideo"); return document.getElementById("carrotRoadVideo") || document.getElementById("rtcVideo");
@@ -94,6 +140,55 @@ function getLegacyRtcVideoElement() {
return document.getElementById("rtcVideo"); return document.getElementById("rtcVideo");
} }
function getRtcVideoHoldElement() {
return document.getElementById("carrotLastFrameCanvas");
}
function getRtcStageElement() {
return document.getElementById("carrotStage");
}
function rtcShowVideoHold(show) {
const stage = getRtcStageElement();
if (!stage) return;
stage.classList.toggle("is-video-held", Boolean(show));
}
function rtcClearVideoHold() {
const hold = getRtcVideoHoldElement();
if (hold) {
const ctx = hold.getContext("2d");
if (ctx) {
ctx.clearRect(0, 0, hold.width || 0, hold.height || 0);
}
}
rtcShowVideoHold(false);
}
function rtcCaptureVideoHoldFrame() {
const video = getRtcVideoElement();
const hold = getRtcVideoHoldElement();
if (!video || !hold || Number(video.readyState || 0) < 2) return false;
const targetWidth = Math.max(1, Number(hold.width || video.videoWidth || 0));
const targetHeight = Math.max(1, Number(hold.height || video.videoHeight || 0));
if (!targetWidth || !targetHeight) return false;
const ctx = hold.getContext("2d");
if (!ctx) return false;
if (hold.width !== targetWidth) hold.width = targetWidth;
if (hold.height !== targetHeight) hold.height = targetHeight;
try {
ctx.clearRect(0, 0, targetWidth, targetHeight);
ctx.drawImage(video, 0, 0, targetWidth, targetHeight);
rtcShowVideoHold(true);
return true;
} catch {
return false;
}
}
function isCarrotPageActive() { function isCarrotPageActive() {
return document.body?.dataset?.page === "carrot"; return document.body?.dataset?.page === "carrot";
} }
@@ -318,6 +413,7 @@ function rtcResetFreezeWatchdog() {
RTC_FREEZE_STATE.lastFramesDecoded = null; RTC_FREEZE_STATE.lastFramesDecoded = null;
RTC_FREEZE_STATE.lastTotalVideoFrames = null; RTC_FREEZE_STATE.lastTotalVideoFrames = null;
RTC_FREEZE_STATE.lastCurrentTime = null; RTC_FREEZE_STATE.lastCurrentTime = null;
RTC_FREEZE_STATE.everDecodedFrame = false;
} }
function rtcCancelResumeCheck() { function rtcCancelResumeCheck() {
@@ -505,6 +601,8 @@ function startRtcPerfPolling(force = false) {
// ===== WebRTC (auto) ===== // ===== WebRTC (auto) =====
let RTC_PC = null; let RTC_PC = null;
let RTC_PENDING_PC = null;
let RTC_STANDBY_PC = null;
let RTC_RETRY_T = null; let RTC_RETRY_T = null;
let RTC_WAIT_TRACK_T = null; let RTC_WAIT_TRACK_T = null;
let RTC_FAIL_COUNT = 0; let RTC_FAIL_COUNT = 0;
@@ -527,7 +625,18 @@ function rtcHasUsableTrack() {
if (typeof stream.getVideoTracks !== "function") return true; if (typeof stream.getVideoTracks !== "function") return true;
const tracks = stream.getVideoTracks(); const tracks = stream.getVideoTracks();
if (!tracks.length) return true; if (!tracks.length) return true;
return tracks.some((track) => track && track.readyState !== "ended" && track.muted !== true); return tracks.some((track) => track && track.readyState !== "ended");
}
function rtcClosePeer(pc) {
if (!pc) return;
try { pc.ontrack = null; } catch {}
try { pc.onconnectionstatechange = null; } catch {}
try { pc.oniceconnectionstatechange = null; } catch {}
try { pc.close(); } catch {}
if (RTC_PC === pc) RTC_PC = null;
if (RTC_PENDING_PC === pc) RTC_PENDING_PC = null;
if (RTC_STANDBY_PC === pc) RTC_STANDBY_PC = null;
} }
function rtcStatusSet(s) { function rtcStatusSet(s) {
@@ -542,26 +651,35 @@ function rtcCancelRetry() {
} }
} }
async function rtcDisconnect() { async function rtcDisconnect(options = {}) {
const keepVideo = Boolean(options.keepVideo);
rtcCancelRetry(); rtcCancelRetry();
rtcDisarmTrackTimeout(); rtcDisarmTrackTimeout();
rtcCancelResumeCheck(); rtcCancelResumeCheck();
rtcCancelFreezeRecovery(); rtcCancelFreezeRecovery();
stopRtcPerfPolling(); stopRtcPerfPolling();
try { if (RTC_PC) RTC_PC.close(); } catch {} const activePc = RTC_PC;
const pendingPc = RTC_PENDING_PC;
const standbyPc = RTC_STANDBY_PC;
RTC_PC = null; RTC_PC = null;
RTC_PENDING_PC = null;
RTC_STANDBY_PC = null;
rtcClosePeer(pendingPc);
rtcClosePeer(activePc);
rtcClosePeer(standbyPc);
resetRtcPerfState(); resetRtcPerfState();
rtcResetFreezeWatchdog(); rtcResetFreezeWatchdog();
const video = getRtcVideoElement(); if (!keepVideo) {
if (video) { rtcClearVideoHold();
video.srcObject = null; const video = getRtcVideoElement();
video.style.display = "none"; if (video) {
} video.srcObject = null;
const legacyVideo = getLegacyRtcVideoElement(); }
if (legacyVideo && legacyVideo !== video) { const legacyVideo = getLegacyRtcVideoElement();
legacyVideo.srcObject = null; if (legacyVideo && legacyVideo !== video) {
legacyVideo.style.display = "none"; legacyVideo.srcObject = null;
}
} }
} }
@@ -574,6 +692,10 @@ function rtcIsWaitingForInitialTrack(pc = RTC_PC) {
return Boolean(RTC_WAIT_TRACK_T && RTC_WAIT_TRACK_PC && RTC_WAIT_TRACK_PC === pc); return Boolean(RTC_WAIT_TRACK_T && RTC_WAIT_TRACK_PC && RTC_WAIT_TRACK_PC === pc);
} }
function rtcIsWaitingForInitialTrack(pc = RTC_PC) {
return Boolean(RTC_WAIT_TRACK_T && RTC_WAIT_TRACK_PC && RTC_WAIT_TRACK_PC === pc);
}
function rtcUpdateFreezeSnapshot(snapshot) { function rtcUpdateFreezeSnapshot(snapshot) {
RTC_FREEZE_STATE.lastFramesDecoded = snapshot.framesDecoded; RTC_FREEZE_STATE.lastFramesDecoded = snapshot.framesDecoded;
RTC_FREEZE_STATE.lastTotalVideoFrames = snapshot.totalVideoFrames; RTC_FREEZE_STATE.lastTotalVideoFrames = snapshot.totalVideoFrames;
@@ -582,15 +704,22 @@ function rtcUpdateFreezeSnapshot(snapshot) {
function rtcScheduleFreezeRecovery(reason, options = {}) { function rtcScheduleFreezeRecovery(reason, options = {}) {
const force = Boolean(options.force); const force = Boolean(options.force);
if (!shouldRunCarrotVisionRealtime() || _rtcConnecting || RTC_FREEZE_RECOVER_T) return; if (!shouldRunCarrotVisionRealtime() || _rtcConnecting || RTC_PENDING_PC || RTC_FREEZE_RECOVER_T) return;
const now = Date.now(); const now = Date.now();
if (!force && (now - RTC_FREEZE_STATE.lastRecoveredAtMs < RTC_FREEZE_RECOVERY_COOLDOWN_MS)) return; if (!force && (now - RTC_FREEZE_STATE.lastRecoveredAtMs < RTC_FREEZE_RECOVERY_COOLDOWN_MS)) return;
RTC_FREEZE_STATE.consecutiveRecoveries++;
RTC_FREEZE_STATE.lastRecoveredAtMs = now; RTC_FREEZE_STATE.lastRecoveredAtMs = now;
RTC_FREEZE_STATE.stallSamples = 0; RTC_FREEZE_STATE.stallSamples = 0;
rtcStatusSet(reason); rtcStatusSet(reason);
rtcTrace("freeze_recovery_scheduled", {
reason,
force,
attempt: RTC_FREEZE_STATE.consecutiveRecoveries,
}, RTC_PC || RTC_PENDING_PC);
console.warn("[RTC] road video stalled, reconnecting", { console.warn("[RTC] road video stalled, reconnecting", {
reason, reason,
attempt: RTC_FREEZE_STATE.consecutiveRecoveries,
connectionState: RTC_PERF_STATE.connectionState, connectionState: RTC_PERF_STATE.connectionState,
iceConnectionState: RTC_PERF_STATE.iceConnectionState, iceConnectionState: RTC_PERF_STATE.iceConnectionState,
inbound: RTC_PERF_STATE.inbound, inbound: RTC_PERF_STATE.inbound,
@@ -600,9 +729,9 @@ function rtcScheduleFreezeRecovery(reason, options = {}) {
RTC_FREEZE_RECOVER_T = setTimeout(async () => { RTC_FREEZE_RECOVER_T = setTimeout(async () => {
RTC_FREEZE_RECOVER_T = null; RTC_FREEZE_RECOVER_T = null;
if (!shouldRunCarrotVisionRealtime()) return; if (!shouldRunCarrotVisionRealtime()) return;
await rtcDisconnect(); rtcCaptureVideoHoldFrame();
RTC_FAIL_COUNT = 0; RTC_FAIL_COUNT = 0;
await rtcConnectOnce().catch(() => {}); await rtcConnectOnce({ force: true }).catch(() => {});
}, 0); }, 0);
} }
@@ -620,11 +749,10 @@ function rtcUpdateFreezeWatchdog(pc, video) {
// PC connected but track dead/muted/inactive → force reconnect // PC connected but track dead/muted/inactive → force reconnect
if (rtcConnectionLooksLive(pc) && !rtcHasUsableTrack() && video.srcObject) { if (rtcConnectionLooksLive(pc) && !rtcHasUsableTrack() && video.srcObject) {
rtcResetFreezeWatchdog(); rtcResetFreezeWatchdog();
rtcScheduleFreezeRecovery("track ended, reconnecting...");
return; return;
} }
if (!rtcConnectionLooksLive(pc) || !rtcHasUsableTrack()) { if (!rtcConnectionLooksLive(pc) || !rtcHasLiveTrack()) {
rtcResetFreezeWatchdog(); rtcResetFreezeWatchdog();
return; return;
} }
@@ -653,11 +781,20 @@ function rtcUpdateFreezeWatchdog(pc, video) {
(snapshot.totalVideoFrames != null && RTC_FREEZE_STATE.lastTotalVideoFrames != null && snapshot.totalVideoFrames > RTC_FREEZE_STATE.lastTotalVideoFrames) || (snapshot.totalVideoFrames != null && RTC_FREEZE_STATE.lastTotalVideoFrames != null && snapshot.totalVideoFrames > RTC_FREEZE_STATE.lastTotalVideoFrames) ||
(snapshot.currentTime != null && RTC_FREEZE_STATE.lastCurrentTime != null && snapshot.currentTime > RTC_FREEZE_STATE.lastCurrentTime + RTC_FREEZE_CURRENT_TIME_EPSILON); (snapshot.currentTime != null && RTC_FREEZE_STATE.lastCurrentTime != null && snapshot.currentTime > RTC_FREEZE_STATE.lastCurrentTime + RTC_FREEZE_CURRENT_TIME_EPSILON);
RTC_FREEZE_STATE.stallSamples = hasProgress ? 0 : RTC_FREEZE_STATE.stallSamples + 1; if (hasProgress) {
RTC_FREEZE_STATE.stallSamples = 0;
RTC_FREEZE_STATE.consecutiveRecoveries = 0;
if (!RTC_FREEZE_STATE.everDecodedFrame) {
RTC_FREEZE_STATE.everDecodedFrame = true;
}
} else {
RTC_FREEZE_STATE.stallSamples++;
}
rtcUpdateFreezeSnapshot(snapshot); rtcUpdateFreezeSnapshot(snapshot);
if (RTC_FREEZE_STATE.stallSamples >= RTC_FREEZE_MAX_STALL_SAMPLES) { const stallLimit = RTC_FREEZE_STATE.everDecodedFrame ? RTC_FREEZE_MAX_STALL_SAMPLES : RTC_INITIAL_FRAME_MAX_STALL_SAMPLES;
rtcScheduleFreezeRecovery("video stalled, reconnecting..."); if (RTC_FREEZE_STATE.stallSamples >= stallLimit) {
rtcScheduleFreezeRecovery(RTC_FREEZE_STATE.everDecodedFrame ? "video stalled, reconnecting..." : "no initial frame, reconnecting...");
} }
} }
@@ -675,6 +812,8 @@ function rtcBindVideoEvents() {
video.addEventListener("playing", () => { video.addEventListener("playing", () => {
RTC_FREEZE_STATE.stallSamples = 0; RTC_FREEZE_STATE.stallSamples = 0;
RTC_FREEZE_STATE.everDecodedFrame = true;
rtcClearVideoHold();
collectRtcPerfStats().catch(() => {}); collectRtcPerfStats().catch(() => {});
}); });
["waiting", "stalled", "suspend", "pause", "ended"].forEach((eventName) => { ["waiting", "stalled", "suspend", "pause", "ended"].forEach((eventName) => {
@@ -695,14 +834,30 @@ function rtcScheduleRetry(ms = 2000) {
} }
function rtcArmTrackTimeout(ms = 5000, expectedPc = RTC_PC) { function rtcArmTrackTimeout(ms = 5000, expectedPc = RTC_PC) {
if (rtcPcSawTrack(expectedPc)) {
rtcTrace("track_timeout_arm_skipped", { timeoutMs: ms, reason: "track already seen" }, expectedPc);
return;
}
if (RTC_WAIT_TRACK_T) clearTimeout(RTC_WAIT_TRACK_T); if (RTC_WAIT_TRACK_T) clearTimeout(RTC_WAIT_TRACK_T);
RTC_WAIT_TRACK_PC = expectedPc; RTC_WAIT_TRACK_PC = expectedPc;
RTC_WAIT_TRACK_T = setTimeout(async () => { RTC_WAIT_TRACK_T = setTimeout(async () => {
RTC_WAIT_TRACK_T = null; RTC_WAIT_TRACK_T = null;
if (RTC_WAIT_TRACK_PC !== expectedPc || RTC_PC !== expectedPc) return; if (RTC_WAIT_TRACK_PC !== expectedPc || (RTC_PC !== expectedPc && RTC_PENDING_PC !== expectedPc)) return;
if (rtcPcSawTrack(expectedPc)) {
RTC_WAIT_TRACK_PC = null;
rtcTrace("track_timeout_ignored", { timeoutMs: ms, reason: "track arrived before timeout fired" }, expectedPc);
return;
}
RTC_WAIT_TRACK_PC = null; RTC_WAIT_TRACK_PC = null;
rtcTrace("track_timeout", { timeoutMs: ms }, expectedPc);
rtcStatusSet("no track, retry..."); rtcStatusSet("no track, retry...");
await rtcDisconnect(); rtcCaptureVideoHoldFrame();
if (RTC_PENDING_PC === expectedPc) {
rtcClosePeer(expectedPc);
rtcScheduleRetry(2000);
return;
}
await rtcDisconnect({ keepVideo: true });
rtcScheduleRetry(2000); rtcScheduleRetry(2000);
}, ms); }, ms);
} }
@@ -720,12 +875,8 @@ function rtcScheduleResumeHealthCheck(reason = "returned visible") {
rtcCancelResumeCheck(); rtcCancelResumeCheck();
RTC_RESUME_CHECK_T = setTimeout(async () => { RTC_RESUME_CHECK_T = setTimeout(async () => {
RTC_RESUME_CHECK_T = null; RTC_RESUME_CHECK_T = null;
if (!shouldRunCarrotVisionRealtime() || _rtcConnecting || !RTC_PC) return; if (!shouldRunCarrotVisionRealtime() || _rtcConnecting || RTC_PENDING_PC || !RTC_PC) return;
const video = getRtcVideoElement(); if (!rtcConnectionLooksLive(RTC_PC) || !rtcHasLiveTrack()) {
const currentTime = Number(video?.currentTime || 0);
const hiddenTime = Number(RTC_VISIBILITY_STATE.currentTimeAtHide || 0);
const progressed = currentTime > hiddenTime + RTC_FREEZE_CURRENT_TIME_EPSILON;
if (!rtcConnectionLooksLive(RTC_PC) || !rtcHasUsableTrack() || !progressed) {
rtcScheduleFreezeRecovery(`${reason}, reconnecting...`, { force: true }); rtcScheduleFreezeRecovery(`${reason}, reconnecting...`, { force: true });
} }
}, RTC_RESUME_PROGRESS_CHECK_MS); }, RTC_RESUME_PROGRESS_CHECK_MS);
@@ -748,23 +899,56 @@ async function waitIceComplete(pc, timeoutMs = 8000) {
let _rtcConnecting = false; let _rtcConnecting = false;
async function rtcConnectOnce() { async function rtcConnectOnce(options = {}) {
const force = Boolean(options.force);
if (!shouldRunCarrotVisionRealtime()) return; if (!shouldRunCarrotVisionRealtime()) return;
if (_rtcConnecting) return; if (_rtcConnecting || RTC_PENDING_PC) return;
if (RTC_PC && (RTC_PC.connectionState === "connected" || RTC_PC.connectionState === "connecting")) return; if (!force && RTC_PC && (RTC_PC.connectionState === "connected" || RTC_PC.connectionState === "connecting") && rtcHasLiveTrack()) return;
_rtcConnecting = true; _rtcConnecting = true;
let previousPc = RTC_PC || RTC_STANDBY_PC;
try { try {
await rtcDisconnect(); rtcCancelRetry();
rtcStatusSet("connecting..."); rtcDisarmTrackTimeout();
rtcCancelResumeCheck();
rtcCancelFreezeRecovery();
rtcTrace("connect_start", {
force,
hasPreviousPc: Boolean(previousPc),
hasLiveTrack: rtcHasLiveTrack(),
}, previousPc || RTC_PC);
const keepPreviousVisible = Boolean(previousPc && rtcHasLiveTrack());
if (keepPreviousVisible) {
if (RTC_STANDBY_PC && RTC_STANDBY_PC !== previousPc) {
rtcClosePeer(RTC_STANDBY_PC);
}
RTC_STANDBY_PC = previousPc;
if (RTC_PC === previousPc) {
RTC_PC = null;
}
stopRtcPerfPolling();
resetRtcPerfState();
rtcResetFreezeWatchdog();
rtcStatusSet("reconnecting...");
} else {
await rtcDisconnect({ keepVideo: true });
previousPc = null;
rtcStatusSet("connecting...");
}
const pc = new RTCPeerConnection({ const pc = new RTCPeerConnection({
iceServers: [], iceServers: [],
sdpSemantics: "unified-plan", sdpSemantics: "unified-plan",
iceCandidatePoolSize: 1, iceCandidatePoolSize: 1,
}); });
RTC_PC = pc; rtcPcLabel(pc);
startRtcPerfPolling(true); pc.__carrotTrackSeen = false;
RTC_PENDING_PC = pc;
rtcTrace("pc_created", {
keepPreviousVisible,
hasPreviousPc: Boolean(previousPc),
}, pc);
const video = getRtcVideoElement(); const video = getRtcVideoElement();
if (video) { if (video) {
@@ -775,9 +959,14 @@ async function rtcConnectOnce() {
pc.addTransceiver("video", { direction: "recvonly" }); pc.addTransceiver("video", { direction: "recvonly" });
pc.ontrack = async (ev) => { pc.ontrack = async (ev) => {
if (RTC_PC !== pc) return; if (RTC_PENDING_PC !== pc) return;
const videoEl = getRtcVideoElement(); const videoEl = getRtcVideoElement();
if (!videoEl) return; if (!videoEl) return;
rtcTrace("track_received", {
kind: ev.track?.kind || null,
streamCount: Array.isArray(ev.streams) ? ev.streams.length : 0,
}, pc);
pc.__carrotTrackSeen = true;
let stream = ev.streams && ev.streams[0]; let stream = ev.streams && ev.streams[0];
if (!stream) { if (!stream) {
@@ -785,44 +974,83 @@ async function rtcConnectOnce() {
} }
videoEl.srcObject = stream; videoEl.srcObject = stream;
if (videoEl.id === "rtcVideo") { RTC_PENDING_PC = null;
videoEl.style.display = "block"; RTC_PC = pc;
} const retiringPc = RTC_STANDBY_PC && RTC_STANDBY_PC !== pc ? RTC_STANDBY_PC : null;
RTC_STANDBY_PC = null;
try { await videoEl.play(); } catch (e) { console.log("[RTC] play() failed", e); } try { await videoEl.play(); } catch (e) { console.log("[RTC] play() failed", e); }
rtcStatusSet("track: " + ev.track.kind); rtcStatusSet("track: " + ev.track.kind);
rtcDisarmTrackTimeout(pc); rtcDisarmTrackTimeout(pc);
RTC_FAIL_COUNT = 0; RTC_FAIL_COUNT = 0;
rtcResetFreezeWatchdog(); rtcResetFreezeWatchdog();
rtcClearVideoHold();
startRtcPerfPolling(true);
collectRtcPerfStats().catch(() => {}); collectRtcPerfStats().catch(() => {});
if (retiringPc) {
rtcClosePeer(retiringPc);
}
// Detect server-side track close → immediate recovery // Detect server-side track close → immediate recovery (guarded by PC identity)
ev.track.addEventListener("ended", () => { ev.track.addEventListener("ended", () => {
rtcTrace("track_ended", {
kind: ev.track?.kind || null,
trackReadyState: ev.track?.readyState || null,
}, pc);
console.warn("[RTC] remote track ended"); console.warn("[RTC] remote track ended");
if (RTC_PC === pc && shouldRunCarrotVisionRealtime() && !_rtcConnecting) { if ((RTC_PC === pc || RTC_PENDING_PC === pc) && shouldRunCarrotVisionRealtime() && !_rtcConnecting) {
rtcScheduleFreezeRecovery("remote track ended"); rtcCaptureVideoHoldFrame();
rtcScheduleFreezeRecovery("remote track ended", { force: true });
} }
}); });
}; };
pc.onconnectionstatechange = () => { pc.onconnectionstatechange = () => {
if (RTC_PC !== pc) return; const isPending = RTC_PENDING_PC === pc;
const isActive = RTC_PC === pc;
if (!isPending && !isActive) return;
const state = pc.connectionState; const state = pc.connectionState;
rtcTrace("connection_state_change", {
isPending,
isActive,
state,
}, pc);
rtcStatusSet("conn: " + state); rtcStatusSet("conn: " + state);
if (state === "connected") RTC_FAIL_COUNT = 0; if (state === "connected") RTC_FAIL_COUNT = 0;
collectRtcPerfStats().catch(() => {}); if (isActive) {
if (state === "failed" || state === "disconnected" || state === "closed") { collectRtcPerfStats().catch(() => {});
rtcDisconnect(); }
if (state === "failed" || state === "closed") {
rtcCaptureVideoHoldFrame();
if (isPending) {
rtcClosePeer(pc);
} else {
rtcDisconnect({ keepVideo: true }).catch(() => {});
}
rtcScheduleRetry(2000); rtcScheduleRetry(2000);
} }
}; };
pc.oniceconnectionstatechange = () => { pc.oniceconnectionstatechange = () => {
if (RTC_PC !== pc) return; const isPending = RTC_PENDING_PC === pc;
const isActive = RTC_PC === pc;
if (!isPending && !isActive) return;
const state = pc.iceConnectionState; const state = pc.iceConnectionState;
rtcTrace("ice_state_change", {
isPending,
isActive,
state,
}, pc);
rtcStatusSet("ice: " + state); rtcStatusSet("ice: " + state);
collectRtcPerfStats().catch(() => {}); if (isActive) {
if (state === "failed" || state === "disconnected" || state === "closed") { collectRtcPerfStats().catch(() => {});
rtcDisconnect(); }
if (state === "failed" || state === "closed") {
rtcCaptureVideoHoldFrame();
if (isPending) {
rtcClosePeer(pc);
} else {
rtcDisconnect({ keepVideo: true }).catch(() => {});
}
rtcScheduleRetry(2000); rtcScheduleRetry(2000);
} }
}; };
@@ -830,6 +1058,9 @@ async function rtcConnectOnce() {
const offer = await pc.createOffer(); const offer = await pc.createOffer();
await pc.setLocalDescription(offer); await pc.setLocalDescription(offer);
await waitIceComplete(pc, 8000); await waitIceComplete(pc, 8000);
rtcTrace("offer_ready", {
localSdpBytes: pc.localDescription?.sdp?.length || 0,
}, pc);
const body = { const body = {
sdp: pc.localDescription.sdp, sdp: pc.localDescription.sdp,
@@ -851,13 +1082,27 @@ async function rtcConnectOnce() {
const answer = await response.json(); const answer = await response.json();
if (!answer || !answer.sdp) throw new Error("bad answer"); if (!answer || !answer.sdp) throw new Error("bad answer");
rtcTrace("answer_received", {
remoteSdpBytes: answer.sdp?.length || 0,
answerType: answer.type || "answer",
}, pc);
await pc.setRemoteDescription({ type: answer.type || "answer", sdp: answer.sdp }); await pc.setRemoteDescription({ type: answer.type || "answer", sdp: answer.sdp });
rtcTrace("answer_applied", {}, pc);
rtcStatusSet("connected (waiting track...)"); rtcStatusSet("connected (waiting track...)");
rtcArmTrackTimeout(6000, pc); rtcArmTrackTimeout(6000, pc);
} catch (e) { } catch (e) {
rtcTrace("connect_error", {
message: e?.message || String(e),
}, RTC_PENDING_PC || previousPc || RTC_PC);
rtcStatusSet("error: " + e.message); rtcStatusSet("error: " + e.message);
await rtcDisconnect(); rtcCaptureVideoHoldFrame();
if (RTC_PENDING_PC) {
rtcClosePeer(RTC_PENDING_PC);
}
if (!RTC_PC && !RTC_STANDBY_PC) {
await rtcDisconnect({ keepVideo: true });
}
rtcScheduleRetry(2000); rtcScheduleRetry(2000);
} finally { } finally {
_rtcConnecting = false; _rtcConnecting = false;
@@ -1639,7 +1884,7 @@ function syncCarrotRealtimeLifecycle(forceFetch = false) {
const nextVisionActive = shouldRunCarrotVisionRealtime(); const nextVisionActive = shouldRunCarrotVisionRealtime();
if (nextHudActive === _carrotHudRealtimeActive && nextVisionActive === _carrotVisionRealtimeActive && !forceFetch) { if (nextHudActive === _carrotHudRealtimeActive && nextVisionActive === _carrotVisionRealtimeActive && !forceFetch) {
if (nextVisionActive && !_rtcConnecting && !rtcHasLiveTrack()) { if (nextVisionActive && !_rtcConnecting && (!RTC_PC || !rtcHasLiveTrack())) {
rtcConnectOnce().catch(() => {}); rtcConnectOnce().catch(() => {});
} }
if (nextVisionActive) scheduleRtcPerfPolling(); if (nextVisionActive) scheduleRtcPerfPolling();
@@ -1669,7 +1914,7 @@ function syncCarrotRealtimeLifecycle(forceFetch = false) {
ensureRawDecodeWorker(); ensureRawDecodeWorker();
rawOverlayConnectAll(); rawOverlayConnectAll();
startRtcPerfPolling(true); startRtcPerfPolling(true);
if (!_rtcConnecting && !rtcHasLiveTrack()) { if (!_rtcConnecting && (!RTC_PC || !rtcHasLiveTrack())) {
rtcCancelRetry(); rtcCancelRetry();
RTC_FAIL_COUNT = 0; RTC_FAIL_COUNT = 0;
rtcConnectOnce().catch(() => {}); rtcConnectOnce().catch(() => {});

39
selfdrive/carrot/web/js/home_drive.js
View File

@@ -3,6 +3,7 @@
window.HomeDrive = (() => { window.HomeDrive = (() => {
const stageEl = document.getElementById("carrotStage"); const stageEl = document.getElementById("carrotStage");
const videoEl = document.getElementById("carrotRoadVideo"); const videoEl = document.getElementById("carrotRoadVideo");
const videoHoldEl = document.getElementById("carrotLastFrameCanvas");
const canvasEl = document.getElementById("carrotOverlayCanvas"); const canvasEl = document.getElementById("carrotOverlayCanvas");
const hudCanvasEl = document.getElementById("carrotHudCanvas"); const hudCanvasEl = document.getElementById("carrotHudCanvas");
const onroadAlertEl = document.getElementById("carrotOnroadAlert"); const onroadAlertEl = document.getElementById("carrotOnroadAlert");
@@ -1614,6 +1615,12 @@ window.HomeDrive = (() => {
overlaySizeSignature = nextOverlaySignature; overlaySizeSignature = nextOverlaySignature;
videoEl.style.width = `${videoWidth}px`; videoEl.style.width = `${videoWidth}px`;
videoEl.style.height = `${videoHeight}px`; videoEl.style.height = `${videoHeight}px`;
if (videoHoldEl) {
videoHoldEl.style.width = `${videoWidth}px`;
videoHoldEl.style.height = `${videoHeight}px`;
videoHoldEl.width = Math.max(1, Math.round(videoWidth * dpr));
videoHoldEl.height = Math.max(1, Math.round(videoHeight * dpr));
}
canvasEl.style.width = `${videoWidth}px`; canvasEl.style.width = `${videoWidth}px`;
canvasEl.style.height = `${videoHeight}px`; canvasEl.style.height = `${videoHeight}px`;
canvasEl.width = Math.max(1, Math.round(videoWidth * dpr)); canvasEl.width = Math.max(1, Math.round(videoWidth * dpr));
@@ -1639,6 +1646,7 @@ window.HomeDrive = (() => {
transformSignature = nextSignature; transformSignature = nextSignature;
const cssMatrix = `matrix(${transform.scale}, 0, 0, ${transform.scale}, ${transform.tx}, ${transform.ty})`; const cssMatrix = `matrix(${transform.scale}, 0, 0, ${transform.scale}, ${transform.tx}, ${transform.ty})`;
videoEl.style.transform = cssMatrix; videoEl.style.transform = cssMatrix;
if (videoHoldEl) videoHoldEl.style.transform = cssMatrix;
canvasEl.style.transform = cssMatrix; canvasEl.style.transform = cssMatrix;
} }
@@ -2075,32 +2083,31 @@ window.HomeDrive = (() => {
function getLeadBoxClampMargins(videoWidth, videoHeight, stageWidth = videoWidth, stageHeight = videoHeight, transform = null, options = {}) { function getLeadBoxClampMargins(videoWidth, videoHeight, stageWidth = videoWidth, stageHeight = videoHeight, transform = null, options = {}) {
const visibleRect = getVisibleSourceRect(videoWidth, videoHeight, stageWidth, stageHeight, transform); const visibleRect = getVisibleSourceRect(videoWidth, videoHeight, stageWidth, stageHeight, transform);
const baseTopMargin = Math.min(videoHeight * 0.28, 200.0); // C3 fixed margins: top=200, bottom=80, marginX=350
const baseBottomMargin = Math.max(videoHeight * 0.14, 80.0); const topMargin = Math.max(200.0, visibleRect.top + 6);
const offsets = getLeadBadgeOffsets(videoWidth, videoHeight);
let bottomReserve = baseBottomMargin;
// C3 base: maxCenterY = fb_h - 80
let maxCenterY = videoHeight - 80.0;
// In crop/fit modes, also keep badges inside visible area
const offsets = getLeadBadgeOffsets(videoWidth, videoHeight);
let badgeReserve = 0;
if (options.includeDistanceBadge !== false) { if (options.includeDistanceBadge !== false) {
bottomReserve = Math.max(bottomReserve, offsets.rectTopOffset + offsets.badgeHeight + 8); badgeReserve = Math.max(badgeReserve, offsets.rectTopOffset + offsets.badgeHeight + 8);
} }
if (options.includeStateText) { if (options.includeStateText) {
const stateBottomReserve = offsets.textBaselineOffset + Math.max(offsets.fontSize * 0.28, 8); badgeReserve = Math.max(badgeReserve, offsets.textBaselineOffset + Math.max(offsets.fontSize * 0.28, 8));
bottomReserve = Math.max(bottomReserve, stateBottomReserve);
} }
maxCenterY = Math.min(maxCenterY, visibleRect.bottom - Math.max(badgeReserve, 80.0));
const topMargin = Math.max(baseTopMargin, visibleRect.top + 6); maxCenterY = Math.max(topMargin, maxCenterY);
const maxCenterY = Math.max(
topMargin,
Math.min(videoHeight - baseBottomMargin, visibleRect.bottom - bottomReserve),
);
return { return {
marginX: Math.min(videoWidth * 0.35, 350.0), marginX: 350.0,
topMargin, topMargin,
bottomMargin: Math.max(baseBottomMargin, videoHeight - maxCenterY), bottomMargin: Math.max(80.0, videoHeight - maxCenterY),
maxCenterY, maxCenterY,
visibleRect, visibleRect,
bottomReserve, bottomReserve: badgeReserve,
}; };
} }

109
system/webrtc/device/audio.py Normal file
View File

@@ -0,0 +1,109 @@
import asyncio
import io
import aiortc
import av
import numpy as np
import pyaudio
class AudioInputStreamTrack(aiortc.mediastreams.AudioStreamTrack):
PYAUDIO_TO_AV_FORMAT_MAP = {
pyaudio.paUInt8: 'u8',
pyaudio.paInt16: 's16',
pyaudio.paInt24: 's24',
pyaudio.paInt32: 's32',
pyaudio.paFloat32: 'flt',
}
def __init__(self, audio_format: int = pyaudio.paInt16, rate: int = 16000, channels: int = 1, packet_time: float = 0.020, device_index: int = None):
super().__init__()
self.p = pyaudio.PyAudio()
chunk_size = int(packet_time * rate)
self.stream = self.p.open(format=audio_format,
channels=channels,
rate=rate,
frames_per_buffer=chunk_size,
input=True,
input_device_index=device_index)
self.format = audio_format
self.rate = rate
self.channels = channels
self.packet_time = packet_time
self.chunk_size = chunk_size
self.pts = 0
async def recv(self):
mic_data = self.stream.read(self.chunk_size)
mic_array = np.frombuffer(mic_data, dtype=np.int16)
mic_array = np.expand_dims(mic_array, axis=0)
layout = 'stereo' if self.channels > 1 else 'mono'
frame = av.AudioFrame.from_ndarray(mic_array, format=self.PYAUDIO_TO_AV_FORMAT_MAP[self.format], layout=layout)
frame.rate = self.rate
frame.pts = self.pts
self.pts += frame.samples
return frame
class AudioOutputSpeaker:
def __init__(self, audio_format: int = pyaudio.paInt16, rate: int = 48000, channels: int = 2, packet_time: float = 0.2, device_index: int = None):
chunk_size = int(packet_time * rate)
self.p = pyaudio.PyAudio()
self.buffer = io.BytesIO()
self.channels = channels
self.stream = self.p.open(format=audio_format,
channels=channels,
rate=rate,
frames_per_buffer=chunk_size,
output=True,
output_device_index=device_index,
stream_callback=self.__pyaudio_callback)
self.tracks_and_tasks: list[tuple[aiortc.MediaStreamTrack, asyncio.Task | None]] = []
def __pyaudio_callback(self, in_data, frame_count, time_info, status):
if self.buffer.getbuffer().nbytes < frame_count * self.channels * 2:
buff = b'\x00\x00' * frame_count * self.channels
elif self.buffer.getbuffer().nbytes > 115200: # 3x the usual read size
self.buffer.seek(0)
buff = self.buffer.read(frame_count * self.channels * 4)
buff = buff[:frame_count * self.channels * 2]
self.buffer.seek(2)
else:
self.buffer.seek(0)
buff = self.buffer.read(frame_count * self.channels * 2)
self.buffer.seek(2)
return (buff, pyaudio.paContinue)
async def __consume(self, track):
while True:
try:
frame = await track.recv()
except aiortc.MediaStreamError:
return
self.buffer.write(bytes(frame.planes[0]))
def hasTrack(self, track: aiortc.MediaStreamTrack) -> bool:
return any(t == track for t, _ in self.tracks_and_tasks)
def addTrack(self, track: aiortc.MediaStreamTrack):
if not self.hasTrack(track):
self.tracks_and_tasks.append((track, None))
def start(self):
for index, (track, task) in enumerate(self.tracks_and_tasks):
if task is None:
self.tracks_and_tasks[index] = (track, asyncio.create_task(self.__consume(track)))
def stop(self):
for _, task in self.tracks_and_tasks:
if task is not None:
task.cancel()
self.tracks_and_tasks = []
self.stream.stop_stream()
self.stream.close()
self.p.terminate()

35
system/webrtc/device/video.py
View File

@@ -6,9 +6,12 @@ from teleoprtc.tracks import TiciVideoStreamTrack
from cereal import messaging from cereal import messaging
from openpilot.common.realtime import DT_MDL, DT_DMON from openpilot.common.realtime import DT_MDL, DT_DMON
from openpilot.common.swaglog import cloudlog
class LiveStreamVideoStreamTrack(TiciVideoStreamTrack): class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
RECOVERY_LOG_THRESHOLD_SECONDS = 1.0
camera_to_sock_mapping = { camera_to_sock_mapping = {
"driver": "livestreamDriverEncodeData", "driver": "livestreamDriverEncodeData",
"wideRoad": "livestreamWideRoadEncodeData", "wideRoad": "livestreamWideRoadEncodeData",
@@ -19,22 +22,37 @@ class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
dt = DT_DMON if camera_type == "driver" else DT_MDL dt = DT_DMON if camera_type == "driver" else DT_MDL
super().__init__(camera_type, dt) super().__init__(camera_type, dt)
self._sock = messaging.sub_sock(self.camera_to_sock_mapping[camera_type], conflate=True) self._source_name = self.camera_to_sock_mapping[camera_type]
self._sock = messaging.sub_sock(self._source_name, conflate=True)
self._pts = 0 self._pts = 0
self._t0_ns = time.monotonic_ns() self._sock_closed = False
self._timeout_started_at = None
async def recv(self): async def recv(self):
waited = 0
while True: while True:
try: try:
msg = messaging.recv_one_or_none(self._sock) msg = messaging.recv_one_or_none(self._sock)
if msg is None: if msg is None:
await asyncio.sleep(0.005) await asyncio.sleep(0.005)
waited += 0.005 waited += 0.005
if self._timeout_started_at is None:
self._timeout_started_at = time.monotonic()
if waited > 1.0: if waited > 1.0:
self._logger.warning("%s frame recv timed out", self.camera_to_sock_mapping.get(self._id.split(':', 1)[0], "video")) elapsed = time.monotonic() - self._timeout_started_at if self._timeout_started_at is not None else waited
self._logger.warning("%s frame recv timed out (elapsed=%.2fs pts=%s sock_closed=%s)",
self._source_name, elapsed, self._pts, self._sock_closed)
cloudlog.warning("[webrtcd-video] %s frame recv timed out (elapsed=%.2fs pts=%s sock_closed=%s)",
self._source_name, elapsed, self._pts, self._sock_closed)
waited = 0 waited = 0
continue continue
if self._timeout_started_at is not None:
elapsed = time.monotonic() - self._timeout_started_at
if elapsed >= self.RECOVERY_LOG_THRESHOLD_SECONDS:
self._logger.info("%s frame recv recovered after %.2fs (pts=%s)", self._source_name, elapsed, self._pts)
cloudlog.info("[webrtcd-video] %s frame recv recovered after %.2fs (pts=%s)", self._source_name, elapsed, self._pts)
self._timeout_started_at = None
waited = 0 waited = 0
evta = getattr(msg, msg.which()) evta = getattr(msg, msg.which())
@@ -49,8 +67,17 @@ class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
except asyncio.CancelledError: except asyncio.CancelledError:
raise raise
except Exception: except Exception:
self._logger.exception("failed to build outgoing video packet") self._logger.exception("failed to build outgoing video packet (%s pts=%s)", self._source_name, self._pts)
cloudlog.exception("[webrtcd-video] failed to build outgoing video packet (%s pts=%s)", self._source_name, self._pts)
await asyncio.sleep(0.01) await asyncio.sleep(0.01)
def close_sock(self):
if not self._sock_closed and self._sock is not None:
try:
self._sock.close()
except Exception:
pass
self._sock_closed = True
def codec_preference(self) -> str | None: def codec_preference(self) -> str | None:
return "H264" return "H264"

77
system/webrtc/webrtcd.py Executable file → Normal file
View File

@@ -18,10 +18,17 @@ from aiohttp import web
if TYPE_CHECKING: if TYPE_CHECKING:
from aiortc.rtcdatachannel import RTCDataChannel from aiortc.rtcdatachannel import RTCDataChannel
from openpilot.common.swaglog import cloudlog
from openpilot.system.webrtc.schema import generate_field from openpilot.system.webrtc.schema import generate_field
from cereal import messaging, log from cereal import messaging, log
def webrtcd_log(level: str, msg: str, *args):
logger = logging.getLogger("webrtcd")
getattr(logger, level)(msg, *args)
getattr(cloudlog, level)("[webrtcd] " + msg, *args)
class CerealOutgoingMessageProxy: class CerealOutgoingMessageProxy:
def __init__(self, sm: messaging.SubMaster): def __init__(self, sm: messaging.SubMaster):
self.sm = sm self.sm = sm
@@ -137,15 +144,26 @@ class StreamSession:
from teleoprtc import WebRTCAnswerBuilder from teleoprtc import WebRTCAnswerBuilder
from teleoprtc.info import parse_info_from_offer from teleoprtc.info import parse_info_from_offer
self.logger = logging.getLogger("webrtcd")
config = parse_info_from_offer(sdp) config = parse_info_from_offer(sdp)
builder = WebRTCAnswerBuilder(sdp) builder = WebRTCAnswerBuilder(sdp)
self._video_tracks = []
assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks" assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks"
for cam in cameras: for cam in cameras:
builder.add_video_stream(cam, LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack()) try:
track = LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack()
builder.add_video_stream(cam, track)
if hasattr(track, 'close_sock'):
self._video_tracks.append(track)
self.logger.info("added camera track: %s", cam)
except Exception:
self.logger.exception("failed to create camera track: %s", cam)
raise
self.stream = builder.stream()
self.identifier = str(uuid.uuid4()) self.identifier = str(uuid.uuid4())
self.stream = builder.stream()
setattr(self.stream, "session_identifier", self.identifier)
self.incoming_bridge: CerealIncomingMessageProxy | None = None self.incoming_bridge: CerealIncomingMessageProxy | None = None
self.incoming_bridge_services = incoming_services self.incoming_bridge_services = incoming_services
@@ -158,17 +176,22 @@ class StreamSession:
self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge) self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge)
self.run_task: asyncio.Task | None = None self.run_task: asyncio.Task | None = None
self.logger = logging.getLogger("webrtcd") self.connected_event = asyncio.Event()
self.logger.info("New stream session (%s), cameras %s, incoming services %s, outgoing services %s", self.logger.info("New stream session (%s), cameras %s, incoming services %s, outgoing services %s",
self.identifier, cameras, incoming_services, outgoing_services) self.identifier, cameras, incoming_services, outgoing_services)
self.logger.info("request cameras=%s", cameras)
def start(self): def start(self):
webrtcd_log("info", "Stream session (%s) start requested", self.identifier)
self.run_task = asyncio.create_task(self.run()) self.run_task = asyncio.create_task(self.run())
async def stop(self): async def stop(self):
if self.run_task is None: if self.run_task is None:
return return
webrtcd_log("info", "Stream session (%s) stop requested (done=%s)", self.identifier, self.run_task.done())
if not self.run_task.done(): if not self.run_task.done():
self.run_task.cancel() self.run_task.cancel()
try: try:
@@ -178,6 +201,7 @@ class StreamSession:
self.run_task = None self.run_task = None
await self.post_run_cleanup() await self.post_run_cleanup()
webrtcd_log("info", "Stream session (%s) stop cleanup complete", self.identifier)
try: try:
if hasattr(self, "stream_dict"): if hasattr(self, "stream_dict"):
@@ -187,7 +211,10 @@ class StreamSession:
async def get_answer(self): async def get_answer(self):
return await self.stream.start() webrtcd_log("info", "Stream session (%s) building answer", self.identifier)
answer = await self.stream.start()
webrtcd_log("info", "Stream session (%s) answer ready", self.identifier)
return answer
async def message_handler(self, message: bytes): async def message_handler(self, message: bytes):
assert self.incoming_bridge is not None assert self.incoming_bridge is not None
@@ -198,6 +225,7 @@ class StreamSession:
async def run(self): async def run(self):
try: try:
webrtcd_log("info", "Stream session (%s) waiting for peer connection", self.identifier)
await self.stream.wait_for_connection() await self.stream.wait_for_connection()
if self.stream.has_messaging_channel(): if self.stream.has_messaging_channel():
if self.incoming_bridge is not None: if self.incoming_bridge is not None:
@@ -207,12 +235,14 @@ class StreamSession:
channel = self.stream.get_messaging_channel() channel = self.stream.get_messaging_channel()
self.outgoing_bridge_runner.proxy.add_channel(channel) self.outgoing_bridge_runner.proxy.add_channel(channel)
self.outgoing_bridge_runner.start() self.outgoing_bridge_runner.start()
self.logger.info("Stream session (%s) connected", self.identifier) self.connected_event.set()
webrtcd_log("info", "Stream session (%s) connected", self.identifier)
webrtcd_log("info", "Stream session (%s) waiting for disconnection", self.identifier)
await self.stream.wait_for_disconnection() await self.stream.wait_for_disconnection()
await self.post_run_cleanup() await self.post_run_cleanup()
self.logger.info("Stream session (%s) ended", self.identifier) webrtcd_log("info", "Stream session (%s) ended", self.identifier)
except Exception: except Exception:
self.logger.exception("Stream session failure") self.logger.exception("Stream session failure")
finally: finally:
@@ -227,6 +257,12 @@ class StreamSession:
await self.stream.stop() await self.stream.stop()
if self.outgoing_bridge is not None: if self.outgoing_bridge is not None:
self.outgoing_bridge_runner.stop() self.outgoing_bridge_runner.stop()
# Close ZMQ sockets held by video tracks
for track in getattr(self, '_video_tracks', []):
try:
track.close_sock()
except Exception:
self.logger.exception("Failed to close video track socket")
@dataclass @dataclass
@@ -237,10 +273,36 @@ class StreamRequestBody:
bridge_services_out: list[str] = field(default_factory=list) bridge_services_out: list[str] = field(default_factory=list)
async def retire_old_sessions_after_handover(old_sessions: list[StreamSession], new_session: StreamSession, timeout_s: float = 12.0):
if not old_sessions:
return
old_ids = [getattr(old, "identifier", "?") for old in old_sessions]
webrtcd_log("info", "Handover: waiting to retire old sessions %s after new session %s", old_ids, new_session.identifier)
try:
await asyncio.wait_for(new_session.connected_event.wait(), timeout=timeout_s)
webrtcd_log("info", "New session %s connected, retiring old sessions %s", new_session.identifier, old_ids)
except TimeoutError:
webrtcd_log("warning", "New session %s did not connect within %.1fs, retiring old sessions %s anyway",
new_session.identifier, timeout_s, old_ids)
for old in old_sessions:
try:
webrtcd_log("info", "Handover: stopping old session %s", getattr(old, 'identifier', '?'))
await old.stop()
webrtcd_log("info", "Handover: stopped old session %s", getattr(old, 'identifier', '?'))
except Exception:
webrtcd_log("exception", "Failed to stop old session %s during handover", getattr(old, 'identifier', '?'))
async def get_stream(request: 'web.Request'): async def get_stream(request: 'web.Request'):
stream_dict, debug_mode = request.app['streams'], request.app['debug'] stream_dict, debug_mode = request.app['streams'], request.app['debug']
old_sessions = list(stream_dict.values())
raw_body = await request.json() raw_body = await request.json()
body = StreamRequestBody(**raw_body) body = StreamRequestBody(**raw_body)
webrtcd_log("info", "get_stream request from %s cameras=%s old_sessions=%s", request.remote, body.cameras,
[getattr(old, "identifier", "?") for old in old_sessions])
session = StreamSession(body.sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, debug_mode) session = StreamSession(body.sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, debug_mode)
session.stream_dict = stream_dict session.stream_dict = stream_dict
@@ -248,6 +310,9 @@ async def get_stream(request: 'web.Request'):
session.start() session.start()
stream_dict[session.identifier] = session stream_dict[session.identifier] = session
webrtcd_log("info", "get_stream created new session %s active_sessions=%d", session.identifier, len(stream_dict))
if old_sessions:
asyncio.create_task(retire_old_sessions_after_handover(old_sessions, session))
return web.json_response({"sdp": answer.sdp, "type": answer.type}, headers={'Access-Control-Allow-Origin': '*'}) return web.json_response({"sdp": answer.sdp, "type": answer.type}, headers={'Access-Control-Allow-Origin': '*'})

50
teleoprtc_repo/teleoprtc/stream.py
View File

@@ -9,6 +9,11 @@ from aiortc.contrib.media import MediaRelay
from teleoprtc.tracks import parse_video_track_id from teleoprtc.tracks import parse_video_track_id
try:
from openpilot.common.swaglog import cloudlog as _cloudlog
except Exception:
_cloudlog = None
@dataclasses.dataclass @dataclasses.dataclass
class StreamingOffer: class StreamingOffer:
@@ -49,8 +54,11 @@ class WebRTCBaseStream(abc.ABC):
self.connection_attempted_event = asyncio.Event() self.connection_attempted_event = asyncio.Event()
self.connection_stopped_event = asyncio.Event() self.connection_stopped_event = asyncio.Event()
self._disconnect_grace_task: Optional[asyncio.Task] = None self._disconnect_grace_task: Optional[asyncio.Task] = None
self._last_connection_state = self.peer_connection.connectionState
self._last_ice_connection_state = getattr(self.peer_connection, "iceConnectionState", None)
self.peer_connection.on("connectionstatechange", self._on_connectionstatechange) self.peer_connection.on("connectionstatechange", self._on_connectionstatechange)
self.peer_connection.on("iceconnectionstatechange", self._on_iceconnectionstatechange)
self.peer_connection.on("datachannel", self._on_incoming_datachannel) self.peer_connection.on("datachannel", self._on_incoming_datachannel)
self.peer_connection.on("track", self._on_incoming_track) self.peer_connection.on("track", self._on_incoming_track)
@@ -59,15 +67,36 @@ class WebRTCBaseStream(abc.ABC):
def _log_debug(self, msg: Any, *args): def _log_debug(self, msg: Any, *args):
self.logger.debug(f"{type(self)}() {msg}", *args) self.logger.debug(f"{type(self)}() {msg}", *args)
def _cancel_disconnect_grace(self): def _stream_label(self) -> str:
return getattr(self, "session_identifier", f"{type(self).__name__}:{id(self):x}")
def _log_info(self, msg: str, *args):
prefix = f"stream {self._stream_label()} "
self.logger.info(prefix + msg, *args)
if _cloudlog is not None:
_cloudlog.info("[webrtcd] " + prefix + msg, *args)
def _log_warning(self, msg: str, *args):
prefix = f"stream {self._stream_label()} "
self.logger.warning(prefix + msg, *args)
if _cloudlog is not None:
_cloudlog.warning("[webrtcd] " + prefix + msg, *args)
def _cancel_disconnect_grace(self, reason: Optional[str] = None):
if self._disconnect_grace_task is not None: if self._disconnect_grace_task is not None:
self._disconnect_grace_task.cancel() self._disconnect_grace_task.cancel()
self._disconnect_grace_task = None self._disconnect_grace_task = None
if reason is not None:
self._log_info("disconnect grace canceled (%s)", reason)
def _schedule_disconnect_grace(self): def _schedule_disconnect_grace(self):
if self._disconnect_grace_task is not None and not self._disconnect_grace_task.done(): if self._disconnect_grace_task is not None and not self._disconnect_grace_task.done():
return return
self._log_warning("disconnect grace scheduled for %.1fs (conn=%s ice=%s)",
self.DISCONNECTED_GRACE_SECONDS,
self.peer_connection.connectionState,
getattr(self.peer_connection, "iceConnectionState", "unknown"))
async def _grace_disconnect(): async def _grace_disconnect():
try: try:
await asyncio.sleep(self.DISCONNECTED_GRACE_SECONDS) await asyncio.sleep(self.DISCONNECTED_GRACE_SECONDS)
@@ -76,10 +105,19 @@ class WebRTCBaseStream(abc.ABC):
if self.peer_connection.connectionState == "disconnected": if self.peer_connection.connectionState == "disconnected":
self._log_debug("disconnect grace expired") self._log_debug("disconnect grace expired")
self._log_warning("disconnect grace expired (conn=%s ice=%s)",
self.peer_connection.connectionState,
getattr(self.peer_connection, "iceConnectionState", "unknown"))
self.connection_stopped_event.set() self.connection_stopped_event.set()
self._disconnect_grace_task = asyncio.create_task(_grace_disconnect()) self._disconnect_grace_task = asyncio.create_task(_grace_disconnect())
def _on_iceconnectionstatechange(self):
prev_state = self._last_ice_connection_state
state = getattr(self.peer_connection, "iceConnectionState", None)
self._last_ice_connection_state = state
self._log_info("ice state changed %s -> %s (conn=%s)",
prev_state, state, self.peer_connection.connectionState)
@property @property
def _number_of_incoming_media(self) -> int: def _number_of_incoming_media(self) -> int:
media = len(self.incoming_camera_tracks) + len(self.incoming_audio_tracks) media = len(self.incoming_camera_tracks) + len(self.incoming_audio_tracks)
@@ -140,19 +178,23 @@ class WebRTCBaseStream(abc.ABC):
transceiver.setCodecPreferences(rtp_codec) transceiver.setCodecPreferences(rtp_codec)
def _on_connectionstatechange(self): def _on_connectionstatechange(self):
prev_state = self._last_connection_state
state = self.peer_connection.connectionState state = self.peer_connection.connectionState
self._last_connection_state = state
self._log_debug("connection state is %s", state) self._log_debug("connection state is %s", state)
self._log_info("connection state changed %s -> %s (ice=%s)",
prev_state, state, getattr(self.peer_connection, "iceConnectionState", "unknown"))
if state in ['connected', 'failed']: if state in ['connected', 'failed']:
self.connection_attempted_event.set() self.connection_attempted_event.set()
if state == 'connected': if state == 'connected':
self._cancel_disconnect_grace() self._cancel_disconnect_grace(reason="connection recovered")
elif state == 'disconnected': elif state == 'disconnected':
self._schedule_disconnect_grace() self._schedule_disconnect_grace()
elif state in ['closed', 'failed']: elif state in ['closed', 'failed']:
self._cancel_disconnect_grace() self._cancel_disconnect_grace(reason=f"connection {state}")
self.connection_stopped_event.set() self.connection_stopped_event.set()
else: else:
self._cancel_disconnect_grace() self._cancel_disconnect_grace(reason=f"connection {state}")
def _on_incoming_track(self, track: aiortc.MediaStreamTrack): def _on_incoming_track(self, track: aiortc.MediaStreamTrack):
self._log_debug("got track: %s %s", track.kind, track.id) self._log_debug("got track: %s %s", track.kind, track.id)