From 3393c99433fb1d3b2a9c2173c6d6dccd3042acba Mon Sep 17 00:00:00 2001 From: Prabhaav Pillai Date: Wed, 26 Aug 2026 15:43:09 -0400 Subject: [PATCH] add logs --- .../controls/lib/longitudinal_planner.py | 47 +++++++++---- .../assets/components/tools/tmux.js | 66 +++++++++++++++++-- starpilot/system/the_galaxy/the_galaxy.py | 60 ++++++++++++++--- 3 files changed, 147 insertions(+), 26 deletions(-) diff --git a/selfdrive/controls/lib/longitudinal_planner.py b/selfdrive/controls/lib/longitudinal_planner.py index 3aaec6017..775084e1e 100755 --- a/selfdrive/controls/lib/longitudinal_planner.py +++ b/selfdrive/controls/lib/longitudinal_planner.py @@ -137,6 +137,24 @@ VISION_LEAD_APPROACH_BRAKING_FULL_LEAD_BRAKE = 1.20 PLANNER_SAFETY_WARNING_INTERVAL = 5.0 HEM_STATUS_LOG_INTERVAL = 10.0 HEM_AUTH_PUB_INTERVAL = 0.5 + + +def _hem_log_timestamp() -> str: + """Wall-clock timestamp with millisecond precision for the [HEM] live log.""" + from datetime import datetime + now = datetime.now() + return now.strftime("%H:%M:%S") + f".{now.microsecond // 1000:03d}" + + +def _hem_format_diag_value(value): + """Compact, deterministic formatting for the per-frame HEM diagnostic dump.""" + if isinstance(value, bool): + return "1" if value else "0" + if isinstance(value, float): + return f"{value:.4f}" + if isinstance(value, (int, np.integer)): + return str(int(value)) + return str(value) VISION_LEAD_APPROACH_BRAKING_FLOOR_MIN_DECEL = 1.30 VISION_LEAD_APPROACH_BRAKING_FLOOR_MAX_DECEL = 1.75 VISION_LEAD_APPROACH_CONFIRM_TIME = 0.25 @@ -1894,22 +1912,20 @@ class LongitudinalPlanner: return floor def _log_hem_status(self, now_t, active, v_ego, a_chill, a_exp, a_fused): + hc = self.hybrid_controller + hc.record_diag = True + ts = _hem_log_timestamp() if active != self._hem_logged_active: self._hem_logged_active = active self._hem_status_log_t = 0.0 - print(f"[HEM] mode {'ON' if active else 'OFF'}") + print(f"[HEM] {ts} mode {'ON' if active else 'OFF'}") if not active: return - if now_t - self._hem_status_log_t < HEM_STATUS_LOG_INTERVAL: - return - self._hem_status_log_t = now_t - hc = self.hybrid_controller - print( - f"[HEM] v={v_ego:5.1f} chill={a_chill:6.2f} exp={a_exp:6.2f} " - + f"fused={a_fused:6.2f} auth={hc.exp_authority:4.2f} " - + f"w_vis={hc.last_w_vision:4.2f} {hc.last_regime}" - + (" stop" if hc.last_standstill else "") - ) + # Rich per-frame dump of every HEM decision variable so a missed stop can be + # diagnosed to the exact frame and signal (see hybrid_experimental_mode diag). + d = hc.diag + if d: + print(f"[HEM] {ts} " + " ".join(f"{k}={_hem_format_diag_value(v)}" for k, v in d.items())) def _publish_hem_status(self, now_t): if self._hem_params_memory is None: @@ -2353,6 +2369,15 @@ class LongitudinalPlanner: a_exp=output_a_target_e2e, t_follow=effective_t_follow, ) + hc = self.hybrid_controller + hc.record_diag = True + # Surface whether Chill/Exp explicitly asked to stop; critical for diagnosing + # cases where Exp wanted to brake but the HEM fusion did not command a stop. + hc.diag["should_stop_mpc"] = bool(output_should_stop_mpc) + hc.diag["should_stop_e2e"] = bool(output_should_stop_e2e) + hc.diag["should_stop_fused"] = bool(output_should_stop_mpc or output_should_stop_e2e) + hc.diag["model_desired_accel"] = float(sm['modelV2'].action.desiredAcceleration) + hc.diag["a_out"] = float(output_a_target) self._log_hem_status(now_t, True, scene_v_ego, output_a_target_mpc, output_a_target_e2e, output_a_target) self._publish_hem_status(now_t) output_should_stop = output_should_stop_mpc or output_should_stop_e2e diff --git a/starpilot/system/the_galaxy/assets/components/tools/tmux.js b/starpilot/system/the_galaxy/assets/components/tools/tmux.js index 45863d679..318667953 100644 --- a/starpilot/system/the_galaxy/assets/components/tools/tmux.js +++ b/starpilot/system/the_galaxy/assets/components/tools/tmux.js @@ -193,6 +193,7 @@ export function TmuxLog() { log: '', selectorAction: null, transport: "connecting", + manualStop: false, }); const lifecycleVersion = ++tmuxLifecycleVersion; let lifecycleTimer = null; @@ -289,6 +290,12 @@ export function TmuxLog() { return; } + if (state.manualStop) { + stopLiveTransport(); + state.transport = "stopped"; + return; + } + if (!isTmuxRouteActive()) { stopLiveTransport(); state.transport = "inactive"; @@ -328,11 +335,55 @@ export function TmuxLog() { return "Tmux Live Log"; } - function togglePause() { - state.paused = !state.paused; - if (!state.paused) { - state.log = state.latest; + function clearLog() { + state.manualStop = false; + state.paused = false; + fetch("/api/tmux_log/clear", { method: "POST" }) + .then(res => { + if (!res.ok) return res.text().then(msg => { throw new Error(msg); }); + state.log = ""; + state.latest = ""; + showSnackbar("Log cleared — ready for a fresh run!", "success"); + }) + .catch(err => showSnackbar(`Clear failed: ${err.message}`, "error")); + } + + function stopLog() { + // Freeze the displayed data and tear down the live transport so nothing more + // can overwrite the captured [HEM] log before the user copies it. + state.manualStop = true; + state.paused = true; + stopLiveTransport(); + state.transport = "stopped"; + showSnackbar("Log stopped — data frozen for copying.", "success"); + } + + function resumeLog() { + state.manualStop = false; + state.paused = false; + state.transport = "connecting"; + updateTransportForLifecycle(); + } + + function copyLog() { + const text = state.log || ""; + if (!text) { + showSnackbar("Nothing to copy yet.", "error"); + return; } + const done = ok => showSnackbar(ok ? `Copied ${text.length} chars to clipboard!` : "Copy failed.", ok ? "success" : "error"); + if (navigator.clipboard && navigator.clipboard.writeText) { + navigator.clipboard.writeText(text).then(() => done(true)).catch(() => done(false)); + return; + } + const ta = document.createElement("textarea"); + ta.value = text; + ta.style.position = "fixed"; + ta.style.left = "-9999px"; + document.body.appendChild(ta); + ta.select(); + try { done(document.execCommand("copy")); } catch (e) { done(false); } + document.body.removeChild(ta); } function captureLog() { @@ -384,11 +435,12 @@ export function TmuxLog() {
- + + + + - -
diff --git a/starpilot/system/the_galaxy/the_galaxy.py b/starpilot/system/the_galaxy/the_galaxy.py index ad064f6cf..230f44cb9 100644 --- a/starpilot/system/the_galaxy/the_galaxy.py +++ b/starpilot/system/the_galaxy/the_galaxy.py @@ -1208,6 +1208,37 @@ NAVIGATION_PERSISTED_LOCATION_MAX_AGE_SECONDS = 24 * 60 * 60 TMUX_LOGS_PATH = Path("/data/tmux_logs") + +def _filter_hem_log_lines(output): + """Reduce a tmux pane capture to only lines emitted by the [HEM] logger. + + The Galaxy terminal is used to debug Hybrid Experimental Mode stopping, so only + [HEM] lines (which now carry a millisecond timestamp and full per-frame diag) + are surfaced; all other process output is dropped. + """ + if not output: + return "" + lines = [line for line in output.splitlines() if "[HEM]" in line] + return "\n".join(lines) + ("\n" if lines else "") + + +# Generous but bounded scrollback/capture depth for the [HEM] diagnostics. This +# is far beyond any stop-signal reproduction run (~20k lines ~= 16 min of HEM +# logging at 20 Hz) while keeping capture cost, tunnel bandwidth, and browser +# render work bounded so it cannot grow without limit or burden the device. +TMUX_HISTORY_LIMIT = 20000 +TMUX_CAPTURE_LINES = 20000 + + +def _ensure_tmux_session(): + """Create the comma session if needed, size it, and raise its scrollback limit + so the captured [HEM] window can hold the full run without overflow.""" + if subprocess.run(["tmux", "has-session", "-t", "comma"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode != 0: + run_cmd(["tmux", "new-session", "-d", "-s", "comma", "-x", "240", "-y", "70", "bash"], "Started tmux session", "Failed to start tmux session") + else: + run_cmd(["tmux", "resize-window", "-t", "comma:0", "-x", "240", "-y", "70"], "Resized tmux window", "Failed to resize tmux window") + run_cmd(["tmux", "set-option", "-t", "comma:0", "history-limit", str(TMUX_HISTORY_LIMIT)], "Set tmux scrollback limit.", "Failed to set tmux scrollback limit.") + MODEL_DOWNLOAD_PARAM = "ModelToDownload" MODEL_DOWNLOAD_ALL_PARAM = "DownloadAllModels" MODEL_DOWNLOAD_PROGRESS_PARAM = "ModelDownloadProgress" @@ -8323,16 +8354,15 @@ def setup(app): @app.route("/api/tmux_log/live", methods=["GET"]) def stream_tmux_log(): - if subprocess.run(["tmux", "has-session", "-t", "comma"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode != 0: - run_cmd(["tmux", "new-session", "-d", "-s", "comma", "-x", "240", "-y", "70", "bash"], "Started tmux session", "Failed to start tmux session") - else: - run_cmd(["tmux", "resize-window", "-t", "comma:0", "-x", "240", "-y", "70"], "Resized tmux window", "Failed to resize tmux window") + _ensure_tmux_session() def generate(): last_output = "" last_keepalive = 0.0 while True: - output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", "-1000"], text=True) + output = _filter_hem_log_lines( + subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", f"-{TMUX_CAPTURE_LINES}"], text=True) + ) if output != last_output: yield "data: " + "\n".join(reversed(output.splitlines())).replace("\n", "\ndata: ") + "\n\n" @@ -8352,19 +8382,33 @@ def setup(app): @app.route("/api/tmux_log/snapshot", methods=["GET"]) def snapshot_tmux_log(): try: - output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", "-1000"], text=True) + output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", f"-{TMUX_CAPTURE_LINES}"], text=True) except subprocess.CalledProcessError: - run_cmd(["tmux", "new-session", "-d", "-s", "comma", "-x", "240", "-y", "70", "bash"], "Started tmux session", "Failed to start tmux session") - output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", "-1000"], text=True) + _ensure_tmux_session() + output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", f"-{TMUX_CAPTURE_LINES}"], text=True) except Exception as e: return jsonify({"error": str(e)}), 500 + output = _filter_hem_log_lines(output) + try: live_text = "\n".join(reversed(output.splitlines())) return jsonify({"data": live_text}), 200 except Exception as e: return jsonify({"error": str(e)}), 500 + @app.route("/api/tmux_log/clear", methods=["POST"]) + def clear_tmux_log(): + """Start a clean slate for a fresh HEM test run: wipe tmux scrollback and the + visible pane so only newly-emitted [HEM] lines are captured.""" + try: + _ensure_tmux_session() + subprocess.run(["tmux", "clear-history", "-t", "comma:0"], check=False) + subprocess.run(["tmux", "send-keys", "-t", "comma:0", "clear", "Enter"], check=False) + return jsonify({"message": "Log cleared!"}), 200 + except Exception as e: + return jsonify({"error": str(e)}), 500 + @app.route("/api/tmux_log/rename//", methods=["PUT"]) def rename_tmux_log_path_params(old, new): old_path = TMUX_LOGS_PATH / old