add logs

selfdrive/controls/lib/longitudinal_planner.py

@@ -137,6 +137,24 @@ VISION_LEAD_APPROACH_BRAKING_FULL_LEAD_BRAKE = 1.20
 PLANNER_SAFETY_WARNING_INTERVAL = 5.0
 HEM_STATUS_LOG_INTERVAL = 10.0
 HEM_AUTH_PUB_INTERVAL = 0.5
+
+
+def _hem_log_timestamp() -> str:
+  """Wall-clock timestamp with millisecond precision for the [HEM] live log."""
+  from datetime import datetime
+  now = datetime.now()
+  return now.strftime("%H:%M:%S") + f".{now.microsecond // 1000:03d}"
+
+
+def _hem_format_diag_value(value):
+  """Compact, deterministic formatting for the per-frame HEM diagnostic dump."""
+  if isinstance(value, bool):
+    return "1" if value else "0"
+  if isinstance(value, float):
+    return f"{value:.4f}"
+  if isinstance(value, (int, np.integer)):
+    return str(int(value))
+  return str(value)
 VISION_LEAD_APPROACH_BRAKING_FLOOR_MIN_DECEL = 1.30
 VISION_LEAD_APPROACH_BRAKING_FLOOR_MAX_DECEL = 1.75
 VISION_LEAD_APPROACH_CONFIRM_TIME = 0.25
@@ -1894,22 +1912,20 @@ class LongitudinalPlanner:
     return floor
 
   def _log_hem_status(self, now_t, active, v_ego, a_chill, a_exp, a_fused):
+    hc = self.hybrid_controller
+    hc.record_diag = True
+    ts = _hem_log_timestamp()
     if active != self._hem_logged_active:
       self._hem_logged_active = active
       self._hem_status_log_t = 0.0
-      print(f"[HEM] mode {'ON' if active else 'OFF'}")
+      print(f"[HEM] {ts} mode {'ON' if active else 'OFF'}")
     if not active:
       return
-    if now_t - self._hem_status_log_t < HEM_STATUS_LOG_INTERVAL:
-      return
-    self._hem_status_log_t = now_t
-    hc = self.hybrid_controller
-    print(
-      f"[HEM] v={v_ego:5.1f} chill={a_chill:6.2f} exp={a_exp:6.2f} "
-      + f"fused={a_fused:6.2f} auth={hc.exp_authority:4.2f} "
-      + f"w_vis={hc.last_w_vision:4.2f} {hc.last_regime}"
-      + (" stop" if hc.last_standstill else "")
-    )
+    # Rich per-frame dump of every HEM decision variable so a missed stop can be
+    # diagnosed to the exact frame and signal (see hybrid_experimental_mode diag).
+    d = hc.diag
+    if d:
+      print(f"[HEM] {ts} " + " ".join(f"{k}={_hem_format_diag_value(v)}" for k, v in d.items()))
 
   def _publish_hem_status(self, now_t):
     if self._hem_params_memory is None:
@@ -2353,6 +2369,15 @@ class LongitudinalPlanner:
         a_exp=output_a_target_e2e,
         t_follow=effective_t_follow,
       )
+      hc = self.hybrid_controller
+      hc.record_diag = True
+      # Surface whether Chill/Exp explicitly asked to stop; critical for diagnosing
+      # cases where Exp wanted to brake but the HEM fusion did not command a stop.
+      hc.diag["should_stop_mpc"] = bool(output_should_stop_mpc)
+      hc.diag["should_stop_e2e"] = bool(output_should_stop_e2e)
+      hc.diag["should_stop_fused"] = bool(output_should_stop_mpc or output_should_stop_e2e)
+      hc.diag["model_desired_accel"] = float(sm['modelV2'].action.desiredAcceleration)
+      hc.diag["a_out"] = float(output_a_target)
       self._log_hem_status(now_t, True, scene_v_ego, output_a_target_mpc, output_a_target_e2e, output_a_target)
       self._publish_hem_status(now_t)
       output_should_stop = output_should_stop_mpc or output_should_stop_e2e

starpilot/system/the_galaxy/assets/components/tools/tmux.js

@@ -193,6 +193,7 @@ export function TmuxLog() {
     log: '',
     selectorAction: null,
     transport: "connecting",
+    manualStop: false,
   });
   const lifecycleVersion = ++tmuxLifecycleVersion;
   let lifecycleTimer = null;
@@ -289,6 +290,12 @@ export function TmuxLog() {
       return;
     }
 
+    if (state.manualStop) {
+      stopLiveTransport();
+      state.transport = "stopped";
+      return;
+    }
+
     if (!isTmuxRouteActive()) {
       stopLiveTransport();
       state.transport = "inactive";
@@ -328,11 +335,55 @@ export function TmuxLog() {
     return "Tmux Live Log";
   }
 
-  function togglePause() {
-    state.paused = !state.paused;
-    if (!state.paused) {
-      state.log = state.latest;
+  function clearLog() {
+    state.manualStop = false;
+    state.paused = false;
+    fetch("/api/tmux_log/clear", { method: "POST" })
+      .then(res => {
+        if (!res.ok) return res.text().then(msg => { throw new Error(msg); });
+        state.log = "";
+        state.latest = "";
+        showSnackbar("Log cleared — ready for a fresh run!", "success");
+      })
+      .catch(err => showSnackbar(`Clear failed: ${err.message}`, "error"));
+  }
+
+  function stopLog() {
+    // Freeze the displayed data and tear down the live transport so nothing more
+    // can overwrite the captured [HEM] log before the user copies it.
+    state.manualStop = true;
+    state.paused = true;
+    stopLiveTransport();
+    state.transport = "stopped";
+    showSnackbar("Log stopped — data frozen for copying.", "success");
+  }
+
+  function resumeLog() {
+    state.manualStop = false;
+    state.paused = false;
+    state.transport = "connecting";
+    updateTransportForLifecycle();
+  }
+
+  function copyLog() {
+    const text = state.log || "";
+    if (!text) {
+      showSnackbar("Nothing to copy yet.", "error");
+      return;
     }
+    const done = ok => showSnackbar(ok ? `Copied ${text.length} chars to clipboard!` : "Copy failed.", ok ? "success" : "error");
+    if (navigator.clipboard && navigator.clipboard.writeText) {
+      navigator.clipboard.writeText(text).then(() => done(true)).catch(() => done(false));
+      return;
+    }
+    const ta = document.createElement("textarea");
+    ta.value = text;
+    ta.style.position = "fixed";
+    ta.style.left = "-9999px";
+    document.body.appendChild(ta);
+    ta.select();
+    try { done(document.execCommand("copy")); } catch (e) { done(false); }
+    document.body.removeChild(ta);
   }
 
   function captureLog() {
@@ -384,11 +435,12 @@ export function TmuxLog() {
       </div>
 
       <div class="tmux-controls">
-        <button class="tmux-control-button" @click="${captureLog}">💾 Capture Log</button>
+        <button class="tmux-control-button" @click="${clearLog}">🧹 Clear Log</button>
+        <button class="tmux-control-button" @click="${() => state.manualStop ? resumeLog() : stopLog()}">${() => state.manualStop ? "▶️ Resume Log" : "⏹️ Stop Log"}</button>
+        <button class="tmux-control-button" @click="${copyLog}">📋 Copy Log</button>
+        <button class="tmux-control-button" @click="${captureLog}">💾 Save Log</button>
         <button class="tmux-control-button" @click="${deleteSession}">🗑️ Delete Log</button>
-        <button class="tmux-control-button" @click="${confirmDeleteAllSessions}">🧨 Delete All Logs</button>
         <button class="tmux-control-button" @click="${downloadSessions}">⬇️ Download Log</button>
-        <button class="tmux-control-button" @click="${togglePause}">${() => state.paused ? "▶️ Resume Log" : "⏸️ Pause Log"}</button>
         <button class="tmux-control-button" @click="${() => state.selectorAction = 'rename'}">✏️ Rename Log</button>
       </div>
 

starpilot/system/the_galaxy/the_galaxy.py

@@ -1208,6 +1208,37 @@ NAVIGATION_PERSISTED_LOCATION_MAX_AGE_SECONDS = 24 * 60 * 60
 
 TMUX_LOGS_PATH = Path("/data/tmux_logs")
 
+
+def _filter_hem_log_lines(output):
+  """Reduce a tmux pane capture to only lines emitted by the [HEM] logger.
+
+  The Galaxy terminal is used to debug Hybrid Experimental Mode stopping, so only
+  [HEM] lines (which now carry a millisecond timestamp and full per-frame diag)
+  are surfaced; all other process output is dropped.
+  """
+  if not output:
+    return ""
+  lines = [line for line in output.splitlines() if "[HEM]" in line]
+  return "\n".join(lines) + ("\n" if lines else "")
+
+
+# Generous but bounded scrollback/capture depth for the [HEM] diagnostics. This
+# is far beyond any stop-signal reproduction run (~20k lines ~= 16 min of HEM
+# logging at 20 Hz) while keeping capture cost, tunnel bandwidth, and browser
+# render work bounded so it cannot grow without limit or burden the device.
+TMUX_HISTORY_LIMIT = 20000
+TMUX_CAPTURE_LINES = 20000
+
+
+def _ensure_tmux_session():
+  """Create the comma session if needed, size it, and raise its scrollback limit
+  so the captured [HEM] window can hold the full run without overflow."""
+  if subprocess.run(["tmux", "has-session", "-t", "comma"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode != 0:
+    run_cmd(["tmux", "new-session", "-d", "-s", "comma", "-x", "240", "-y", "70", "bash"], "Started tmux session", "Failed to start tmux session")
+  else:
+    run_cmd(["tmux", "resize-window", "-t", "comma:0", "-x", "240", "-y", "70"], "Resized tmux window", "Failed to resize tmux window")
+  run_cmd(["tmux", "set-option", "-t", "comma:0", "history-limit", str(TMUX_HISTORY_LIMIT)], "Set tmux scrollback limit.", "Failed to set tmux scrollback limit.")
+
 MODEL_DOWNLOAD_PARAM = "ModelToDownload"
 MODEL_DOWNLOAD_ALL_PARAM = "DownloadAllModels"
 MODEL_DOWNLOAD_PROGRESS_PARAM = "ModelDownloadProgress"
@@ -8323,16 +8354,15 @@ def setup(app):
 
   @app.route("/api/tmux_log/live", methods=["GET"])
   def stream_tmux_log():
-    if subprocess.run(["tmux", "has-session", "-t", "comma"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode != 0:
-      run_cmd(["tmux", "new-session", "-d", "-s", "comma", "-x", "240", "-y", "70", "bash"], "Started tmux session", "Failed to start tmux session")
-    else:
-      run_cmd(["tmux", "resize-window", "-t", "comma:0", "-x", "240", "-y", "70"], "Resized tmux window", "Failed to resize tmux window")
+    _ensure_tmux_session()
 
     def generate():
       last_output = ""
       last_keepalive = 0.0
       while True:
-        output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", "-1000"], text=True)
+        output = _filter_hem_log_lines(
+          subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", f"-{TMUX_CAPTURE_LINES}"], text=True)
+        )
 
         if output != last_output:
           yield "data: " + "\n".join(reversed(output.splitlines())).replace("\n", "\ndata: ") + "\n\n"
@@ -8352,19 +8382,33 @@ def setup(app):
   @app.route("/api/tmux_log/snapshot", methods=["GET"])
   def snapshot_tmux_log():
     try:
-      output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", "-1000"], text=True)
+      output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", f"-{TMUX_CAPTURE_LINES}"], text=True)
     except subprocess.CalledProcessError:
-      run_cmd(["tmux", "new-session", "-d", "-s", "comma", "-x", "240", "-y", "70", "bash"], "Started tmux session", "Failed to start tmux session")
-      output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", "-1000"], text=True)
+      _ensure_tmux_session()
+      output = subprocess.check_output(["tmux", "capture-pane", "-t", "comma:0", "-p", "-S", f"-{TMUX_CAPTURE_LINES}"], text=True)
     except Exception as e:
       return jsonify({"error": str(e)}), 500
 
+    output = _filter_hem_log_lines(output)
+
     try:
       live_text = "\n".join(reversed(output.splitlines()))
       return jsonify({"data": live_text}), 200
     except Exception as e:
       return jsonify({"error": str(e)}), 500
 
+  @app.route("/api/tmux_log/clear", methods=["POST"])
+  def clear_tmux_log():
+    """Start a clean slate for a fresh HEM test run: wipe tmux scrollback and the
+    visible pane so only newly-emitted [HEM] lines are captured."""
+    try:
+      _ensure_tmux_session()
+      subprocess.run(["tmux", "clear-history", "-t", "comma:0"], check=False)
+      subprocess.run(["tmux", "send-keys", "-t", "comma:0", "clear", "Enter"], check=False)
+      return jsonify({"message": "Log cleared!"}), 200
+    except Exception as e:
+      return jsonify({"error": str(e)}), 500
+
   @app.route("/api/tmux_log/rename/<old>/<new>", methods=["PUT"])
   def rename_tmux_log_path_params(old, new):
     old_path = TMUX_LOGS_PATH / old