Add ability to download route logs

starpilot/system/the_galaxy/assets/components/recordings/dashcam_routes.css

@@ -244,6 +244,47 @@
   transform: var(--hover-scale-sm);
 }
 
+.route-logs {
+  margin-top: var(--padding-base);
+  max-height: 40vh;
+  overflow-y: auto;
+  text-align: left;
+}
+
+.route-logs-message {
+  margin: 0;
+  padding: var(--padding-sm) 0;
+  font-size: var(--font-size-sm);
+}
+
+.route-logs-header {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: var(--padding-sm);
+  padding-bottom: var(--padding-sm);
+  font-size: var(--font-size-sm);
+}
+
+.route-logs-list {
+  list-style: none;
+  margin: 0;
+  padding: 0;
+}
+
+.route-logs-list li {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: var(--padding-sm);
+  padding: var(--padding-sm) 0;
+  font-size: var(--font-size-sm);
+}
+
+.route-logs a {
+  text-decoration: underline;
+}
+
 @media only screen and (max-width: 768px) and (orientation: portrait) {
   .media-player-content {
     width: 90%;
@@ -266,4 +307,11 @@
     text-align: center;
     width: 100%;
   }
+
+  .route-logs-header,
+  .route-logs-list li {
+    flex-direction: column;
+    align-items: flex-start;
+    gap: 0.25rem;
+  }
 }

starpilot/system/the_galaxy/assets/components/recordings/dashcam_routes.js

@@ -298,8 +298,10 @@ async function openOverlay(route) {
         <button class="close-button camera-button" data-camera="wide">Wide</button>
         <button class="close-button camera-button" data-camera="driver">Driver</button>
         <button class="close-button action-download">Download</button>
+        <button class="close-button action-logs">Logs</button>
         <button class="close-button action-delete">Delete</button>
       </div>
+      <div class="route-logs" hidden></div>
     </div>`;
   document.body.appendChild(overlay);
 
@@ -312,6 +314,50 @@ async function openOverlay(route) {
 
   const vid = overlay.querySelector("video");
   const downloadButton = overlay.querySelector(".action-download");
+  const logsButton = overlay.querySelector(".action-logs");
+  const logsPanel = overlay.querySelector(".route-logs");
+
+  const formatBytes = bytes => {
+    if (!bytes) return "0 MB";
+    const mb = bytes / 1e6;
+    return mb >= 1000 ? `${(mb / 1000).toFixed(2)} GB` : `${mb.toFixed(1)} MB`;
+  };
+
+  let logsLoaded = false;
+  logsButton.onclick = async () => {
+    if (logsLoaded) {
+      logsPanel.hidden = !logsPanel.hidden;
+      return;
+    }
+
+    logsPanel.hidden = false;
+    logsPanel.innerHTML = `<p class="route-logs-message">Looking for full logs...</p>`;
+    try {
+      const response = await fetch(`/api/routes/${route.name}/logs`);
+      const data = await response.json();
+      if (!response.ok) {
+        logsPanel.innerHTML = `<p class="route-logs-message">${data.error || "Could not read logs."}</p>`;
+        return;
+      }
+
+      // sizes are shown up front so a metered connection is a deliberate choice
+      logsPanel.innerHTML = `
+        <div class="route-logs-header">
+          <span>${data.segments.length} segment${data.segments.length === 1 ? "" : "s"} &middot; ${formatBytes(data.totalBytes)} total</span>
+          <a class="route-logs-all" href="/api/routes/${route.name}/logs/download" download>Download all (.tar)</a>
+        </div>
+        <ul class="route-logs-list">
+          ${data.segments.map(segment => `
+            <li>
+              <span>Segment ${segment.segmentNum} &middot; ${formatBytes(segment.bytes)}</span>
+              <a href="${segment.url}" download>${segment.filename}</a>
+            </li>`).join("")}
+        </ul>`;
+      logsLoaded = true;
+    } catch (error) {
+      logsPanel.innerHTML = `<p class="route-logs-message">Could not reach the device: ${error.message}</p>`;
+    }
+  };
 
   let segments;
   let current = 0;

starpilot/system/the_galaxy/tests/test_route_logs.py

@@ -0,0 +1,111 @@
+import io
+import tarfile
+
+from test_dashboard_stats import MODULE_DIR, _install_server_import_stubs
+
+
+def _load_server_module():
+  import importlib.util
+  import sys
+
+  _install_server_import_stubs()
+  spec = importlib.util.spec_from_file_location("route_logs_server", MODULE_DIR / "the_galaxy.py")
+  module = importlib.util.module_from_spec(spec)
+  sys.modules["route_logs_server"] = module
+  spec.loader.exec_module(module)
+  return module
+
+
+the_galaxy = _load_server_module()
+
+
+def _make_route(root, name, segments, filename="rlog.zst", size=32):
+  for segment_num in segments:
+    segment_dir = root / f"{name}--{segment_num}"
+    segment_dir.mkdir(parents=True)
+    (segment_dir / filename).write_bytes(bytes([segment_num]) * size)
+    # a sibling that must never be offered as a full log
+    (segment_dir / "qlog.zst").write_bytes(b"q")
+
+
+def _use_footage_root(monkeypatch, root):
+  monkeypatch.setattr(the_galaxy, "FOOTAGE_PATHS", [str(root) + "/"])
+
+
+def test_route_log_files_are_ordered_numerically(monkeypatch, tmp_path):
+  _make_route(tmp_path, "0000006a--9f0a7bdf9c", [0, 1, 2, 10])
+  _use_footage_root(monkeypatch, tmp_path)
+
+  logs = the_galaxy._route_log_files("0000006a--9f0a7bdf9c")
+
+  # 10 must sort after 2, not lexically between 1 and 2
+  assert [segment for segment, _, _, _ in logs] == [
+    "0000006a--9f0a7bdf9c--0",
+    "0000006a--9f0a7bdf9c--1",
+    "0000006a--9f0a7bdf9c--2",
+    "0000006a--9f0a7bdf9c--10",
+  ]
+  assert {filename for _, filename, _, _ in logs} == {"rlog.zst"}
+  assert [size for _, _, _, size in logs] == [32, 32, 32, 32]
+
+
+def test_route_log_files_prefers_newest_available_format(monkeypatch, tmp_path):
+  _make_route(tmp_path, "0000006a--9f0a7bdf9c", [0], filename="rlog.bz2")
+  (tmp_path / "0000006a--9f0a7bdf9c--0" / "rlog.zst").write_bytes(b"zstd")
+  _use_footage_root(monkeypatch, tmp_path)
+
+  logs = the_galaxy._route_log_files("0000006a--9f0a7bdf9c")
+
+  assert [filename for _, filename, _, _ in logs] == ["rlog.zst"]
+
+
+def test_route_log_files_rejects_names_that_are_not_routes(monkeypatch, tmp_path):
+  _make_route(tmp_path, "0000006a--9f0a7bdf9c", [0])
+  _use_footage_root(monkeypatch, tmp_path)
+
+  for name in ("", None, "..", "../..", "0000006a--9f0a7bdf9c--0", "0000006a--9f0a7bdf9cx", "/etc"):
+    assert the_galaxy._route_log_files(name) == [], name
+
+
+def test_route_log_files_skips_segments_without_logs(monkeypatch, tmp_path):
+  _make_route(tmp_path, "0000006a--9f0a7bdf9c", [0, 1])
+  (tmp_path / "0000006a--9f0a7bdf9c--1" / "rlog.zst").unlink()
+  _use_footage_root(monkeypatch, tmp_path)
+
+  logs = the_galaxy._route_log_files("0000006a--9f0a7bdf9c")
+
+  assert [segment for segment, _, _, _ in logs] == ["0000006a--9f0a7bdf9c--0"]
+
+
+def test_tar_buffer_hands_back_each_write_once():
+  buffer = the_galaxy._TarBuffer()
+
+  buffer.write(b"one")
+  buffer.write(b"two")
+
+  assert buffer.pop() == b"onetwo"
+  assert buffer.pop() == b""
+
+
+def test_streamed_archive_is_a_readable_tar(monkeypatch, tmp_path):
+  _make_route(tmp_path, "0000006a--9f0a7bdf9c", [0, 1], size=4096)
+  _use_footage_root(monkeypatch, tmp_path)
+  logs = the_galaxy._route_log_files("0000006a--9f0a7bdf9c")
+
+  buffer = the_galaxy._TarBuffer()
+  chunks = []
+  with tarfile.open(fileobj=buffer, mode="w|") as archive:
+    for segment, filename, path, _ in logs:
+      archive.add(path, arcname=f"{segment}/{filename}")
+      chunks.append(buffer.pop())
+  chunks.append(buffer.pop())
+
+  # more than one chunk means a long route never has to be buffered whole
+  assert sum(1 for chunk in chunks if chunk) > 1
+
+  with tarfile.open(fileobj=io.BytesIO(b"".join(chunks)), mode="r:") as archive:
+    assert archive.getnames() == [
+      "0000006a--9f0a7bdf9c--0/rlog.zst",
+      "0000006a--9f0a7bdf9c--1/rlog.zst",
+    ]
+    assert archive.extractfile("0000006a--9f0a7bdf9c--1/rlog.zst").read() == bytes([1]) * 4096

starpilot/system/the_galaxy/the_galaxy.py

@@ -10,6 +10,7 @@ import sys
 import sysconfig
 import tarfile
 
+import io
 from io import BytesIO
 from pathlib import Path
 
@@ -1116,6 +1117,24 @@ def _get_toggle_backup_keys():
   return keys
 
 
+def _route_log_files(name):
+  """Full logs for a route as [(segment, filename, path, size)], oldest segment first."""
+  if not utilities.ROUTE_RE.match(name or ""):
+    return []
+
+  for footage_path in FOOTAGE_PATHS:
+    logs = []
+    for segment in sorted(utilities.get_segments_in_route(name, footage_path), key=lambda s: int(s.rsplit("--", 1)[1])):
+      for filename in ROUTE_LOG_CANDIDATES:
+        path = os.path.join(footage_path, segment, filename)
+        if os.path.isfile(path):
+          logs.append((segment, filename, path, os.path.getsize(path)))
+          break
+    if logs:
+      return logs
+  return []
+
+
 def _coerce_toggle_restore_value(key, value):
   value_type = _get_param_key_type(_params_raw, key)
 
@@ -1192,6 +1211,29 @@ except TypeError:
     str(Paths.log_root()),
   ]
 
+# Full drive logs, newest format first. comma only accepts qlog/qcamera uploads, so these come off the device directly.
+ROUTE_LOG_CANDIDATES = ("rlog.zst", "rlog.bz2", "rlog")
+
+
+class _TarBuffer(io.RawIOBase):
+  """Collects tarfile output so a route archive can be streamed out instead of built on disk."""
+
+  def __init__(self):
+    self._chunks = []
+
+  def writable(self):
+    return True
+
+  def write(self, data):
+    self._chunks.append(bytes(data))
+    return len(data)
+
+  def pop(self):
+    data = b"".join(self._chunks)
+    self._chunks.clear()
+    return data
+
+
 KEYS = {
   "amap1": ("amap1", "", "AMapKey1", "AMap / Gaode key #1", 39),
   "amap2": ("amap2", "", "AMapKey2", "AMap / Gaode key #2", 39),
@@ -6258,6 +6300,60 @@ def setup(app):
         }, 200
     return {"error": "Route not found"}, 404
 
+  @app.route("/api/routes/<name>/logs", methods=["GET"])
+  def list_route_logs(name):
+    logs = _route_log_files(name)
+    if not logs:
+      return jsonify({"error": "No full logs are stored on the device for this route."}), 404
+
+    return jsonify({
+      "name": name,
+      "totalBytes": sum(size for *_, size in logs),
+      "segments": [
+        {
+          "segment": segment,
+          "segmentNum": int(segment.rsplit("--", 1)[1]),
+          "filename": filename,
+          "bytes": size,
+          "url": f"/api/routes/{name}/logs/{int(segment.rsplit('--', 1)[1])}",
+        }
+        for segment, filename, _, size in logs
+      ],
+    }), 200
+
+  @app.route("/api/routes/<name>/logs/<int:segment_num>", methods=["GET"])
+  def download_route_log(name, segment_num):
+    for segment, filename, path, _ in _route_log_files(name):
+      if int(segment.rsplit("--", 1)[1]) == segment_num:
+        return send_file(path, as_attachment=True, download_name=f"{segment}-{filename}")
+    return jsonify({"error": "No full log is stored on the device for this segment."}), 404
+
+  @app.route("/api/routes/<name>/logs/download", methods=["GET"])
+  def download_route_logs_archive(name):
+    logs = _route_log_files(name)
+    if not logs:
+      return jsonify({"error": "No full logs are stored on the device for this route."}), 404
+
+    def generate():
+      buffer = _TarBuffer()
+      # streamed a file at a time so a long route never needs its whole archive in memory
+      with tarfile.open(fileobj=buffer, mode="w|") as archive:
+        for segment, filename, path, _ in logs:
+          try:
+            archive.add(path, arcname=f"{segment}/{filename}")
+          except OSError:
+            continue
+          chunk = buffer.pop()
+          if chunk:
+            yield chunk
+      chunk = buffer.pop()
+      if chunk:
+        yield chunk
+
+    response = Response(generate(), mimetype="application/x-tar")
+    response.headers["Content-Disposition"] = f'attachment; filename="{name}-logs.tar"'
+    return response
+
   @app.route("/api/routes/clear_name", methods=["POST"])
   def clear_route_name():
     data = request.get_json()