IQ.Pilot Prebuilt Release @ 0bfb3ae

tools/scripts/adb_ssh.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Forward all openpilot service ports
+mapfile -t SERVICE_PORTS < <(python3 - <<'PY'
+from cereal.services import SERVICE_LIST
+
+FNV_PRIME = 0x100000001b3
+FNV_OFFSET_BASIS = 0xcbf29ce484222325
+START_PORT = 8023
+MAX_PORT = 65535
+PORT_RANGE = MAX_PORT - START_PORT
+MASK = 0xffffffffffffffff
+
+def fnv1a(endpoint: str) -> int:
+  h = FNV_OFFSET_BASIS
+  for b in endpoint.encode():
+    h ^= b
+    h = (h * FNV_PRIME) & MASK
+  return h
+
+ports = set()
+for name in SERVICE_LIST.keys():
+  port = START_PORT + fnv1a(name) % PORT_RANGE
+  ports.add((name, port))
+
+for name, port in sorted(ports):
+  print(f"{name} {port}")
+PY
+)
+
+for entry in "${SERVICE_PORTS[@]}"; do
+  name="${entry% *}"
+  port="${entry##* }"
+  adb forward "tcp:${port}" "tcp:${port}" > /dev/null
+done
+
+# Forward SSH port first for interactive shell access.
+adb forward tcp:2222 tcp:22
+
+# SSH!
+ssh comma@localhost -p 2222 "$@"

tools/scripts/extract_audio.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+import os
+import sys
+import wave
+import argparse
+import numpy as np
+
+from openpilot.tools.lib.logreader import LogReader, ReadMode
+
+
+def extract_audio(route_or_segment_name, output_file=None, play=False):
+  lr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACTIVE)
+  audio_messages = list(lr.filter("rawAudioData"))
+  if not audio_messages:
+    print("No rawAudioData messages found in logs")
+    return
+  sample_rate = audio_messages[0].sampleRate
+
+  audio_chunks = []
+  total_frames = 0
+  for msg in audio_messages:
+    audio_array = np.frombuffer(msg.data, dtype=np.int16)
+    audio_chunks.append(audio_array)
+    total_frames += len(audio_array)
+  full_audio = np.concatenate(audio_chunks)
+
+  print(f"Found {total_frames} frames from {len(audio_messages)} audio messages at {sample_rate} Hz")
+
+  if output_file:
+    if write_wav_file(output_file, full_audio, sample_rate):
+      print(f"Audio written to {output_file}")
+    else:
+      print("Audio extraction canceled.")
+  if play:
+    play_audio(full_audio, sample_rate)
+
+
+def write_wav_file(filename, audio_data, sample_rate):
+  if os.path.exists(filename):
+    if input(f"File '{filename}' exists. Overwrite? (y/N): ").lower() not in ['y', 'yes']:
+      return False
+
+  with wave.open(filename, 'wb') as wav_file:
+    wav_file.setnchannels(1)  # Mono
+    wav_file.setsampwidth(2)  # 16-bit
+    wav_file.setframerate(sample_rate)
+    wav_file.writeframes(audio_data.tobytes())
+  return True
+
+
+def play_audio(audio_data, sample_rate):
+  try:
+    import sounddevice as sd
+
+    print("Playing audio... Press Ctrl+C to stop")
+    sd.play(audio_data, sample_rate)
+    sd.wait()
+  except KeyboardInterrupt:
+    print("\nPlayback stopped")
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="Extract audio data from openpilot logs")
+  parser.add_argument("-o", "--output", help="Output WAV file path")
+  parser.add_argument("--play", action="store_true", help="Play audio with sounddevice")
+  parser.add_argument("route_or_segment_name", nargs='?', help="The route or segment name")
+
+  if len(sys.argv) == 1:
+    parser.print_help()
+    sys.exit()
+  args = parser.parse_args()
+
+  output_file = args.output
+  if not args.output and not args.play:
+    output_file = "extracted_audio.wav"
+
+  extract_audio(args.route_or_segment_name.strip(), output_file, args.play)

tools/scripts/fetch_image_from_route.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+if len(sys.argv) < 4:
+  print(f"{sys.argv[0]} <route> <segment> <frame number> [front|wide|driver]")
+  print('example: ./fetch_image_from_route.py "02c45f73a2e5c6e9|2020-06-01--18-03-08" 3 500 driver')
+  exit(0)
+
+cameras = {
+  "front": "cameras",
+  "wide": "ecameras",
+  "driver": "dcameras"
+}
+
+import requests
+from PIL import Image
+from openpilot.tools.lib.auth_config import get_token
+from openpilot.tools.lib.framereader import FrameReader
+
+jwt = get_token()
+
+route = sys.argv[1]
+segment = int(sys.argv[2])
+frame = int(sys.argv[3])
+camera = cameras[sys.argv[4]] if len(sys.argv) > 4 and sys.argv[4] in cameras else "cameras"
+
+api_host = os.getenv("API_HOST", "https://api-iqlabs.konn3kt.com")
+url = f'{api_host}/v1/route/{route}/files'
+r = requests.get(url, headers={"Authorization": f"JWT {jwt}"}, timeout=10)
+assert r.status_code == 200
+print("got api response")
+
+segments = r.json()[camera]
+if segment >= len(segments):
+  raise Exception(f"segment {segment} not found, got {len(segments)} segments")
+
+fr = FrameReader(segments[segment])
+if frame >= fr.frame_count:
+  raise Exception("frame {frame} not found, got {fr.frame_count} frames")
+
+im = Image.fromarray(fr.get(frame))
+fn = f"uxxx_{route.replace('|', '_')}_{segment}_{frame}.png"
+im.save(fn)
+print(f"saved {fn}")

tools/scripts/io_stall_repro.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""
+io_stall_repro.py — bench reproduction + fix-validation rig for the VW PQ EPS
+HCA fault (control loop stalling on a /data read under eMMC write saturation).
+
+The real fault chain (proven from rlog b29ee8c5a0a735d1/000000e4--8a8ba97b54):
+  loggerd buffered writes saturate eMMC -> ext4 jbd2 journal commits ->
+  controlsd's inline Params.get (util::read_file on /data) blocks ~300-630ms ->
+  controlsd stops publishing carControl -> card's all_alive guard withholds
+  HCA_1 -> EPS LH2_Sta_HCA 7->2.
+
+This reproduces the *proximate* cause WITHOUT driving, WITHOUT the EPS, and
+WITHOUT touching the real control stack. Two roles:
+
+  --writer : emulate loggerd. Buffered (no-fsync) writes to /data at a target
+             MB/s, with an optional periodic big flush to mimic 60s segment
+             rotation. This is what saturates the eMMC.
+
+  --probe  : emulate controlsd's I/O exposure. A 100Hz loop that every
+             --param-period seconds reads a real param (util::read_file on
+             /data). It records per-iteration loop gaps and param-read
+             durations. A 300ms gap here == the stall that drops HCA.
+             --threaded moves the param read to a background thread (the
+             proposed fix, mirroring card.py's params_thread) so you can A/B it.
+
+USAGE (run parked, ignition on, on the device):
+  # 1) baseline: probe alone -> gaps should be tiny
+  python3 io_stall_repro.py --probe --secs 120
+
+  # 2) reproduce: writer in one shell, probe in another
+  python3 io_stall_repro.py --writer --mbps 25 --rotate 60
+  python3 io_stall_repro.py --probe --secs 180          # expect big gaps
+
+  # 3) validate fix A (params off control thread):
+  python3 io_stall_repro.py --probe --secs 180 --threaded   # gaps should vanish
+
+  # 4) validate fix B (smoother writeback) — set before step 2, as root:
+  #   echo 5  > /proc/sys/vm/dirty_background_ratio
+  #   echo 10 > /proc/sys/vm/dirty_ratio
+  #   then re-run step 2 inline probe and compare gap distribution.
+
+Cleanup: writer deletes its scratch files on exit. Read-only wrt openpilot.
+"""
+import argparse, os, sys, time, threading, statistics, signal
+
+SCRATCH_DEFAULT = "/data/media/0/io_repro_scratch"
+
+# ----------------------------------------------------------------------------- writer
+def run_writer(args):
+  os.makedirs(args.scratch, exist_ok=True)
+  chunk = os.urandom(1 << 20)  # 1 MiB
+  bytes_per_s = int(args.mbps * (1 << 20))
+  print(f"[writer] buffered no-fsync writes to {args.scratch} at ~{args.mbps} MB/s, "
+        f"rotate every {args.rotate}s (big flush). Ctrl-C to stop.", file=sys.stderr)
+
+  stop = {"v": False}
+  signal.signal(signal.SIGINT, lambda *_: stop.update(v=True))
+  signal.signal(signal.SIGTERM, lambda *_: stop.update(v=True))
+
+  files = []
+  seg = 0
+  try:
+    while not stop["v"]:
+      seg_start = time.monotonic()
+      path = os.path.join(args.scratch, f"seg_{seg}.bin")
+      f = open(path, "wb", buffering=1 << 20)
+      files.append(path)
+      written = 0
+      # write at target rate using buffered fwrite, NO fsync (exactly loggerd)
+      while not stop["v"] and (time.monotonic() - seg_start) < args.rotate:
+        t0 = time.monotonic()
+        f.write(chunk)
+        written += len(chunk)
+        # pace to target MB/s
+        target_t = written / bytes_per_s
+        elapsed = time.monotonic() - seg_start
+        if target_t > elapsed:
+          time.sleep(min(0.1, target_t - elapsed))
+      # "segment rotation": flush+close a big buffered file at once -> writeback burst
+      f.flush()
+      f.close()
+      seg += 1
+      # keep only a few recent files so we don't fill the disk
+      while len(files) > 3:
+        old = files.pop(0)
+        try: os.remove(old)
+        except OSError: pass
+  finally:
+    for p in files:
+      try: os.remove(p)
+      except OSError: pass
+    print("[writer] stopped, scratch cleaned.", file=sys.stderr)
+
+# ----------------------------------------------------------------------------- probe
+def _get_param(key):
+  # real /data read, same syscall path as controlsd's get_params_iq
+  try:
+    from openpilot.common.params import Params
+    return Params().get_bool(key)
+  except Exception:
+    # fallback: plain file read of a param file if openpilot import unavailable
+    p = os.path.join(os.getenv("PARAMS_ROOT", "/data/params"), "d", key)
+    try:
+      with open(p, "rb") as fh:
+        return fh.read()
+    except OSError:
+      return None
+
+class ThreadedParam:
+  """Mirror card.py params_thread: refresh the param on a bg thread, control
+  loop reads the cached value (non-blocking)."""
+  def __init__(self, key, period):
+    self.key = key; self.period = period; self.val = None
+    self.stop = False
+    self.t = threading.Thread(target=self._loop, daemon=True); self.t.start()
+  def _loop(self):
+    while not self.stop:
+      self.val = _get_param(self.key)
+      time.sleep(self.period)
+  def read(self):  # O(1), no I/O on the control thread
+    return self.val
+
+def run_probe(args):
+  # pin like controlsd (core 4) so we share the same iowait domain if possible
+  try:
+    os.sched_setaffinity(0, {args.core})
+  except (OSError, AttributeError):
+    pass
+
+  interval = 0.01  # 100Hz, like the control loop
+  gaps = []          # ms, per-iteration loop overrun beyond 10ms
+  read_ms = []       # ms, time spent in the param read on the control thread
+  worst = 0.0
+  threaded = ThreadedParam(args.param_key, args.param_period) if args.threaded else None
+
+  print(f"[probe] 100Hz loop for {args.secs}s, param '{args.param_key}' every "
+        f"{args.param_period}s, threaded={args.threaded}, core={args.core}", file=sys.stderr)
+  t_end = time.monotonic() + args.secs
+  next_t = time.monotonic()
+  last_param = 0.0
+  while time.monotonic() < t_end:
+    loop_start = time.monotonic()
+
+    # the I/O exposure: read param on the control thread (inline) every period
+    if loop_start - last_param >= args.param_period:
+      r0 = time.monotonic()
+      if threaded is not None:
+        _ = threaded.read()            # cached, no I/O on this thread (the FIX)
+      else:
+        _ = _get_param(args.param_key)  # inline /data read (current behavior)
+      dr = (time.monotonic() - r0) * 1000
+      read_ms.append(dr)
+      last_param = loop_start
+
+    # measure scheduling/lag: how late did this iteration actually fire?
+    next_t += interval
+    lag = (time.monotonic() - next_t) * 1000  # ms behind schedule
+    if lag > 5:
+      gaps.append(lag)
+      worst = max(worst, lag)
+    sleep = next_t - time.monotonic()
+    if sleep > 0:
+      time.sleep(sleep)
+    else:
+      next_t = time.monotonic()  # don't spiral after a big stall
+
+  if threaded:
+    threaded.stop = True
+
+  def pct(xs, p):
+    return sorted(xs)[int(p/100*(len(xs)-1))] if xs else 0.0
+  print("\n================ PROBE RESULT ================")
+  print(f"loop-lag events >5ms : {len(gaps)}")
+  print(f"loop-lag p50/p99/max : {pct(gaps,50):.0f} / {pct(gaps,99):.0f} / {worst:.0f} ms")
+  print(f"param-read p50/p99/max: {pct(read_ms,50):.1f} / {pct(read_ms,99):.1f} / {max(read_ms+[0]):.1f} ms  (n={len(read_ms)})")
+  hca_class = max(gaps + [0])
+  verdict = ("FAULT-CLASS STALL REPRODUCED (>250ms -> would drop HCA)" if hca_class > 250
+             else "marginal (100-250ms)" if hca_class > 100
+             else "clean (<100ms)")
+  print(f"VERDICT: {verdict}")
+  print("=============================================")
+
+# ----------------------------------------------------------------------------- main
+def main():
+  ap = argparse.ArgumentParser()
+  ap.add_argument("--writer", action="store_true", help="emulate loggerd eMMC saturation")
+  ap.add_argument("--probe", action="store_true", help="emulate controlsd I/O exposure")
+  ap.add_argument("--mbps", type=float, default=25.0, help="writer target MB/s (loggerd ~10-30)")
+  ap.add_argument("--rotate", type=float, default=60.0, help="writer segment/flush period s")
+  ap.add_argument("--scratch", default=SCRATCH_DEFAULT)
+  ap.add_argument("--secs", type=float, default=180.0, help="probe duration s")
+  ap.add_argument("--param-key", default="IsMetric", help="a real param key to read")
+  ap.add_argument("--param-period", type=float, default=3.0, help="controlsd reads every 3s")
+  ap.add_argument("--threaded", action="store_true", help="probe: read param off control thread (the FIX)")
+  ap.add_argument("--core", type=int, default=4, help="probe cpu affinity (control core)")
+  args = ap.parse_args()
+  if args.writer == args.probe:
+    ap.error("pick exactly one of --writer / --probe (run them in separate shells)")
+  run_writer(args) if args.writer else run_probe(args)
+
+if __name__ == "__main__":
+  main()

tools/scripts/io_stall_tracer.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""
+io_stall_tracer.py — continuous low-overhead per-process disk-I/O tracer.
+
+WHY: the VW PQ EPS HCA faults are caused by a control process (controlsd/card)
+blocking ~300-630ms in disk I/O (iowait) on /data, which stops HCA_1 TX. The
+stall is far too short to catch with a manual `iostat`/`iotop` run. This sampler
+runs for the whole drive at 100ms cadence and records, per process:
+
+  - write_bytes      (/proc/<pid>/io)   -> identifies the WRITER saturating eMMC
+  - delayacct_blkio  (/proc/<pid>/stat) -> per-process cumulative block-I/O wait
+  - state            (/proc/<pid>/stat) -> catches 'D' (uninterruptible disk wait)
+
+plus whole-device /proc/diskstats. After a fault, find the wall-clock time of the
+LH2_Sta_HCA->2 event (from the rlog) and look at the rows around it: the process
+whose write_bytes delta spikes is the bully; the control process whose blkio
+delta jumps / state == 'D' is the victim.
+
+Deploy: copy to the device, run alongside openpilot during a drive:
+    python3 io_stall_tracer.py --out /data/media/0/io_trace.csv
+Overhead: reading /proc for ~40 procs every 100ms is well under 1% of one core,
+and it pins itself to CPU 0 (away from the control cores 4/5) at low priority.
+
+Read-only. Writes a single CSV. No openpilot deps.
+"""
+import argparse, os, time, glob, sys
+
+CLK_TCK = os.sysconf("SC_CLK_TCK")  # usually 100 -> blkio ticks are 10ms each
+
+def read_proc_io(pid):
+  # wchar/rchar = bytes moved via read()/write() syscalls (catches BUFFERED writers
+  # like loggerd, which never appear in write_bytes because the kernel flushes their
+  # page-cache dirty pages asynchronously via kworker). write_bytes = bytes actually
+  # sent to the block device. Track both.
+  try:
+    with open(f"/proc/{pid}/io") as f:
+      d = {}
+      for line in f:
+        k, _, v = line.partition(":")
+        d[k] = int(v)
+      return (d.get("wchar", 0), d.get("rchar", 0),
+              d.get("write_bytes", 0), d.get("read_bytes", 0))
+  except (OSError, ValueError):
+    return None
+
+def read_proc_stat(pid):
+  # state is field 3; delayacct_blkio_ticks is field 42 (1-indexed). comm may
+  # contain spaces/parens, so split on the last ')'.
+  try:
+    with open(f"/proc/{pid}/stat") as f:
+      data = f.read()
+    rparen = data.rfind(")")
+    comm = data[data.find("(") + 1:rparen]
+    rest = data[rparen + 2:].split()
+    state = rest[0]                       # field 3
+    blkio_ticks = int(rest[39]) if len(rest) > 39 else 0  # field 42
+    return comm, state, blkio_ticks
+  except (OSError, ValueError, IndexError):
+    return None
+
+def read_diskstats():
+  # returns {dev: (sectors_written, ms_doing_io)} for whole-disk devices
+  out = {}
+  try:
+    with open("/proc/diskstats") as f:
+      for line in f:
+        p = line.split()
+        if len(p) < 14:
+          continue
+        dev = p[2]
+        # field 10 (idx 9) = sectors written; field 13 (idx 12) = ms doing I/O
+        out[dev] = (int(p[9]), int(p[12]))
+  except (OSError, AttributeError):
+    pass
+  return out
+
+# These counters tell us WHICH kernel mechanism caused a stall, which decides
+# the fix: compact_stall jumping -> memory compaction (texture-pool fix);
+# allocstall/pgsteal jumping -> direct reclaim; high nr_dirty/nr_writeback ->
+# loggerd writeback bomb (loggerd sync_file_range fix). meminfo Dirty/Writeback
+# are absolute kB; vmstat ones are cumulative event counts (we delta them).
+VMSTAT_KEYS = ("compact_stall", "compact_fail", "allocstall_normal", "allocstall_movable",
+               "pgsteal_direct", "pgscan_direct", "pgmajfault", "nr_dirty", "nr_writeback")
+MEMINFO_KEYS = ("MemFree", "MemAvailable", "Dirty", "Writeback")
+
+def read_vmstat():
+  out = {}
+  try:
+    with open("/proc/vmstat") as f:
+      for line in f:
+        k, _, v = line.partition(" ")
+        if k in VMSTAT_KEYS:
+          out[k] = int(v)
+  except OSError:
+    pass
+  return out
+
+def read_meminfo():
+  out = {}
+  try:
+    with open("/proc/meminfo") as f:
+      for line in f:
+        k, _, v = line.partition(":")
+        if k in MEMINFO_KEYS:
+          out[k] = int(v.split()[0])  # kB
+  except (OSError, IndexError):
+    pass
+  return out
+
+def main():
+  ap = argparse.ArgumentParser()
+  ap.add_argument("--out", default="/data/media/0/io_trace.csv")
+  ap.add_argument("--hz", type=float, default=10.0, help="sample rate (default 10Hz/100ms)")
+  ap.add_argument("--disk", default="sda", help="comma-separated disk devices to track (default sda)")
+  ap.add_argument("--names", default="controlsd,car.c,selfd,ui,loggerd,encoderd,modeld,camerad,locationd,paramsd,navd,mapd",
+                  help="substring match of process comm to record (others summed as 'other')")
+  args = ap.parse_args()
+
+  # be a good citizen: low priority, off the control cores
+  try:
+    os.nice(10)
+    os.sched_setaffinity(0, {0})
+  except OSError:
+    pass
+
+  watch = [n.strip() for n in args.names.split(",") if n.strip()]
+  disks = [d.strip() for d in args.disk.split(",") if d.strip()]
+  interval = 1.0 / args.hz
+
+  prev_io = {}      # pid -> (wchar, rchar, write_bytes, read_bytes)
+  prev_blkio = {}   # pid -> blkio_ticks
+  prev_disk = read_diskstats()
+  prev_vm = read_vmstat()
+
+  f = open(args.out, "w", buffering=1)
+  # per-proc rows fill the first block; one SYS row per tick fills the trailing
+  # mechanism columns (deltas for the vmstat counts, absolute kB for meminfo).
+  f.write("wall,mono,proc,pid,state,d_wchar_kB,d_wbytes_kB,d_blkio_ms,disk_d_write_kB,disk_d_busy_ms,"
+          "compact_stall,allocstall,pgmajfault,dirty_kB,writeback_kB,memfree_kB,memavail_kB\n")
+  print(f"[io_stall_tracer] writing {args.out} at {args.hz}Hz, tracking {watch}", file=sys.stderr)
+
+  while True:
+    t_wall = time.time()
+    t_mono = time.monotonic()
+
+    # whole-disk delta (write kB + busy ms) for the named disks
+    disk = read_diskstats()
+    disk_dw = disk_db = 0
+    for dev in disks:
+      if dev in disk and dev in prev_disk:
+        disk_dw += (disk[dev][0] - prev_disk[dev][0]) * 512 / 1024.0  # sectors->kB
+        disk_db += (disk[dev][1] - prev_disk[dev][1])
+    prev_disk = disk
+
+    seen = set()
+    rows = []
+    for path in glob.glob("/proc/[0-9]*"):
+      pid = path.rsplit("/", 1)[1]
+      st = read_proc_stat(pid)
+      if st is None:
+        continue
+      comm, state, blkio = st
+      label = next((w for w in watch if w in comm), None)
+      if label is None:
+        # still track D-state of anything to catch surprise writers/blockers
+        if state != "D":
+          continue
+        label = comm
+      io = read_proc_io(pid)
+      if io is None:
+        continue
+      wchar, rchar, wbytes, rbytes = io
+      pw = prev_io.get(pid, io)
+      pblk = prev_blkio.get(pid, blkio)
+      d_wchar = (wchar - pw[0]) / 1024.0     # syscall write volume (catches loggerd)
+      d_wbytes = (wbytes - pw[2]) / 1024.0   # bytes hitting the block device
+      d_blk = (blkio - pblk) * (1000.0 / CLK_TCK)  # ticks -> ms blocked on block I/O
+      prev_io[pid] = io
+      prev_blkio[pid] = blkio
+      seen.add(pid)
+      # only emit rows that carry signal (writing, blocked, or in D) to keep file small
+      if d_wchar > 4 or d_wbytes > 4 or d_blk > 5 or state == "D":
+        rows.append((label, pid, state, d_wchar, d_wbytes, d_blk))
+
+    # drop dead pids from prev maps occasionally
+    if len(prev_io) > 4000:
+      prev_io = {p: v for p, v in prev_io.items() if p in seen}
+      prev_blkio = {p: v for p, v in prev_blkio.items() if p in seen}
+
+    for label, pid, state, d_wchar, d_wbytes, d_blk in rows:
+      f.write(f"{t_wall:.3f},{t_mono:.3f},{label},{pid},{state},{d_wchar:.0f},{d_wbytes:.0f},{d_blk:.0f},{disk_dw:.0f},{disk_db:.0f},,,,,,,\n")
+
+    # one SYS row per tick: the kernel-mechanism counters (compaction vs reclaim
+    # vs writeback). Compare these against the stall's wall-clock to see which
+    # one spiked.
+    vm = read_vmstat(); mi = read_meminfo()
+    d_compact = vm.get("compact_stall", 0) - prev_vm.get("compact_stall", 0)
+    d_alloc = ((vm.get("allocstall_normal", 0) + vm.get("allocstall_movable", 0))
+               - (prev_vm.get("allocstall_normal", 0) + prev_vm.get("allocstall_movable", 0)))
+    d_majflt = vm.get("pgmajfault", 0) - prev_vm.get("pgmajfault", 0)
+    prev_vm = vm
+    f.write(f"{t_wall:.3f},{t_mono:.3f},SYS,0,-,,,,,,"
+            f"{d_compact},{d_alloc},{d_majflt},{mi.get('Dirty',0)},{mi.get('Writeback',0)},"
+            f"{mi.get('MemFree',0)},{mi.get('MemAvailable',0)}\n")
+
+    time.sleep(max(0.0, interval - (time.monotonic() - t_mono)))
+
+if __name__ == "__main__":
+  main()

tools/scripts/iqpilot_model_compile_guide.md

@@ -0,0 +1,482 @@
+# IQPilot Model Compile Guide
+
+This guide documents the current, working IQPilot flow for compiling and publishing tinygrad `supercombo` model artifacts for comma 3X/4 class devices.
+
+It is written to be usable by either a human operator or another agent without needing outside context.
+
+## Scope
+
+This guide covers:
+
+- compiling `supercombo` ONNX models into prebuilt `QCOM` tinygrad `.pkl` artifacts
+- matching IQPilot's current pinned compile path
+- validating artifacts on-device before publishing
+- updating the `IQModels` repo and selector manifests
+- common failure modes seen in current IQPilot
+
+This guide is specifically for the current IQPilot model stack:
+
+- compiler script: [iqpilot/selfdrive/iqmodeld/tools/compile_supercombo.py](/Users/t/Developer/iqpilot/iqpilot/iqpilot/selfdrive/iqmodeld/tools/compile_supercombo.py)
+- runtime: [iqpilot/selfdrive/iqmodeld/daemon.py](/Users/t/Developer/iqpilot/iqpilot/iqpilot/selfdrive/iqmodeld/daemon.py)
+- tinygrad compile flags: [iqpilot/selfdrive/iqmodeld/SConscript](/Users/t/Developer/iqpilot/iqpilot/iqpilot/selfdrive/iqmodeld/SConscript)
+- on-device harness: [iqpilot/selfdrive/iqmodeld/tools/force_onroad_iqmodeld_test.py](/Users/t/Developer/iqpilot/iqpilot/iqpilot/selfdrive/iqmodeld/tools/force_onroad_iqmodeld_test.py)
+
+## Current Known-Good QCOM Compile Flags
+
+IQPilot should compile QCOM `supercombo` artifacts with the same important flags stock openpilot uses for the proven QCOM tinygrad path:
+
+```bash
+DEV=QCOM
+WARP_DEV=QCOM
+IMAGE=2
+FLOAT16=1
+NOLOCALS=1
+JIT_BATCH_SIZE=0
+OPENPILOT_HACKS=1
+```
+
+The critical correction was `OPENPILOT_HACKS=1`.
+
+Without that, models can compile into artifacts that look valid but behave differently from stock openpilot builds, including:
+
+- PoseNET/JIT shape mismatches
+- freezes when going onroad
+- `iqmodeld` death or stall after model startup
+- artifacts that drive on stock but fail on IQPilot
+
+## Current Expected Device Context
+
+These instructions assume:
+
+- IQPilot lives at `/data/openpilot`
+- the repo on your Mac is `/Users/t/Developer/iqpilot/iqpilot`
+- the target device is reachable over SSH as `iq@<ip>`
+- the device has a working venv at `/usr/local/venv`
+- tinygrad is available under `/data/openpilot/tinygrad_repo`
+
+If your paths differ, adjust the commands accordingly.
+
+## 1. Confirm IQPilot Source Is Using the Correct Flags
+
+Before compiling anything, verify [iqpilot/selfdrive/iqmodeld/SConscript](/Users/t/Developer/iqpilot/iqpilot/iqpilot/selfdrive/iqmodeld/SConscript) is returning the current QCOM flags for `larch64`.
+
+The effective flag set for the QCOM path should include:
+
+```python
+DEV=QCOM IMAGE=2 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1
+```
+
+If you edit the file locally, copy it to the device before recompiling:
+
+```bash
+scp /Users/t/Developer/iqpilot/iqpilot/iqpilot/selfdrive/iqmodeld/SConscript \
+  iq@<ip>:/data/openpilot/iqpilot/selfdrive/iqmodeld/SConscript
+```
+
+## 2. Identify the Source ONNX and Target Artifact Name
+
+For each model, determine:
+
+- source ONNX file
+- published artifact filename
+- selector entry in `IQModels`
+
+Examples from recent working flow:
+
+- `TobyRL`
+  - source: `driving_supercombo.onnx`
+  - artifact: `driving_supercombo_tobyrl.pkl`
+- `NoPP`
+  - source: `driving_supercombo.onnx`
+  - artifact: `driving_supercombo_nopp.pkl`
+- `DRLV3`
+  - source: `driving_supercombo.onnx`
+  - artifact: `driving_supercombo_drl3.pkl`
+
+## 3. Copy the Source ONNX to the Device
+
+Copy the chosen ONNX to a neutral working path on-device:
+
+```bash
+scp /path/to/driving_supercombo.onnx \
+  iq@<ip>:/data/media/0/driving_supercombo_<name>_source.onnx
+```
+
+Example:
+
+```bash
+scp '/Volumes/New New Vault/IQModels/models/recompiled16/model-No PP (June 26, 2026)-165/driving_supercombo.onnx' \
+  iq@192.168.173.10:/data/media/0/driving_supercombo_nopp_source.onnx
+```
+
+## 4. Prepare Writable Cache Directories on the Device
+
+Do this once per session before compiling.
+
+This avoids tinygrad falling back to unwritable root-owned cache paths and prevents failures caused by:
+
+- `/root/.cache` write problems
+- temporary directory permission failures
+- unstable shell environments
+
+```bash
+ssh iq@<ip> "sudo mkdir -p /data/root-home /data/root-cache /data/root-uv-cache /data/root-tmp"
+```
+
+## 5. Compile the Model On-Device
+
+Use the exact compile environment below.
+
+This is the current known-good command:
+
+```bash
+ssh iq@<ip> "sudo env \
+HOME=/data/root-home \
+XDG_CACHE_HOME=/data/root-cache \
+UV_CACHE_DIR=/data/root-uv-cache \
+TMPDIR=/data/root-tmp \
+PYTHONPATH=/data/openpilot:/data/openpilot/tinygrad_repo \
+PATH=/usr/local/venv/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+VIRTUAL_ENV=/usr/local/venv \
+DEV=QCOM \
+WARP_DEV=QCOM \
+IMAGE=2 \
+FLOAT16=1 \
+NOLOCALS=1 \
+JIT_BATCH_SIZE=0 \
+OPENPILOT_HACKS=1 \
+/usr/local/venv/bin/python \
+/data/openpilot/iqpilot/selfdrive/iqmodeld/tools/compile_supercombo.py \
+  --model-size 512x256 \
+  --camera-resolutions 1344x760 1928x1208 \
+  --onnx /data/media/0/driving_supercombo_<name>_source.onnx \
+  --output /data/media/0/models/driving_supercombo_<name>.recompiled.pkl \
+  --frame-skip 4 \
+  --expected-device QCOM"
+```
+
+### Example: NoPP
+
+```bash
+ssh iq@192.168.173.10 "sudo env \
+HOME=/data/root-home \
+XDG_CACHE_HOME=/data/root-cache \
+UV_CACHE_DIR=/data/root-uv-cache \
+TMPDIR=/data/root-tmp \
+PYTHONPATH=/data/openpilot:/data/openpilot/tinygrad_repo \
+PATH=/usr/local/venv/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+VIRTUAL_ENV=/usr/local/venv \
+DEV=QCOM \
+WARP_DEV=QCOM \
+IMAGE=2 \
+FLOAT16=1 \
+NOLOCALS=1 \
+JIT_BATCH_SIZE=0 \
+OPENPILOT_HACKS=1 \
+/usr/local/venv/bin/python \
+/data/openpilot/iqpilot/selfdrive/iqmodeld/tools/compile_supercombo.py \
+  --model-size 512x256 \
+  --camera-resolutions 1344x760 1928x1208 \
+  --onnx /data/media/0/driving_supercombo_nopp_source.onnx \
+  --output /data/media/0/models/driving_supercombo_nopp.recompiled.pkl \
+  --frame-skip 4 \
+  --expected-device QCOM"
+```
+
+## 6. What Success Looks Like
+
+On success, the compile ends with output similar to:
+
+```text
+Saved JITs to /data/media/0/models/driving_supercombo_<name>.recompiled.pkl (119.07 MB)
+```
+
+Current good rebuilt `supercombo` artifacts have been around `114M` to `119 MB`.
+
+If you get a tiny file, do not publish it.
+
+Always verify both size and hash:
+
+```bash
+ssh iq@<ip> "ls -lh /data/media/0/models/driving_supercombo_<name>.recompiled.pkl"
+ssh iq@<ip> "sha256sum /data/media/0/models/driving_supercombo_<name>.recompiled.pkl"
+```
+
+## 7. Expected Warnings During Compile
+
+These warnings have appeared during successful builds and are not by themselves proof of a bad artifact:
+
+```text
+input desire_pulse has mismatch on dtype. Expected dtypes.half, received dtypes.float.
+input traffic_convention has mismatch on dtype. Expected dtypes.half, received dtypes.float.
+input action_t has mismatch on dtype. Expected dtypes.half, received dtypes.float.
+input features_buffer has mismatch on dtype. Expected dtypes.half, received dtypes.float.
+```
+
+Treat them as informational unless the resulting model fails runtime validation.
+
+## 8. Install the Rebuilt Artifact On the Device
+
+Before publishing to `IQModels`, install the rebuilt artifact locally on the device being tested.
+
+Back up the current file first:
+
+```bash
+ssh iq@<ip> "cp /data/media/0/models/driving_supercombo_<name>.pkl /data/media/0/models/driving_supercombo_<name>.pkl.bak.\$(date +%s)"
+```
+
+Then replace it:
+
+```bash
+ssh iq@<ip> "sudo cp /data/media/0/models/driving_supercombo_<name>.recompiled.pkl /data/media/0/models/driving_supercombo_<name>.pkl"
+```
+
+Verify the active file hash:
+
+```bash
+ssh iq@<ip> "sha256sum /data/media/0/models/driving_supercombo_<name>.pkl"
+```
+
+## 9. Validate the Model Before Publishing
+
+Do not push a model just because it compiled.
+
+Current IQPilot validation should include both of these:
+
+### 9.1 Offroad/on-device `iqmodeld` harness validation
+
+Use the current harness:
+
+[iqpilot/selfdrive/iqmodeld/tools/force_onroad_iqmodeld_test.py](/Users/t/Developer/iqpilot/iqpilot/iqpilot/selfdrive/iqmodeld/tools/force_onroad_iqmodeld_test.py)
+
+This script creates a minimal fake onroad environment and starts `iqmodeld`.
+
+Run it on-device from the normal IQPilot tree:
+
+```bash
+ssh iq@<ip> "cd /data/openpilot && /usr/local/venv/bin/python /data/openpilot/iqpilot/selfdrive/iqmodeld/tools/force_onroad_iqmodeld_test.py --speed 15.0"
+```
+
+What you want:
+
+- `iqmodeld` stays alive
+- no PoseNET/JIT mismatch crash
+- no immediate deadlock
+- no model startup freeze
+
+### 9.2 Real launcher / real device validation
+
+The harness is necessary, but not sufficient.
+
+Also validate through the actual launcher/runtime:
+
+- boot IQPilot normally
+- select the model through the selector
+- clear model cache if required by your workflow
+- confirm the model loads
+- confirm it survives transition to onroad
+- confirm `modelV2`, `livePose`, `iqPlan`, and related services stay alive
+- confirm the UI does not freeze
+
+For any model intended for release, a real onroad validation pass is strongly recommended before publishing.
+
+## 10. Runtime Signals to Watch
+
+When a model is bad, the system often does not fail cleanly.
+
+Useful checks:
+
+```bash
+ssh iq@<ip> "tmux capture-pane -pt 0"
+ssh iq@<ip> "cat /data/community/crashes/error.log"
+ssh iq@<ip> "ls -1t /data/community/crashes | head"
+```
+
+Common bad signals:
+
+- `tinygrad.engine.jit.JitError: args mismatch in JIT`
+- `expected_input_info` showing `CPU` on one side and `QCOM` on the other
+- shape mismatch like `arg=2` vs `arg=5`
+- `iqmodeld` disappearing or zombifying
+- `modelV2`, `livePose`, or `iqPlan` dying
+- UI frozen onroad
+- `commIssue` spam caused by model pipeline collapse
+
+## 11. Common Failure Modes and Meaning
+
+### Failure: `args mismatch in JIT`
+
+Typical causes:
+
+- compiled artifact does not match current runtime expectations
+- wrong `frame_skip`
+- wrong device capture path
+- CPU-built artifact being used on QCOM runtime
+- stale/old artifact still being loaded
+
+### Failure: artifact references `CPU` in crash output
+
+Typical cause:
+
+- artifact compiled on the wrong backend or with the wrong compile environment
+
+Fix:
+
+- rebuild on-device with `DEV=QCOM`, `WARP_DEV=QCOM`, and `OPENPILOT_HACKS=1`
+
+### Failure: tiny file after `scp`
+
+Typical cause:
+
+- interrupted copy
+- partial transfer
+
+Fix:
+
+- verify local file size
+- verify local hash
+- do not publish until the copied file matches the device hash exactly
+
+### Failure: tinygrad cache or permission errors
+
+Typical causes:
+
+- `/root/.cache` unwritable
+- shell launched outside the expected manager environment
+
+Fix:
+
+- use the writable cache dirs from this guide
+- use the exact `sudo env ...` compile command above
+
+### Failure: model compiles but freezes onroad
+
+Possible causes:
+
+- wrong compile flags despite a successful build
+- artifact differs from stock QCOM compile assumptions
+- runtime deadlock only visible under real launch conditions
+
+Fix:
+
+- recompile with the current known-good flags
+- run the harness
+- then validate through normal IQPilot startup and onroad transition
+
+## 12. Pull the Final Artifact Back to the Workstation
+
+Once the device artifact is validated, pull it back locally:
+
+```bash
+scp -o StrictHostKeyChecking=no \
+  iq@<ip>:/data/media/0/models/driving_supercombo_<name>.pkl \
+  /tmp/driving_supercombo_<name>.recompiled.full
+```
+
+Verify locally:
+
+```bash
+ls -lh /tmp/driving_supercombo_<name>.recompiled.full
+sha256sum /tmp/driving_supercombo_<name>.recompiled.full
+```
+
+The local hash must match the device hash exactly.
+
+## 13. Publish to IQModels
+
+Mounted example path:
+
+```text
+/Volumes/New New Vault/IQModels
+```
+
+Replace the existing artifact in the model directory:
+
+```bash
+cp /tmp/driving_supercombo_<name>.recompiled.full \
+  '/Volumes/New New Vault/IQModels/models/recompiled16/model-<Model Name>/driving_supercombo_<name>.pkl'
+```
+
+Then update both:
+
+- `docs/model_selector_b.json`
+- `docs/model_fetcher_b.json`
+
+Update the `sha256` values to the rebuilt artifact hash. If the file path and artifact filename stay the same, the URL does not need to change.
+
+## 14. Verify Published Repo State
+
+Before committing:
+
+```bash
+git -C '/Volumes/New New Vault/IQModels' status --short
+git -C '/Volumes/New New Vault/IQModels' diff -- docs/model_selector_b.json docs/model_fetcher_b.json
+sha256sum '/Volumes/New New Vault/IQModels/models/recompiled16/model-<Model Name>/driving_supercombo_<name>.pkl'
+```
+
+Make sure:
+
+- repo artifact hash matches the rebuilt local hash
+- selector SHA matches the repo artifact hash
+- only intended model files and manifest files are changed
+
+## 15. Commit and Push
+
+Example:
+
+```bash
+git -C '/Volumes/New New Vault/IQModels' add -- \
+  docs/model_selector_b.json \
+  docs/model_fetcher_b.json \
+  'models/recompiled16/model-<Model Name>/driving_supercombo_<name>.pkl'
+
+git -C '/Volumes/New New Vault/IQModels' commit -m 'Recompile <Model Name> with corrected QCOM flags'
+git -C '/Volumes/New New Vault/IQModels' push
+```
+
+## 16. Post-Publish Validation
+
+After pushing:
+
+- redownload the model through the selector on a device
+- confirm the downloaded file SHA matches the published selector SHA
+- confirm the device loads the rebuilt model
+- confirm the model survives onroad entry
+
+Useful command:
+
+```bash
+python3 - <<'PY'
+from openpilot.common.params import Params
+print(Params().get("ModelManager_ActiveBundle"))
+PY
+```
+
+## 17. Recommended Release Checklist
+
+Use this every time.
+
+1. Confirm `SConscript` contains the current QCOM flags, especially `OPENPILOT_HACKS=1`.
+2. Copy the source ONNX to the device.
+3. Compile on-device with the exact environment from this guide.
+4. Verify artifact size is full-sized, not truncated.
+5. Verify artifact hash.
+6. Install the rebuilt artifact on the test device.
+7. Run the `force_onroad_iqmodeld_test.py` harness.
+8. Validate under the real launcher path.
+9. Confirm no crash logs, no PoseNET/JIT mismatch, no freeze.
+10. Pull the artifact back locally and verify hash match.
+11. Replace the published IQModels artifact.
+12. Update selector and fetcher SHA values.
+13. Verify repo hash, manifest hash, and git diff.
+14. Commit and push.
+15. Redownload through selector and validate one more time.
+
+## 18. Notes for Future Agents
+
+- Do not assume a successful compile means the model is good.
+- Do not assume a selector SHA mismatch is the only cause of failure.
+- Do not publish partially copied `.pkl` files.
+- Do not skip the real launcher/onroad validation path.
+- If a model works on stock openpilot but fails on IQPilot, compare the QCOM compile path first.
+- When in doubt, rebuild on-device with the exact current flags in this guide and validate again.

tools/scripts/save_ubloxraw_stream.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+from openpilot.common.basedir import BASEDIR
+from openpilot.tools.lib.logreader import LogReader
+
+os.environ['BASEDIR'] = BASEDIR
+
+
+def get_arg_parser():
+  parser = argparse.ArgumentParser(
+      description="Unlogging and save to file",
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+  parser.add_argument("route", type=(lambda x: x.replace("#", "|")), nargs="?",
+                      help="The route whose messages will be published.")
+  parser.add_argument("--out_path", nargs='?', default='/data/ubloxRaw.stream',
+                      help="Output pickle file path")
+  return parser
+
+
+def main():
+  args = get_arg_parser().parse_args(sys.argv[1:])
+
+  lr = LogReader(args.route)
+
+  with open(args.out_path, 'wb') as f:
+    try:
+      done = False
+      i = 0
+      while not done:
+        msg = next(lr)
+        if not msg:
+          break
+        smsg = msg.as_builder()
+        typ = smsg.which()
+        if typ == 'ubloxRaw':
+          f.write(smsg.to_bytes())
+          i += 1
+    except StopIteration:
+      print('All done')
+  print(f'Writed {i} msgs')
+
+
+if __name__ == "__main__":
+  main()

tools/scripts/sendcan_gap_audit.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""
+sendcan_gap_audit.py — measure the openpilot->car actuator TX cadence from an rlog.
+
+WHY: VW PQ random long/lat disengages are caused by control-loop STALLS that
+freeze the `sendcan` publish for >100ms. The car's ECU runs a counter/checksum
+watchdog on the actuator messages (ACC_System ADR, HCA_1) and FAULTS when frames
+arrive late/missing — it does NOT care about the payload value. So the single
+metric that predicts the fault is the inter-frame GAP in sendcan, not anything
+about accel/torque. (Reference: route 20e3cd4f0d5f39d1|00000038--0f69286335 had a
+103ms gap at ~373s -> engine MO2_Sta_GRA->0 -> main switch off -> disengage. A
+separate 60ms gap did NOT disengage: the ECU timeout sits ~60-100ms.)
+
+This is the pass/fail metric for the mlockall / loggerd-writeback fix (5324c46)
+and, later, the decoupled in-card heartbeat TX. Run it on a BASELINE route to see
+the offending gaps, then on POST-FIX drives to confirm they're gone.
+
+  python3 tools/scripts/sendcan_gap_audit.py <route_or_segment> [--warn-ms 30] [--fault-ms 100]
+
+Exit code 0 if no gap >= --fault-ms, else 1 (so it can gate CI / a smoke test).
+Read-only; pulls rlogs via the normal LogReader (konn3kt for IQ.Pilot routes).
+"""
+import argparse
+import sys
+
+from openpilot.tools.lib.logreader import LogReader
+
+
+def audit(route: str, warn_ms: float, fault_ms: float) -> int:
+  lr = LogReader(route, sort_by_time=True)
+
+  last = None
+  gaps = []          # (t_end, dt_ms) for every gap >= warn_ms
+  n = 0
+  worst = 0.0
+  for m in lr:
+    if m.which() != "sendcan":
+      continue
+    t = m.logMonoTime / 1e9
+    n += 1
+    if last is not None:
+      dt = (t - last) * 1000.0
+      worst = max(worst, dt)
+      if dt >= warn_ms:
+        gaps.append((t, dt))
+    last = t
+
+  faults = [(t, dt) for t, dt in gaps if dt >= fault_ms]
+
+  print(f"route            : {route}")
+  print(f"sendcan frames   : {n}")
+  print(f"worst gap        : {worst:.1f} ms")
+  print(f"gaps >= {warn_ms:.0f}ms     : {len(gaps)}")
+  print(f"gaps >= {fault_ms:.0f}ms (FAULT-RISK): {len(faults)}")
+  if gaps:
+    print("\n  t(s)        gap(ms)   risk")
+    for t, dt in gaps:
+      print(f"  {t:10.3f}  {dt:7.1f}   {'<-- FAULT RISK' if dt >= fault_ms else ''}")
+
+  if faults:
+    print(f"\nFAIL: {len(faults)} gap(s) >= {fault_ms:.0f}ms can trip the car's "
+          f"actuator counter watchdog (late/missing frames).")
+    return 1
+  print(f"\nPASS: no sendcan gap >= {fault_ms:.0f}ms.")
+  return 0
+
+
+def main() -> int:
+  p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+  p.add_argument("route", help="route name, segment, or URL (e.g. dongle|time--hash or .../5:8)")
+  p.add_argument("--warn-ms", type=float, default=30.0, help="list gaps >= this (default 30)")
+  p.add_argument("--fault-ms", type=float, default=100.0, help="fail on gaps >= this (default 100)")
+  args = p.parse_args()
+  return audit(args.route, args.warn_ms, args.fault_ms)
+
+
+if __name__ == "__main__":
+  sys.exit(main())

tools/scripts/serial.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+while true; do
+  if ls /dev/serial/by-id/usb-FTDI_FT230X* 2> /dev/null; then
+    sudo screen /dev/serial/by-id/usb-FTDI_FT230X* 115200
+  fi
+  sleep 0.005
+done

tools/scripts/setup_ssh_keys.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import requests
+from openpilot.common.params import Params
+import sys
+
+
+if __name__ == "__main__":
+  if len(sys.argv) < 2:
+    print(f"{sys.argv[0]} <github username>")
+    exit(1)
+
+  username = sys.argv[1]
+  keys = requests.get(f"https://github.com/{username}.keys", timeout=10)
+
+  if keys.status_code == 200:
+    params = Params()
+    params.put_bool("SshEnabled", True)
+    params.put("GithubSshKeys", keys.text)
+    params.put("GithubUsername", username)
+    print("Set up ssh keys successfully")
+  else:
+    print("Error getting public keys from github")

tools/scripts/ssh.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+import os
+import sys
+import argparse
+import re
+
+from openpilot.common.basedir import BASEDIR
+from openpilot.tools.lib.auth_config import get_token
+from openpilot.tools.lib.api import CommaApi
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="A helper for connecting to devices over the comma prime SSH proxy.\
+      Adding your SSH key to your SSH config is recommended for more convenient use; see https://docs.comma.ai/how-to/connect-to-comma/.")
+  parser.add_argument("device", help="device name or dongle id")
+  parser.add_argument("--host", help="ssh jump server host", default="ssh.comma.ai")
+  parser.add_argument("--port", help="ssh jump server port", default=22, type=int)
+  parser.add_argument("--key", help="ssh key", default=os.path.join(BASEDIR, "system/hardware/tici/id_rsa"))
+  parser.add_argument("--debug", help="enable debug output", action="store_true")
+  args = parser.parse_args()
+
+  r = CommaApi(get_token()).get("v1/me/devices")
+  devices = {x['dongle_id']: x['alias'] for x in r}
+
+  if not re.match("[0-9a-zA-Z]{16}", args.device):
+    user_input = args.device.replace(" ", "").lower()
+    matches = { k: v for k, v in devices.items() if isinstance(v, str) and user_input in v.replace(" ", "").lower() }
+    if len(matches) == 1:
+      dongle_id = list(matches.keys())[0]
+    else:
+      print(f"failed to look up dongle id for \"{args.device}\"", file=sys.stderr)
+      if len(matches) > 1:
+        print("found multiple matches:", file=sys.stderr)
+        for k, v in matches.items():
+          print(f"  \"{v}\" ({k})", file=sys.stderr)
+      exit(1)
+  else:
+    dongle_id = args.device
+
+  name = dongle_id
+  if dongle_id in devices:
+    name = f"{devices[dongle_id]} ({dongle_id})"
+  print(f"connecting to {name} through {args.host}:{args.port} ...")
+
+  command = [
+    "ssh",
+    "-i", args.key,
+    "-o", f"ProxyCommand=ssh -i {args.key} -W %h:%p -p %p %h@{args.host}",
+    "-p", str(args.port),
+  ]
+  if args.debug:
+    command += ["-v"]
+  command += [
+    f"comma@comma-{dongle_id}",
+  ]
+  if args.debug:
+    print(" ".join([f"'{c}'" if " " in c else c for c in command]))
+  os.execvp(command[0], command)