IQ.Pilot Release Commit @ b36e2e3

tools/scripts/adb_ssh.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Forward all openpilot service ports
+mapfile -t SERVICE_PORTS < <(python3 - <<'PY'
+from cereal.services import SERVICE_LIST
+
+FNV_PRIME = 0x100000001b3
+FNV_OFFSET_BASIS = 0xcbf29ce484222325
+START_PORT = 8023
+MAX_PORT = 65535
+PORT_RANGE = MAX_PORT - START_PORT
+MASK = 0xffffffffffffffff
+
+def fnv1a(endpoint: str) -> int:
+  h = FNV_OFFSET_BASIS
+  for b in endpoint.encode():
+    h ^= b
+    h = (h * FNV_PRIME) & MASK
+  return h
+
+ports = set()
+for name in SERVICE_LIST.keys():
+  port = START_PORT + fnv1a(name) % PORT_RANGE
+  ports.add((name, port))
+
+for name, port in sorted(ports):
+  print(f"{name} {port}")
+PY
+)
+
+for entry in "${SERVICE_PORTS[@]}"; do
+  name="${entry% *}"
+  port="${entry##* }"
+  adb forward "tcp:${port}" "tcp:${port}" > /dev/null
+done
+
+# Forward SSH port first for interactive shell access.
+adb forward tcp:2222 tcp:22
+
+# SSH!
+ssh comma@localhost -p 2222 "$@"

tools/scripts/extract_audio.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+import os
+import sys
+import wave
+import argparse
+import numpy as np
+
+from openpilot.tools.lib.logreader import LogReader, ReadMode
+
+
+def extract_audio(route_or_segment_name, output_file=None, play=False):
+  lr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACTIVE)
+  audio_messages = list(lr.filter("rawAudioData"))
+  if not audio_messages:
+    print("No rawAudioData messages found in logs")
+    return
+  sample_rate = audio_messages[0].sampleRate
+
+  audio_chunks = []
+  total_frames = 0
+  for msg in audio_messages:
+    audio_array = np.frombuffer(msg.data, dtype=np.int16)
+    audio_chunks.append(audio_array)
+    total_frames += len(audio_array)
+  full_audio = np.concatenate(audio_chunks)
+
+  print(f"Found {total_frames} frames from {len(audio_messages)} audio messages at {sample_rate} Hz")
+
+  if output_file:
+    if write_wav_file(output_file, full_audio, sample_rate):
+      print(f"Audio written to {output_file}")
+    else:
+      print("Audio extraction canceled.")
+  if play:
+    play_audio(full_audio, sample_rate)
+
+
+def write_wav_file(filename, audio_data, sample_rate):
+  if os.path.exists(filename):
+    if input(f"File '{filename}' exists. Overwrite? (y/N): ").lower() not in ['y', 'yes']:
+      return False
+
+  with wave.open(filename, 'wb') as wav_file:
+    wav_file.setnchannels(1)  # Mono
+    wav_file.setsampwidth(2)  # 16-bit
+    wav_file.setframerate(sample_rate)
+    wav_file.writeframes(audio_data.tobytes())
+  return True
+
+
+def play_audio(audio_data, sample_rate):
+  try:
+    import sounddevice as sd
+
+    print("Playing audio... Press Ctrl+C to stop")
+    sd.play(audio_data, sample_rate)
+    sd.wait()
+  except KeyboardInterrupt:
+    print("\nPlayback stopped")
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="Extract audio data from openpilot logs")
+  parser.add_argument("-o", "--output", help="Output WAV file path")
+  parser.add_argument("--play", action="store_true", help="Play audio with sounddevice")
+  parser.add_argument("route_or_segment_name", nargs='?', help="The route or segment name")
+
+  if len(sys.argv) == 1:
+    parser.print_help()
+    sys.exit()
+  args = parser.parse_args()
+
+  output_file = args.output
+  if not args.output and not args.play:
+    output_file = "extracted_audio.wav"
+
+  extract_audio(args.route_or_segment_name.strip(), output_file, args.play)

tools/scripts/fetch_image_from_route.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+if len(sys.argv) < 4:
+  print(f"{sys.argv[0]} <route> <segment> <frame number> [front|wide|driver]")
+  print('example: ./fetch_image_from_route.py "02c45f73a2e5c6e9|2020-06-01--18-03-08" 3 500 driver')
+  exit(0)
+
+cameras = {
+  "front": "cameras",
+  "wide": "ecameras",
+  "driver": "dcameras"
+}
+
+import requests
+from PIL import Image
+from openpilot.tools.lib.auth_config import get_token
+from openpilot.tools.lib.framereader import FrameReader
+
+jwt = get_token()
+
+route = sys.argv[1]
+segment = int(sys.argv[2])
+frame = int(sys.argv[3])
+camera = cameras[sys.argv[4]] if len(sys.argv) > 4 and sys.argv[4] in cameras else "cameras"
+
+api_host = os.getenv("API_HOST", "https://api-iqlabs.konn3kt.com")
+url = f'{api_host}/v1/route/{route}/files'
+r = requests.get(url, headers={"Authorization": f"JWT {jwt}"}, timeout=10)
+assert r.status_code == 200
+print("got api response")
+
+segments = r.json()[camera]
+if segment >= len(segments):
+  raise Exception(f"segment {segment} not found, got {len(segments)} segments")
+
+fr = FrameReader(segments[segment])
+if frame >= fr.frame_count:
+  raise Exception("frame {frame} not found, got {fr.frame_count} frames")
+
+im = Image.fromarray(fr.get(frame))
+fn = f"uxxx_{route.replace('|', '_')}_{segment}_{frame}.png"
+im.save(fn)
+print(f"saved {fn}")

tools/scripts/io_stall_repro.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""
+io_stall_repro.py — bench reproduction + fix-validation rig for the VW PQ EPS
+HCA fault (control loop stalling on a /data read under eMMC write saturation).
+
+The real fault chain (proven from rlog b29ee8c5a0a735d1/000000e4--8a8ba97b54):
+  loggerd buffered writes saturate eMMC -> ext4 jbd2 journal commits ->
+  controlsd's inline Params.get (util::read_file on /data) blocks ~300-630ms ->
+  controlsd stops publishing carControl -> card's all_alive guard withholds
+  HCA_1 -> EPS LH2_Sta_HCA 7->2.
+
+This reproduces the *proximate* cause WITHOUT driving, WITHOUT the EPS, and
+WITHOUT touching the real control stack. Two roles:
+
+  --writer : emulate loggerd. Buffered (no-fsync) writes to /data at a target
+             MB/s, with an optional periodic big flush to mimic 60s segment
+             rotation. This is what saturates the eMMC.
+
+  --probe  : emulate controlsd's I/O exposure. A 100Hz loop that every
+             --param-period seconds reads a real param (util::read_file on
+             /data). It records per-iteration loop gaps and param-read
+             durations. A 300ms gap here == the stall that drops HCA.
+             --threaded moves the param read to a background thread (the
+             proposed fix, mirroring card.py's params_thread) so you can A/B it.
+
+USAGE (run parked, ignition on, on the device):
+  # 1) baseline: probe alone -> gaps should be tiny
+  python3 io_stall_repro.py --probe --secs 120
+
+  # 2) reproduce: writer in one shell, probe in another
+  python3 io_stall_repro.py --writer --mbps 25 --rotate 60
+  python3 io_stall_repro.py --probe --secs 180          # expect big gaps
+
+  # 3) validate fix A (params off control thread):
+  python3 io_stall_repro.py --probe --secs 180 --threaded   # gaps should vanish
+
+  # 4) validate fix B (smoother writeback) — set before step 2, as root:
+  #   echo 5  > /proc/sys/vm/dirty_background_ratio
+  #   echo 10 > /proc/sys/vm/dirty_ratio
+  #   then re-run step 2 inline probe and compare gap distribution.
+
+Cleanup: writer deletes its scratch files on exit. Read-only wrt openpilot.
+"""
+import argparse, os, sys, time, threading, statistics, signal
+
+SCRATCH_DEFAULT = "/data/media/0/io_repro_scratch"
+
+# ----------------------------------------------------------------------------- writer
+def run_writer(args):
+  os.makedirs(args.scratch, exist_ok=True)
+  chunk = os.urandom(1 << 20)  # 1 MiB
+  bytes_per_s = int(args.mbps * (1 << 20))
+  print(f"[writer] buffered no-fsync writes to {args.scratch} at ~{args.mbps} MB/s, "
+        f"rotate every {args.rotate}s (big flush). Ctrl-C to stop.", file=sys.stderr)
+
+  stop = {"v": False}
+  signal.signal(signal.SIGINT, lambda *_: stop.update(v=True))
+  signal.signal(signal.SIGTERM, lambda *_: stop.update(v=True))
+
+  files = []
+  seg = 0
+  try:
+    while not stop["v"]:
+      seg_start = time.monotonic()
+      path = os.path.join(args.scratch, f"seg_{seg}.bin")
+      f = open(path, "wb", buffering=1 << 20)
+      files.append(path)
+      written = 0
+      # write at target rate using buffered fwrite, NO fsync (exactly loggerd)
+      while not stop["v"] and (time.monotonic() - seg_start) < args.rotate:
+        t0 = time.monotonic()
+        f.write(chunk)
+        written += len(chunk)
+        # pace to target MB/s
+        target_t = written / bytes_per_s
+        elapsed = time.monotonic() - seg_start
+        if target_t > elapsed:
+          time.sleep(min(0.1, target_t - elapsed))
+      # "segment rotation": flush+close a big buffered file at once -> writeback burst
+      f.flush()
+      f.close()
+      seg += 1
+      # keep only a few recent files so we don't fill the disk
+      while len(files) > 3:
+        old = files.pop(0)
+        try: os.remove(old)
+        except OSError: pass
+  finally:
+    for p in files:
+      try: os.remove(p)
+      except OSError: pass
+    print("[writer] stopped, scratch cleaned.", file=sys.stderr)
+
+# ----------------------------------------------------------------------------- probe
+def _get_param(key):
+  # real /data read, same syscall path as controlsd's get_params_iq
+  try:
+    from openpilot.common.params import Params
+    return Params().get_bool(key)
+  except Exception:
+    # fallback: plain file read of a param file if openpilot import unavailable
+    p = os.path.join(os.getenv("PARAMS_ROOT", "/data/params"), "d", key)
+    try:
+      with open(p, "rb") as fh:
+        return fh.read()
+    except OSError:
+      return None
+
+class ThreadedParam:
+  """Mirror card.py params_thread: refresh the param on a bg thread, control
+  loop reads the cached value (non-blocking)."""
+  def __init__(self, key, period):
+    self.key = key; self.period = period; self.val = None
+    self.stop = False
+    self.t = threading.Thread(target=self._loop, daemon=True); self.t.start()
+  def _loop(self):
+    while not self.stop:
+      self.val = _get_param(self.key)
+      time.sleep(self.period)
+  def read(self):  # O(1), no I/O on the control thread
+    return self.val
+
+def run_probe(args):
+  # pin like controlsd (core 4) so we share the same iowait domain if possible
+  try:
+    os.sched_setaffinity(0, {args.core})
+  except (OSError, AttributeError):
+    pass
+
+  interval = 0.01  # 100Hz, like the control loop
+  gaps = []          # ms, per-iteration loop overrun beyond 10ms
+  read_ms = []       # ms, time spent in the param read on the control thread
+  worst = 0.0
+  threaded = ThreadedParam(args.param_key, args.param_period) if args.threaded else None
+
+  print(f"[probe] 100Hz loop for {args.secs}s, param '{args.param_key}' every "
+        f"{args.param_period}s, threaded={args.threaded}, core={args.core}", file=sys.stderr)
+  t_end = time.monotonic() + args.secs
+  next_t = time.monotonic()
+  last_param = 0.0
+  while time.monotonic() < t_end:
+    loop_start = time.monotonic()
+
+    # the I/O exposure: read param on the control thread (inline) every period
+    if loop_start - last_param >= args.param_period:
+      r0 = time.monotonic()
+      if threaded is not None:
+        _ = threaded.read()            # cached, no I/O on this thread (the FIX)
+      else:
+        _ = _get_param(args.param_key)  # inline /data read (current behavior)
+      dr = (time.monotonic() - r0) * 1000
+      read_ms.append(dr)
+      last_param = loop_start
+
+    # measure scheduling/lag: how late did this iteration actually fire?
+    next_t += interval
+    lag = (time.monotonic() - next_t) * 1000  # ms behind schedule
+    if lag > 5:
+      gaps.append(lag)
+      worst = max(worst, lag)
+    sleep = next_t - time.monotonic()
+    if sleep > 0:
+      time.sleep(sleep)
+    else:
+      next_t = time.monotonic()  # don't spiral after a big stall
+
+  if threaded:
+    threaded.stop = True
+
+  def pct(xs, p):
+    return sorted(xs)[int(p/100*(len(xs)-1))] if xs else 0.0
+  print("\n================ PROBE RESULT ================")
+  print(f"loop-lag events >5ms : {len(gaps)}")
+  print(f"loop-lag p50/p99/max : {pct(gaps,50):.0f} / {pct(gaps,99):.0f} / {worst:.0f} ms")
+  print(f"param-read p50/p99/max: {pct(read_ms,50):.1f} / {pct(read_ms,99):.1f} / {max(read_ms+[0]):.1f} ms  (n={len(read_ms)})")
+  hca_class = max(gaps + [0])
+  verdict = ("FAULT-CLASS STALL REPRODUCED (>250ms -> would drop HCA)" if hca_class > 250
+             else "marginal (100-250ms)" if hca_class > 100
+             else "clean (<100ms)")
+  print(f"VERDICT: {verdict}")
+  print("=============================================")
+
+# ----------------------------------------------------------------------------- main
+def main():
+  ap = argparse.ArgumentParser()
+  ap.add_argument("--writer", action="store_true", help="emulate loggerd eMMC saturation")
+  ap.add_argument("--probe", action="store_true", help="emulate controlsd I/O exposure")
+  ap.add_argument("--mbps", type=float, default=25.0, help="writer target MB/s (loggerd ~10-30)")
+  ap.add_argument("--rotate", type=float, default=60.0, help="writer segment/flush period s")
+  ap.add_argument("--scratch", default=SCRATCH_DEFAULT)
+  ap.add_argument("--secs", type=float, default=180.0, help="probe duration s")
+  ap.add_argument("--param-key", default="IsMetric", help="a real param key to read")
+  ap.add_argument("--param-period", type=float, default=3.0, help="controlsd reads every 3s")
+  ap.add_argument("--threaded", action="store_true", help="probe: read param off control thread (the FIX)")
+  ap.add_argument("--core", type=int, default=4, help="probe cpu affinity (control core)")
+  args = ap.parse_args()
+  if args.writer == args.probe:
+    ap.error("pick exactly one of --writer / --probe (run them in separate shells)")
+  run_writer(args) if args.writer else run_probe(args)
+
+if __name__ == "__main__":
+  main()

tools/scripts/io_stall_tracer.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""
+io_stall_tracer.py — continuous low-overhead per-process disk-I/O tracer.
+
+WHY: the VW PQ EPS HCA faults are caused by a control process (controlsd/card)
+blocking ~300-630ms in disk I/O (iowait) on /data, which stops HCA_1 TX. The
+stall is far too short to catch with a manual `iostat`/`iotop` run. This sampler
+runs for the whole drive at 100ms cadence and records, per process:
+
+  - write_bytes      (/proc/<pid>/io)   -> identifies the WRITER saturating eMMC
+  - delayacct_blkio  (/proc/<pid>/stat) -> per-process cumulative block-I/O wait
+  - state            (/proc/<pid>/stat) -> catches 'D' (uninterruptible disk wait)
+
+plus whole-device /proc/diskstats. After a fault, find the wall-clock time of the
+LH2_Sta_HCA->2 event (from the rlog) and look at the rows around it: the process
+whose write_bytes delta spikes is the bully; the control process whose blkio
+delta jumps / state == 'D' is the victim.
+
+Deploy: copy to the device, run alongside openpilot during a drive:
+    python3 io_stall_tracer.py --out /data/media/0/io_trace.csv
+Overhead: reading /proc for ~40 procs every 100ms is well under 1% of one core,
+and it pins itself to CPU 0 (away from the control cores 4/5) at low priority.
+
+Read-only. Writes a single CSV. No openpilot deps.
+"""
+import argparse, os, time, glob, sys
+
+CLK_TCK = os.sysconf("SC_CLK_TCK")  # usually 100 -> blkio ticks are 10ms each
+
+def read_proc_io(pid):
+  # wchar/rchar = bytes moved via read()/write() syscalls (catches BUFFERED writers
+  # like loggerd, which never appear in write_bytes because the kernel flushes their
+  # page-cache dirty pages asynchronously via kworker). write_bytes = bytes actually
+  # sent to the block device. Track both.
+  try:
+    with open(f"/proc/{pid}/io") as f:
+      d = {}
+      for line in f:
+        k, _, v = line.partition(":")
+        d[k] = int(v)
+      return (d.get("wchar", 0), d.get("rchar", 0),
+              d.get("write_bytes", 0), d.get("read_bytes", 0))
+  except (OSError, ValueError):
+    return None
+
+def read_proc_stat(pid):
+  # state is field 3; delayacct_blkio_ticks is field 42 (1-indexed). comm may
+  # contain spaces/parens, so split on the last ')'.
+  try:
+    with open(f"/proc/{pid}/stat") as f:
+      data = f.read()
+    rparen = data.rfind(")")
+    comm = data[data.find("(") + 1:rparen]
+    rest = data[rparen + 2:].split()
+    state = rest[0]                       # field 3
+    blkio_ticks = int(rest[39]) if len(rest) > 39 else 0  # field 42
+    return comm, state, blkio_ticks
+  except (OSError, ValueError, IndexError):
+    return None
+
+def read_diskstats():
+  # returns {dev: (sectors_written, ms_doing_io)} for whole-disk devices
+  out = {}
+  try:
+    with open("/proc/diskstats") as f:
+      for line in f:
+        p = line.split()
+        if len(p) < 14:
+          continue
+        dev = p[2]
+        # field 10 (idx 9) = sectors written; field 13 (idx 12) = ms doing I/O
+        out[dev] = (int(p[9]), int(p[12]))
+  except (OSError, AttributeError):
+    pass
+  return out
+
+# These counters tell us WHICH kernel mechanism caused a stall, which decides
+# the fix: compact_stall jumping -> memory compaction (texture-pool fix);
+# allocstall/pgsteal jumping -> direct reclaim; high nr_dirty/nr_writeback ->
+# loggerd writeback bomb (loggerd sync_file_range fix). meminfo Dirty/Writeback
+# are absolute kB; vmstat ones are cumulative event counts (we delta them).
+VMSTAT_KEYS = ("compact_stall", "compact_fail", "allocstall_normal", "allocstall_movable",
+               "pgsteal_direct", "pgscan_direct", "pgmajfault", "nr_dirty", "nr_writeback")
+MEMINFO_KEYS = ("MemFree", "MemAvailable", "Dirty", "Writeback")
+
+def read_vmstat():
+  out = {}
+  try:
+    with open("/proc/vmstat") as f:
+      for line in f:
+        k, _, v = line.partition(" ")
+        if k in VMSTAT_KEYS:
+          out[k] = int(v)
+  except OSError:
+    pass
+  return out
+
+def read_meminfo():
+  out = {}
+  try:
+    with open("/proc/meminfo") as f:
+      for line in f:
+        k, _, v = line.partition(":")
+        if k in MEMINFO_KEYS:
+          out[k] = int(v.split()[0])  # kB
+  except (OSError, IndexError):
+    pass
+  return out
+
+def main():
+  ap = argparse.ArgumentParser()
+  ap.add_argument("--out", default="/data/media/0/io_trace.csv")
+  ap.add_argument("--hz", type=float, default=10.0, help="sample rate (default 10Hz/100ms)")
+  ap.add_argument("--disk", default="sda", help="comma-separated disk devices to track (default sda)")
+  ap.add_argument("--names", default="controlsd,car.c,selfd,ui,loggerd,encoderd,modeld,camerad,locationd,paramsd,navd,mapd",
+                  help="substring match of process comm to record (others summed as 'other')")
+  args = ap.parse_args()
+
+  # be a good citizen: low priority, off the control cores
+  try:
+    os.nice(10)
+    os.sched_setaffinity(0, {0})
+  except OSError:
+    pass
+
+  watch = [n.strip() for n in args.names.split(",") if n.strip()]
+  disks = [d.strip() for d in args.disk.split(",") if d.strip()]
+  interval = 1.0 / args.hz
+
+  prev_io = {}      # pid -> (wchar, rchar, write_bytes, read_bytes)
+  prev_blkio = {}   # pid -> blkio_ticks
+  prev_disk = read_diskstats()
+  prev_vm = read_vmstat()
+
+  f = open(args.out, "w", buffering=1)
+  # per-proc rows fill the first block; one SYS row per tick fills the trailing
+  # mechanism columns (deltas for the vmstat counts, absolute kB for meminfo).
+  f.write("wall,mono,proc,pid,state,d_wchar_kB,d_wbytes_kB,d_blkio_ms,disk_d_write_kB,disk_d_busy_ms,"
+          "compact_stall,allocstall,pgmajfault,dirty_kB,writeback_kB,memfree_kB,memavail_kB\n")
+  print(f"[io_stall_tracer] writing {args.out} at {args.hz}Hz, tracking {watch}", file=sys.stderr)
+
+  while True:
+    t_wall = time.time()
+    t_mono = time.monotonic()
+
+    # whole-disk delta (write kB + busy ms) for the named disks
+    disk = read_diskstats()
+    disk_dw = disk_db = 0
+    for dev in disks:
+      if dev in disk and dev in prev_disk:
+        disk_dw += (disk[dev][0] - prev_disk[dev][0]) * 512 / 1024.0  # sectors->kB
+        disk_db += (disk[dev][1] - prev_disk[dev][1])
+    prev_disk = disk
+
+    seen = set()
+    rows = []
+    for path in glob.glob("/proc/[0-9]*"):
+      pid = path.rsplit("/", 1)[1]
+      st = read_proc_stat(pid)
+      if st is None:
+        continue
+      comm, state, blkio = st
+      label = next((w for w in watch if w in comm), None)
+      if label is None:
+        # still track D-state of anything to catch surprise writers/blockers
+        if state != "D":
+          continue
+        label = comm
+      io = read_proc_io(pid)
+      if io is None:
+        continue
+      wchar, rchar, wbytes, rbytes = io
+      pw = prev_io.get(pid, io)
+      pblk = prev_blkio.get(pid, blkio)
+      d_wchar = (wchar - pw[0]) / 1024.0     # syscall write volume (catches loggerd)
+      d_wbytes = (wbytes - pw[2]) / 1024.0   # bytes hitting the block device
+      d_blk = (blkio - pblk) * (1000.0 / CLK_TCK)  # ticks -> ms blocked on block I/O
+      prev_io[pid] = io
+      prev_blkio[pid] = blkio
+      seen.add(pid)
+      # only emit rows that carry signal (writing, blocked, or in D) to keep file small
+      if d_wchar > 4 or d_wbytes > 4 or d_blk > 5 or state == "D":
+        rows.append((label, pid, state, d_wchar, d_wbytes, d_blk))
+
+    # drop dead pids from prev maps occasionally
+    if len(prev_io) > 4000:
+      prev_io = {p: v for p, v in prev_io.items() if p in seen}
+      prev_blkio = {p: v for p, v in prev_blkio.items() if p in seen}
+
+    for label, pid, state, d_wchar, d_wbytes, d_blk in rows:
+      f.write(f"{t_wall:.3f},{t_mono:.3f},{label},{pid},{state},{d_wchar:.0f},{d_wbytes:.0f},{d_blk:.0f},{disk_dw:.0f},{disk_db:.0f},,,,,,,\n")
+
+    # one SYS row per tick: the kernel-mechanism counters (compaction vs reclaim
+    # vs writeback). Compare these against the stall's wall-clock to see which
+    # one spiked.
+    vm = read_vmstat(); mi = read_meminfo()
+    d_compact = vm.get("compact_stall", 0) - prev_vm.get("compact_stall", 0)
+    d_alloc = ((vm.get("allocstall_normal", 0) + vm.get("allocstall_movable", 0))
+               - (prev_vm.get("allocstall_normal", 0) + prev_vm.get("allocstall_movable", 0)))
+    d_majflt = vm.get("pgmajfault", 0) - prev_vm.get("pgmajfault", 0)
+    prev_vm = vm
+    f.write(f"{t_wall:.3f},{t_mono:.3f},SYS,0,-,,,,,,"
+            f"{d_compact},{d_alloc},{d_majflt},{mi.get('Dirty',0)},{mi.get('Writeback',0)},"
+            f"{mi.get('MemFree',0)},{mi.get('MemAvailable',0)}\n")
+
+    time.sleep(max(0.0, interval - (time.monotonic() - t_mono)))
+
+if __name__ == "__main__":
+  main()

tools/scripts/save_ubloxraw_stream.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+from openpilot.common.basedir import BASEDIR
+from openpilot.tools.lib.logreader import LogReader
+
+os.environ['BASEDIR'] = BASEDIR
+
+
+def get_arg_parser():
+  parser = argparse.ArgumentParser(
+      description="Unlogging and save to file",
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+  parser.add_argument("route", type=(lambda x: x.replace("#", "|")), nargs="?",
+                      help="The route whose messages will be published.")
+  parser.add_argument("--out_path", nargs='?', default='/data/ubloxRaw.stream',
+                      help="Output pickle file path")
+  return parser
+
+
+def main():
+  args = get_arg_parser().parse_args(sys.argv[1:])
+
+  lr = LogReader(args.route)
+
+  with open(args.out_path, 'wb') as f:
+    try:
+      done = False
+      i = 0
+      while not done:
+        msg = next(lr)
+        if not msg:
+          break
+        smsg = msg.as_builder()
+        typ = smsg.which()
+        if typ == 'ubloxRaw':
+          f.write(smsg.to_bytes())
+          i += 1
+    except StopIteration:
+      print('All done')
+  print(f'Writed {i} msgs')
+
+
+if __name__ == "__main__":
+  main()

tools/scripts/sendcan_gap_audit.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""
+sendcan_gap_audit.py — measure the openpilot->car actuator TX cadence from an rlog.
+
+WHY: VW PQ random long/lat disengages are caused by control-loop STALLS that
+freeze the `sendcan` publish for >100ms. The car's ECU runs a counter/checksum
+watchdog on the actuator messages (ACC_System ADR, HCA_1) and FAULTS when frames
+arrive late/missing — it does NOT care about the payload value. So the single
+metric that predicts the fault is the inter-frame GAP in sendcan, not anything
+about accel/torque. (Reference: route 20e3cd4f0d5f39d1|00000038--0f69286335 had a
+103ms gap at ~373s -> engine MO2_Sta_GRA->0 -> main switch off -> disengage. A
+separate 60ms gap did NOT disengage: the ECU timeout sits ~60-100ms.)
+
+This is the pass/fail metric for the mlockall / loggerd-writeback fix (5324c46)
+and, later, the decoupled in-card heartbeat TX. Run it on a BASELINE route to see
+the offending gaps, then on POST-FIX drives to confirm they're gone.
+
+  python3 tools/scripts/sendcan_gap_audit.py <route_or_segment> [--warn-ms 30] [--fault-ms 100]
+
+Exit code 0 if no gap >= --fault-ms, else 1 (so it can gate CI / a smoke test).
+Read-only; pulls rlogs via the normal LogReader (konn3kt for IQ.Pilot routes).
+"""
+import argparse
+import sys
+
+from openpilot.tools.lib.logreader import LogReader
+
+
+def audit(route: str, warn_ms: float, fault_ms: float) -> int:
+  lr = LogReader(route, sort_by_time=True)
+
+  last = None
+  gaps = []          # (t_end, dt_ms) for every gap >= warn_ms
+  n = 0
+  worst = 0.0
+  for m in lr:
+    if m.which() != "sendcan":
+      continue
+    t = m.logMonoTime / 1e9
+    n += 1
+    if last is not None:
+      dt = (t - last) * 1000.0
+      worst = max(worst, dt)
+      if dt >= warn_ms:
+        gaps.append((t, dt))
+    last = t
+
+  faults = [(t, dt) for t, dt in gaps if dt >= fault_ms]
+
+  print(f"route            : {route}")
+  print(f"sendcan frames   : {n}")
+  print(f"worst gap        : {worst:.1f} ms")
+  print(f"gaps >= {warn_ms:.0f}ms     : {len(gaps)}")
+  print(f"gaps >= {fault_ms:.0f}ms (FAULT-RISK): {len(faults)}")
+  if gaps:
+    print("\n  t(s)        gap(ms)   risk")
+    for t, dt in gaps:
+      print(f"  {t:10.3f}  {dt:7.1f}   {'<-- FAULT RISK' if dt >= fault_ms else ''}")
+
+  if faults:
+    print(f"\nFAIL: {len(faults)} gap(s) >= {fault_ms:.0f}ms can trip the car's "
+          f"actuator counter watchdog (late/missing frames).")
+    return 1
+  print(f"\nPASS: no sendcan gap >= {fault_ms:.0f}ms.")
+  return 0
+
+
+def main() -> int:
+  p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+  p.add_argument("route", help="route name, segment, or URL (e.g. dongle|time--hash or .../5:8)")
+  p.add_argument("--warn-ms", type=float, default=30.0, help="list gaps >= this (default 30)")
+  p.add_argument("--fault-ms", type=float, default=100.0, help="fail on gaps >= this (default 100)")
+  args = p.parse_args()
+  return audit(args.route, args.warn_ms, args.fault_ms)
+
+
+if __name__ == "__main__":
+  sys.exit(main())

tools/scripts/serial.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+while true; do
+  if ls /dev/serial/by-id/usb-FTDI_FT230X* 2> /dev/null; then
+    sudo screen /dev/serial/by-id/usb-FTDI_FT230X* 115200
+  fi
+  sleep 0.005
+done

tools/scripts/setup_ssh_keys.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import requests
+from openpilot.common.params import Params
+import sys
+
+
+if __name__ == "__main__":
+  if len(sys.argv) < 2:
+    print(f"{sys.argv[0]} <github username>")
+    exit(1)
+
+  username = sys.argv[1]
+  keys = requests.get(f"https://github.com/{username}.keys", timeout=10)
+
+  if keys.status_code == 200:
+    params = Params()
+    params.put_bool("SshEnabled", True)
+    params.put("GithubSshKeys", keys.text)
+    params.put("GithubUsername", username)
+    print("Set up ssh keys successfully")
+  else:
+    print("Error getting public keys from github")

tools/scripts/ssh.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+import os
+import sys
+import argparse
+import re
+
+from openpilot.common.basedir import BASEDIR
+from openpilot.tools.lib.auth_config import get_token
+from openpilot.tools.lib.api import CommaApi
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="A helper for connecting to devices over the comma prime SSH proxy.\
+      Adding your SSH key to your SSH config is recommended for more convenient use; see https://docs.comma.ai/how-to/connect-to-comma/.")
+  parser.add_argument("device", help="device name or dongle id")
+  parser.add_argument("--host", help="ssh jump server host", default="ssh.comma.ai")
+  parser.add_argument("--port", help="ssh jump server port", default=22, type=int)
+  parser.add_argument("--key", help="ssh key", default=os.path.join(BASEDIR, "system/hardware/tici/id_rsa"))
+  parser.add_argument("--debug", help="enable debug output", action="store_true")
+  args = parser.parse_args()
+
+  r = CommaApi(get_token()).get("v1/me/devices")
+  devices = {x['dongle_id']: x['alias'] for x in r}
+
+  if not re.match("[0-9a-zA-Z]{16}", args.device):
+    user_input = args.device.replace(" ", "").lower()
+    matches = { k: v for k, v in devices.items() if isinstance(v, str) and user_input in v.replace(" ", "").lower() }
+    if len(matches) == 1:
+      dongle_id = list(matches.keys())[0]
+    else:
+      print(f"failed to look up dongle id for \"{args.device}\"", file=sys.stderr)
+      if len(matches) > 1:
+        print("found multiple matches:", file=sys.stderr)
+        for k, v in matches.items():
+          print(f"  \"{v}\" ({k})", file=sys.stderr)
+      exit(1)
+  else:
+    dongle_id = args.device
+
+  name = dongle_id
+  if dongle_id in devices:
+    name = f"{devices[dongle_id]} ({dongle_id})"
+  print(f"connecting to {name} through {args.host}:{args.port} ...")
+
+  command = [
+    "ssh",
+    "-i", args.key,
+    "-o", f"ProxyCommand=ssh -i {args.key} -W %h:%p -p %p %h@{args.host}",
+    "-p", str(args.port),
+  ]
+  if args.debug:
+    command += ["-v"]
+  command += [
+    f"comma@comma-{dongle_id}",
+  ]
+  if args.debug:
+    print(" ".join([f"'{c}'" if " " in c else c for c in command]))
+  os.execvp(command[0], command)